@blario/react-native 0.1.4 → 0.1.6

package/README.md

@@ -63,6 +63,7 @@ export default function App() {
 Screenshots use `react-native-view-shot` when it is installed, otherwise `@shopify/react-native-skia`
 (`makeImageFromView`) on iOS. Android requires view-shot: Skia's Android capture can raise a fatal native error
 when a view unmounts during capture. With no supported capture module, a screenshot answers `sdk_error`.
+Android on the New Architecture needs view-shot 4 or later: 3.x fails every capture there, while actions keep working.
 Each module is resolved once when the SDK loads:
 
 - view-shot leaves a host `Modal` out (it is a separate window), so the response carries `partial: true` and
@@ -230,7 +231,7 @@ A run looks and behaves like the web widget's agent mode (`web-sdk/src/managers/
   installed (both optional peers); without one this is a no-op, reported once with `debug`.
 
 Known differences from the web: the pending dim covers the whole screen (the web dims only the chat window); the
-layer takes window focus, so an open keyboard closes when a run starts; `session_closed` ends the run (the web only
+keyboard stays closed during a run, unless the chat is open; `session_closed` ends the run (the web only
 tears it down). Visual effects (dot grid, glow, slime) come in a later milestone.
 
 ### Test nudges without a real delivery
@@ -284,6 +285,16 @@ npx expo start --clear                     # Metro; the simulator reaches 127.0.
 xcrun simctl launch "iPhone 17 iOS 26" io.blar.example
 ```
 
+### On an Android emulator
+
+```bash
+npx expo prebuild -p android --no-install   # it rewrites package.json's scripts: revert that
+# newArchEnabled=true in android/gradle.properties, and sdk.dir in android/local.properties
+(cd android && ./gradlew app:assembleDebug)
+adb install -r android/app/build/outputs/apk/debug/app-debug.apk
+adb reverse tcp:8081 tcp:8081 && adb reverse tcp:8790 tcp:8790
+```
+
 `.maestro/` holds [Maestro](https://maestro.dev) flows that tap through the app, chat WebView included
 (`brew install mobile-dev-inc/tap/maestro`, needs Java 17): `maestro test .maestro/chat-keyboard.yaml`.
 
@@ -315,6 +326,7 @@ Wire envelope (mirrors the backend): `agent_command` / `request_dom_state` / `re
 | `click` | | `btn-N` (a switch toggles, an input is focused, a wheel picker's row scrolls to the center) |
 | `type` | `value`, `clear_first` (default true) | `inp-N` text input |
 | `check` | `checked` (omit to toggle) | `inp-N` switch |
+| `select` | `value` | `scr-N` or one of its rows: centers the row with that text, in one list at a time |
 | `press_key` | `key` (`Enter` only) | `inp-N`, or the last input used |
 | `scroll` | `direction` up/down/left/right, `amount` px | `scr-N` |
 | `scroll_to` | | any element, uses its nearest scroll container |
@@ -322,8 +334,11 @@ Wire envelope (mirrors the backend): `agent_command` / `request_dom_state` / `re
 | `highlight` | `duration_ms` (default 3000) | any element |
 
 `click`, `type`, `check` and `highlight` scroll the target into view first. Elements that sit beyond their scroll
-viewport are listed with `offscreen=above|below|left|right` (at most 30, nearest first). A wheel picker is a snapping
-list (`snapToInterval` or `snapToOffsets`) of plain text rows: each row is a `btn-N`, and all of them stay listed.
+viewport are listed with `offscreen=above|below|left|right`, however many there are. The page text also includes what a
+list holds off screen in any direction; only blocks parked beyond the window with no list behind them are left out. A wheel picker is a snapping
+list (`snapToInterval` or `snapToOffsets`) of plain text rows: each row is a `btn-N`, and all of them stay listed. Lists
+carry `list=<name>` from their own or their wrapper's `accessibilityLabel` or `testID`, and so do those rows. `select`
+needs no snap props: it reaches any list, so a wheel that snaps in JS works through it.
 
 ## Develop
 

package/dist/BlarProvider.d.ts

@@ -42,6 +42,7 @@ export declare class BlarProvider extends Component<BlarProviderProps, BlarProvi
     private chatControls;
     private shownNudge;
     private readonly releases;
+    private readonly keyboardGuard;
     private stackingCheckQueued;
     private loggedPhase;
     private readonly stacking;
@@ -62,6 +63,7 @@ export declare class BlarProvider extends Component<BlarProviderProps, BlarProvi
     private createTransport;
     private createChat;
     private attachBackend;
+    private syncKeyboardGuard;
     private queueStackingCheck;
     private keepLayerOnTop;
     private countHostModals;

package/dist/actions/scrolling.d.ts

@@ -1,12 +1,14 @@
 import { type Band } from '../host/rect.ts';
+import { type ScrollGeometry } from '../snapshot/scroll.ts';
 import type { ResolvedTarget } from './target.ts';
 import type { AgentAction, HandlerResult } from './types.ts';
 type ScrollAction = Extract<AgentAction, {
     type: 'scroll';
 }>;
+export declare function applyOffset(target: ResolvedTarget, geometry: ScrollGeometry, offset: number, animated: boolean): HandlerResult;
 export declare function scrollContainer(target: ResolvedTarget, action: ScrollAction): Promise<HandlerResult>;
+export declare function nearestScrollAncestor(target: ResolvedTarget): ResolvedTarget['fiber'] | null;
 export declare function isInWheel(target: ResolvedTarget): boolean;
-export declare function selectWheelOption(target: ResolvedTarget): Promise<HandlerResult>;
 export declare function scrollIntoView(target: ResolvedTarget, options?: {
     animated: boolean;
 }): Promise<HandlerResult>;

package/dist/actions/select.d.ts

@@ -0,0 +1,16 @@
+import type { Fiber } from '../fiber/types.ts';
+import type { ActionServices } from './services.ts';
+import type { ResolvedTarget } from './target.ts';
+import type { HandlerResult } from './types.ts';
+interface ListRow {
+    host: Fiber;
+    text: string;
+}
+export declare function centerInList(target: ResolvedTarget, list: Fiber, row: Fiber): Promise<HandlerResult>;
+export declare function scrollListTo(target: ResolvedTarget, list: Fiber, offset: number): Promise<HandlerResult>;
+export declare function selectWheelOption(target: ResolvedTarget): Promise<HandlerResult>;
+export declare function rowsOf(list: Fiber): ListRow[];
+export declare function matchesRow(text: string, value: string): boolean;
+export declare function selectValue(target: ResolvedTarget, value: string, services: ActionServices, isCancelled?: () => boolean): Promise<HandlerResult>;
+export declare function walk(from: number, to: number, step: number): number[];
+export {};

package/dist/actions/text-input.d.ts

@@ -4,6 +4,7 @@ import type { AgentAction, HandlerResult } from './types.ts';
 type TypeAction = Extract<AgentAction, {
     type: 'type';
 }>;
+export declare function restoreKeyboards(): void;
 export declare function typeText(target: ResolvedTarget, action: TypeAction, services: ActionServices): HandlerResult;
 export declare function submitInput(target: ResolvedTarget): HandlerResult;
 export {};

package/dist/actions/types.d.ts

@@ -9,6 +9,9 @@ export type AgentAction = {
 } | {
     type: 'check';
     checked: boolean | null;
+} | {
+    type: 'select';
+    value: string;
 } | {
     type: 'press_key';
     key: string;

package/dist/agent/compose.d.ts

@@ -11,6 +11,7 @@ export interface AgentEnvironment {
     transport: AgentTransport;
     getAnchorFiber: () => Fiber | null;
     getViewport: () => Size;
+    getWindow: () => Size;
     getNavigationRef: () => NavigationRefLike | null;
     getCaptureTarget: () => unknown;
     reading: ReadingPort;

package/dist/agent-mode/keyboard-guard.d.ts

@@ -0,0 +1,13 @@
+export interface KeyboardPort {
+    addListener: (event: 'keyboardWillShow' | 'keyboardDidShow', listener: () => void) => {
+        remove: () => void;
+    };
+    dismiss: () => void;
+}
+export declare class KeyboardGuard {
+    private readonly keyboard;
+    private readonly onRelease;
+    private subscriptions;
+    constructor(keyboard: KeyboardPort, onRelease?: () => void);
+    setActive(active: boolean): void;
+}

package/dist/backend/app-info.d.ts

@@ -1,5 +1,6 @@
 export interface AppInfo {
     platform: 'react-native';
+    sdk_version: string;
     os: string;
     os_version: string;
     rn_version: string;

package/dist/fiber/text.d.ts

@@ -1,4 +1,5 @@
 import type { Fiber } from './types.ts';
+export declare const TEXT_HOST_TYPE = "RCTText";
 export declare function normalizeText(value: string): string;
 export declare function collectText(fiber: Fiber, limit: number): string;
 export declare function precedingText(ancestors: Fiber[], fiber: Fiber, limit: number): string;

package/dist/snapshot/capture.d.ts

@@ -13,6 +13,7 @@ export interface SnapshotDependencies {
     getAnchorFiber: () => Fiber | null;
     registry: ElementRegistry;
     getViewport: () => Size;
+    getWindow: () => Size;
     getExtraRoutes: () => string[];
 }
 export type CaptureMode = 'register' | 'peek';

package/dist/snapshot/view.d.ts

@@ -17,8 +17,10 @@ export interface ElementView {
     selected: boolean | null;
     scroll: string | null;
     offscreen: Offscreen | null;
+    list: string | null;
     rect: Rect;
 }
+export declare function listNameOf(list: Fiber, ancestors: Fiber[]): string | null;
 export interface ViewSource {
     kind: ElementKind;
     fiber: Fiber;
@@ -26,5 +28,6 @@ export interface ViewSource {
     rect: Rect;
     scroll: string | null;
     offscreen?: Offscreen | null;
+    wheelOption?: boolean;
 }
 export declare function buildView(source: ViewSource): ElementView;

package/dist/snapshot/visibility.d.ts

@@ -9,9 +9,9 @@ export type Visibility = {
 } | {
     kind: 'offscreen';
     direction: Offscreen;
-    distance: number;
 } | {
     kind: 'hidden';
 };
 export declare function classifyVisibility(rect: Rect, window: Size, containers: Fiber[], geometries: ScrollGeometries): Visibility;
+export declare function markOffscreenText(visibility: Visibility, text: string): string;
 export declare function keepInTextTree(visibility: Visibility, rect: Rect, window: Size, viewportOnly: boolean): boolean;

package/dist/snapshot/wrappers.d.ts

@@ -5,7 +5,5 @@ export interface MeasuredCandidate {
     candidate: Candidate;
     rect: Rect;
     offscreen: Offscreen | null;
-    distance: number;
 }
 export declare function dropWrapperDuplicates(measured: MeasuredCandidate[]): MeasuredCandidate[];
-export declare function limitOffscreen(measured: MeasuredCandidate[]): MeasuredCandidate[];

package/dist/version.d.ts

@@ -1 +1 @@
-export declare const SDK_VERSION = "0.1.4";
+export declare const SDK_VERSION = "0.1.6";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blario/react-native",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "description": "Blar agent SDK for React Native",
   "license": "MIT",
   "repository": {

package/src/BlarProvider.tsx

@@ -13,6 +13,8 @@ import type { AgentModePhase } from './agent-mode/state.ts';
 import { KEEP_AWAKE_ADAPTER, KeepAwake } from './agent-mode/keep-awake.ts';
 import { LayerStacking } from './agent-mode/layer-stacking.ts';
 import { AgentLayer } from './agent-mode/ui/AgentLayer.tsx';
+import { restoreKeyboards } from './actions/text-input.ts';
+import { KeyboardGuard } from './agent-mode/keyboard-guard.ts';
 import { AgentLayerHost } from './agent-mode/ui/AgentLayerHost.tsx';
 import { BackendTransport, type WidgetProof } from './backend/backend-transport.ts';
 import type { BlarIdentity } from './backend/identity.ts';
@@ -104,6 +106,8 @@ export class BlarProvider extends Component<BlarProviderProps, BlarProviderState
 
   private readonly releases: Array<() => void> = [];
 
+  private readonly keyboardGuard = new KeyboardGuard(Keyboard, restoreKeyboards);
+
   private stackingCheckQueued = false;
 
   private loggedPhase: AgentModePhase = 'idle';
@@ -163,6 +167,7 @@ export class BlarProvider extends Component<BlarProviderProps, BlarProviderState
       transport,
       getAnchorFiber: () => (this as WithReactInternals)._reactInternals ?? null,
       getViewport: () => visibleViewport(Dimensions.get('window'), Keyboard.metrics()?.height ?? 0),
+      getWindow: () => Dimensions.get('window'),
       getNavigationRef: () => this.props.navigationRef ?? null,
       getCaptureTarget: () => this.contentRef.current,
       reading: this.spotlight,
@@ -316,8 +321,10 @@ export class BlarProvider extends Component<BlarProviderProps, BlarProviderState
           this.debug(`agent mode ${snapshot.phase}`);
         }
         this.setState({ agentMode: snapshot });
+        this.syncKeyboardGuard();
       }),
     );
+    this.releases.push(() => this.keyboardGuard.setActive(false));
     return agentMode;
   }
 
@@ -373,6 +380,7 @@ export class BlarProvider extends Component<BlarProviderProps, BlarProviderState
       coordinator.subscribe((chat) => {
         this.noteNudge(chat);
         this.setState({ chat });
+        this.syncKeyboardGuard();
       }),
     );
     this.releases.push(backend.onFrame((frame) => coordinator.handleFrame(frame)));
@@ -395,6 +403,11 @@ export class BlarProvider extends Component<BlarProviderProps, BlarProviderState
     Blar.attach(backend, (options) => controller.readMessageMetadata({ ...options, tabId: backend.getTabId() }));
   }
 
+  private syncKeyboardGuard(): void {
+    const running = this.agentMode?.getSnapshot().phase === 'active';
+    this.keyboardGuard.setActive(running && this.chatControls?.isOpen() !== true);
+  }
+
   private queueStackingCheck(): void {
     if (this.stackingCheckQueued || (this.agentMode?.getSnapshot().phase ?? 'idle') === 'idle') {
       return;

package/src/actions/click.ts

@@ -6,7 +6,8 @@ import { buildPressEvent } from './events.ts';
 import { checkInteractable } from './interaction-guards.ts';
 import { failed, failedFromException, succeeded } from './result.ts';
 import type { ActionServices } from './services.ts';
-import { isInWheel, selectWheelOption } from './scrolling.ts';
+import { isInWheel } from './scrolling.ts';
+import { selectWheelOption } from './select.ts';
 import type { ResolvedTarget } from './target.ts';
 import type { HandlerResult } from './types.ts';
 import { toggleSwitch } from './switch.ts';

package/src/actions/executor.ts

@@ -5,6 +5,7 @@ import { highlight } from './highlight.ts';
 import { tapsTarget, waitForLanding, waitsForLanding } from './landing.ts';
 import { failed, failedFromException } from './result.ts';
 import { scrollContainer, scrollIntoView } from './scrolling.ts';
+import { selectValue } from './select.ts';
 import { SYSTEM_CLOCK } from './settle.ts';
 import type { ActionServices } from './services.ts';
 import { waitForSettle } from './settle.ts';
@@ -40,6 +41,9 @@ export class ActionExecutor {
     if (command.action.type === 'navigate') {
       return { ok: true, target: null };
     }
+    if (command.action.type === 'press_key' && command.action.key !== 'Enter') {
+      return { ok: false, outcome: outcomeOf(failed('unsupported_action', `key ${command.action.key} is not supported: only Enter is. To move a list use scroll or select on its scr-N`)) };
+    }
     const target = this.resolveTarget(command);
     if (target === null) {
       return { ok: false, outcome: outcomeOf(this.missingTarget(command)) };
@@ -78,7 +82,7 @@ export class ActionExecutor {
       if (options.isCancelled()) {
         return failed('user_cancelled', CANCELLED_MESSAGE);
       }
-      return await this.dispatch(command, target);
+      return await this.dispatch(command, target, options);
     } catch (error) {
       return failedFromException(error);
     }
@@ -123,7 +127,7 @@ export class ActionExecutor {
     return failed('element_not_found', reason);
   }
 
-  private dispatch(command: AgentCommandData, target: ResolvedTarget): Promise<HandlerResult> | HandlerResult {
+  private dispatch(command: AgentCommandData, target: ResolvedTarget, options: ExecutionOptions): Promise<HandlerResult> | HandlerResult {
     const { action } = command;
     switch (action.type) {
       case 'click':
@@ -132,6 +136,8 @@ export class ActionExecutor {
         return typeText(target, action, this.services);
       case 'check':
         return toggleSwitch(target, action.checked);
+      case 'select':
+        return selectValue(target, action.value, this.services, options.isCancelled);
       case 'press_key':
         return action.key === 'Enter' ? submitInput(target) : failed('unsupported_action', `key ${action.key} is not supported`);
       case 'scroll':

package/src/actions/landing.ts

@@ -9,9 +9,9 @@ export interface CursorPort {
 
 export const LANDING_MAX_MS = 1800;
 
-const TAP_ACTIONS: readonly AgentAction['type'][] = ['click', 'check', 'type'];
+const TAP_ACTIONS: readonly AgentAction['type'][] = ['click', 'check', 'select', 'type'];
 
-const LANDING_ACTIONS: readonly AgentAction['type'][] = ['click', 'check'];
+const LANDING_ACTIONS: readonly AgentAction['type'][] = ['click', 'check', 'select'];
 
 export function tapsTarget(action: AgentAction): boolean {
   return TAP_ACTIONS.includes(action.type);

package/src/actions/parse.ts

@@ -32,6 +32,8 @@ function parseAction(raw: unknown): AgentAction | null {
       return typeof raw.value === 'string' ? { type: 'type', value: raw.value, clear_first: raw.clear_first !== false } : null;
     case 'check':
       return { type: 'check', checked: typeof raw.checked === 'boolean' ? raw.checked : null };
+    case 'select':
+      return typeof raw.value === 'string' ? { type: 'select', value: raw.value } : null;
     case 'press_key':
       return typeof raw.key === 'string' ? { type: 'press_key', key: raw.key } : null;
     case 'scroll':

package/src/actions/scrolling.ts

@@ -1,7 +1,7 @@
 import { measureHost } from '../host/measure.ts';
 import { type Band, scrollDeltaIntoBand } from '../host/rect.ts';
 import { scrollInstanceOf } from '../snapshot/classify.ts';
-import { isWheel, maxScrollOffsetOf, measureScrollGeometry, offsetCentering, scrollOffsetOf, type ScrollGeometry, viewportExtentOf } from '../snapshot/scroll.ts';
+import { isWheel, maxScrollOffsetOf, measureScrollGeometry, scrollOffsetOf, type ScrollGeometry, viewportExtentOf } from '../snapshot/scroll.ts';
 import { failed, failedFromException, succeeded } from './result.ts';
 import type { ResolvedTarget } from './target.ts';
 import type { AgentAction, HandlerResult } from './types.ts';
@@ -16,7 +16,7 @@ function clamp(value: number, minimum: number, maximum: number): number {
   return Math.min(Math.max(value, minimum), maximum);
 }
 
-function applyOffset(target: ResolvedTarget, geometry: ScrollGeometry, offset: number, animated: boolean): HandlerResult {
+export function applyOffset(target: ResolvedTarget, geometry: ScrollGeometry, offset: number, animated: boolean): HandlerResult {
   const instance = scrollInstanceOf(target.fiber);
   if (instance === null) {
     return failed('element_not_interactable', 'the scroll container has no scrollTo');
@@ -51,7 +51,7 @@ export async function scrollContainer(target: ResolvedTarget, action: ScrollActi
   return applyOffset(target, geometry, next, true);
 }
 
-function nearestScrollAncestor(target: ResolvedTarget): ResolvedTarget['fiber'] | null {
+export function nearestScrollAncestor(target: ResolvedTarget): ResolvedTarget['fiber'] | null {
   for (let index = target.ancestors.length - 1; index >= 0; index -= 1) {
     const ancestor = target.ancestors[index];
     if (ancestor !== undefined && scrollInstanceOf(ancestor) !== null) {
@@ -66,23 +66,6 @@ export function isInWheel(target: ResolvedTarget): boolean {
   return container !== null && isWheel(container);
 }
 
-// The animated scroll ends in onMomentumScrollEnd, the event a wheel commits its value on, as a user's swipe does.
-export async function selectWheelOption(target: ResolvedTarget): Promise<HandlerResult> {
-  const container = nearestScrollAncestor(target);
-  if (container === null || target.host === null) {
-    return failed('element_not_interactable', 'the option is not inside a wheel');
-  }
-  const [geometry, rect] = await Promise.all([measureScrollGeometry(container), measureHost(target.host)]);
-  if (geometry === null || rect === null) {
-    return failed('element_not_found', 'the option or its wheel could not be measured');
-  }
-  const next = offsetCentering(rect, geometry);
-  if (Math.abs(next - scrollOffsetOf(geometry)) < MOVEMENT_TOLERANCE) {
-    return succeeded('the option is already selected');
-  }
-  return applyOffset({ ...target, fiber: container }, geometry, next, true);
-}
-
 function offsetDeltaToReveal(start: number, end: number, viewportStart: number, viewportEnd: number): number {
   const startShortfall = start - (viewportStart + VISIBLE_MARGIN);
   if (startShortfall < 0) {

package/src/actions/select.ts

@@ -0,0 +1,176 @@
+import { Platform } from 'react-native';
+
+import { isFunction } from '../fiber/narrow.ts';
+import { childrenOf, isHostComponent, propsOf } from '../fiber/reader.ts';
+import { collectText, TEXT_HOST_TYPE } from '../fiber/text.ts';
+import type { Fiber } from '../fiber/types.ts';
+import { measureHost } from '../host/measure.ts';
+import { scrollInstanceOf } from '../snapshot/classify.ts';
+import { maxScrollOffsetOf, measureScrollGeometry, offsetCentering, type ScrollGeometry, scrollOffsetOf, viewportExtentOf } from '../snapshot/scroll.ts';
+import { runBatched } from './batched.ts';
+import { failed, succeeded } from './result.ts';
+import { applyOffset, isInWheel, nearestScrollAncestor } from './scrolling.ts';
+import { SYSTEM_CLOCK } from './settle.ts';
+import type { ActionServices } from './services.ts';
+import type { ResolvedTarget } from './target.ts';
+import type { HandlerResult } from './types.ts';
+
+const ROW_TEXT_LIMIT = 80;
+const MOUNT_WAIT_MS = 600;
+const WALK_STEP_FRACTION = 3;
+const WALK_SETTLE_MS = 120;
+const MOUNT_POLL_MS = 150;
+const LISTED_OPTIONS = 30;
+const CANCELLED_MESSAGE = 'Cancelled by the user before the action ran';
+const MOVEMENT_TOLERANCE = 1;
+const SCROLL_ANIMATION_MS = 400;
+
+interface ListRow {
+  host: Fiber;
+  text: string;
+}
+
+// iOS ends an animated scrollTo in onMomentumScrollEnd, the event a wheel commits its value on, as a user's swipe does.
+// Android only fires it after a fling, so the wheel is told here.
+function commitScrollEnd(list: Fiber, geometry: ScrollGeometry, offset: number): void {
+  const onMomentumScrollEnd = propsOf(list).onMomentumScrollEnd;
+  if (Platform.OS !== 'android' || !isFunction(onMomentumScrollEnd)) {
+    return;
+  }
+  const { viewport, content, horizontal } = geometry;
+  runBatched(() =>
+    onMomentumScrollEnd({
+      persist: () => undefined,
+      nativeEvent: {
+        contentInset: { top: 0, left: 0, bottom: 0, right: 0 },
+        contentOffset: horizontal ? { x: offset, y: 0 } : { x: 0, y: offset },
+        contentSize: { width: content.width, height: content.height },
+        layoutMeasurement: { width: viewport.width, height: viewport.height },
+      },
+    }),
+  );
+}
+
+export async function centerInList(target: ResolvedTarget, list: Fiber, row: Fiber): Promise<HandlerResult> {
+  const [geometry, rect] = await Promise.all([measureScrollGeometry(list), measureHost(row)]);
+  if (geometry === null || rect === null) {
+    return failed('element_not_found', 'the option or its list could not be measured');
+  }
+  const next = offsetCentering(rect, geometry);
+  if (Math.abs(next - scrollOffsetOf(geometry)) < MOVEMENT_TOLERANCE) {
+    commitScrollEnd(list, geometry, next);
+    return succeeded('the option is already selected');
+  }
+  const moved = applyOffset({ ...target, fiber: list }, geometry, next, true);
+  if (moved.ok) {
+    await SYSTEM_CLOCK.sleep(SCROLL_ANIMATION_MS);
+    commitScrollEnd(list, geometry, next);
+  }
+  return moved;
+}
+
+export async function scrollListTo(target: ResolvedTarget, list: Fiber, offset: number): Promise<HandlerResult> {
+  const geometry = await measureScrollGeometry(list);
+  return geometry === null ? failed('element_not_found', 'the list could not be measured') : applyOffset({ ...target, fiber: list }, geometry, offset, false);
+}
+
+export async function selectWheelOption(target: ResolvedTarget): Promise<HandlerResult> {
+  const container = nearestScrollAncestor(target);
+  if (container === null || target.host === null) {
+    return failed('element_not_interactable', 'the option is not inside a wheel');
+  }
+  return centerInList(target, container, target.host);
+}
+
+export function rowsOf(list: Fiber): ListRow[] {
+  const rows: ListRow[] = [];
+  const visit = (fiber: Fiber): void => {
+    if (fiber !== list && scrollInstanceOf(fiber) !== null) {
+      return;
+    }
+    if (isHostComponent(fiber) && fiber.type === TEXT_HOST_TYPE) {
+      const text = collectText(fiber, ROW_TEXT_LIMIT);
+      if (text.length > 0) {
+        rows.push({ host: fiber, text });
+      }
+      return;
+    }
+    childrenOf(fiber).forEach(visit);
+  };
+  visit(list);
+  return rows;
+}
+
+export function matchesRow(text: string, value: string): boolean {
+  const [left, right] = [text.trim().toLowerCase(), value.trim().toLowerCase()];
+  return left === right || (/^\d+$/.test(left) && /^\d+$/.test(right) && Number(left) === Number(right));
+}
+
+// Any list counts when the agent names it, not only a snapping one: this is what reaches a wheel that snaps in JS.
+function listOf(target: ResolvedTarget): Fiber | null {
+  if (target.element.kind === 'scroll') {
+    return target.fiber;
+  }
+  return isInWheel(target) ? nearestScrollAncestor(target) : null;
+}
+
+// A virtualized list mounts its rows shortly after it opens, and a distant row only once scrolling gets near it: wait
+// a moment, then walk the list. A value that isn't there leaves the list where it was.
+export async function selectValue(target: ResolvedTarget, value: string, services: ActionServices, isCancelled: () => boolean = () => false): Promise<HandlerResult> {
+  const first = listOf(target);
+  if (first === null) {
+    return failed('element_not_interactable', 'select works on one open list: click this field to open its picker, then select each value on its list (scr-N)');
+  }
+  let current = target;
+  let list = first;
+  const seen = new Set<string>();
+  const findRow = (): ListRow | undefined => {
+    current = services.resolver.resolveElement(target.element) ?? current;
+    list = listOf(current) ?? list;
+    const rows = rowsOf(list);
+    rows.forEach((row) => seen.add(row.text));
+    return rows.find((row) => matchesRow(row.text, value));
+  };
+  let row = findRow();
+  for (let waited = 0; row === undefined && waited < MOUNT_WAIT_MS && !isCancelled(); waited += MOUNT_POLL_MS) {
+    await SYSTEM_CLOCK.sleep(MOUNT_POLL_MS);
+    row = findRow();
+  }
+  if (isCancelled()) {
+    return failed('user_cancelled', CANCELLED_MESSAGE);
+  }
+  if (row !== undefined) {
+    return centerInList(current, list, row.host);
+  }
+  const geometry = await measureScrollGeometry(list);
+  if (geometry !== null) {
+    const start = scrollOffsetOf(geometry);
+    const step = Math.max(1, viewportExtentOf(geometry) * WALK_STEP_FRACTION);
+    const end = maxScrollOffsetOf(geometry);
+    const stops = [...walk(start, end, step), ...walk(start, 0, step)];
+    for (const stop of stops) {
+      if (isCancelled()) {
+        await scrollListTo(current, list, start);
+        return failed('user_cancelled', CANCELLED_MESSAGE);
+      }
+      await scrollListTo(current, list, stop);
+      await SYSTEM_CLOCK.sleep(WALK_SETTLE_MS);
+      row = findRow();
+      if (row !== undefined) {
+        return centerInList(current, list, row.host);
+      }
+    }
+    await scrollListTo(current, list, start);
+  }
+  const options = [...seen].slice(0, LISTED_OPTIONS).join(', ');
+  return failed('element_not_found', `"${value}" is not in this list. Options: ${options}`);
+}
+
+export function walk(from: number, to: number, step: number): number[] {
+  const stops: number[] = [];
+  const direction = to >= from ? 1 : -1;
+  for (let offset = from + direction * step; direction > 0 ? offset < to : offset > to; offset += direction * step) {
+    stops.push(offset);
+  }
+  return from === to ? stops : [...stops, to];
+}

package/src/actions/text-input.ts

@@ -25,6 +25,33 @@ function isControlled(target: ResolvedTarget): boolean {
   return typeof target.props.value === 'string';
 }
 
+const noKeyboard = new Map<NativeInput, boolean>();
+
+interface NativeInput {
+  setNativeProps?: (props: { showSoftInputOnFocus: boolean }) => void;
+}
+
+// Focus without a keyboard: blurring a field to close it would dismiss what depends on its focus, like the suggestions
+// of a search box. The field gets its keyboard back when the run ends.
+function focusWithoutKeyboard(instance: NativeInput & { focus?: () => void }, target: ResolvedTarget): void {
+  if (!noKeyboard.has(instance)) {
+    noKeyboard.set(instance, target.props.showSoftInputOnFocus !== false);
+    instance.setNativeProps?.({ showSoftInputOnFocus: false });
+  }
+  instance.focus?.();
+}
+
+export function restoreKeyboards(): void {
+  noKeyboard.forEach((enabled, instance) => {
+    try {
+      instance.setNativeProps?.({ showSoftInputOnFocus: enabled });
+    } catch {
+      // The field is gone: nothing left to restore.
+    }
+  });
+  noKeyboard.clear();
+}
+
 export function typeText(target: ResolvedTarget, action: TypeAction, services: ActionServices): HandlerResult {
   if (target.element.kind !== 'text_input') {
     return failed('element_not_interactable', 'type needs a text input');
@@ -36,7 +63,9 @@ export function typeText(target: ResolvedTarget, action: TypeAction, services: A
   const next = action.clear_first ? action.value : currentTextOf(target) + action.value;
   const instance = target.host === null ? null : hostInstanceOf(target.host);
   try {
-    instance?.focus?.();
+    if (instance !== null) {
+      focusWithoutKeyboard(instance as NativeInput & { focus?: () => void }, target);
+    }
     if (!isControlled(target)) {
       if (Platform.OS === 'android') {
         const tag = findNodeHandle(instance as Parameters<typeof findNodeHandle>[0]);

package/src/actions/types.ts

@@ -6,6 +6,7 @@ export type AgentAction =
   | { type: 'click' }
   | { type: 'type'; value: string; clear_first: boolean }
   | { type: 'check'; checked: boolean | null }
+  | { type: 'select'; value: string }
   | { type: 'press_key'; key: string }
   | { type: 'scroll'; direction: ScrollDirectionName; amount: number | null }
   | { type: 'scroll_to' }