mobile-debug-mcp 0.34.0 → 0.36.0

package/src/interact/input-service.ts

@@ -0,0 +1,11 @@
+import type { AndroidInteract } from './android.js'
+import type { ActionTargetResolved, DeviceInfo } from '../types.js'
+
+type AdjustmentResult = { success?: boolean; device?: DeviceInfo; error?: string }
+type AdjustmentTarget = { deviceId: string; target: ActionTargetResolved | null; property: string }
+
+/** Common native input surface, with optional semantic control capabilities. */
+export type InputService = Pick<AndroidInteract, 'tap' | 'swipe' | 'scrollToElement'> & {
+  setAdjustableValue?(input: AdjustmentTarget & { value: number }): Promise<AdjustmentResult>
+  adjustAccessibleValue?(input: AdjustmentTarget & { direction: 'increment' | 'decrement' }): Promise<AdjustmentResult>
+}

package/src/interact/interaction-trace.ts

@@ -0,0 +1,36 @@
+import { systemClock, type Clock } from '../runtime/clock.js'
+import type { ActionTrace, TraceStep } from '../types.js'
+import { createTraceStep, nextActionId } from '../runtime/action-results.js'
+
+export function buildObservationTrace({
+  actionType,
+  stage,
+  success,
+  attempts,
+  metadata
+}: {
+  actionType: string
+  stage: 'verify' | 'stabilize' | 'resolve'
+  success: boolean
+  attempts: number
+  metadata?: Record<string, unknown>
+}, clock: Clock = systemClock): ActionTrace {
+  const now = clock.now()
+  const actionId = nextActionId(actionType, now)
+  const steps: TraceStep[] = [
+    createTraceStep({
+      stage,
+      timestamp: now,
+      result: success ? 'success' : 'failure',
+      attemptIndex: 0,
+      metadata
+    })
+  ]
+
+  return {
+    action_id: actionId,
+    steps,
+    final_outcome: success ? 'success' : 'failure',
+    attempts: Math.max(1, Math.floor(attempts || 1))
+  }
+}

package/src/interact/model.ts

@@ -0,0 +1,109 @@
+import type { UIElementState, UIElementSemanticMetadata } from '../types.js'
+export interface ScreenFingerprintResponse { fingerprint: string | null }
+
+export interface UiElement {
+  text?: string | null
+  label?: string | null
+  value?: string | null
+  contentDescription?: string | null
+  contentDesc?: string | null
+  accessibilityLabel?: string | null
+  resourceId?: string | null
+  resourceID?: string | null
+  id?: string | null
+  type?: string | null
+  class?: string | null
+  bounds?: number[] | null
+  clickable?: boolean
+  enabled?: boolean
+  focusable?: boolean
+  visible?: boolean
+  parentId?: number | string | null
+  children?: number[]
+  _index?: number
+  _interactable?: boolean
+  _sliderLike?: boolean
+  state?: UIElementState | null
+  stable_id?: string | null
+  role?: string | null
+  test_tag?: string | null
+  selector?: { value: string | null, confidence: { score: number, reason: string } | null } | null
+  semantic?: UIElementSemanticMetadata | null
+  actions?: string[] | null
+  accessibilityActions?: string[] | null
+  runtimeActions?: string[] | null
+  platformActions?: string[] | null
+}
+
+export interface ResolvedUiElementContext {
+  elementId: string
+  platform: 'android' | 'ios'
+  deviceId?: string
+  bounds: [number, number, number, number] | null
+  index: number
+  stable_id?: string | null
+}
+
+export interface UiResolution {
+  width?: number
+  height?: number
+}
+
+export interface UiChangeSignatureSet {
+  hierarchy: string | null
+  text: string | null
+  state: string | null
+}
+
+export interface UiChangeScopeResolution {
+  scope: 'screen' | 'subtree'
+  target: string | null
+  resolved: boolean
+  resolvedIndex: number | null
+  resolvedStableId: string | null
+  reason: string
+}
+
+export interface UiChangeScopeResult {
+  elements: UiElement[]
+  resolution: UiChangeScopeResolution
+  error?: {
+    code: 'INVALID_SCOPE' | 'ELEMENT_NOT_FOUND'
+    message: string
+  }
+}
+
+export interface RankedResolutionCandidate {
+  el: UiElement
+  idx: number
+  score: number
+  reason: string
+  interactable: boolean
+}
+
+export interface InferredLabeledSliderState {
+  current: number
+  min: number
+  max: number
+  value_range: { min: number; max: number }
+  touch_bounds: [number, number, number, number]
+  source: 'nearby_labels'
+}
+
+export interface FindElementResolutionSummary {
+  confidence: number
+  reason: string
+  fallback_available: boolean
+  matched_count: number
+  alternates: Array<{
+    text: string | null
+    resource_id: string | null
+    accessibility_id: string | null
+    class: string | null
+    bounds: { left: number; top: number; right: number; bottom: number } | null
+    clickable: boolean
+    enabled: boolean
+    score: number
+    reason: string
+  }>
+}

package/src/interact/observation.ts

@@ -0,0 +1,7 @@
+import type { GetUITreeResponse } from '../types.js'
+import type { UiElement } from './model.js'
+
+export type UiTree = Omit<Partial<GetUITreeResponse>, 'elements'> & { elements?: UiElement[] }
+import type { DeviceTarget } from '../contracts/target.js'
+export type { DeviceTarget } from '../contracts/target.js'
+export type ObserveUI = (target: DeviceTarget) => Promise<UiTree>

package/src/interact/targeting/candidate-ancestor.ts

@@ -0,0 +1,49 @@
+import type { UiElement, RankedResolutionCandidate, UiResolution } from '../model.js'
+import { _isSemanticActionable } from './element-helpers.js'
+import { nearbyControl } from './nearby-control.js'
+
+type Match = { el: UiElement; idx: number }
+function parentOf(elements: UiElement[], parent: number | string) {
+  const index = typeof parent === 'number' ? parent : /^\d+$/.test(parent) ? Number(parent) : null
+  if (index !== null && elements[index]) return { el: elements[index], idx: index }
+  if (typeof parent !== 'string') return null
+  const found = elements.findIndex(el => el.resourceId === parent || el.id === parent)
+  return found >= 0 && elements[found] ? { el: elements[found], idx: found } : null
+}
+function ancestor(elements: UiElement[], chosen: Match): Match | null {
+  if (!Array.isArray(chosen.el.bounds) || chosen.el.parentId == null) return null
+  let current = chosen, safety = 0
+  while (safety < 20 && !(current.el.clickable || current.el.focusable) && current.el.parentId != null) {
+    const parent = parentOf(elements, current.el.parentId)
+    if (!parent) break
+    current = parent
+    if (current.el.clickable || current.el.enabled || current.el.focusable) return current
+    safety++
+  }
+  return null
+}
+function containing(elements: UiElement[], chosen: Match): Match | null {
+  const child = chosen.el.bounds
+  if (!Array.isArray(child)) return null
+  let best: Match | null = null, area = Infinity
+  for (let idx = 0; idx < elements.length; idx++) {
+    const el = elements[idx]
+    if (!el || !(el.clickable || el.focusable) || !Array.isArray(el.bounds) || el.bounds.length < 4) continue
+    const [l, t, r, b] = el.bounds
+    if (l <= child[0] && t <= child[1] && r >= child[2] && b >= child[3]) {
+      const size = (r - l) * (b - t)
+      if (size < area) { area = size; best = { el, idx } }
+    }
+  }
+  return best
+}
+export function preferActionable(best: RankedResolutionCandidate, elements: UiElement[], screen: UiResolution | null) {
+  const parent = ancestor(elements, best) ?? containing(elements, best)
+  if (parent) best = { ...parent, score: Math.min(1, best.score + .02), reason: 'clickable_parent_preferred', interactable: true }
+  if (!(best.el.clickable || best.el.focusable || _isSemanticActionable(best.el))) {
+    const nearby = nearbyControl(elements, best, screen)
+    if (nearby) best = { el: nearby.el, idx: nearby.idx, score: Math.min(1, best.score + .02),
+      reason: nearby.sliderLike ? 'slider_track_preferred' : 'nearby_actionable_control', interactable: true }
+  }
+  return best
+}

package/src/interact/targeting/candidate-result.ts

@@ -0,0 +1,24 @@
+import type { FindElementResponse, FindElementElement } from '../../types.js'
+import type { RankedResolutionCandidate } from '../model.js'
+import { _summarizeResolutionCandidate } from './element-helpers.js'
+
+export function candidateResult(best: RankedResolutionCandidate, candidates: RankedResolutionCandidate[]): FindElementResponse {
+  const b = best.el.bounds
+  const bounds = Array.isArray(b) ? { left: b[0], top: b[1], right: b[2], bottom: b[3] } : null
+  const unique = candidates.filter((candidate, index, array) => index === array.findIndex(other => other.idx === candidate.idx && other.el === candidate.el))
+  const alternates = unique.filter(candidate => candidate.idx !== best.idx || candidate.el !== best.el).slice(0, 3).map(_summarizeResolutionCandidate)
+  const element: FindElementElement = {
+    text: best.el.text ?? null, resourceId: best.el.resourceId ?? null,
+    contentDesc: best.el.contentDescription ?? best.el.contentDesc ?? null, class: best.el.type ?? best.el.class ?? null,
+    bounds, clickable: !!best.el.clickable, enabled: !!best.el.enabled,
+    stable_id: best.el.stable_id ?? null, role: best.el.role ?? null, test_tag: best.el.test_tag ?? null,
+    selector: best.el.selector ?? null, semantic: best.el.semantic ?? null,
+    tapCoordinates: bounds ? { x: Math.floor((bounds.left + bounds.right) / 2), y: Math.floor((bounds.top + bounds.bottom) / 2) } : null,
+    telemetry: { matchedIndex: best.idx ?? null, matchedInteractable: !!best.interactable, sliderLike: best.reason === 'slider_track_preferred' }
+  }
+  if (best.reason === 'slider_track_preferred') element.interactionHint = { kind: 'slider',
+    axis: bounds && (bounds.bottom - bounds.top) > (bounds.right - bounds.left) ? 'vertical' : 'horizontal', trackBounds: bounds }
+  const score = Math.min(1, Number(best.score.toFixed(3)))
+  return { found: true, element, score, confidence: score, resolution: {
+    confidence: score, reason: best.reason, fallback_available: alternates.length > 0, matched_count: unique.length, alternates } }
+}

package/src/interact/targeting/candidate-scoring.ts

@@ -0,0 +1,42 @@
+import type { UiElement, RankedResolutionCandidate } from '../model.js'
+import { _normalize, _isSemanticActionable } from './element-helpers.js'
+
+type Score = { score: number; reason: string }
+function textScore(el: UiElement, query: string, exact: boolean): Score {
+  const text = _normalize(el.text ?? el.label ?? el.value ?? '')
+  const content = _normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? '')
+  const resource = _normalize(el.resourceId ?? el.resourceID ?? el.id ?? '')
+  const type = _normalize(el.type ?? el.class ?? '')
+  const choices: Array<[string, boolean, number, string]> = [
+    [text, text === query, 1, 'exact_text_match'], [content, content === query, .95, 'exact_content_desc_match'],
+    [resource, resource === query, .92, 'exact_resource_id_match']
+  ]
+  if (!exact) choices.push([text, text.includes(query), .6, 'partial_text_match'], [content, content.includes(query), .55, 'partial_content_desc_match'],
+    [resource, resource.includes(query), .7, 'partial_resource_id_match'])
+  choices.push([type, exact ? type === query : type.includes(query), .3, exact ? 'exact_class_match' : 'partial_class_match'])
+  for (const [value, matched, score, reason] of choices) if (value && matched) return { score, reason }
+  return { score: 0, reason: 'best_scoring_candidate' }
+}
+function semanticScore(el: UiElement, query: string, exact: boolean, initial: Score): Score {
+  const role = _normalize(el.semantic?.semantic_role ?? '')
+  const actions = Array.isArray(el.semantic?.supported_actions) ? el.semantic.supported_actions.map(_normalize).filter(Boolean) : []
+  const result = { ...initial }
+  if (!result.score && role && (exact ? role === query : role.includes(query))) { result.score = .5; result.reason = 'semantic_role_match' }
+  if (actions.some(action => exact ? action === query : action.includes(query))) {
+    result.score = Math.max(result.score, result.score > 0 ? .65 : .6); result.reason = 'semantic_action_match'
+  }
+  if (!exact && result.score === 0 && el.semantic?.adjustable && /slider|stepper|dropdown|segment|control|adjust/.test(query)) {
+    result.score = .45; result.reason = 'semantic_control_match'
+  }
+  return result
+}
+export function scoreCandidate(el: UiElement | null, idx: number, query: string, exact: boolean): RankedResolutionCandidate | null {
+  if (!el || !el.visible) return null
+  const bounds = el.bounds || [0, 0, 0, 0]
+  if (!Array.isArray(bounds) || bounds.length < 4) return null
+  if (bounds[2] <= bounds[0] || bounds[3] <= bounds[1]) return null
+  const interactable = !!(el.clickable || el.enabled || el.focusable || _isSemanticActionable(el))
+  const result = semanticScore(el, query, exact, textScore(el, query, exact))
+  if (result.score > 0 && interactable) result.score += .05
+  return result.score > 0 ? { el, idx, ...result, interactable } : null
+}

package/src/interact/targeting/element-helpers.ts

@@ -0,0 +1,243 @@
+import type { FindElementResolutionSummary } from '../model.js'
+import type { ActionFailureCode, ActionTargetResolved } from '../../types.js'
+import type { UiElement, RankedResolutionCandidate } from '../model.js'
+import { createHash } from 'node:crypto'
+
+export function _normalize(s: unknown): string {
+    if (s === null || s === undefined) return ''
+    try { return String(s).toLowerCase().trim() } catch { return '' }
+  }
+
+export function _normalizeBounds(bounds: unknown): [number, number, number, number] | null {
+    if (!Array.isArray(bounds) || bounds.length < 4) return null
+    const normalized = bounds.slice(0, 4).map((value: unknown) => Number(value))
+    if (normalized.some((value: number) => Number.isNaN(value))) return null
+    return normalized as [number, number, number, number]
+  }
+
+export function _hash(value: unknown): string {
+    return createHash('sha256').update(JSON.stringify(value)).digest('hex')
+  }
+
+function matchesField(value: string, query: string | null | undefined, contains: boolean) {
+  if (query === undefined || query === null) return true
+  const normalized = _normalize(query)
+  return contains ? value.includes(normalized) : value === normalized
+}
+export function _matchesSelector(el: UiElement, selector?: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean }): boolean {
+  if (!selector) return false
+  const contains = !!selector.contains
+  return matchesField(_normalize(el.text ?? el.label ?? el.value ?? ''), selector.text, contains) &&
+    matchesField(_normalize(el.resourceId ?? el.resourceID ?? el.id ?? ''), selector.resource_id, contains) &&
+    matchesField(_normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? el.label ?? ''), selector.accessibility_id, contains)
+}
+
+export function _findFirstMatchingElement(
+    elements: UiElement[],
+    selector?: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean }
+  ): { el: UiElement, idx: number } | null {
+    if (!selector) return null
+    for (let i = 0; i < elements.length; i++) {
+      const el = elements[i]
+      if (!el) continue
+      if (_matchesSelector(el, selector)) return { el, idx: i }
+    }
+    return null
+  }
+
+export function _resolveParentIndex(elements: UiElement[], parentId: number | string | null | undefined): number | null {
+    if (parentId === undefined || parentId === null) return null
+
+    if (typeof parentId === 'number' && Number.isInteger(parentId) && parentId >= 0 && parentId < elements.length) {
+      return parentId
+    }
+
+    if (typeof parentId === 'string') {
+      const normalized = _normalize(parentId)
+      if (!normalized) return null
+
+      if (/^\d+$/.test(normalized)) {
+        const index = Number(normalized)
+        if (index >= 0 && index < elements.length) return index
+      }
+
+      const foundIndex = elements.findIndex((el) => {
+        if (!el) return false
+        return _normalize(el.resourceId ?? el.resourceID ?? el.id ?? '') === normalized ||
+          _normalize(el.stable_id ?? '') === normalized
+      })
+
+      return foundIndex >= 0 ? foundIndex : null
+    }
+
+    return null
+  }
+
+export function _isVisibleElement(el: UiElement): boolean {
+    const bounds = _normalizeBounds(el.bounds)
+    return !!el.visible && !!bounds && bounds[2] > bounds[0] && bounds[3] > bounds[1]
+  }
+
+export function _isTapActionable(
+    el: UiElement,
+    storedStableId?: string | null,
+    platform?: 'android' | 'ios'
+  ): { actionable: boolean, failureCode?: ActionFailureCode, reason?: string } {
+    if (!_isVisibleElement(el)) {
+      return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' }
+    }
+
+    if (el.enabled === false) {
+      return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' }
+    }
+
+    const semanticTapActionable = !!el.semantic && (
+      el.semantic.is_clickable ||
+      (Array.isArray(el.semantic.supported_actions) && el.semantic.supported_actions.some((action) => _normalize(action) === 'tap'))
+    )
+
+    if (!el.clickable && !(platform === 'ios' && semanticTapActionable)) {
+      return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not clickable' }
+    }
+
+    if (storedStableId) {
+      if (!el.stable_id || el.stable_id !== storedStableId) {
+        return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' }
+      }
+    }
+
+    return { actionable: true }
+  }
+
+export function _isAdjustableActionable(el: UiElement, storedStableId?: string | null): { actionable: boolean, failureCode?: ActionFailureCode, reason?: string } {
+    if (!_isVisibleElement(el)) {
+      return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' }
+    }
+
+    if (el.enabled === false) {
+      return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' }
+    }
+
+    if (storedStableId) {
+      if (!el.stable_id || el.stable_id !== storedStableId) {
+        return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' }
+      }
+    }
+
+    return { actionable: true }
+  }
+
+export function _computeElementId(platform: 'android' | 'ios', deviceId: string | undefined, el: UiElement, index: number): string {
+    const identity = {
+      platform,
+      deviceId: deviceId || '',
+      text: _normalize(el.text ?? el.label ?? el.value ?? ''),
+      resourceId: _normalize(el.resourceId ?? el.resourceID ?? el.id ?? ''),
+      accessibilityId: _normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? el.label ?? ''),
+      class: _normalize(el.type ?? el.class ?? ''),
+      bounds: _normalizeBounds(el.bounds) ?? [0, 0, 0, 0],
+      index
+    }
+    return `el_${createHash('sha1').update(JSON.stringify(identity)).digest('hex').slice(0, 24)}`
+  }
+
+export function _resolvedTargetFromElement(
+    elementId: string,
+    element: UiElement,
+    index: number
+  ): ActionTargetResolved {
+    return {
+      elementId,
+      text: element.text ?? null,
+      resource_id: element.resourceId ?? element.resourceID ?? element.id ?? null,
+      accessibility_id: element.contentDescription ?? element.contentDesc ?? element.accessibilityLabel ?? element.label ?? null,
+      class: element.type ?? element.class ?? null,
+      bounds: _normalizeBounds(element.bounds),
+      index,
+      state: element.state ?? null,
+      stable_id: element.stable_id ?? null,
+      role: element.role ?? null,
+      test_tag: element.test_tag ?? null,
+      selector: element.selector ?? null,
+      semantic: element.semantic ?? null
+    }
+  }
+
+export function _summarizeResolutionCandidate(candidate: RankedResolutionCandidate): FindElementResolutionSummary['alternates'][number] {
+    const bounds = _normalizeBounds(candidate.el.bounds)
+    return {
+      text: candidate.el.text ?? null,
+      resource_id: candidate.el.resourceId ?? candidate.el.resourceID ?? candidate.el.id ?? null,
+      accessibility_id: candidate.el.contentDescription ?? candidate.el.contentDesc ?? candidate.el.accessibilityLabel ?? candidate.el.label ?? null,
+      class: candidate.el.type ?? candidate.el.class ?? null,
+      bounds: bounds
+        ? { left: bounds[0], top: bounds[1], right: bounds[2], bottom: bounds[3] }
+        : null,
+      clickable: !!candidate.el.clickable,
+      enabled: !!candidate.el.enabled,
+      score: candidate.score,
+      reason: candidate.reason
+    }
+  }
+
+export function _isAdjustableControl(el: UiElement | null): boolean {
+    if (!el) return false
+    const type = _normalize(el.type ?? el.class ?? '')
+    const role = _normalize(el.role ?? '')
+    return !!el.state?.value_range || /slider|seekbar|stepper|adjustable|range/.test(type) || /slider|seekbar|stepper|adjustable|range/.test(role)
+  }
+
+export function _isSemanticActionable(el: UiElement | null): boolean {
+    if (!el?.semantic) return false
+    if (el.semantic.adjustable) return true
+    return Array.isArray(el.semantic.supported_actions) && el.semantic.supported_actions.length > 0
+  }
+
+export function _resolveActionableAncestor(elements: UiElement[], chosen: { el: UiElement, idx: number } | null): { el: UiElement, idx: number } | null {
+    if (!chosen) return null
+    if (hasActionableTrait(chosen.el)) return chosen
+
+    let current = chosen
+    let safety = 0
+
+    while (safety < 20 && !(hasActionableTrait(current.el)) && current.el.parentId !== undefined && current.el.parentId !== null) {
+      const parentIndex = _resolveParentIndex(elements, current.el.parentId)
+
+      if (parentIndex !== null && elements[parentIndex]) {
+        current = { el: elements[parentIndex], idx: parentIndex }
+        if (hasActionableTrait(current.el)) return current
+      } else {
+        break
+      }
+
+      safety++
+    }
+
+    const childBounds = _normalizeBounds(chosen.el.bounds)
+    if (!childBounds) return null
+    const [cl, ct, cr, cb] = childBounds
+
+    let best: { el: UiElement, idx: number } | null = null
+    let bestArea = Infinity
+
+    for (let i = 0; i < elements.length; i++) {
+      const el = elements[i]
+      if (!el || !(hasActionableTrait(el))) continue
+      const bounds = _normalizeBounds(el.bounds)
+      if (!bounds) continue
+      const [pl, pt, pr, pb] = bounds
+      if (pl <= cl && pt <= ct && pr >= cr && pb >= cb) {
+        const area = (pr - pl) * (pb - pt)
+        if (area < bestArea) {
+          bestArea = area
+          best = { el, idx: i }
+        }
+      }
+    }
+
+    return best
+  }
+
+function hasActionableTrait(el: UiElement): boolean {
+  return !!(el.clickable || el.focusable || _isSemanticActionable(el))
+}

package/src/interact/targeting/find-element.ts

@@ -0,0 +1,47 @@
+import { systemClock, type Clock } from '../../runtime/clock.js'
+import type { FindElementResponse } from '../../types.js'
+import type { RankedResolutionCandidate } from '../model.js'
+import type { UiTree, ObserveUI, DeviceTarget } from '../observation.js'
+import { _normalize } from './element-helpers.js'
+import { checkDebuggerGuard } from '../../debugger/guard.js'
+import { scoreCandidate } from './candidate-scoring.js'
+import { preferActionable } from './candidate-ancestor.js'
+import { candidateResult } from './candidate-result.js'
+
+export interface FindInput extends DeviceTarget { query: string; exact?: boolean; timeoutMs?: number }
+interface Search { best: RankedResolutionCandidate | null; tree: UiTree | null; candidates: RankedResolutionCandidate[]; stop: boolean }
+function scoreTree(search: Search, tree: UiTree, query: string, exact: boolean) {
+  if (!tree || !Array.isArray(tree.elements)) return
+  const candidates: RankedResolutionCandidate[] = []
+  let improved = false
+  for (let i = 0; i < tree.elements.length; i++) {
+    try {
+      const candidate = scoreCandidate(tree.elements[i], i, query, exact)
+      if (!candidate) continue
+      candidates.push(candidate)
+      if (!search.best || candidate.score > search.best.score) {
+        search.best = candidate; search.tree = tree; improved = true
+        if (candidate.score >= .95) { search.stop = true; break }
+      }
+    } catch (error) { checkDebuggerGuard(); console.error('Error scoring element:', error) }
+  }
+  if (improved) search.candidates = candidates.slice()
+}
+export async function findElement({ query, exact = false, timeoutMs = 3000, platform, deviceId }: FindInput, observe: ObserveUI, clock: Clock = systemClock): Promise<FindElementResponse> {
+  const start = clock.now(), deadline = start + timeoutMs, normalized = _normalize(query)
+  if (!normalized) return { found: false, error: 'Empty query' }
+  const search: Search = { best: null, tree: null, candidates: [], stop: false }
+  while (clock.now() <= deadline) {
+    try { scoreTree(search, await observe({ platform, deviceId }), normalized, exact) }
+    catch (error) { checkDebuggerGuard(); console.error('Error fetching UI tree:', error) }
+    if (search.stop || clock.now() > deadline) break
+    await clock.sleep(100)
+  }
+  if (!search.best) return { found: false, error: 'Element not found' }
+  try {
+    const elements = search.tree && Array.isArray(search.tree.elements) ? search.tree.elements : []
+    const screen = search.tree?.resolution && typeof search.tree.resolution === 'object' ? search.tree.resolution : null
+    search.best = preferActionable(search.best, elements, screen)
+  } catch (error) { checkDebuggerGuard(); console.error('Error resolving ancestor:', error) }
+  return candidateResult(search.best, search.candidates)
+}

package/src/interact/targeting/nearby-control.ts

@@ -0,0 +1,46 @@
+import type { UiElement, UiResolution } from '../model.js'
+import { _normalize, _normalizeBounds } from './element-helpers.js'
+
+type Match = { el: UiElement; idx: number; sliderLike?: boolean }
+function trackLimits(label: [number, number, number, number], screen?: UiResolution | null) {
+  const width = Number(screen?.width) > 0 ? Number(screen?.width) : 0
+  const height = Number(screen?.height) > 0 ? Number(screen?.height) : 0
+  return { length: Math.max(220, width > 0 ? Math.floor(width * .18) : 0, height > 0 ? Math.floor(height * .18) : 0),
+    thickness: Math.max(180, width > 0 ? Math.floor(width * .08) : 0, height > 0 ? Math.floor(height * .08) : 0),
+    labelWidth: label[2] - label[0], labelHeight: label[3] - label[1] }
+}
+function nearbyScore(el: UiElement, label: [number, number, number, number], limits: ReturnType<typeof trackLimits>) {
+  if (!el || !(el.clickable || el.focusable) || el.visible === false) return null
+  const b = _normalizeBounds(el.bounds)
+  if (!b) return null
+  const width = b[2] - b[0], height = b[3] - b[1], gap = b[1] - label[3]
+  if (gap < -32 || gap > 640 || Math.min(label[2], b[2]) - Math.max(label[0], b[0]) < -32) return null
+  const horizontal = width >= Math.max(limits.length, Math.floor(limits.labelWidth * 1.5)) && height <= limits.thickness
+  const vertical = height >= Math.max(limits.length, Math.floor(limits.labelHeight * 1.5)) && width <= limits.thickness
+  const sliderLike = /slider|seek|range/i.test(_normalize(el.type ?? el.class ?? '')) || horizontal || vertical
+  return { score: appearanceScore(gap, el, width, limits.labelWidth, sliderLike), sliderLike }
+}
+function appearanceScore(gap: number, el: UiElement, width: number, labelWidth: number, sliderLike: boolean) {
+  const text = _normalize(el.text ?? el.label ?? el.value ?? '')
+  const content = _normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? '')
+  const type = _normalize(el.type ?? el.class ?? '')
+  let score = gap
+  if (!text && !content) score -= 18
+  if (sliderLike) score -= 30
+  if (/view|layout|group|frame/i.test(type)) score -= 10
+  if (width > labelWidth * 1.5) score -= 8
+  if (text || content) score += 20
+  return score
+}
+export function nearbyControl(elements: UiElement[], chosen: Match | null, screen?: UiResolution | null): Match | null {
+  if (!chosen) return null
+  const label = _normalizeBounds(chosen.el.bounds)
+  if (!label) return null
+  const limits = trackLimits(label, screen)
+  let best: Match | null = null, score = Infinity
+  for (let i = chosen.idx + 1; i < Math.min(elements.length, chosen.idx + 8); i++) {
+    const candidate = nearbyScore(elements[i], label, limits)
+    if (candidate && candidate.score < score) { score = candidate.score; best = { el: elements[i], idx: i, sliderLike: candidate.sliderLike } }
+  }
+  return best
+}

package/src/interact/targeting/resolved-elements.ts

@@ -0,0 +1,20 @@
+import type { ResolvedUiElementContext } from '../model.js'
+
+/** Bounded, insertion-ordered handles; repeated observations refresh retention order. */
+export class ResolvedElements {
+  private readonly elements = new Map<string, ResolvedUiElementContext>()
+  constructor(private readonly maximum = 256) {}
+
+  get(id: string) { return this.elements.get(id) }
+  clear() { this.elements.clear() }
+
+  remember(id: string, context: ResolvedUiElementContext) {
+    if (this.elements.has(id)) this.elements.delete(id)
+    this.elements.set(id, context)
+    while (this.elements.size > this.maximum) {
+      const oldest = this.elements.keys().next().value
+      if (!oldest) break
+      this.elements.delete(oldest)
+    }
+  }
+}

package/src/interact/targeting/tap-target.ts

@@ -0,0 +1,22 @@
+import type { ActionTargetResolved, ActionFailureCode } from '../../types.js'
+import type { UiElement, ResolvedUiElementContext } from '../model.js'
+import type { UiTree } from '../observation.js'
+import { _isTapActionable, _normalizeBounds } from './element-helpers.js'
+interface TapResolution {
+  current(elements: UiElement[], platform: 'android' | 'ios', deviceId: string | undefined, handle: ResolvedUiElementContext): { el: UiElement; index: number } | null
+  target(element: UiElement, index: number): ActionTargetResolved
+}
+type TapTarget = { failure: ActionFailureCode; target: ActionTargetResolved | null } | { target: ActionTargetResolved; point: { x: number; y: number } }
+export function resolveTap(tree: UiTree, stored: ResolvedUiElementContext, deps: TapResolution): TapTarget {
+  const platform = tree?.device?.platform === 'ios' ? 'ios' : stored.platform
+  const deviceId = tree?.device?.id || stored.deviceId
+  const elements = Array.isArray(tree?.elements) ? tree.elements : []
+  const current = deps.current(elements, platform, deviceId, stored)
+  if (!current) return { failure: 'STALE_REFERENCE', target: null }
+  const target = deps.target(current.el, current.index)
+  const actionable = _isTapActionable(current.el, stored.stable_id, stored.platform)
+  if (!actionable.actionable) return { failure: actionable.failureCode ?? 'ELEMENT_NOT_INTERACTABLE', target }
+  const bounds = _normalizeBounds(current.el.bounds) ?? stored.bounds
+  if (!bounds || bounds[2] <= bounds[0] || bounds[3] <= bounds[1]) return { failure: 'ELEMENT_NOT_INTERACTABLE', target }
+  return { target, point: { x: Math.floor((bounds[0] + bounds[2]) / 2), y: Math.floor((bounds[1] + bounds[3]) / 2) } }
+}