mobile-debug-mcp 0.26.2 → 0.26.4

package/src/interact/index.ts

@@ -10,6 +10,8 @@ import { buildActionExecutionResult } from '../server/common.js'
 import type {
   ActionFailureCode,
   ActionTargetResolved,
+  AdjustControlResponse,
+  FindElementResponse,
   ExpectElementVisibleResponse,
   ExpectStateResponse,
   ExpectScreenResponse,
@@ -68,6 +70,32 @@ interface UiChangeSignatureSet {
   state: string | null
 }
 
+interface RankedResolutionCandidate {
+  el: UiElement
+  idx: number
+  score: number
+  reason: string
+  interactable: boolean
+}
+
+interface FindElementResolutionSummary {
+  confidence: number
+  reason: string
+  fallback_available: boolean
+  matched_count: number
+  alternates: Array<{
+    text: string | null
+    resource_id: string | null
+    accessibility_id: string | null
+    class: string | null
+    bounds: { left: number; top: number; right: number; bottom: number } | null
+    clickable: boolean
+    enabled: boolean
+    score: number
+    reason: string
+  }>
+}
+
 
 export class ToolsInteract {
   private static readonly _maxResolvedUiElements = 256
@@ -290,6 +318,106 @@ export class ToolsInteract {
     }
   }
 
+  private static _summarizeResolutionCandidate(candidate: RankedResolutionCandidate): FindElementResolutionSummary['alternates'][number] {
+    const bounds = ToolsInteract._normalizeBounds(candidate.el.bounds)
+    return {
+      text: candidate.el.text ?? null,
+      resource_id: candidate.el.resourceId ?? candidate.el.resourceID ?? candidate.el.id ?? null,
+      accessibility_id: candidate.el.contentDescription ?? candidate.el.contentDesc ?? candidate.el.accessibilityLabel ?? candidate.el.label ?? null,
+      class: candidate.el.type ?? candidate.el.class ?? null,
+      bounds: bounds
+        ? { left: bounds[0], top: bounds[1], right: bounds[2], bottom: bounds[3] }
+        : null,
+      clickable: !!candidate.el.clickable,
+      enabled: !!candidate.el.enabled,
+      score: candidate.score,
+      reason: candidate.reason
+    }
+  }
+
+  private static _isAdjustableControl(el: UiElement | null): boolean {
+    if (!el) return false
+    const type = ToolsInteract._normalize(el.type ?? el.class ?? '')
+    const role = ToolsInteract._normalize(el.role ?? '')
+    return !!el.state?.value_range || /slider|seekbar|stepper|adjustable|range/.test(type) || /slider|seekbar|stepper|adjustable|range/.test(role)
+  }
+
+  private static _readNumericControlValue(el: UiElement | null, property: string): number | null {
+    if (!el?.state) return null
+    const stateValue = el.state[property as keyof UIElementState]
+    if (typeof stateValue === 'number' && Number.isFinite(stateValue)) return stateValue
+    if (property === 'value' || property === 'raw_value') {
+      const fallback = el.state.raw_value ?? el.state.value
+      if (typeof fallback === 'number' && Number.isFinite(fallback)) return fallback
+    }
+    return null
+  }
+
+  private static _buildControlPoint(bounds: [number, number, number, number], ratio: number, axis: 'horizontal' | 'vertical') {
+    const clampedRatio = Math.max(0, Math.min(1, ratio))
+    const [left, top, right, bottom] = bounds
+    const width = Math.max(1, right - left)
+    const height = Math.max(1, bottom - top)
+    const insetX = Math.max(8, Math.floor(width * 0.08))
+    const insetY = Math.max(8, Math.floor(height * 0.08))
+    if (axis === 'vertical') {
+      const usableHeight = Math.max(1, height - (insetY * 2))
+      return {
+        x: Math.floor((left + right) / 2),
+        y: Math.floor(bottom - insetY - (usableHeight * clampedRatio))
+      }
+    }
+    const usableWidth = Math.max(1, width - (insetX * 2))
+    return {
+      x: Math.floor(left + insetX + (usableWidth * clampedRatio)),
+      y: Math.floor((top + bottom) / 2)
+    }
+  }
+
+  private static _buildConservativeControlPoint(
+    bounds: [number, number, number, number],
+    targetValue: number,
+    currentValue: number | null,
+    min: number,
+    max: number,
+    axis: 'horizontal' | 'vertical'
+  ) {
+    const range = Math.max(1, max - min)
+    const targetRatio = (targetValue - min) / range
+    const stepRatio = 1 / range
+    const centerBias = stepRatio / 2
+    const direction = currentValue === null ? 0 : Math.sign(targetValue - currentValue)
+    const controlLengthPx = axis === 'vertical' ? Math.max(1, bounds[3] - bounds[1]) : Math.max(1, bounds[2] - bounds[0])
+    const edgeWindow = Math.max(3, Math.floor(range * 0.1))
+    const isNearLowEdge = targetValue - min <= edgeWindow
+    const isNearHighEdge = max - targetValue <= edgeWindow
+    const directionBias = direction > 0
+      ? -stepRatio * 0.15
+      : direction < 0
+        ? stepRatio * 0.65
+        : 0
+    const pixelBasedMargin = Math.min(0.03, Math.max(0.005, 2 / controlLengthPx))
+    const endpointMargin = Math.max(stepRatio * 0.5, pixelBasedMargin)
+    const edgeBias = isNearLowEdge
+      ? endpointMargin
+      : isNearHighEdge
+        ? Math.max(stepRatio * 0.4, endpointMargin * 0.75)
+        : 0
+    const safeRatio = Math.min(
+      1 - (endpointMargin * 0.25),
+      Math.max(endpointMargin, targetRatio + centerBias + directionBias + edgeBias)
+    )
+    return ToolsInteract._buildControlPoint(bounds, safeRatio, axis)
+  }
+
+  private static _controlAxis(el: UiElement, bounds: [number, number, number, number]): 'horizontal' | 'vertical' {
+    const type = ToolsInteract._normalize(el.type ?? el.class ?? '')
+    const role = ToolsInteract._normalize(el.role ?? '')
+    if (/vertical/.test(type) || /vertical/.test(role)) return 'vertical'
+    if (/horizontal/.test(type) || /horizontal/.test(role)) return 'horizontal'
+    return (bounds[3] - bounds[1]) > (bounds[2] - bounds[0]) ? 'vertical' : 'horizontal'
+  }
+
   private static _actionFailure(
     actionType: string,
     selector: Record<string, unknown> | null,
@@ -526,6 +654,507 @@ export class ToolsInteract {
     })
   }
 
+  static async adjustControlHandler({
+    selector,
+    element_id,
+    property = 'value',
+    targetValue,
+    tolerance = 0,
+    maxAttempts = 3,
+    platform,
+    deviceId
+  }: {
+    selector?: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean },
+    element_id?: string,
+    property?: string,
+    targetValue: number,
+    tolerance?: number,
+    maxAttempts?: number,
+    platform?: 'android' | 'ios',
+    deviceId?: string
+  }): Promise<AdjustControlResponse> {
+    const actionType = 'adjust_control'
+    const targetSelector = selector ?? (element_id ? { elementId: element_id } : null)
+    const normalizedTolerance = Number.isFinite(tolerance) ? Math.max(0, tolerance) : 0
+    const attemptsLimit = Math.max(1, Math.floor(Number(maxAttempts) || 1))
+    const sourcePlatform: 'android' | 'ios' = platform || 'android'
+    let resolvedPlatform = sourcePlatform
+    let resolvedDeviceId = deviceId
+    const fingerprintBefore = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
+    let semanticFallbackElement: FindElementResponse['element'] | null = null
+
+    const buildFailure = (
+      failureCode: ActionFailureCode,
+      reason: string,
+      resolved: ActionTargetResolved | null,
+      device: any,
+      actualState: { property: string; value: number | null; raw_value?: number | null } | null,
+      attempts: number,
+      adjustmentMode: 'semantic' | 'gesture' | 'coordinate' = 'gesture',
+      retryable = false,
+      uiFingerprintAfter: string | null = null
+    ): AdjustControlResponse => {
+      const base = buildActionExecutionResult({
+        actionType,
+        sourceModule: 'interact',
+        device,
+        selector: targetSelector,
+        resolved,
+        success: false,
+        uiFingerprintBefore: fingerprintBefore,
+        uiFingerprintAfter,
+        failure: { failureCode, retryable },
+        details: {
+          target_value: targetValue,
+          tolerance: normalizedTolerance,
+          property,
+          attempts,
+          adjustment_mode: adjustmentMode,
+          actual_state: actualState,
+          converged: false,
+          within_tolerance: false,
+          reason
+        }
+      }) as AdjustControlResponse
+
+      return {
+        ...base,
+        target_state: {
+          property,
+          target_value: targetValue,
+          tolerance: normalizedTolerance
+        },
+        actual_state: actualState,
+        within_tolerance: false,
+        converged: false,
+        attempts,
+        adjustment_mode: adjustmentMode
+      }
+    }
+
+    const resolveCurrentMatch = async (): Promise<{
+      tree: any
+      device: any
+      match: { el: UiElement, idx: number } | null
+      resolvedTarget: ActionTargetResolved | null
+    } | null> => {
+      const tree = await ToolsObserve.getUITreeHandler({ platform: resolvedPlatform, deviceId: resolvedDeviceId }) as any
+      resolvedPlatform = tree?.device?.platform === 'ios' ? 'ios' : resolvedPlatform
+      resolvedDeviceId = tree?.device?.id || resolvedDeviceId
+      const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
+
+      if (element_id) {
+        const stored = ToolsInteract._resolvedUiElements.get(element_id)
+        if (!stored) {
+          return null
+        }
+        const current = ToolsInteract._findCurrentResolvedElement(elements, resolvedPlatform, resolvedDeviceId, stored)
+        if (!current) {
+          return null
+        }
+        return {
+          tree,
+          device: tree?.device,
+          match: { el: current.el, idx: current.index },
+          resolvedTarget: ToolsInteract._resolvedTargetFromElement(
+            ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, current.el, current.index),
+            current.el,
+            current.index
+          )
+        }
+      }
+
+      if (semanticFallbackElement) {
+        const fallbackBounds = ToolsInteract._normalizeBounds(
+          Array.isArray(semanticFallbackElement.bounds)
+            ? semanticFallbackElement.bounds
+            : semanticFallbackElement.bounds && typeof semanticFallbackElement.bounds === 'object'
+              ? [
+                Number((semanticFallbackElement.bounds as any).left),
+                Number((semanticFallbackElement.bounds as any).top),
+                Number((semanticFallbackElement.bounds as any).right),
+                Number((semanticFallbackElement.bounds as any).bottom)
+              ]
+              : null
+        )
+
+        let matchedIndex = -1
+        if (fallbackBounds) {
+          matchedIndex = elements.findIndex((el) => {
+            const bounds = ToolsInteract._normalizeBounds(el.bounds)
+            return !!bounds && bounds[0] === fallbackBounds[0] && bounds[1] === fallbackBounds[1] && bounds[2] === fallbackBounds[2] && bounds[3] === fallbackBounds[3]
+          })
+        }
+
+        if (matchedIndex === -1 && fallbackBounds) {
+          const fallbackCenterX = Math.floor((fallbackBounds[0] + fallbackBounds[2]) / 2)
+          const fallbackCenterY = Math.floor((fallbackBounds[1] + fallbackBounds[3]) / 2)
+          let bestDistance = Infinity
+          for (let i = 0; i < elements.length; i++) {
+            const el = elements[i]
+            if (!ToolsInteract._isAdjustableControl(el)) continue
+            const bounds = ToolsInteract._normalizeBounds(el.bounds)
+            if (!bounds) continue
+            const centerX = Math.floor((bounds[0] + bounds[2]) / 2)
+            const centerY = Math.floor((bounds[1] + bounds[3]) / 2)
+            const distance = Math.abs(centerX - fallbackCenterX) + Math.abs(centerY - fallbackCenterY)
+            if (distance < bestDistance) {
+              bestDistance = distance
+              matchedIndex = i
+            }
+          }
+        }
+
+        if (matchedIndex >= 0 && elements[matchedIndex]) {
+          const matched = { el: elements[matchedIndex], idx: matchedIndex }
+          return {
+            tree,
+            device: tree?.device,
+            match: matched,
+            resolvedTarget: ToolsInteract._resolvedTargetFromElement(
+              ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, matched.el, matched.idx),
+              matched.el,
+              matched.idx
+            )
+          }
+        }
+      }
+
+      if (selector) {
+        const matched = ToolsInteract._findFirstMatchingElement(elements, selector)
+        if (!matched) {
+          return null
+        }
+        return {
+          tree,
+          device: tree?.device,
+          match: matched,
+          resolvedTarget: ToolsInteract._resolvedTargetFromElement(
+            ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, matched.el, matched.idx),
+            matched.el,
+            matched.idx
+          )
+        }
+      }
+
+      return null
+    }
+
+    if (!selector && !element_id) {
+      return buildFailure('ELEMENT_NOT_FOUND', 'selector or element_id is required', null, undefined, null, 0, 'gesture', false)
+    }
+
+    if (selector && !element_id) {
+      const waitResult = await ToolsInteract.waitForUIHandler({
+        selector,
+        condition: 'clickable',
+        timeout_ms: 5000,
+        poll_interval_ms: 300,
+        platform: resolvedPlatform,
+        deviceId: resolvedDeviceId
+      }) as any
+
+      if (waitResult?.status !== 'success' || !waitResult?.element?.elementId) {
+        const semanticQuery = selector.text ?? selector.resource_id ?? selector.accessibility_id ?? ''
+        if (!semanticQuery) {
+          return buildFailure(
+            waitResult?.error?.code === 'ELEMENT_NOT_FOUND' ? 'ELEMENT_NOT_FOUND' : 'TIMEOUT',
+            waitResult?.error?.message ?? 'adjustable control not found',
+            null,
+            waitResult?.device,
+            null,
+            0,
+            'gesture',
+            waitResult?.error?.code === 'ELEMENT_NOT_FOUND'
+          )
+        }
+
+        const fallback = await ToolsInteract.findElementHandler({
+          query: semanticQuery,
+          exact: false,
+          timeoutMs: 3000,
+          platform: resolvedPlatform,
+          deviceId: resolvedDeviceId
+        })
+
+        if (!fallback.found || !fallback.element) {
+          return buildFailure(
+            'ELEMENT_NOT_FOUND',
+            waitResult?.error?.message ?? 'adjustable control not found',
+            null,
+            waitResult?.device,
+            null,
+            0,
+            'gesture',
+            true
+          )
+        }
+
+        semanticFallbackElement = fallback.element
+      } else {
+        element_id = waitResult.element.elementId
+        semanticFallbackElement = null
+      }
+    }
+
+    let lastObservedState: { property: string; value: number | null; raw_value?: number | null } | null = null
+    let lastAdjustmentMode: 'semantic' | 'gesture' | 'coordinate' = 'gesture'
+    let resolvedTarget: ActionTargetResolved | null = null
+    let currentDevice: any = undefined
+    let attemptCount = 0
+    let cachedResolvedMatch: { el: UiElement, idx: number } | null = null
+
+    for (let attempt = 0; attempt < attemptsLimit; attempt++) {
+      const resolved: {
+        tree: any
+        device: any
+        match: { el: UiElement, idx: number } | null
+        resolvedTarget: ActionTargetResolved | null
+      } | null = cachedResolvedMatch
+        ? {
+          tree: null,
+          device: currentDevice,
+          match: cachedResolvedMatch,
+          resolvedTarget: ToolsInteract._resolvedTargetFromElement(
+            ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, cachedResolvedMatch.el, cachedResolvedMatch.idx),
+            cachedResolvedMatch.el,
+            cachedResolvedMatch.idx
+          )
+        }
+        : await resolveCurrentMatch()
+      if (!resolved || !resolved.match || !resolved.resolvedTarget) {
+        return buildFailure('STALE_REFERENCE', 'adjustable control could not be resolved', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true)
+      }
+
+      currentDevice = resolved.device
+      resolvedTarget = resolved.resolvedTarget
+      const currentEl: UiElement = resolved.match.el
+      cachedResolvedMatch = resolved.match
+      const bounds = ToolsInteract._normalizeBounds(currentEl.bounds)
+      const valueRange = currentEl.state?.value_range ?? null
+      const currentValue = ToolsInteract._readNumericControlValue(currentEl, property)
+      const actualState = currentValue !== null
+        ? { property, value: currentValue, raw_value: typeof currentEl.state?.raw_value === 'number' ? currentEl.state.raw_value : undefined }
+        : null
+
+      lastObservedState = actualState
+
+      if (property !== 'value' && property !== 'raw_value') {
+        return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjust_control currently supports numeric value and raw_value properties only', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
+      }
+
+      if (currentValue !== null && Math.abs(currentValue - targetValue) <= normalizedTolerance) {
+        const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
+        const base = buildActionExecutionResult({
+          actionType,
+          sourceModule: 'interact',
+          device: currentDevice,
+          selector: targetSelector,
+          resolved: resolvedTarget,
+          success: true,
+          uiFingerprintBefore: fingerprintBefore,
+          uiFingerprintAfter,
+          details: {
+            target_value: targetValue,
+            tolerance: normalizedTolerance,
+            property,
+            attempts: attemptCount,
+            adjustment_mode: 'semantic',
+            actual_state: actualState,
+            converged: true,
+            within_tolerance: true,
+            reason: 'control already within tolerance'
+          }
+        }) as AdjustControlResponse
+
+        return {
+          ...base,
+          target_state: {
+            property,
+            target_value: targetValue,
+            tolerance: normalizedTolerance
+          },
+          actual_state: actualState,
+          within_tolerance: true,
+          converged: true,
+          attempts: attemptCount,
+          adjustment_mode: 'semantic'
+        }
+      }
+
+      if (!ToolsInteract._isAdjustableControl(currentEl)) {
+        return buildFailure('ELEMENT_NOT_INTERACTABLE', 'target is not an adjustable control', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
+      }
+
+      if (!bounds) {
+        return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjustable control has no bounds', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
+      }
+
+      const min = typeof valueRange?.min === 'number' ? valueRange.min : null
+      const max = typeof valueRange?.max === 'number' ? valueRange.max : null
+      if (min === null || max === null || !Number.isFinite(min) || !Number.isFinite(max) || max <= min) {
+        return buildFailure('ELEMENT_NOT_INTERACTABLE', 'value_range unavailable', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
+      }
+
+      if (targetValue < min || targetValue > max) {
+        return buildFailure('UNKNOWN', `targetValue ${targetValue} is outside the control range ${min}..${max}`, resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
+      }
+
+      const axis = ToolsInteract._controlAxis(currentEl, bounds)
+      const targetPoint = ToolsInteract._buildConservativeControlPoint(bounds, targetValue, currentValue, min, max, axis)
+      const currentPoint = currentValue !== null
+        ? ToolsInteract._buildControlPoint(bounds, (currentValue - min) / (max - min), axis)
+        : ToolsInteract._buildControlPoint(bounds, 0.5, axis)
+
+      const runVerification = async (): Promise<{
+        verification: any
+        observedState: { property: string; value: number | null; raw_value?: number | null } | null
+        withinTolerance: boolean
+      }> => {
+        const verification = await ToolsInteract.expectStateHandler({
+          element_id: resolvedTarget?.elementId ?? element_id,
+          selector: selector ?? undefined,
+          property,
+          expected: targetValue,
+          platform: resolvedPlatform,
+          deviceId: resolvedDeviceId
+        }) as any
+
+        const observedValue = typeof verification?.observed_state?.value === 'number'
+          ? verification.observed_state.value
+          : typeof verification?.observed_state?.raw_value === 'number'
+            ? verification.observed_state.raw_value
+            : null
+        const observedState = observedValue !== null
+          ? {
+            property,
+            value: observedValue,
+            raw_value: typeof verification?.observed_state?.raw_value === 'number' ? verification.observed_state.raw_value : undefined
+          }
+          : actualState
+
+        return {
+          verification,
+          observedState,
+          withinTolerance: observedValue !== null && Math.abs(observedValue - targetValue) <= normalizedTolerance
+        }
+      }
+
+      lastAdjustmentMode = 'coordinate'
+      const primaryActionResult = await ToolsInteract.tapHandler({
+        platform: resolvedPlatform,
+        x: targetPoint.x,
+        y: targetPoint.y,
+        deviceId: resolvedDeviceId
+      })
+      let actionDevice = primaryActionResult.device ?? currentDevice
+      attemptCount++
+
+      if (!primaryActionResult.success) {
+        lastAdjustmentMode = 'gesture'
+        const fallbackActionResult = await ToolsInteract.swipeHandler({
+          platform: resolvedPlatform,
+          x1: currentPoint.x,
+          y1: currentPoint.y,
+          x2: targetPoint.x,
+          y2: targetPoint.y,
+          duration: 220,
+          deviceId: resolvedDeviceId
+        })
+        attemptCount++
+
+        if (!fallbackActionResult.success) {
+          return buildFailure('UNKNOWN', fallbackActionResult.error ?? primaryActionResult.error ?? 'adjustment gesture failed', resolvedTarget, fallbackActionResult.device ?? primaryActionResult.device, actualState, attemptCount, lastAdjustmentMode, false)
+        }
+
+        actionDevice = fallbackActionResult.device ?? actionDevice
+      }
+
+      let verificationResult = await runVerification()
+      let observedState = verificationResult.observedState
+      lastObservedState = observedState
+
+      if (!verificationResult.withinTolerance && currentValue !== null) {
+        lastAdjustmentMode = 'gesture'
+        const fallbackActionResult = await ToolsInteract.swipeHandler({
+          platform: resolvedPlatform,
+          x1: currentPoint.x,
+          y1: currentPoint.y,
+          x2: targetPoint.x,
+          y2: targetPoint.y,
+          duration: 220,
+          deviceId: resolvedDeviceId
+        })
+        attemptCount++
+        if (!fallbackActionResult.success) {
+          return buildFailure('UNKNOWN', fallbackActionResult.error ?? 'adjustment gesture failed', resolvedTarget, fallbackActionResult.device, observedState ?? actualState, attemptCount, lastAdjustmentMode, false)
+        }
+
+        verificationResult = await runVerification()
+        observedState = verificationResult.observedState
+      }
+
+      const verification = verificationResult.verification
+      lastObservedState = observedState
+
+      if (verificationResult.withinTolerance) {
+        const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
+        const base = buildActionExecutionResult({
+          actionType,
+          sourceModule: 'interact',
+          device: actionDevice ?? currentDevice,
+          selector: targetSelector,
+          resolved: resolvedTarget,
+          success: true,
+          uiFingerprintBefore: fingerprintBefore,
+          uiFingerprintAfter,
+          details: {
+            target_value: targetValue,
+            tolerance: normalizedTolerance,
+            property,
+            attempts: attemptCount,
+            adjustment_mode: lastAdjustmentMode,
+            actual_state: observedState,
+            converged: true,
+            within_tolerance: true,
+            reason: verification?.reason ?? 'control converged to target value'
+          }
+        }) as AdjustControlResponse
+
+        return {
+          ...base,
+          target_state: {
+            property,
+            target_value: targetValue,
+            tolerance: normalizedTolerance
+          },
+          actual_state: observedState,
+          within_tolerance: true,
+          converged: true,
+          attempts: attemptCount,
+          adjustment_mode: lastAdjustmentMode
+        }
+      }
+
+      cachedResolvedMatch = {
+        el: {
+          ...currentEl,
+          state: {
+            ...(currentEl.state ?? null),
+            ...(observedState ? {
+              [observedState.property]: observedState.value,
+              raw_value: observedState.raw_value ?? observedState.value
+            } : {})
+          }
+        },
+        idx: resolved.match.idx
+      }
+    }
+
+    const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
+    return buildFailure('TIMEOUT', 'control did not converge within the allotted attempts', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true, uiFingerprintAfter)
+  }
+
   static async swipeHandler({ platform = 'android', x1, y1, x2, y2, duration, deviceId }: { platform?: 'android' | 'ios', x1: number, y1: number, x2: number, y2: number, duration: number, deviceId?: string }) {
     const { interact, resolved } = await ToolsInteract.getInteractionService(platform, deviceId)
     return await interact.swipe(x1, y1, x2, y2, duration, resolved.id)
@@ -546,7 +1175,7 @@ export class ToolsInteract {
     return await interact.scrollToElement(selector, direction, maxScrolls, scrollAmount, resolved.id)
   }
 
-  static async findElementHandler({ query, exact = false, timeoutMs = 3000, platform, deviceId }: { query: string, exact?: boolean, timeoutMs?: number, platform?: 'android' | 'ios', deviceId?: string }) {
+  static async findElementHandler({ query, exact = false, timeoutMs = 3000, platform, deviceId }: { query: string, exact?: boolean, timeoutMs?: number, platform?: 'android' | 'ios', deviceId?: string }): Promise<FindElementResponse> {
     // Try to use observe layer to fetch the current UI tree and perform a fast semantic search
     const start = Date.now()
     const deadline = start + timeoutMs
@@ -555,16 +1184,17 @@ export class ToolsInteract {
     const q = normalize(query)
     if (!q) return { found: false, error: 'Empty query' }
 
-    let best: UiElement | null = null
-    let bestScore = 0
-    let lastTree: any = null
+    let best: RankedResolutionCandidate | null = null
+    let bestTree: any = null
+    let bestIterationCandidates: RankedResolutionCandidate[] = []
+    let shouldStop = false
 
-    const scoreElement = (el: UiElement | null) => {
-      if (!el || !el.visible) return 0
+    const scoreElement = (el: UiElement | null, idx: number): RankedResolutionCandidate | null => {
+      if (!el || !el.visible) return null
       const bounds = el.bounds || [0,0,0,0]
-      if (!Array.isArray(bounds) || bounds.length < 4) return 0
+      if (!Array.isArray(bounds) || bounds.length < 4) return null
       const [l,t,r,b] = bounds
-      if (r <= l || b <= t) return 0
+      if (r <= l || b <= t) return null
       // Do not early-return on non-interactable elements — score them so we can locate their clickable ancestor later
       const interactable = !!(el.clickable || el.enabled || el.focusable)
 
@@ -574,44 +1204,80 @@ export class ToolsInteract {
       const className = normalize(el.type ?? el.class ?? '')
 
       let score = 0
+      let reason = 'best_scoring_candidate'
       if (exact) {
-        if (text && text === q) score = 1.0
-        else if (content && content === q) score = 0.95
+        if (text && text === q) {
+          score = 1.0
+          reason = 'exact_text_match'
+        } else if (content && content === q) {
+          score = 0.95
+          reason = 'exact_content_desc_match'
+        } else if (resourceId && resourceId === q) {
+          score = 0.92
+          reason = 'exact_resource_id_match'
+        } else if (className && className === q) {
+          score = 0.3
+          reason = 'exact_class_match'
+        }
       } else {
-        if (text && text === q) score = 1.0
-        else if (content && content === q) score = 0.95
-        else if (text && text.includes(q)) score = 0.6
-        else if (content && content.includes(q)) score = 0.55
-        else if (resourceId && resourceId.includes(q)) score = 0.7
-        else if (className && className.includes(q)) score = 0.3
+        if (text && text === q) {
+          score = 1.0
+          reason = 'exact_text_match'
+        } else if (content && content === q) {
+          score = 0.95
+          reason = 'exact_content_desc_match'
+        } else if (resourceId && resourceId === q) {
+          score = 0.92
+          reason = 'exact_resource_id_match'
+        } else if (text && text.includes(q)) {
+          score = 0.6
+          reason = 'partial_text_match'
+        } else if (content && content.includes(q)) {
+          score = 0.55
+          reason = 'partial_content_desc_match'
+        } else if (resourceId && resourceId.includes(q)) {
+          score = 0.7
+          reason = 'partial_resource_id_match'
+        } else if (className && className.includes(q)) {
+          score = 0.3
+          reason = 'partial_class_match'
+        }
       }
       if (score > 0 && interactable) score += 0.05
-      return score
+      if (score <= 0) return null
+      return { el, idx, score, reason, interactable }
     }
 
     while (Date.now() <= deadline) {
       try {
-      const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId })
-      lastTree = tree
+        const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId })
         if (tree && Array.isArray((tree as any).elements)) {
           const elements = ((tree as any).elements as UiElement[])
+          const iterationCandidates: RankedResolutionCandidate[] = []
+          let iterationImprovedBest = false
           for (let i = 0; i < elements.length; i++) {
             const el = elements[i]
             try {
-              const s = scoreElement(el)
-              const interactable = !!(el.clickable || el.enabled || (el as any).focusable)
-              if (s > bestScore) {
-                bestScore = s
-                best = el as UiElement
-                if (best) { best._index = i; best._interactable = interactable }
+              const candidate = scoreElement(el, i)
+              if (!candidate) continue
+              iterationCandidates.push(candidate)
+              if (!best || candidate.score > best.score) {
+                best = candidate
+                bestTree = tree
+                iterationImprovedBest = true
+                if (best.score >= 0.95) {
+                  shouldStop = true
+                  break
+                }
               }
-              if (bestScore >= 0.95) break
             } catch (e) { console.error('Error scoring element:', e) }
           }
-          if (bestScore >= 0.95) break
+          if (iterationImprovedBest) {
+            bestIterationCandidates = iterationCandidates.slice()
+          }
         }
       } catch (e) { console.error('Error fetching UI tree:', e) }
-      if (Date.now() > deadline) break
+      if (shouldStop || Date.now() > deadline) break
       await new Promise(r => setTimeout(r, 100))
     }
 
@@ -619,31 +1285,32 @@ export class ToolsInteract {
 
     // If the best match is not interactable, try to resolve an actionable ancestor.
     try {
-      const elements = (lastTree && Array.isArray(lastTree.elements)) ? (lastTree.elements as UiElement[]) : []
-      const screen = lastTree?.resolution && typeof lastTree.resolution === 'object' ? lastTree.resolution as UiResolution : null
-      let chosen = best as any
-      const childBounds = Array.isArray(chosen?.bounds) ? chosen.bounds : null
+      const elements = (bestTree && Array.isArray(bestTree.elements)) ? (bestTree.elements as UiElement[]) : []
+      const screen = bestTree?.resolution && typeof bestTree.resolution === 'object' ? bestTree.resolution as UiResolution : null
+      let chosen = best as { el: UiElement, idx: number }
+      const childBounds = Array.isArray(chosen?.el?.bounds) ? chosen.el.bounds : null
 
       // Strategy 1: if parentId references an index, climb that chain
-      let resolvedAncestor: any = null
-      if (childBounds && (chosen.parentId !== undefined && chosen.parentId !== null)) {
+      let resolvedAncestor: { el: UiElement, idx: number } | null = null
+      if (childBounds && (chosen.el.parentId !== undefined && chosen.el.parentId !== null)) {
         let cur = chosen
         let safety = 0
-        while (cur && safety < 20 && !(cur.clickable || cur.focusable) && (cur.parentId !== undefined && cur.parentId !== null)) {
-          let pid = cur.parentId
+        while (cur && safety < 20 && !(cur.el.clickable || cur.el.focusable) && (cur.el.parentId !== undefined && cur.el.parentId !== null)) {
+          let pid = cur.el.parentId
           let idx: number | null = null
           if (typeof pid === 'number') idx = pid
           else if (typeof pid === 'string' && /^\d+$/.test(pid)) idx = Number(pid)
           // If parentId is not an index, try to find by matching resourceId or id field
           if (idx !== null && elements[idx]) {
-            cur = elements[idx]
-            if (cur && (cur.clickable || cur.enabled || cur.focusable)) { resolvedAncestor = cur; break }
+            cur = { el: elements[idx], idx }
+            if (cur && (cur.el.clickable || cur.el.enabled || cur.el.focusable)) { resolvedAncestor = cur; break }
           } else if (typeof pid === 'string') {
             // fallback: search elements for matching resourceId or id
-            const found = elements.find((el: UiElement)=> (el.resourceId === pid || el.id === pid))
+            const foundIndex = elements.findIndex((el: UiElement)=> (el.resourceId === pid || el.id === pid))
+            const found = foundIndex >= 0 ? elements[foundIndex] : null
             if (found) {
-              cur = found
-              if (cur && (cur.clickable || cur.enabled || cur.focusable)) { resolvedAncestor = cur; break }
+              cur = { el: found, idx: foundIndex }
+              if (cur && (cur.el.clickable || cur.el.enabled || cur.el.focusable)) { resolvedAncestor = cur; break }
               // otherwise continue climbing if this found element has its own parentId
             } else {
               break
@@ -659,62 +1326,77 @@ export class ToolsInteract {
       if (!resolvedAncestor && childBounds) {
         const [cl,ct,cr,cb] = childBounds
         // find candidates that are clickable and contain the child bounds
-        const candidates = elements.filter((el: UiElement)=> el && (el.clickable || el.focusable) && Array.isArray(el.bounds) && el.bounds!.length>=4).map((el: UiElement)=>({el, bounds: el.bounds! as number[]}))
-        let bestCandidate: any = null
+        const candidates = elements
+          .map((el: UiElement, idx: number) => ({ el, idx }))
+          .filter(({ el }) => el && (el.clickable || el.focusable) && Array.isArray(el.bounds) && el.bounds!.length >= 4)
+        let bestCandidate: { el: UiElement, idx: number } | null = null
         let bestCandidateArea = Infinity
         for (const c of candidates) {
-          const [pl,pt,pr,pb] = c.bounds
+          const bounds = c.el.bounds as number[]
+          const [pl,pt,pr,pb] = bounds
           if (pl <= cl && pt <= ct && pr >= cr && pb >= cb) {
             const area = (pr-pl) * (pb-pt)
-            if (area < bestCandidateArea) { bestCandidateArea = area; bestCandidate = c.el }
+            if (area < bestCandidateArea) { bestCandidateArea = area; bestCandidate = c }
           }
         }
         if (bestCandidate) resolvedAncestor = bestCandidate
       }
 
       if (resolvedAncestor) {
-        best = resolvedAncestor
-        // small score bump to reflect actionability
-        bestScore = Math.min(1, bestScore + 0.02)
+        best = {
+          el: resolvedAncestor.el,
+          idx: resolvedAncestor.idx,
+          score: Math.min(1, best.score + 0.02),
+          reason: 'clickable_parent_preferred',
+          interactable: true
+        }
       }
 
-      if (best && !(best.clickable || best.focusable)) {
-        const nearbyActionable = ToolsInteract._resolveNearbyActionableControl(elements, { el: best, idx: best._index ?? elements.indexOf(best) }, screen)
+      if (best && !(best.el.clickable || best.el.focusable)) {
+        const nearbyActionable = ToolsInteract._resolveNearbyActionableControl(elements, { el: best.el, idx: best.idx }, screen)
         if (nearbyActionable) {
-          best = nearbyActionable.el
-          best._index = nearbyActionable.idx
-          best._interactable = true
-          best._sliderLike = nearbyActionable.sliderLike
+          best = {
+            el: nearbyActionable.el,
+            idx: nearbyActionable.idx,
+            score: Math.min(1, best.score + 0.02),
+            reason: nearbyActionable.sliderLike ? 'slider_track_preferred' : 'nearby_actionable_control',
+            interactable: true
+          }
         }
       }
     } catch (e) { console.error('Error resolving ancestor:', e) }
 
     if (!best) return { found: false, error: 'Element not found' }
 
-    const boundsObj = Array.isArray(best.bounds) ? { left: best.bounds[0], top: best.bounds[1], right: best.bounds[2], bottom: best.bounds[3] } : null
+    const boundsObj = Array.isArray(best.el.bounds) ? { left: best.el.bounds[0], top: best.el.bounds[1], right: best.el.bounds[2], bottom: best.el.bounds[3] } : null
     const tapCoordinates = boundsObj ? { x: Math.floor((boundsObj.left + boundsObj.right) / 2), y: Math.floor((boundsObj.top + boundsObj.bottom) / 2) } : null
+    const uniqueRanked = bestIterationCandidates.filter((candidate, index, array) => index === array.findIndex((other) => other.idx === candidate.idx && other.el === candidate.el))
+    const alternateCandidates = uniqueRanked
+      .filter((candidate) => candidate.idx !== best.idx || candidate.el !== best.el)
+      .slice(0, 3)
+      .map((candidate) => ToolsInteract._summarizeResolutionCandidate(candidate))
 
     const outEl = {
-      text: best.text ?? null,
-      resourceId: best.resourceId ?? null,
-      contentDesc: best.contentDescription ?? best.contentDesc ?? null,
-      class: best.type ?? best.class ?? null,
+      text: best.el.text ?? null,
+      resourceId: best.el.resourceId ?? null,
+      contentDesc: best.el.contentDescription ?? best.el.contentDesc ?? null,
+      class: best.el.type ?? best.el.class ?? null,
       bounds: boundsObj,
-      clickable: !!best.clickable,
-      enabled: !!best.enabled,
-      stable_id: best.stable_id ?? null,
-      role: best.role ?? null,
-      test_tag: best.test_tag ?? null,
-      selector: best.selector ?? null,
-      semantic: best.semantic ?? null,
+      clickable: !!best.el.clickable,
+      enabled: !!best.el.enabled,
+      stable_id: best.el.stable_id ?? null,
+      role: best.el.role ?? null,
+      test_tag: best.el.test_tag ?? null,
+      selector: best.el.selector ?? null,
+      semantic: best.el.semantic ?? null,
       tapCoordinates,
       telemetry: {
-        matchedIndex: best?._index ?? null,
-        matchedInteractable: !!best?._interactable,
-        sliderLike: !!best?._sliderLike
+        matchedIndex: best.idx ?? null,
+        matchedInteractable: !!best.interactable,
+        sliderLike: best.reason === 'slider_track_preferred'
       }
     }
-    if (best?._sliderLike) {
+    if (best.reason === 'slider_track_preferred') {
       const isVertical = !!boundsObj && (boundsObj.bottom - boundsObj.top) > (boundsObj.right - boundsObj.left)
       const interactionHint = {
         kind: 'slider',
@@ -723,8 +1405,15 @@ export class ToolsInteract {
       }
       ;(outEl as any).interactionHint = interactionHint
     }
-    const scoreVal = Math.min(1, Number(bestScore.toFixed(3)))
-    return { found: true, element: outEl, score: scoreVal, confidence: scoreVal }
+    const scoreVal = Math.min(1, Number(best.score.toFixed(3)))
+    const resolution: FindElementResolutionSummary = {
+      confidence: scoreVal,
+      reason: best.reason,
+      fallback_available: alternateCandidates.length > 0,
+      matched_count: uniqueRanked.length,
+      alternates: alternateCandidates
+    }
+    return { found: true, element: outEl, score: scoreVal, confidence: scoreVal, resolution }
   }
 
   static async waitForUIHandler({ selector, condition = 'exists', timeout_ms = 60000, poll_interval_ms = 300, match, retry = { max_attempts: 1, backoff_ms: 0 }, platform, deviceId }: { selector?: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean }, condition?: 'exists'|'not_exists'|'visible'|'clickable', timeout_ms?: number, poll_interval_ms?: number, match?: { index?: number }, retry?: { max_attempts?: number, backoff_ms?: number }, platform?: 'android'|'ios', deviceId?: string }) {