mobile-debug-mcp 0.33.0 → 0.35.0

package/src/debugger/worker.ts

@@ -0,0 +1,135 @@
+import { DebuggerError, classify, diagnostic, atStage } from './diagnostics.js'
+export { DebuggerError } from './diagnostics.js'
+import { parseWorkerMessage, type WorkerEvent, type WorkerReply } from './protocol.js'
+export type { WorkerEvent } from './protocol.js'
+import { AsyncLocalStorage } from 'node:async_hooks'
+import { execFile, spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'
+import { promisify } from 'node:util'
+import { readFile, mkdtemp, chmod, rm } from 'node:fs/promises'
+import { createHash } from 'node:crypto'
+import path from 'node:path'
+import os from 'node:os'
+import { fileURLToPath } from 'node:url'
+import { detectJavaHome } from '../utils/java.js'
+
+const exec = promisify(execFile)
+const operations = new AsyncLocalStorage<{ deadline: number; signal?: AbortSignal }>()
+export function withDebuggerBudget<T>(timeout: number, signal: AbortSignal | undefined, operation: () => Promise<T>) {
+  return operations.run({ deadline: performance.now() + timeout, signal }, operation)
+}
+export function debuggerRemaining(fallback: number) {
+  const context = operations.getStore()
+  if (context?.signal?.aborted) throw new DebuggerError('DEBUGGER_CANCELLED')
+  const remaining = Math.floor((context?.deadline ?? Infinity) - performance.now())
+  if (remaining <= 0) throw new DebuggerError('DEBUGGER_TIMEOUT')
+  return Math.min(fallback, remaining)
+}
+export function debuggerSignal() { return operations.getStore()?.signal }
+export function nativeError(error: unknown, fallback: string): DebuggerError {
+  if (error instanceof DebuggerError) return error
+  if (debuggerSignal()?.aborted) return new DebuggerError('DEBUGGER_CANCELLED')
+  if (typeof error === 'object' && error !== null && 'killed' in error && error.killed) return new DebuggerError('DEBUGGER_TIMEOUT')
+  return new DebuggerError(fallback)
+}
+export const workerSource = fileURLToPath(new URL('../../src/debugger/java/DebuggerWorker.java', import.meta.url))
+let compiled: Promise<{ java: string; directory: string }> | undefined
+export async function prepareWorker() {
+  let stage: 'prerequisite' | 'compilation' = 'prerequisite'
+  compiled ??= (async () => {
+    const home = await detectJavaHome({ quiet: true }).catch(e => { throw classify(e, 'prerequisite', 'DEBUGGER_ATTACH_FAILED', 'unavailable') })
+    const java = home ? path.join(home, 'bin', 'java') : 'java'
+    const javac = home ? path.join(home, 'bin', 'javac') : 'javac'
+    const version = await exec(java, ['-version'], { timeout: debuggerRemaining(5000), signal: debuggerSignal() }).catch(e => { throw classify(e, 'prerequisite', 'DEBUGGER_ATTACH_FAILED', debuggerSignal()?.aborted ? 'cancelled' : typeof e === 'object' && e !== null && 'killed' in e && e.killed ? 'timeout' : 'unavailable') })
+    const major = Number(/version "(\d+)/.exec(version.stderr)?.[1])
+    if (!(major >= 17)) throw new DebuggerError('DEBUGGER_UNSUPPORTED', diagnostic('prerequisite', 'unsupported'))
+    stage = 'compilation'
+    const source = await readFile(workerSource).catch(e => { throw classify(e, 'compilation', 'DEBUGGER_ATTACH_FAILED', 'unavailable') })
+    const key = createHash('sha256').update(source).update(version.stderr).digest('hex').slice(0, 16)
+    const directory = await atStage('compilation', 'DEBUGGER_ATTACH_FAILED', () => mkdtemp(path.join(os.tmpdir(), `mobile-debug-jdi-${key}-`)))
+    try { await atStage('compilation', 'DEBUGGER_ATTACH_FAILED', () => chmod(directory, 0o700)); await exec(javac, ['--release', '17', '--add-modules', 'jdk.jdi', '-d', directory, workerSource], { timeout: debuggerRemaining(15000), signal: debuggerSignal(), maxBuffer: 1024 * 1024 }) }
+    catch (e) { await rm(directory, { recursive: true, force: true }); throw classify(nativeError(e, 'DEBUGGER_UNSUPPORTED'), 'compilation', 'DEBUGGER_UNSUPPORTED', 'operation_failed') }
+    return { java, directory }
+  })().catch(error => { compiled = undefined; throw classify(error, stage, 'DEBUGGER_ATTACH_FAILED') })
+  return compiled
+}
+function commandStage(command: string) { return command === 'attach' ? 'attach' as const : command === 'detach' ? 'cleanup' as const : 'command' as const }
+export interface DebugWorker {
+  call(command: string, args?: Record<string, unknown>, timeout?: number): Promise<WorkerReply>
+  close(): Promise<void>
+}
+type WorkerFailureReason = 'spawn_failed' | 'process_exited' | 'write_failed' | 'invalid_message' | 'message_limit'
+export class JdiWorker implements DebugWorker {
+  private child: ChildProcessWithoutNullStreams
+  private pending = new Map<number, { resolve(value: WorkerReply): void; reject(error: Error): void; timer: ReturnType<typeof setTimeout>; command: string }>()
+  private sequence = 0
+  private dead = false
+  private failure?: DebuggerError
+  private replied = false
+  private exited: Promise<void>
+  constructor(java: string, directory: string, private onEvent: (event: WorkerEvent) => void) {
+    this.child = spawn(java, ['-Xmx128m', '--add-modules', 'jdk.jdi', '-cp', directory, 'DebuggerWorker'], { stdio: ['pipe', 'pipe', 'pipe'] })
+    let buffer = ''
+    this.exited = new Promise(resolve => { this.child.once('exit', () => { this.fail('process_exited'); resolve() }); this.child.once('error', () => { this.fail('spawn_failed'); resolve() }) })
+    this.child.stdin.on('error', () => this.fail('write_failed'))
+    this.child.stderr.resume() // Never copy worker data or VM diagnostics into MCP stdout/logs.
+    this.child.stdout.setEncoding('utf8')
+    this.child.stdout.on('data', (chunk: string) => {
+      buffer += chunk
+      if (Buffer.byteLength(buffer) > 2 * 1024 * 1024) { this.fail('message_limit'); this.child.kill('SIGKILL'); return }
+      let end: number
+      while ((end = buffer.indexOf('\n')) >= 0) {
+        const line = buffer.slice(0, end); buffer = buffer.slice(end + 1)
+        try {
+          if (Buffer.byteLength(line) > 1024 * 1024) throw new Error('IPC_LIMIT')
+          const msg = parseWorkerMessage(JSON.parse(line))
+          if (msg.kind === 'event') onEvent(msg.value)
+          else {
+            this.replied = true
+            const request = this.pending.get(msg.id)
+            if (!request) continue
+            clearTimeout(request.timer); this.pending.delete(msg.id)
+            if (msg.error) request.reject(classify(new DebuggerError(msg.error.code ?? 'DEBUGGER_WORKER_FAILED', msg.error.diagnostic), commandStage(request.command), 'DEBUGGER_WORKER_FAILED'))
+            else request.resolve(msg.result)
+          }
+        } catch { this.fail(Buffer.byteLength(line) > 1024 * 1024 ? 'message_limit' : 'invalid_message'); this.child.kill('SIGKILL'); return }
+      }
+    })
+  }
+  private fail(reason: WorkerFailureReason) {
+    if (this.dead) return
+    this.dead = true
+    const stage = reason === 'spawn_failed' || (reason === 'process_exited' && !this.replied) ? 'worker_start' : 'ipc'
+    this.failure = new DebuggerError('DEBUGGER_WORKER_FAILED', diagnostic(stage, reason))
+    for (const request of this.pending.values()) {
+      clearTimeout(request.timer)
+      request.reject(this.failure)
+    }
+    this.pending.clear()
+    this.onEvent({ event: 'worker_failed', diagnostic: this.failure.diagnostic })
+  }
+  call(command: string, args: Record<string, unknown> = {}, timeout = 10000): Promise<WorkerReply> {
+    if (this.dead) return Promise.reject(this.failure ?? new DebuggerError('DEBUGGER_WORKER_FAILED', diagnostic('ipc', 'unknown')))
+    try { timeout = Math.min(timeout, debuggerRemaining(timeout)) } catch (e) { return Promise.reject(classify(e, commandStage(command), 'DEBUGGER_WORKER_FAILED')) }
+    const id = ++this.sequence
+    const message = JSON.stringify({ ...args, command, id }) + '\n'
+    if (Buffer.byteLength(message) > 1024 * 1024) return Promise.reject(new DebuggerError('INVALID_ARGUMENT'))
+    return new Promise((resolve, reject) => {
+      const signal = debuggerSignal()
+      const abort = () => { this.pending.delete(id); clearTimeout(timer); done(); reject(new DebuggerError('DEBUGGER_CANCELLED', diagnostic(commandStage(command), 'cancelled'))) }
+      const done = () => signal?.removeEventListener('abort', abort)
+      const timer = setTimeout(() => { this.pending.delete(id); done(); reject(new DebuggerError('DEBUGGER_TIMEOUT', diagnostic(commandStage(command), 'timeout'))) }, timeout)
+      this.pending.set(id, { command, resolve: value => { done(); resolve(value) }, reject: error => { done(); reject(error) }, timer })
+      signal?.addEventListener('abort', abort, { once: true })
+      try { this.child.stdin.write(message) } catch { this.fail('write_failed') }
+    })
+  }
+  async close() {
+    this.child.stdin.end()
+    const kill = setTimeout(() => this.child.kill('SIGKILL'), 2000)
+    try { await this.exited } finally { clearTimeout(kill) }
+  }
+}
+export async function createWorker(onEvent: (event: WorkerEvent) => void) {
+  const { java, directory } = await prepareWorker()
+  return new JdiWorker(java, directory, onEvent)
+}

package/src/interact/adjustment/adjustment-context.ts

@@ -0,0 +1,83 @@
+import { systemClock, type Clock } from '../../runtime/clock.js'
+import type { ActionFailureCode, ActionTargetResolved, AdjustControlResponse, DeviceInfo, FindElementResponse, TraceStep } from '../../types.js'
+import type { UiSelector } from '../../contracts/selector.js'
+import type { UiElement, ResolvedUiElementContext } from '../model.js'
+import type { InputService } from '../input-service.js'
+import type { AdjustmentVerificationDependencies } from './adjustment-verification.js'
+import { createTraceStep, buildActionExecutionResult } from '../../runtime/action-results.js'
+import { completeAdjustment } from './adjustment-result.js'
+import { planAdjustment } from './adjustment-plan.js'
+
+export interface AdjustmentInput { selector?: UiSelector; element_id?: string; property?: string; targetValue: number; tolerance?: number; maxAttempts?: number; platform?: 'android' | 'ios'; deviceId?: string }
+export interface ControlTree { elements?: UiElement[]; device?: DeviceInfo }
+export type Match = { el: UiElement; idx: number }
+export interface ControlMatch { tree: ControlTree | null; device?: DeviceInfo; match: Match | null; resolvedTarget: ActionTargetResolved | null }
+export interface ControlWait { status?: string; element?: { elementId?: string | null } | null; device?: DeviceInfo; error?: { code?: string; message?: string } }
+export interface AdjustmentDependencies {
+  clock?: Clock
+  observe(platform: 'android' | 'ios', deviceId?: string): Promise<ControlTree>
+  fingerprint(platform: 'android' | 'ios', deviceId?: string): Promise<string | null>
+  getHandle(id: string): ResolvedUiElementContext | undefined
+  currentHandle(elements: UiElement[], platform: 'android' | 'ios', deviceId: string | undefined, handle: ResolvedUiElementContext): { el: UiElement; index: number } | null
+  target(platform: 'android' | 'ios', deviceId: string | undefined, match: Match): ActionTargetResolved
+  wait(input: { selector: UiSelector; condition: 'clickable'; timeout_ms: number; poll_interval_ms: number; platform: 'android' | 'ios'; deviceId?: string }): Promise<ControlWait | undefined>
+  find(input: { query: string; exact: boolean; timeoutMs: number; platform: 'android' | 'ios'; deviceId?: string }): Promise<FindElementResponse>
+  expectState: AdjustmentVerificationDependencies['expectState']
+  service(platform: 'android' | 'ios', deviceId?: string): Promise<{ interact: Pick<InputService, 'setAdjustableValue' | 'adjustAccessibleValue'>; resolved: { id: string } }>
+  tap(input: { platform: 'android' | 'ios'; x: number; y: number; deviceId?: string }): Promise<{ success: boolean; device?: DeviceInfo }>
+  swipe(input: { platform: 'android' | 'ios'; x1: number; y1: number; x2: number; y2: number; duration: number; deviceId?: string }): Promise<{ success: boolean; device?: DeviceInfo; error?: string }>
+}
+export type ActualState = AdjustControlResponse['actual_state']
+export type Mode = AdjustControlResponse['adjustment_mode']
+export type Verification = { verification: { reason?: string } | null; observedState: ActualState; withinTolerance: boolean }
+export interface AdjustmentState {
+  clock: Clock
+  input: AdjustmentInput & { property: string }
+  platform: 'android' | 'ios'; deviceId?: string; stored: ResolvedUiElementContext | null
+  selector: Record<string, unknown> | null; tolerance: number | null; limit: number; before: string | null
+  fallback: FindElementResponse['element'] | null; trace: TraceStep[]; sequence: number
+  observed: ActualState; mode: Mode; target: ActionTargetResolved | null; device?: DeviceInfo
+  attempts: number; cached: Match | null; coordinatePending: boolean
+}
+export type Iteration = ReturnType<typeof planAdjustment> & {
+  element: UiElement; index: number; actionDevice?: DeviceInfo; observed: ActualState; verification: Verification
+}
+export type StrategyOutcome = 'next' | 'retry' | AdjustControlResponse
+
+export async function createAdjustment(input: AdjustmentInput, deps: AdjustmentDependencies): Promise<AdjustmentState> {
+  const platform = input.platform || 'android'
+  const tolerance = typeof input.tolerance === 'number' && Number.isFinite(input.tolerance) ? Math.max(0, input.tolerance) : null
+  const limit = input.maxAttempts == null ? 5 : Math.max(1, Math.floor(Number(input.maxAttempts) || 1))
+  const stored = input.element_id ? deps.getHandle(input.element_id) ?? null : null
+  const before = await deps.fingerprint(platform, input.deviceId)
+  return { clock: deps.clock ?? systemClock, input: { ...input, property: input.property === undefined ? 'value' : input.property }, platform, deviceId: input.deviceId, stored,
+    selector: input.selector ?? (input.element_id ? { elementId: input.element_id } : null), tolerance, limit, before,
+    fallback: null, trace: [], sequence: 0, observed: null, mode: 'gesture', target: null,
+    attempts: 0, cached: null, coordinatePending: false }
+}
+export function recordAdjustment(s: AdjustmentState, stage: TraceStep['stage'], result: TraceStep['result'], metadata?: Record<string, unknown>) {
+  s.trace.push(createTraceStep({ stage, timestamp: s.clock.now(), result, attemptIndex: s.sequence++, metadata }))
+}
+export function failAdjustment(s: AdjustmentState, code: ActionFailureCode, reason: string, retryable = false,
+  target = s.target, device = s.device, observed = s.observed, attempts = s.attempts, mode: Mode = s.mode, after: string | null = null): AdjustControlResponse {
+  if (!s.trace.some(step => step.stage === 'resolve')) recordAdjustment(s, 'resolve', 'failure', { reason, failure_code: code })
+  if (!s.trace.some(step => step.stage === 'recover')) recordAdjustment(s, 'recover', retryable ? 'retry' : 'failure', {
+    reason, failure_code: code, retry_allowed: retryable, recovery_attempts: attempts, retry_depth: attempts })
+  const tolerance = s.tolerance ?? 0
+  const base = buildActionExecutionResult({ actionType: 'adjust_control', sourceModule: 'interact', device,
+    selector: s.selector, resolved: target, success: false, uiFingerprintBefore: s.before, uiFingerprintAfter: after,
+    failure: { failureCode: code, retryable }, traceSteps: s.trace,
+    details: { target_value: s.input.targetValue, tolerance, property: s.input.property, attempts,
+      adjustment_mode: mode, actual_state: observed, converged: false, within_tolerance: false, reason } })
+  return { ...base, target_state: { property: s.input.property, target_value: s.input.targetValue, tolerance },
+    actual_state: observed, within_tolerance: false, converged: false, attempts, adjustment_mode: mode }
+}
+export async function succeedAdjustment(s: AdjustmentState, p: Iteration, deps: AdjustmentDependencies,
+  reason = 'control converged to target value', mode = s.mode, device = p.actionDevice ?? s.device) {
+  const after = await deps.fingerprint(s.platform, s.deviceId)
+  return completeAdjustment({ actionType: 'adjust_control', sourceModule: 'interact', device,
+    selector: s.selector, resolved: s.target, success: true, uiFingerprintBefore: s.before, uiFingerprintAfter: after,
+    traceSteps: s.trace, details: { target_value: p.normalizedTargetValue, tolerance: p.effectiveTolerance,
+      property: s.input.property, attempts: s.attempts, adjustment_mode: mode, actual_state: p.observed,
+      converged: true, within_tolerance: true, reason } })
+}

package/src/interact/adjustment/adjustment-gestures.ts

@@ -0,0 +1,48 @@
+import { _controlAxis, _buildConservativeControlPoint, _buildControlPoint, _buildAdjustmentProbePoints } from './control-geometry.js'
+import { recordAdjustment, succeedAdjustment, failAdjustment, type AdjustmentState, type AdjustmentDependencies, type Iteration, type StrategyOutcome } from './adjustment-context.js'
+import { verifyControl } from './adjustment-semantic.js'
+
+function points(p: Iteration) {
+  const bounds = p.bounds!, range = p.valueRange!, value = p.currentValue!
+  const axis = _controlAxis(p.element, bounds)
+  return { axis, target: _buildConservativeControlPoint(bounds, p.normalizedTargetValue, value, range.min, range.max, axis),
+    current: _buildControlPoint(bounds, (value - range.min) / (range.max - range.min), axis) }
+}
+export async function coordinateAdjustment(s: AdjustmentState, p: Iteration, deps: AdjustmentDependencies): Promise<StrategyOutcome> {
+  if (s.coordinatePending) return 'next'
+  const geometry = points(p)
+  const probes = p.inferredSliderState ? [geometry.target] : _buildAdjustmentProbePoints(p.bounds!, p.normalizedTargetValue, p.currentValue!, p.valueRange!.min, p.valueRange!.max, geometry.axis)
+  for (let i = 0; i < probes.length && s.attempts < s.limit; i++) {
+    const point = probes[i]
+    s.mode = 'coordinate'
+    recordAdjustment(s, 'execute', 'retry', { attempt: s.attempts + 1, mode: 'coordinate', point })
+    const action = await deps.tap({ platform: s.platform, x: point.x, y: point.y, deviceId: s.deviceId })
+    s.attempts++; p.actionDevice = action.device ?? p.actionDevice
+    if (!action.success) {
+      recordAdjustment(s, 'execute', 'retry', { attempt: s.attempts, mode: 'coordinate', point, success: false }); continue
+    }
+    await verifyControl(s, p, deps, 'control did not converge yet')
+    if (p.verification.withinTolerance) return succeedAdjustment(s, p, deps, p.verification.verification?.reason ?? 'control converged to target value')
+    s.coordinatePending = true; s.cached = null
+    if (s.attempts < s.limit) return 'retry'
+  }
+  return 'next'
+}
+export async function gestureAdjustment(s: AdjustmentState, p: Iteration, deps: AdjustmentDependencies): Promise<StrategyOutcome> {
+  if (s.attempts >= s.limit) return 'next'
+  const geometry = points(p)
+  s.coordinatePending = false; s.mode = 'gesture'
+  recordAdjustment(s, 'execute', 'retry', { attempt: s.attempts + 1, mode: 'gesture', start: geometry.current, end: geometry.target })
+  const action = await deps.swipe({ platform: s.platform, x1: geometry.current.x, y1: geometry.current.y,
+    x2: geometry.target.x, y2: geometry.target.y, duration: p.inferredSliderState ? 520 : 220, deviceId: s.deviceId })
+  s.attempts++
+  if (!action.success) {
+    recordAdjustment(s, 'execute', 'failure', { attempt: s.attempts, mode: 'gesture', start: geometry.current, end: geometry.target, success: false })
+    return failAdjustment(s, 'UNKNOWN', action.error ?? 'adjustment gesture failed', false, s.target,
+      action.device ?? p.actionDevice, p.observed ?? p.actualState)
+  }
+  p.actionDevice = action.device ?? p.actionDevice
+  await verifyControl(s, p, deps, 'gesture adjustment did not converge yet')
+  if (p.verification.withinTolerance) return succeedAdjustment(s, p, deps, p.verification.verification?.reason ?? 'control converged to target value')
+  return 'next'
+}

package/src/interact/adjustment/adjustment-plan.ts

@@ -0,0 +1,23 @@
+import type { UiElement } from '../model.js'
+import type { AdjustControlResponse } from '../../types.js'
+import { _normalizeBounds } from '../targeting/element-helpers.js'
+import { _inferLabeledSliderState } from './slider-labels.js'
+import { _readNumericControlRange, _readControlStep, _quantizeTargetValue,
+  _defaultAdjustTolerance, _readNumericControlValue } from './control-values.js'
+
+export function planAdjustment(currentEl: UiElement, elements: UiElement[], targetValue: number,
+  property: string, requestedTolerance: number | null) {
+  const inferredSliderState = _inferLabeledSliderState(elements, currentEl)
+  const nativeBounds = _normalizeBounds(currentEl.bounds)
+  const bounds = inferredSliderState?.touch_bounds ?? nativeBounds
+  const valueRange = _readNumericControlRange(currentEl) ?? inferredSliderState?.value_range ?? null
+  const step = _readControlStep(currentEl)
+  const normalizedTargetValue = valueRange ? _quantizeTargetValue(targetValue, valueRange.min, valueRange.max, step) : targetValue
+  const effectiveTolerance = requestedTolerance ?? _defaultAdjustTolerance(valueRange, step)
+  const currentValue = _readNumericControlValue(currentEl, property) ?? inferredSliderState?.current ?? null
+  const actualState: AdjustControlResponse['actual_state'] = currentValue !== null ? {
+    property, value: currentValue,
+    raw_value: typeof currentEl.state?.raw_value === 'number' ? currentEl.state.raw_value : inferredSliderState?.current
+  } : null
+  return { inferredSliderState, bounds, valueRange, normalizedTargetValue, effectiveTolerance, currentValue, actualState }
+}

package/src/interact/adjustment/adjustment-resolution.ts

@@ -0,0 +1,104 @@
+import type { UiSelector } from '../../contracts/selector.js'
+import type { ControlWait } from './adjustment-context.js'
+import type { FindElementResponse } from '../../types.js'
+import type { UiElement } from '../model.js'
+import { _normalizeBounds, _isAdjustableControl, _findFirstMatchingElement, _isAdjustableActionable } from '../targeting/element-helpers.js'
+import { planAdjustment } from './adjustment-plan.js'
+import { failAdjustment, recordAdjustment, succeedAdjustment, type AdjustmentState, type AdjustmentDependencies, type ControlMatch, type Iteration } from './adjustment-context.js'
+
+function fallbackBounds(fallback: FindElementResponse['element']) {
+  const bounds = fallback?.bounds
+  if (Array.isArray(bounds)) return _normalizeBounds(bounds)
+  return bounds && typeof bounds === 'object' ? _normalizeBounds([Number(bounds.left), Number(bounds.top), Number(bounds.right), Number(bounds.bottom)]) : null
+}
+function findFallback(elements: UiElement[], fallback: FindElementResponse['element']) {
+  const bounds = fallbackBounds(fallback)
+  if (!bounds) return null
+  let index = elements.findIndex(el => {
+    const current = _normalizeBounds(el.bounds)
+    return !!current && current.every((value, i) => value === bounds[i])
+  })
+  if (index === -1) index = nearestControl(elements, bounds)
+  return index >= 0 && elements[index] ? { el: elements[index], idx: index } : null
+}
+function nearestControl(elements: UiElement[], bounds: [number, number, number, number]) {
+  const x = Math.floor((bounds[0] + bounds[2]) / 2), y = Math.floor((bounds[1] + bounds[3]) / 2)
+  let best = Infinity, index = -1
+  for (let i = 0; i < elements.length; i++) {
+    const el = elements[i]
+    if (!_isAdjustableControl(el)) continue
+    const current = _normalizeBounds(el.bounds)
+    if (!current) continue
+    const distance = Math.abs(Math.floor((current[0] + current[2]) / 2) - x) + Math.abs(Math.floor((current[1] + current[3]) / 2) - y)
+    if (distance < best) { best = distance; index = i }
+  }
+  return index
+}
+export async function resolveControl(s: AdjustmentState, deps: AdjustmentDependencies): Promise<ControlMatch | null> {
+  const tree = await deps.observe(s.platform, s.deviceId)
+  s.platform = tree?.device?.platform === 'ios' ? 'ios' : s.platform
+  s.deviceId = tree?.device?.id || s.deviceId
+  const elements = Array.isArray(tree?.elements) ? tree.elements : []
+  let match = null
+  if (s.input.element_id) {
+    const stored = deps.getHandle(s.input.element_id)
+    if (!stored) return null
+    const current = deps.currentHandle(elements, s.platform, s.deviceId, stored)
+    if (!current) return null
+    match = { el: current.el, idx: current.index }
+  } else {
+    if (s.fallback) match = findFallback(elements, s.fallback)
+    if (!match && s.input.selector) match = _findFirstMatchingElement(elements, s.input.selector)
+  }
+  return match ? { tree, device: tree?.device, match, resolvedTarget: deps.target(s.platform, s.deviceId, match) } : null
+}
+export async function initializeControl(s: AdjustmentState, deps: AdjustmentDependencies) {
+  const { selector, element_id } = s.input
+  if ((!selector && !element_id) || (selector && element_id)) {
+    return failAdjustment(s, 'ELEMENT_NOT_FOUND', 'exactly one of selector or element_id is required', false, null, undefined, null, 0, 'gesture')
+  }
+  if (!selector || element_id) return null
+  const wait = await deps.wait({ selector, condition: 'clickable', timeout_ms: 5000, poll_interval_ms: 300, platform: s.platform, deviceId: s.deviceId })
+  if (wait?.status === 'success' && wait?.element?.elementId) {
+    s.input.element_id = wait.element.elementId; s.fallback = null; return null
+  }
+  return initializeFallback(s, deps, wait, selector)
+}
+async function initializeFallback(s: AdjustmentState, deps: AdjustmentDependencies, wait: ControlWait | undefined, selector: UiSelector) {
+  const query = selector.text ?? selector.resource_id ?? selector.accessibility_id ?? ''
+  if (!query) return failAdjustment(s, wait?.error?.code === 'ELEMENT_NOT_FOUND' ? 'ELEMENT_NOT_FOUND' : 'TIMEOUT',
+    wait?.error?.message ?? 'adjustable control not found', wait?.error?.code === 'ELEMENT_NOT_FOUND', null, wait?.device, null, 0, 'gesture')
+  const fallback = await deps.find({ query, exact: false, timeoutMs: 3000, platform: s.platform, deviceId: s.deviceId })
+  if (!fallback.found || !fallback.element) return failAdjustment(s, 'ELEMENT_NOT_FOUND', wait?.error?.message ?? 'adjustable control not found', true, null, wait?.device, null, 0, 'gesture')
+  s.fallback = fallback.element
+  return null
+}
+export async function prepareIteration(s: AdjustmentState, deps: AdjustmentDependencies) {
+  const resolved = s.cached ? { tree: null, device: s.device, match: s.cached, resolvedTarget: deps.target(s.platform, s.deviceId, s.cached) } : await resolveControl(s, deps)
+  if (!resolved?.match || !resolved.resolvedTarget) return failAdjustment(s, 'STALE_REFERENCE', 'adjustable control could not be resolved', true)
+  s.device = resolved.device; s.target = resolved.resolvedTarget; s.cached = resolved.match
+  const element = resolved.match.el
+  const actionable = _isAdjustableActionable(element, s.stored?.stable_id)
+  if (!actionable.actionable) return failAdjustment(s, actionable.failureCode ?? 'ELEMENT_NOT_INTERACTABLE', actionable.reason ?? 'adjustable control is not actionable', true)
+  const elements = Array.isArray(resolved.tree?.elements) ? resolved.tree.elements : s.cached ? [s.cached.el] : [element]
+  const plan = planAdjustment(element, elements, s.input.targetValue, s.input.property, s.tolerance)
+  s.observed = plan.actualState
+  if (!s.trace.some(step => step.stage === 'resolve')) recordAdjustment(s, 'resolve', 'success', { resolved_target: s.target, current_value: plan.currentValue, adjustment_mode: s.mode })
+  const iteration: Iteration = { ...plan, element, index: resolved.match.idx, actionDevice: s.device, observed: plan.actualState,
+    verification: { verification: null, observedState: plan.actualState, withinTolerance: false } }
+  const invalid = validateControl(s, iteration)
+  if (invalid) return invalid
+  if (plan.currentValue !== null && Math.abs(plan.currentValue - plan.normalizedTargetValue) <= plan.effectiveTolerance) {
+    recordAdjustment(s, 'verify', 'success', { property: s.input.property, target_value: plan.normalizedTargetValue, actual_state: plan.actualState, reason: 'control already within tolerance' })
+    return succeedAdjustment(s, iteration, deps, 'control already within tolerance', 'semantic', s.device)
+  }
+  if (!plan.bounds) return failAdjustment(s, 'ELEMENT_NOT_INTERACTABLE', 'adjustable control has no bounds')
+  return iteration
+}
+function validateControl(s: AdjustmentState, p: Iteration) {
+  if (s.input.property !== 'value' && s.input.property !== 'raw_value') return failAdjustment(s, 'ELEMENT_NOT_INTERACTABLE', 'adjust_control currently supports numeric value and raw_value properties only')
+  if (!_isAdjustableControl(p.element) && !p.inferredSliderState) return failAdjustment(s, 'ELEMENT_NOT_INTERACTABLE', 'target is not an adjustable control')
+  if (p.currentValue === null) return failAdjustment(s, 'UNKNOWN', 'current numeric value is unavailable')
+  if (p.valueRange && (s.input.targetValue < p.valueRange.min || s.input.targetValue > p.valueRange.max)) return failAdjustment(s, 'CONTROL_CONVERGENCE_FAILED', `targetValue ${s.input.targetValue} is outside the control range ${p.valueRange.min}..${p.valueRange.max}`)
+  return null
+}

package/src/interact/adjustment/adjustment-result.ts

@@ -0,0 +1,34 @@
+import type { AdjustControlResponse } from '../../types.js'
+import { buildActionExecutionResult } from '../../runtime/action-results.js'
+
+type SuccessDetails = {
+  property: string
+  target_value: number
+  tolerance: number
+  attempts: number
+  adjustment_mode: AdjustControlResponse['adjustment_mode']
+  actual_state: AdjustControlResponse['actual_state']
+  reason: string
+  converged: true
+  within_tolerance: true
+}
+type AdjustmentSuccess = Parameters<typeof buildActionExecutionResult>[0] & { details: SuccessDetails }
+
+/** Keep the action envelope and top-level control result in agreement. */
+export function completeAdjustment(input: AdjustmentSuccess): AdjustControlResponse {
+  const base = buildActionExecutionResult(input)
+  const details = input.details
+  return {
+    ...base,
+    target_state: {
+      property: details.property,
+      target_value: details.target_value,
+      tolerance: details.tolerance
+    },
+    actual_state: details.actual_state,
+    within_tolerance: true,
+    converged: true,
+    attempts: details.attempts,
+    adjustment_mode: details.adjustment_mode
+  }
+}

package/src/interact/adjustment/adjustment-semantic.ts

@@ -0,0 +1,53 @@
+import { _hasDirectAdjustmentAction, _adjustmentDirections } from './control-values.js'
+import { verifyAdjustment } from './adjustment-verification.js'
+import { recordAdjustment, succeedAdjustment, type AdjustmentState, type AdjustmentDependencies, type Iteration, type StrategyOutcome } from './adjustment-context.js'
+
+export async function verifyControl(s: AdjustmentState, p: Iteration, deps: AdjustmentDependencies, fallback: string) {
+  p.verification = await verifyAdjustment({ labeled: !!p.inferredSliderState, currentEl: p.element, actualState: p.actualState,
+    property: s.input.property, target: p.normalizedTargetValue, tolerance: p.effectiveTolerance,
+    expectation: { element_id: s.target?.elementId ?? s.input.element_id, selector: s.input.selector ?? undefined,
+      property: s.input.property, expected: p.normalizedTargetValue, platform: s.platform, deviceId: s.deviceId }
+  }, { observe: () => deps.observe(s.platform, s.deviceId), expectState: deps.expectState })
+  p.observed = p.verification.observedState; s.observed = p.observed
+  recordAdjustment(s, 'verify', p.verification.withinTolerance ? 'success' : 'retry', {
+    attempt: s.attempts, property: s.input.property, target_value: p.normalizedTargetValue,
+    actual_state: p.observed, reason: p.verification.verification?.reason ?? fallback })
+}
+export async function directAdjustment(s: AdjustmentState, p: Iteration, deps: AdjustmentDependencies): Promise<StrategyOutcome> {
+  if (!_hasDirectAdjustmentAction(p.element) || s.attempts >= s.limit) return 'next'
+  const { interact, resolved } = await deps.service(s.platform, s.deviceId)
+  if (typeof interact.setAdjustableValue !== 'function') return 'next'
+  s.mode = 'semantic'
+  recordAdjustment(s, 'execute', 'retry', { attempt: s.attempts + 1, mode: 'semantic', strategy: 'direct_set_value', target_value: p.normalizedTargetValue })
+  const action = await interact.setAdjustableValue({ deviceId: resolved.id, target: s.target, property: s.input.property, value: p.normalizedTargetValue })
+  s.attempts++; p.actionDevice = action?.device ?? p.actionDevice
+  if (action?.success === false) return 'next'
+  await verifyControl(s, p, deps, 'direct adjustment did not converge yet')
+  if (p.verification.withinTolerance) return succeedAdjustment(s, p, deps, p.verification.verification?.reason ?? 'control converged to target value')
+  s.cached = null
+  return s.attempts < s.limit ? 'retry' : 'next'
+}
+export async function incrementalAdjustment(s: AdjustmentState, p: Iteration, deps: AdjustmentDependencies): Promise<StrategyOutcome> {
+  const directions = _adjustmentDirections(p.element)
+  if ((!directions.increment && !directions.decrement) || s.attempts >= s.limit) return 'next'
+  const { interact, resolved } = await deps.service(s.platform, s.deviceId)
+  const adjust = typeof interact.adjustAccessibleValue === 'function' ? interact.adjustAccessibleValue.bind(interact) : null
+  while (adjust && s.attempts < s.limit) {
+    const direction = p.normalizedTargetValue > (p.observed?.value ?? p.currentValue!) ? 'increment' : 'decrement'
+    if (!directionAllowed(direction, directions)) break
+    s.mode = 'semantic'
+    recordAdjustment(s, 'execute', 'retry', { attempt: s.attempts + 1, mode: 'semantic', strategy: 'accessibility_increment_decrement', direction })
+    const action = await adjust({ deviceId: resolved.id, target: s.target, property: s.input.property, direction })
+    s.attempts++; p.actionDevice = action?.device ?? p.actionDevice
+    if (action?.success === false) break
+    await verifyControl(s, p, deps, 'incremental adjustment did not converge yet')
+    if (p.verification.withinTolerance) return succeedAdjustment(s, p, deps, p.verification.verification?.reason ?? 'control converged to target value')
+    s.cached = null
+    if (s.attempts < s.limit) return 'retry'
+  }
+  return 'next'
+}
+
+function directionAllowed(direction: 'increment' | 'decrement', directions: { increment: boolean; decrement: boolean }) {
+  return !((direction === 'increment' && !directions.increment) || (direction === 'decrement' && !directions.decrement))
+}

package/src/interact/adjustment/adjustment-verification.ts

@@ -0,0 +1,60 @@
+import type { AdjustControlResponse, ExpectStateResponse } from '../../types.js'
+import type { UiElement } from '../model.js'
+import type { UiSelector } from '../../contracts/selector.js'
+import { _inferLabeledSliderState } from './slider-labels.js'
+import { _parseNumericControlValue } from './control-values.js'
+
+interface StateExpectation {
+  element_id?: string
+  selector?: UiSelector
+  property: string
+  expected: number
+  platform: 'android' | 'ios'
+  deviceId?: string
+}
+export interface AdjustmentVerificationInput {
+  labeled: boolean
+  currentEl: UiElement
+  actualState: AdjustControlResponse['actual_state']
+  property: string
+  target: number
+  tolerance: number
+  expectation: StateExpectation
+}
+export interface AdjustmentVerificationDependencies {
+  observe(): Promise<{ elements?: UiElement[] } | null>
+  expectState(input: StateExpectation): Promise<ExpectStateResponse>
+}
+
+/** Every verification obtains fresh evidence through its supplied observation boundary. */
+export async function verifyAdjustment(input: AdjustmentVerificationInput, dependencies: AdjustmentVerificationDependencies) {
+  if (input.labeled) return verifyLabeledControl(input, dependencies)
+  const verification = await dependencies.expectState(input.expectation)
+  const observedValue = _parseNumericControlValue(verification?.observed_state?.value) ??
+    _parseNumericControlValue(verification?.observed_state?.raw_value)
+  const observedState = observedValue !== null ? {
+    property: input.property, value: observedValue,
+    raw_value: typeof verification?.observed_state?.raw_value === 'number' ? verification.observed_state.raw_value : undefined
+  } : input.actualState
+  return { verification, observedState, withinTolerance: withinTolerance(observedValue, input) }
+}
+async function verifyLabeledControl(input: AdjustmentVerificationInput, dependencies: AdjustmentVerificationDependencies) {
+  const tree = await dependencies.observe()
+  const elements = Array.isArray(tree?.elements) ? tree.elements : []
+  const refreshed = _inferLabeledSliderState(elements, input.currentEl)
+  const observedValue = refreshed?.current ?? null
+  const observedState = observedValue !== null ? {
+    property: input.property, value: observedValue, raw_value: observedValue
+  } : input.actualState
+  const matched = withinTolerance(observedValue, input)
+  return {
+    verification: {
+      success: matched, observed_state: observedState,
+      reason: observedValue !== null ? 'visible labeled value read from nearby slider labels' : 'visible labeled value unavailable'
+    },
+    observedState, withinTolerance: matched
+  }
+}
+function withinTolerance(value: number | null, input: AdjustmentVerificationInput) {
+  return value !== null && Math.abs(value - input.target) <= input.tolerance
+}

package/src/interact/adjustment/adjustment.ts

@@ -0,0 +1,49 @@
+import type { AdjustControlResponse } from '../../types.js'
+import { createAdjustment, failAdjustment, recordAdjustment, succeedAdjustment,
+  type AdjustmentInput, type AdjustmentDependencies, type AdjustmentState, type Iteration } from './adjustment-context.js'
+import { initializeControl, prepareIteration } from './adjustment-resolution.js'
+import { directAdjustment, incrementalAdjustment } from './adjustment-semantic.js'
+import { coordinateAdjustment, gestureAdjustment } from './adjustment-gestures.js'
+
+function cacheObservation(s: AdjustmentState, p: Iteration) {
+  s.cached = p.inferredSliderState ? null : {
+    el: { ...p.element, state: { ...(p.element.state ?? null), ...(p.observed ? {
+      [p.observed.property]: p.observed.value, raw_value: p.observed.raw_value ?? p.observed.value
+    } : {}) } }, idx: p.index
+  }
+}
+async function attempt(s: AdjustmentState, p: Iteration, deps: AdjustmentDependencies) {
+  for (const strategy of [directAdjustment, incrementalAdjustment]) {
+    const outcome = await strategy(s, p, deps)
+    if (outcome !== 'next') return outcome
+  }
+  if (!p.bounds) return failAdjustment(s, 'ELEMENT_NOT_INTERACTABLE', 'adjustable control has no bounds')
+  if (!p.valueRange) return failAdjustment(s, 'UNKNOWN', 'value_range unavailable')
+  for (const strategy of [coordinateAdjustment, gestureAdjustment]) {
+    const outcome = await strategy(s, p, deps)
+    if (outcome !== 'next') return outcome
+  }
+  s.observed = p.observed
+  if (p.verification.withinTolerance) {
+    const reason = p.verification.verification?.reason ?? 'control converged to target value'
+    recordAdjustment(s, 'verify', 'success', { attempt: s.attempts, property: s.input.property,
+      target_value: p.normalizedTargetValue, actual_state: p.observed, reason })
+    return succeedAdjustment(s, p, deps, reason)
+  }
+  cacheObservation(s, p)
+  return 'next'
+}
+export async function adjustControl(input: AdjustmentInput, deps: AdjustmentDependencies): Promise<AdjustControlResponse> {
+  const state = await createAdjustment(input, deps)
+  const invalid = await initializeControl(state, deps)
+  if (invalid) return invalid
+  for (let index = 0; index < state.limit && state.attempts < state.limit; index++) {
+    const prepared = await prepareIteration(state, deps)
+    if ('success' in prepared) return prepared
+    const outcome = await attempt(state, prepared, deps)
+    if (outcome !== 'retry' && outcome !== 'next') return outcome
+  }
+  const after = await deps.fingerprint(state.platform, state.deviceId)
+  return failAdjustment(state, 'CONTROL_CONVERGENCE_FAILED', 'control did not converge within the allotted attempts', true,
+    state.target, state.device, state.observed, state.attempts, state.mode, after)
+}