mobile-debug-mcp 0.34.0 → 0.35.0

package/src/observe/logs/model.ts

@@ -0,0 +1,11 @@
+/** Fields consumed from persisted NDJSON log events; other fields pass through. */
+export interface StreamLogEntry {
+  timestamp?: string | number | null
+  _iso?: string | null
+  level?: string
+  message?: string
+  exception?: string
+  crash?: boolean
+}
+
+export interface LogFilters { appId?: string; deviceId?: string; pid?: number; tag?: string; level?: string; contains?: string; since_seconds?: number; limit?: number }

package/src/{network → observe/network}/index.ts

@@ -1,5 +1,5 @@
-import { execAdb, parseLogLine } from '../utils/android/utils.js'
-import { execCommand } from '../utils/ios/utils.js'
+import { execAdb, parseLogLine } from '../../utils/android/utils.js'
+import { execCommand } from '../../utils/ios/utils.js'
 
 export type NetworkErrorCode =
   | 'timeout'

package/src/observe/readiness-service.ts

@@ -42,7 +42,7 @@ export async function getAppReadiness(args: Target & { timeoutMs?: number; stabi
             const tree = await ToolsObserve.getUITreeHandler({ platform: device!.platform as 'android' | 'ios', deviceId: device!.id })
             remainingBudget()
             tracker.observe(tree, captureStarted)
-            if (isDeviceUnavailable((tree as any)?.error)) { failureCode = 'DEVICE_UNAVAILABLE'; return }
+            if (isDeviceUnavailable(tree?.error)) { failureCode = 'DEVICE_UNAVAILABLE'; return }
           } catch (error) {
             if (/ACTION_TIMEOUT/.test(String(error))) throw error
             tracker.unavailable()
@@ -64,8 +64,10 @@ export async function getAppReadiness(args: Target & { timeoutMs?: number; stabi
   if (debugPause) return { success: !wait, device, readiness: debugPause, elapsed_ms: performance.now() - started, ...(wait ? { failure_code: 'DEBUGGER_PAUSED' } : {}) }
   const state = tracker.summary.state
   const success = !failureCode && (wait ? state === 'settled' : state !== 'unknown')
-  return { success, device, readiness: tracker.summary, elapsed_ms: performance.now() - started,
-    ...(!success ? { failure_code: failureCode ?? (wait ? 'READINESS_TIMEOUT' : 'READINESS_UNAVAILABLE') } : {}) }
+  return {
+    success, device, readiness: tracker.summary, elapsed_ms: performance.now() - started,
+    ...(!success ? { failure_code: failureCode ?? (wait ? 'READINESS_TIMEOUT' : 'READINESS_UNAVAILABLE') } : {})
+  }
 }
 
 export async function withReadyWait<T extends object | undefined, F extends object>(target: Target, timeoutMs: number, operation: (device: DeviceInfo) => Promise<T>, failure: (code: string, elapsedMs: number, samples: number) => F) {

package/src/observe/readiness.ts

@@ -1,3 +1,4 @@
+import type { GetUITreeResponse } from '../types.js'
 import { pausedReadiness } from '../debugger/guard.js'
 import { AsyncLocalStorage } from 'node:async_hooks'
 import type { ReadinessSummary } from '../types.js'
@@ -17,19 +18,21 @@ export class ReadinessTracker {
   summary: ReadinessSummary
 
   constructor(private platform: string, private deviceId: string, private stabilityMs = 500) {
-  this.summary = {
-    state: 'unknown', scope: 'device_ui', basis: 'insufficient', captured_at_ms: null,
-    sample_count: 0, stable_for_ms: 0, snapshot_session_id: null, snapshot_revision: null,
-    evidence: [], limitations: [...baselineLimitations, this.platform === 'android' ? 'MANAGED_DEBUGGER_ONLY' : 'DEBUGGER_SIGNAL_UNSUPPORTED', 'INSUFFICIENT_SAMPLES'], capabilities: { ...capabilities, debugger_paused: this.platform === 'android' }
-  }
+    this.summary = {
+      state: 'unknown', scope: 'device_ui', basis: 'insufficient', captured_at_ms: null,
+      sample_count: 0, stable_for_ms: 0, snapshot_session_id: null, snapshot_revision: null,
+      evidence: [], limitations: [...baselineLimitations, this.platform === 'android' ? 'MANAGED_DEBUGGER_ONLY' : 'DEBUGGER_SIGNAL_UNSUPPORTED', 'INSUFFICIENT_SAMPLES'], capabilities: { ...capabilities, debugger_paused: this.platform === 'android' }
+    }
 
   }
 
   unavailable(code = 'OBSERVATION_FAILED'): ReadinessSummary {
     this.signature = null
-    this.summary = { ...this.summary, state: 'unknown', basis: 'insufficient', stable_for_ms: 0,
+    this.summary = {
+      ...this.summary, state: 'unknown', basis: 'insufficient', stable_for_ms: 0,
       sample_count: this.summary.sample_count + 1, limitations: [...baselineLimitations, this.platform === 'android' ? 'MANAGED_DEBUGGER_ONLY' : 'DEBUGGER_SIGNAL_UNSUPPORTED', code],
-      evidence: [...this.summary.evidence, { code, source: 'ui_tree', captured_at_ms: null, snapshot_revision: null }].slice(-8) }
+      evidence: [...this.summary.evidence, { code, source: 'ui_tree', captured_at_ms: null, snapshot_revision: null }].slice(-8)
+    }
     return this.summary
   }
 
@@ -38,27 +41,24 @@ export class ReadinessTracker {
     return this.summary
   }
 
-  observe(tree: any, acquisitionStartedAtMs: number, now = performance.now()): ReadinessSummary {
+  observe(sample: unknown, acquisitionStartedAtMs: number, now = performance.now()): ReadinessSummary {
     const paused = this.platform === 'android' ? pausedReadiness(this.deviceId) : null
     if (paused) { this.summary = paused; return this.summary }
+    const tree = sample as GetUITreeResponse | null | undefined
     if (tree?.error) return this.unavailable('OBSERVATION_FAILED')
-    if (!tree || tree.device?.platform !== this.platform || tree.device?.id !== this.deviceId ||
-        tree.snapshot_session_id !== getSnapshotSessionId() || !Number.isSafeInteger(tree.snapshot_revision) || tree.snapshot_revision <= 0 ||
-        !Number.isFinite(tree.captured_at_ms) || tree.captured_at_ms < acquisitionStartedAtMs || tree.captured_at_ms <= 0 ||
-        !Array.isArray(tree.elements) || !tree.elements.length || tree.elements.some((element: unknown) => !element || typeof element !== 'object') ||
-        !Number.isFinite(tree.resolution?.width) || !Number.isFinite(tree.resolution?.height) || tree.resolution.width <= 0 || tree.resolution.height <= 0) return this.unavailable('OBSERVATION_INVALID')
+    if (!validReadinessSample(tree, this.platform, this.deviceId, acquisitionStartedAtMs)) return this.unavailable('OBSERVATION_INVALID')
     const signature = computeSnapshotSignature(tree)
     if (!signature) return this.unavailable('OBSERVATION_INVALID')
-    const nativeLoading = tree.elements.some((element: any) => element.visible === true && ['android.widget.ProgressBar', 'ActivityIndicator', 'ProgressIndicator'].includes(element.type))
-    const textLoading = tree.elements.some((element: any) => element.visible === true && [element.text, element.contentDescription].some(label =>
+    const nativeLoading = tree.elements.some((element) => element.visible === true && ['android.widget.ProgressBar', 'ActivityIndicator', 'ProgressIndicator'].includes(element.type))
+    const textLoading = tree.elements.some((element) => element.visible === true && [element.text, element.contentDescription].some(label =>
       typeof label === 'string' && /^(loading|please\s+wait)[.\s…]*$/i.test(label.trim())))
     const previous = this.signature
     const unchanged = previous === signature
     if (!unchanged || nativeLoading || textLoading) this.stableSince = now
     this.signature = signature
     const stableFor = unchanged && !nativeLoading && !textLoading ? Math.max(0, now - this.stableSince) : 0
-    const state = nativeLoading || textLoading ? 'loading' : !previous ? 'unknown' : !unchanged ? 'transitioning' : stableFor >= this.stabilityMs ? 'settled' : 'unknown'
-    const code = nativeLoading ? 'NATIVE_LOADING_INDICATOR' : textLoading ? 'LOADING_LABEL' : state === 'transitioning' ? 'SNAPSHOT_CHANGED' : state === 'settled' ? 'SNAPSHOT_STABLE' : 'INSUFFICIENT_SAMPLES'
+    const state = readinessState(nativeLoading || textLoading, previous, unchanged, stableFor, this.stabilityMs)
+    const code = readinessCode(nativeLoading, textLoading, state)
     this.summary = {
       state, scope: 'device_ui', basis: textLoading && !nativeLoading ? 'heuristic' : state === 'unknown' ? 'insufficient' : 'observed',
       captured_at_ms: tree.captured_at_ms, snapshot_session_id: tree.snapshot_session_id, snapshot_revision: tree.snapshot_revision,
@@ -70,6 +70,20 @@ export class ReadinessTracker {
   }
 }
 
+function validReadinessSample(tree: GetUITreeResponse | null | undefined, platform: string, deviceId: string, acquisitionStartedAtMs: number): tree is GetUITreeResponse & { snapshot_session_id: string } {
+  return !!tree && validIdentity(tree, platform, deviceId) && validFreshness(tree, acquisitionStartedAtMs) && validGeometry(tree)
+}
+function validIdentity(tree: GetUITreeResponse, platform: string, deviceId: string) {
+  return tree.device?.platform === platform && tree.device?.id === deviceId &&
+    tree.snapshot_session_id === getSnapshotSessionId() && Number.isSafeInteger(tree.snapshot_revision) && tree.snapshot_revision > 0
+}
+function validFreshness(tree: GetUITreeResponse, startedAtMs: number) {
+  return Number.isFinite(tree.captured_at_ms) && !(tree.captured_at_ms < startedAtMs) && tree.captured_at_ms > 0
+}
+function validGeometry(tree: GetUITreeResponse) {
+  return Array.isArray(tree.elements) && tree.elements.length > 0 && !tree.elements.some((element: unknown) => !element || typeof element !== 'object') &&
+    Number.isFinite(tree.resolution?.width) && Number.isFinite(tree.resolution?.height) && tree.resolution.width > 0 && tree.resolution.height > 0
+}
 const waits = new AsyncLocalStorage<ReadinessTracker>()
 export function currentReadinessWait() { return waits.getStore() }
 export function withReadinessTracker<T>(tracker: ReadinessTracker, operation: () => Promise<T>) { return waits.run(tracker, operation) }
@@ -79,6 +93,13 @@ export function observeReadinessWait(tree: unknown, startedAtMs: number) {
   if (!tracker) return
   remainingBudget()
   const summary = tracker.observe(tree, startedAtMs)
-  if (isDeviceUnavailable((tree as any)?.error)) throw new Error('READINESS_DEVICE_UNAVAILABLE')
+  if (isDeviceUnavailable((tree as { error?: unknown } | null)?.error)) throw new Error('READINESS_DEVICE_UNAVAILABLE')
   if (summary.limitations.some(code => ['OBSERVATION_INVALID', 'OBSERVATION_FAILED'].includes(code))) throw new Error('READINESS_SAMPLE_INVALID')
 }
+
+function readinessState(loading: boolean, previous: string | null, unchanged: boolean, stableFor: number, stabilityMs: number): ReadinessSummary['state'] {
+  return loading ? 'loading' : !previous ? 'unknown' : !unchanged ? 'transitioning' : stableFor >= stabilityMs ? 'settled' : 'unknown'
+}
+function readinessCode(nativeLoading: boolean, textLoading: boolean, state: ReadinessSummary['state']) {
+  return nativeLoading ? 'NATIVE_LOADING_INDICATOR' : textLoading ? 'LOADING_LABEL' : state === 'transitioning' ? 'SNAPSHOT_CHANGED' : state === 'settled' ? 'SNAPSHOT_STABLE' : 'INSUFFICIENT_SAMPLES'
+}

package/src/observe/screenshot-encoding.ts

@@ -0,0 +1,32 @@
+import type { Metadata } from 'sharp'
+interface EncodedScreenshot { screenshot: string; screenshot_mime: string; screenshot_fallback?: string | null; screenshot_fallback_mime?: string }
+async function convert(buffer: Buffer, platform: 'android' | 'ios') {
+  const { default: sharp } = await import('sharp')
+  const image = sharp(buffer)
+  const meta = await image.metadata().catch((error: unknown) => {
+    console.error(platform === 'android' ? 'sharp.metadata failed (Android):' : 'sharp.metadata failed:', error)
+    return {} as Metadata
+  })
+  const hasAlpha = !!meta.hasAlpha || !!(meta.channels && meta.channels > 3)
+  let webp: Buffer | null = null, jpeg: Buffer | null = null
+  try { webp = await image.webp({ quality: 80 }).toBuffer() }
+  catch (error) { console.error(`WebP conversion failed (${platform === 'android' ? 'Android' : 'iOS'}):`, error instanceof Error ? error.message : String(error)) }
+  try { jpeg = await image.jpeg({ quality: 80 }).toBuffer() }
+  catch (error) { console.error(`JPEG conversion failed (${platform === 'android' ? 'Android' : 'iOS'}):`, error instanceof Error ? error.message : String(error)) }
+  return { hasAlpha, webp, jpeg }
+}
+export async function encodeScreenshot(buffer: Buffer, platform: 'android' | 'ios'): Promise<EncodedScreenshot> {
+  const png = buffer.toString('base64')
+  try {
+    const { hasAlpha, webp, jpeg } = await convert(buffer, platform)
+    if (webp) {
+      if (hasAlpha && platform === 'ios') return { screenshot: webp.toString('base64'), screenshot_mime: 'image/webp', screenshot_fallback: png, screenshot_fallback_mime: 'image/png' }
+      const fallback = jpeg ? jpeg.toString('base64') : platform === 'android' ? null : undefined
+      return { screenshot: webp.toString('base64'), screenshot_mime: 'image/webp', screenshot_fallback: fallback, screenshot_fallback_mime: jpeg ? 'image/jpeg' : undefined }
+    }
+    if (!hasAlpha && jpeg) return { screenshot: jpeg.toString('base64'), screenshot_mime: 'image/jpeg' }
+  } catch (error) {
+    console.error(`Screenshot conversion pipeline failed (${platform === 'android' ? 'Android' : 'iOS'}):`, error instanceof Error ? error.message : String(error))
+  }
+  return { screenshot: png, screenshot_mime: 'image/png' }
+}

package/src/observe/snapshot-collection.ts

@@ -0,0 +1,69 @@
+import type { CaptureDebugSnapshotRawResponse } from '../types.js'
+import type { ToolsObserve } from './index.js'
+import { omitEvidence } from './evidence.js'
+import { measure } from '../utils/timing.js'
+import { deriveSnapshotMetadata } from './snapshot-metadata.js'
+import { deriveSnapshotSemantic } from './snapshot-semantic.js'
+import { snapshotLog, streamLog, type SnapshotLogEntry } from './snapshot-logs.js'
+export interface SnapshotInput { reason?: string; includeLogs?: boolean; includeScreenshot?: boolean; logLines?: number; platform?: 'android' | 'ios'; appId?: string; deviceId?: string; sessionId?: string }
+type Dependencies = Pick<typeof ToolsObserve, 'captureScreenshotHandler' | 'getCurrentScreenHandler' | 'getScreenFingerprintHandler' | 'getUITreeHandler' | 'readLogStreamHandler' | 'getLogsHandler'>
+async function settle<T extends Record<string, Promise<unknown>>>(tasks: T) {
+  const results = await Promise.allSettled(Object.values(tasks))
+  return Object.fromEntries(Object.keys(tasks).map((key, index) => [key, results[index]])) as { [K in keyof T]: PromiseSettledResult<Awaited<T[K]>> }
+}
+function apply<T>(result: PromiseSettledResult<T>, success: (value: T) => void, failure: (message: string) => void) {
+  if (result.status === 'fulfilled') success(result.value)
+  else failure(result.reason instanceof Error ? result.reason.message : String(result.reason))
+}
+async function collect(raw: CaptureDebugSnapshotRawResponse, input: SnapshotInput, deps: Dependencies) {
+  const { platform, deviceId, includeScreenshot = true, includeLogs = true, sessionId, logLines = 200 } = input
+  const results = await settle({
+    screenshot: includeScreenshot ? measure('screenshot_ms', () => deps.captureScreenshotHandler({ platform, deviceId })) : Promise.resolve(null),
+    currentScreen: !platform || platform === 'android' ? deps.getCurrentScreenHandler({ deviceId }) : Promise.resolve(null),
+    fingerprint: deps.getScreenFingerprintHandler({ platform, deviceId }),
+    uiTree: deps.getUITreeHandler({ platform, deviceId }),
+    readLogStream: includeLogs ? deps.readLogStreamHandler({ platform, deviceId, sessionId: sessionId || 'default', limit: logLines }) : Promise.resolve({ entries: [] })
+  })
+  let entries: unknown[] = []
+  apply(results.screenshot, value => { raw.screenshot = value && value.screenshot ? value.screenshot : null }, error => { raw.screenshot_error = error })
+  apply(results.currentScreen, value => { raw.activity = value && (value.activity || value.shortActivity) ? value.activity || value.shortActivity : raw.activity || '' }, error => { raw.activity_error = error })
+  apply(results.fingerprint, value => {
+    if (value && value.fingerprint) raw.fingerprint = value.fingerprint
+    if (value && value.activity) raw.activity = raw.activity || value.activity
+    if (value && value.error) raw.fingerprint_error = value.error
+  }, error => { raw.fingerprint = null; raw.fingerprint_error = error })
+  apply(results.uiTree, value => { raw.ui_tree = value; if (value && value.error) raw.ui_tree_error = value.error }, error => { raw.ui_tree = null; raw.ui_tree_error = error })
+  apply(results.readLogStream, value => { entries = Array.isArray(value?.entries) ? value.entries : [] }, error => { raw.logs_error = error })
+  return entries
+}
+async function collectLogs(entries: unknown[], input: SnapshotInput, deps: Dependencies): Promise<SnapshotLogEntry[]> {
+  if (entries.length) return entries.map(streamLog)
+  const { platform, appId, deviceId, logLines = 200 } = input
+  const response = await deps.getLogsHandler({ platform, appId, deviceId, lines: logLines })
+  return (response && response.logs ? response.logs : []).slice(-Math.max(0, logLines)).map(snapshotLog)
+}
+function metadata(raw: CaptureDebugSnapshotRawResponse, input: SnapshotInput) {
+  const key = raw.ui_tree?.device ? `${raw.ui_tree.device.platform}:${raw.ui_tree.device.id}` : `${input.platform || 'unknown'}:${input.deviceId || 'default'}`
+  const value = deriveSnapshotMetadata(key, raw.ui_tree, 'snapshot', raw.ui_tree?.snapshot_revision ? null : raw.fingerprint || raw.activity || null)
+  raw.snapshot_revision = raw.ui_tree?.snapshot_revision ?? value.snapshot_revision
+  raw.captured_at_ms = raw.ui_tree?.captured_at_ms ?? value.captured_at_ms
+  raw.snapshot_delta = raw.ui_tree?.snapshot_delta ?? value.snapshot_delta ?? null
+  raw.loading_state = raw.ui_tree?.loading_state ?? value.loading_state
+}
+export async function collectDebugSnapshot(input: SnapshotInput = {}, deps: Dependencies) {
+  if (input.includeScreenshot === false) omitEvidence('screenshot')
+  if (input.includeLogs === false) omitEvidence('logs')
+  const timestamp = Date.now()
+  const raw: CaptureDebugSnapshotRawResponse = {
+    timestamp, snapshot_revision: 0, captured_at_ms: timestamp,
+    reason: input.reason || '', activity: null, fingerprint: null, screenshot: null, ui_tree: null, logs: []
+  }
+  const entries = await collect(raw, input, deps)
+  if (input.includeLogs !== false) {
+    try { raw.logs = await collectLogs(entries, input, deps) }
+    catch (error) { raw.logs = []; raw.logs_error = error instanceof Error ? error.message : String(error) }
+  }
+  metadata(raw, input)
+  const semantic = deriveSnapshotSemantic(raw)
+  return semantic ? { raw, semantic } : { raw }
+}

package/src/observe/snapshot-logs.ts

@@ -0,0 +1,26 @@
+export interface SnapshotLogEntry { timestamp: number | null; level: string; message: string }
+interface LogFields { message?: string; msg?: string; level?: string; levelName?: string; _level?: string; timestamp?: string | number; _iso?: string }
+function levelOf(message: string, raw: string | undefined, stream: boolean) {
+  return (raw && String(raw).toUpperCase()) || (stream
+    ? /\bERROR\b/i.test(message) ? 'ERROR' : /\bWARN\b/i.test(message) ? 'WARN' : 'INFO'
+    : /\b(FATAL EXCEPTION|ERROR| E )\b/i.test(message) ? 'ERROR' : /\b(WARN| W )\b/i.test(message) ? 'WARN' : 'INFO')
+}
+export function snapshotLog(item: unknown): SnapshotLogEntry {
+  if (!item) return { timestamp: null, level: 'INFO', message: '' }
+  if (typeof item === 'string') return { timestamp: null, level: levelOf(item, undefined, false), message: item }
+  const fields = item as LogFields
+  const message = fields.message || fields.msg || JSON.stringify(item)
+  const raw = fields.level || fields.levelName || fields._level || ''
+  const time = fields.timestamp || fields._iso || null
+  const timestamp = time && typeof time === 'string' ? (isNaN(new Date(time).getTime()) ? null : new Date(time).getTime()) : typeof time === 'number' ? time : null
+  return { timestamp, level: levelOf(message, raw, true), message }
+}
+export function streamLog(item: unknown): SnapshotLogEntry {
+  const fields = item as LogFields | null
+  const message = fields && (fields.message || fields.msg) ? fields.message || fields.msg! : typeof item === 'string' ? item : JSON.stringify(item)
+  const raw = fields && (fields.level || fields.levelName || fields._level) ? fields.level || fields.levelName || fields._level : ''
+  const iso = fields && (fields._iso || fields.timestamp)
+  let timestamp: number | null = null
+  if (iso && typeof iso === 'string') { const date = new Date(iso); if (!isNaN(date.getTime())) timestamp = date.getTime() }
+  return { timestamp, level: levelOf(message, raw, true), message }
+}

package/src/observe/snapshot-metadata.ts

@@ -38,7 +38,10 @@ function normalizeBounds(bounds: unknown): [number, number, number, number] | nu
   return normalized as [number, number, number, number]
 }
 
-function stableElementSignature(element: UIElement) {
+type SignatureElement = Partial<Pick<UIElement, 'text' | 'contentDescription' | 'resourceId' | 'stable_id' | 'role' | 'test_tag' | 'selector' | 'clickable' | 'enabled' | 'visible' | 'state'>> & { bounds?: number[] | null; type?: string | null }
+export interface SnapshotSignatureInput { elements?: SignatureElement[]; screen?: unknown; resolution?: unknown; error?: unknown }
+
+function stableElementSignature(element: SignatureElement) {
   return {
     text: normalize(element.text),
     contentDescription: normalize(element.contentDescription),
@@ -115,7 +118,7 @@ function summarizeSnapshotDelta(previous: SnapshotState | undefined, currentElem
   }
 }
 
-export function computeSnapshotSignature(tree: Pick<GetUITreeResponse, 'elements' | 'screen' | 'resolution' | 'error'> | null | undefined): string | null {
+export function computeSnapshotSignature(tree: SnapshotSignatureInput | null | undefined): string | null {
   if (!tree || tree.error) return null
 
   const payload = {
@@ -162,34 +165,10 @@ export function deriveSnapshotMetadata(
   const existing = snapshotStateByDevice.get(deviceKey)
   const previous = existing && Date.now() - existing.updatedAt <= 10 * 60 * 1000 ? existing : undefined
   if (!previous && existing) snapshotStateByDevice.delete(deviceKey)
-  const hasValidTree = !!tree && !tree.error
-  const records = hasValidTree ? buildElementRecords(tree) : null
-  const currentElementSignatures = records?.signatures ?? previous?.elementSignatures ?? new Map<string, string>()
-  const currentElements = records?.elementsByIdentity ?? previous?.elements ?? new Map<string, UIElement>()
+  const { hasValidTree, currentElementSignatures, currentElements } = snapshotRecords(tree, previous)
 
   const revision = previous && (signature === null || previous.signature === signature) ? previous.revision : nextRevision++
-  const history = previous?.history ?? new Map<number, Map<string, UIElement>>()
-  const historyTimes = previous?.historyTimes ?? new Map<number, number>()
-  const oversized = previous?.oversized ?? new Map<number, number>()
-  if (hasValidTree) {
-    history.set(revision, currentElements)
-    historyTimes.set(revision, Date.now())
-    if ((tree.elements?.length ?? 0) > 500) oversized.set(revision, (tree.elements?.length ?? 0) - 500)
-  }
-  for (const [id, timestamp] of historyTimes) {
-    if (Date.now() - timestamp > 10 * 60 * 1000) {
-      history.delete(id)
-      historyTimes.delete(id)
-      oversized.delete(id)
-    }
-  }
-  while (history.size > 8) {
-    const oldest = history.keys().next().value
-    if (oldest === undefined) break
-    history.delete(oldest)
-    historyTimes.delete(oldest)
-    oversized.delete(oldest)
-  }
+  const { history, historyTimes, oversized } = retainHistory(previous, revision, currentElements, tree, hasValidTree)
 
   snapshotStateByDevice.set(deviceKey, {
     revision,
@@ -226,19 +205,20 @@ export function getStateDelta(deviceKey: string, baseRevision: number, currentRe
   }
   let index = 0
   for (const [identity, element] of base) {
-    if (!current.has(identity)) removed.push({ stable_id: element.stable_id ?? undefined, resourceId: element.resourceId ?? undefined, contentDescription: element.contentDescription ?? undefined, text: element.text ?? undefined, index })
+    if (!current.has(identity)) removed.push(removedElement(element, index))
     index++
   }
   const all = added.length + changed.length + removed.length
   const limit = 200
+  const truncated = all > limit || state.oversized.has(baseRevision) || state.oversized.has(currentRevision)
   return {
     base_snapshot_revision: baseRevision,
     snapshot_revision: currentRevision,
     added: added.slice(0, limit),
     changed: changed.slice(0, Math.max(0, limit - added.length)),
     removed: removed.slice(0, Math.max(0, limit - added.length - changed.length)),
-    truncated: all > limit || state.oversized.has(baseRevision) || state.oversized.has(currentRevision),
-    ...((all > limit || state.oversized.has(baseRevision) || state.oversized.has(currentRevision))
+    truncated,
+    ...((truncated)
       ? { omitted_changes: Math.max(all - limit, state.oversized.get(baseRevision) ?? 0, state.oversized.get(currentRevision) ?? 0) } : {})
   }
 }
@@ -262,3 +242,43 @@ export function resetSnapshotMetadataForTests() {
   snapshotStateByDevice.clear()
   nextRevision = 1
 }
+
+function retainHistory(previous: SnapshotState | undefined, revision: number, currentElements: Map<string, UIElement>, tree: Parameters<typeof deriveSnapshotMetadata>[1], hasValidTree: boolean) {
+  const history = previous?.history ?? new Map<number, Map<string, UIElement>>()
+  const historyTimes = previous?.historyTimes ?? new Map<number, number>()
+  const oversized = previous?.oversized ?? new Map<number, number>()
+  if (hasValidTree) {
+    history.set(revision, currentElements)
+    historyTimes.set(revision, Date.now())
+    if ((tree!.elements?.length ?? 0) > 500) oversized.set(revision, (tree!.elements?.length ?? 0) - 500)
+  }
+  for (const [id, timestamp] of historyTimes) {
+    if (Date.now() - timestamp > 10 * 60 * 1000) {
+      history.delete(id)
+      historyTimes.delete(id)
+      oversized.delete(id)
+    }
+  }
+  while (history.size > 8) {
+    const oldest = history.keys().next().value
+    if (oldest === undefined) break
+    history.delete(oldest)
+    historyTimes.delete(oldest)
+    oversized.delete(oldest)
+  }
+
+  return { history, historyTimes, oversized }
+}
+
+function snapshotRecords(tree: Parameters<typeof deriveSnapshotMetadata>[1], previous: SnapshotState | undefined) {
+  const hasValidTree = !!tree && !tree.error
+  const records = hasValidTree ? buildElementRecords(tree) : null
+  const currentElementSignatures = records?.signatures ?? previous?.elementSignatures ?? new Map<string, string>()
+  const currentElements = records?.elementsByIdentity ?? previous?.elements ?? new Map<string, UIElement>()
+
+  return { hasValidTree, currentElementSignatures, currentElements }
+}
+
+function removedElement(element: UIElement, index: number) {
+  return { stable_id: element.stable_id ?? undefined, resourceId: element.resourceId ?? undefined, contentDescription: element.contentDescription ?? undefined, text: element.text ?? undefined, index }
+}

package/src/observe/snapshot-semantic.ts

@@ -0,0 +1,153 @@
+import type {
+  CaptureDebugSnapshotRawResponse,
+  SnapshotSemanticResponse
+} from '../types.js'
+export interface SnapshotTreeElementLike {
+  text?: string | null
+  contentDescription?: string | null
+  contentDesc?: string | null
+  accessibilityLabel?: string | null
+  resourceId?: string | null
+  id?: string | null
+  type?: string | null
+  class?: string | null
+  clickable?: boolean
+  enabled?: boolean
+  visible?: boolean
+  state?: unknown
+  stable_id?: string | null
+  role?: string | null
+  test_tag?: string | null
+  selector?: unknown
+  semantic?: unknown
+}
+
+export interface SnapshotTreeLike {
+  screen?: string | null
+  elements?: SnapshotTreeElementLike[]
+}
+
+export function normalizeHint(value: unknown): string {
+  if (value === null || value === undefined) return ''
+  return String(value).trim().replace(/\s+/g, ' ').toLowerCase()
+}
+
+export function titleCase(value: string): string {
+  return value
+    .replace(/[_-]+/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim()
+    .replace(/\b\w/g, (match) => match.toUpperCase())
+}
+
+export function shortActivityName(activity: string | null | undefined): string | null {
+  if (!activity) return null
+  const trimmed = String(activity).trim()
+  if (!trimmed) return null
+  const lastSegment = trimmed.split('.').pop() || trimmed
+  const withoutSuffix = lastSegment.replace(/Activity$/, '')
+  return withoutSuffix ? titleCase(withoutSuffix) : titleCase(lastSegment)
+}
+
+function elementText(element: SnapshotTreeElementLike | undefined) {
+  return normalizeHint(element?.text ?? element?.contentDescription ?? element?.contentDesc ?? element?.accessibilityLabel ?? element?.resourceId ?? element?.id ?? '')
+}
+export function collectSnapshotTexts(tree: SnapshotTreeLike | null | undefined) {
+  const elements = Array.isArray(tree?.elements) ? tree!.elements! : []
+  const texts: string[] = []
+  const actionables: string[] = []
+
+  for (const element of elements) {
+    const text = elementText(element)
+    if (text) texts.push(text)
+    if (element?.clickable && element?.enabled !== false && text) {
+      actionables.push(text)
+    }
+  }
+
+  return {
+    texts: Array.from(new Set(texts)),
+    actionables: Array.from(new Set(actionables))
+  }
+}
+
+export function inferSnapshotScreen(raw: CaptureDebugSnapshotRawResponse): string | null {
+  const tree = raw.ui_tree as SnapshotTreeLike | null | undefined
+  const treeScreen = normalizeHint(tree?.screen)
+  if (treeScreen) return titleCase(treeScreen)
+
+  const activity = shortActivityName(raw.activity)
+  if (activity) return activity
+
+  const { texts } = collectSnapshotTexts(tree)
+  if (texts.length > 0) return titleCase(texts[0])
+
+  return null
+}
+
+export function deriveSnapshotSemantic(raw: CaptureDebugSnapshotRawResponse): SnapshotSemanticResponse | null {
+  const tree = raw.ui_tree as SnapshotTreeLike | null | undefined
+  const { texts, actionables } = collectSnapshotTexts(tree)
+  const screenFromTree = normalizeHint(tree?.screen)
+  const activityHint = normalizeHint(raw.activity)
+  const screen = inferSnapshotScreen(raw)
+
+  if (!screen && !activityHint && texts.length === 0 && !raw.logs.length) return null
+
+  const hasErrorLogs = raw.logs.some((entry) => /error|fatal exception|exception|failed/i.test(entry.message))
+  const hasLoadingSignals = texts.some((text) => /loading|please wait|spinner|progress/i.test(text))
+  const hasPrimaryText = texts.some((text) => /sign in|log in|login|home|checkout|settings|menu|profile|search/i.test(text))
+  const hasScreenshot = typeof raw.screenshot === 'string' && raw.screenshot.length > 0
+  const hasUiTree = !!tree && Array.isArray(tree.elements)
+
+  const signals: Record<string, string | number | boolean> = {
+    has_activity: !!activityHint,
+    has_ui_tree: hasUiTree,
+    has_screenshot: hasScreenshot,
+    has_visible_text: texts.length > 0,
+    has_clickable_elements: actionables.length > 0,
+    has_error_logs: hasErrorLogs,
+    has_loading_signals: hasLoadingSignals,
+    has_primary_text: hasPrimaryText
+  }
+
+  const warnings = snapshotWarnings(screenFromTree, activityHint, hasUiTree, hasErrorLogs)
+  const confidence = snapshotConfidence(hasUiTree, screen, activityHint, actionables.length, texts.length, hasScreenshot, hasErrorLogs, hasLoadingSignals)
+
+  if (!screen && confidence < 0.3) return null
+
+  return {
+    screen,
+    signals,
+    actions_available: actionables.length > 0 ? actionables.slice(0, 10) : null,
+    confidence,
+    warnings
+  }
+}
+
+function snapshotWarnings(screenFromTree: string, activityHint: string, hasUiTree: boolean, hasErrorLogs: boolean) {
+  const warnings: string[] = []
+  if (screenFromTree && activityHint && screenFromTree !== activityHint) {
+    warnings.push('ui_tree.screen and activity hints differ')
+  }
+  if (!hasUiTree) warnings.push('ui tree unavailable')
+  if (!activityHint) warnings.push('activity unavailable')
+  if (hasErrorLogs) warnings.push('error signals present in logs')
+
+  return warnings
+}
+
+function snapshotConfidence(hasUiTree: boolean, screen: string | null, activityHint: string, actionableCount: number, textCount: number, hasScreenshot: boolean, hasErrorLogs: boolean, hasLoadingSignals: boolean) {
+  const evidenceScore =
+    (hasUiTree ? 0.35 : 0) +
+    (screen ? 0.2 : 0) +
+    (activityHint ? 0.15 : 0) +
+    (actionableCount > 0 ? 0.15 : 0) +
+    (textCount > 0 ? 0.1 : 0) +
+    (hasScreenshot ? 0.05 : 0) +
+    (hasErrorLogs ? -0.15 : 0) +
+    (hasLoadingSignals ? -0.05 : 0)
+
+  const confidence = Math.max(0, Math.min(1, Number(evidenceScore.toFixed(2))))
+  return confidence
+}

package/src/observe/system-trace.ts

@@ -37,7 +37,7 @@ const clock: TraceClock = {
 export class SystemTraceSessions {
   private sessions = new Map<string, Session>()
   private closing = false
-  constructor(private native: TraceNative = new AndroidTraceNative(), private time: TraceClock = clock) {}
+  constructor(private native: TraceNative = new AndroidTraceNative(), private time: TraceClock = clock) { }
   private prune() {
     const terminal = [...this.sessions.values()].filter(s => s.terminalAt !== undefined && s.result.cleanup.status === 'complete')
     for (const s of terminal) if (this.time.mono() - s.terminalAt! >= 600000) this.sessions.delete(s.result.trace_id)
@@ -49,7 +49,7 @@ export class SystemTraceSessions {
   private snapshot(s: Session): TraceResult { return structuredClone(s.result) }
   private serial<T>(s: Session, fn: () => Promise<T>): Promise<T> {
     const result = s.queue.then(fn)
-    s.queue = result.catch(() => {})
+    s.queue = result.catch(() => { })
     return result
   }
   async capabilities(device: string) { return this.native.capabilities(device) }
@@ -59,11 +59,15 @@ export class SystemTraceSessions {
     const reserved = [...this.sessions.values()].filter(s => s.result.cleanup.status !== 'complete')
     if (this.closing || reserved.length >= 4 || reserved.some(s => s.result.device.id === device)) throw new TraceError('TRACE_BUSY')
     const id = randomUUID()
-    const result: TraceResult = { trace_id: id, session_id: evidenceSessionId(), device: { platform: 'android', id: device }, backend: 'perfetto',
+    const result: TraceResult = {
+      trace_id: id, session_id: evidenceSessionId(), device: { platform: 'android', id: device }, backend: 'perfetto',
       state: 'starting', reason: null, completion_reason: null, capture_started_at_ms: null, capture_completed_at_ms: null,
-      artifact: null, cleanup: { status: 'pending', remaining_resources: [] }, correlation_ids: [], enabled_sources: [], unavailable_categories: [] }
-    const s: Session = { result, remote: `/data/misc/perfetto-traces/mcp-${id}.perfetto-trace`, owner: null, stopped: true,
-      discard: false, deadline: Infinity, queue: Promise.resolve(), linked: new Set(), partial: new Set() }
+      artifact: null, cleanup: { status: 'pending', remaining_resources: [] }, correlation_ids: [], enabled_sources: [], unavailable_categories: []
+    }
+    const s: Session = {
+      result, remote: `/data/misc/perfetto-traces/mcp-${id}.perfetto-trace`, owner: null, stopped: true,
+      discard: false, deadline: Infinity, queue: Promise.resolve(), linked: new Set(), partial: new Set()
+    }
     this.sessions.set(id, s)
     return this.serial(s, async () => {
       let cap: TraceCapabilities
@@ -81,7 +85,7 @@ export class SystemTraceSessions {
         await this.native.start(device, s.remote, duration, cap.enabled_sources)
         s.owner = await this.native.find(device, s.remote)
         s.result.state = 'recording'
-        s.timer = this.time.schedule(() => { void this.control(id, 'duration').catch(() => {}) }, Math.max(0, s.deadline - this.time.mono()))
+        s.timer = this.time.schedule(() => { void this.control(id, 'duration').catch(() => { }) }, Math.max(0, s.deadline - this.time.mono()))
       } catch (error) {
         s.discard = true
         this.fail(s, this.code(error, 'TRACE_START_FAILED'))
@@ -173,15 +177,18 @@ export class SystemTraceSessions {
       }
       try { await this.remove(s) }
       catch { this.fail(s, 'TRACE_STOP_FAILED'); s.result.cleanup.status = 'failed'; return this.snapshot(s) }
-      if (hadStartFailure) s.result.state = 'failed'
-      else {
-        s.result.state = s.discard ? 'cancelled' : 'completed'
-        s.result.reason = s.discard ? 'TRACE_CANCELLED' : null
-        s.result.completion_reason = s.discard ? 'cancelled' : operation === 'duration' || this.time.mono() >= s.deadline ? 'duration_limit' : 'requested'
-      }
+      this.complete(s, hadStartFailure, operation)
       return this.snapshot(s)
     })
   }
+  private complete(s: Session, hadStartFailure: boolean, operation: string) {
+    if (hadStartFailure) s.result.state = 'failed'
+    else {
+      s.result.state = s.discard ? 'cancelled' : 'completed'
+      s.result.reason = s.discard ? 'TRACE_CANCELLED' : null
+      s.result.completion_reason = s.discard ? 'cancelled' : operation === 'duration' || this.time.mono() >= s.deadline ? 'duration_limit' : 'requested'
+    }
+  }
   async link(id: string, device: { platform: string; id: string }, correlation: string) {
     const s = this.get(id)
     if (device.platform !== 'android' || device.id !== s.result.device.id) throw new TraceError('TRACE_NOT_FOUND')
@@ -201,8 +208,10 @@ export class SystemTraceSessions {
         try { recording = (await this.control(id, 'status')).state === 'recording' }
         catch { /* Unavailable state cannot establish complete coverage. */ }
         s.linked.delete(correlation)
-        return { trace_id: id, coverage: !recording || s.partial.has(correlation) || this.time.mono() >= s.deadline ? 'partial' : 'covered',
-          clock_alignment: 'unknown' }
+        return {
+          trace_id: id, coverage: !recording || s.partial.has(correlation) || this.time.mono() >= s.deadline ? 'partial' : 'covered',
+          clock_alignment: 'unknown'
+        }
       })()
     })
   }