agent-device 0.21.19 → 0.21.21

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift

@@ -5,7 +5,7 @@ final class TextInputProbeIssues {
   var count = 0
 }
 
-enum TextInputProbeFailure: String {
+enum TextInputProbeFailure: String, Error {
   case recordedIssue = "text_input_probe_recorded_issue"
   case exception = "text_input_probe_exception"
 }
@@ -38,7 +38,60 @@ extension RunnerTests {
     }
   }
 
+
+
+  func coordinateTapTextInputIdentityAt(app: XCUIApplication, x: Double, y: Double) -> TextInputAtPoint? {
+    let probed = containingTextInputProbeIssues(fallback: nil) { shouldStop -> TextInputAtPoint? in
+      guard let element = queryTextInputs(app: app, point: CGPoint(x: x, y: y), shouldStop: shouldStop).first,
+            case .input(let identity) = probeTextEntryInput(element)
+      else {
+        return nil
+      }
+      return TextInputAtPoint(element: element, identity: identity)
+    }
+    guard case .success(let input) = probed else { return nil }
+    return input
+  }
+
+
+
+  func textInputStillResolves(_ input: TextInputAtPoint) -> Bool {
+    let probed = containingTextInputProbeIssues(fallback: false) { _ in
+      input.element.exists && probeTextEntryInput(input.element) == .input(input.identity)
+    }
+    guard case .success(let resolves) = probed else { return false }
+    return resolves
+  }
+
+
+
+  func clearedFieldRead(app: XCUIApplication, target: TextEntryTarget) -> ClearedFieldRead {
+    let probed = containingTextInputProbeIssues(fallback: ClearedFieldRead.unavailable) { _ in
+      guard let input = resolveTextEntryElement(app: app, target: target) else { return .unavailable }
+      if input.elementType == .secureTextField { return .unreadable }
+      return editableTextValue(for: input, treatingPlaceholderAsEmpty: true).map { .text($0) } ?? .unavailable
+    }
+    guard case .success(let read) = probed else { return .unavailable }
+    return read
+  }
+
   func probeTextInputs(app: XCUIApplication, point: CGPoint) -> TextInputProbeOutcome {
+    switch containingTextInputProbeIssues(fallback: [], { shouldStop in
+      queryTextInputs(app: app, point: point, shouldStop: shouldStop)
+    }) {
+    case .success(let elements):
+      return elements.isEmpty ? .absent : .matches(elements)
+    case .failure(let failure):
+      return .unavailable(failure)
+    }
+  }
+
+
+
+  private func containingTextInputProbeIssues<T>(
+    fallback: T,
+    _ probe: (_ shouldStop: () -> Bool) -> T
+  ) -> Result<T, TextInputProbeFailure> {
     precondition(Thread.isMainThread)
     let issues = TextInputProbeIssues()
     suppressedIssueLock.lock()
@@ -50,12 +103,12 @@ extension RunnerTests {
       textInputProbeIssues = previous
       suppressedIssueLock.unlock()
     }
-    let (elements, exception) = catchingObjCException(fallback: []) {
-      queryTextInputs(app: app, point: point, shouldStop: { self.hasTextInputProbeIssues(issues) })
+    let (value, exception) = catchingObjCException(fallback: fallback) {
+      probe({ self.hasTextInputProbeIssues(issues) })
     }
-    if hasTextInputProbeIssues(issues) { return .unavailable(.recordedIssue) }
-    if exception != nil { return .unavailable(.exception) }
-    return elements.isEmpty ? .absent : .matches(elements)
+    if hasTextInputProbeIssues(issues) { return .failure(.recordedIssue) }
+    if exception != nil { return .failure(.exception) }
+    return .success(value)
   }
 
   private func hasTextInputProbeIssues(_ scope: TextInputProbeIssues) -> Bool {

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift

@@ -117,6 +117,7 @@ extension RunnerTests {
       )
     }
 
+    var entryBaseline: TextEntryObservation?
     if repairMode == .replacement {
       guard let replacementTarget = initialTarget else {
         logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: text.count, mode: repairMode)
@@ -134,6 +135,7 @@ extension RunnerTests {
           mode: repairMode
         )
       }
+      entryBaseline = textEntryObservation(for: replacementTarget)
     }
 
 
@@ -194,7 +196,7 @@ extension RunnerTests {
         textEntryRoute = "xctest-element"
         currentTarget.typeText(value)
         return (currentTarget, nil)
-      } else if activeTarget.prefersFocusedElement && isKeyboardVisible(app: app) {
+      } else if activeTarget.boundIdentity == nil && activeTarget.prefersFocusedElement && isKeyboardVisible(app: app) {
 #if os(iOS)
 
 
@@ -344,7 +346,8 @@ extension RunnerTests {
       app: app,
       target: activeTarget,
       expectedText: expectedText,
-      repairMode: repairMode
+      repairMode: repairMode,
+      baseline: entryBaseline
     )
     logTextEntryPhase(
       commandId: commandId,
@@ -390,14 +393,40 @@ extension RunnerTests {
     app: XCUIApplication,
     target: TextEntryTarget,
     expectedText: String?,
-    repairMode: TextTypingRepairMode
+    repairMode: TextTypingRepairMode,
+    baseline: TextEntryObservation?
   ) -> TextEntryResult {
+    if target.boundIdentity != nil, resolveTextEntryElement(app: app, target: target) == nil {
+      guard boundTextEntryInputIsGone(app: app, target: target) else {
+        return TextEntryResult(
+          verified: nil,
+          repaired: false,
+          expectedText: expectedText,
+          observedText: nil,
+          failure: .commitNotObserved
+        )
+      }
+
+
+      NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_INPUT_REMOVED_AFTER_DELIVERY")
+      return TextEntryResult(verified: nil, repaired: false, expectedText: expectedText, observedText: nil)
+    }
     let initialResult = verifyTextEntry(
       app: app,
       target: target,
       expectedText: expectedText,
       repaired: false
     )
+
+
+    if let unconfirmed = settledUnconfirmedTextEntry(
+      app: app,
+      target: target,
+      result: initialResult,
+      baseline: baseline
+    ) {
+      return unconfirmed
+    }
 #if os(iOS)
     guard initialResult.verified == false,
           let expectedText = initialResult.expectedText
@@ -417,6 +446,15 @@ extension RunnerTests {
         repaired: false
       )
     }
+    guard Self.textEntryRepairCanTarget(boundIdentity: target.boundIdentity) else {
+      NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_REPAIR_REFUSED reason=unidentified-input")
+      return verifyTextEntry(
+        app: app,
+        target: target,
+        expectedText: expectedText,
+        repaired: false
+      )
+    }
 
     guard let repairTarget = resolveTextEntryElement(app: app, target: target) else {
       return initialResult
@@ -429,17 +467,83 @@ extension RunnerTests {
     )
     clearTextInput(repairTarget)
     repairTarget.typeText(expectedText)
-    return verifyTextEntry(
+    let repairedTarget = target.withElement(repairTarget)
+    let repairedResult = verifyTextEntry(
       app: app,
-      target: target.withElement(repairTarget),
+      target: repairedTarget,
       expectedText: expectedText,
       repaired: true
     )
+    return settledUnconfirmedTextEntry(
+      app: app,
+      target: repairedTarget,
+      result: repairedResult,
+      baseline: baseline
+    ) ?? repairedResult
 #else
     return initialResult
 #endif
   }
 
+
+
+  private func settledUnconfirmedTextEntry(
+    app: XCUIApplication,
+    target: TextEntryTarget,
+    result: TextEntryResult,
+    baseline: TextEntryObservation?
+  ) -> TextEntryResult? {
+    guard result.verified == false, let requested = result.expectedText, let baseline else {
+      return nil
+    }
+    func observe() -> TextEntryObservation? {
+      textEntryObservation(for: resolveTextEntryElement(app: app, target: target))
+    }
+    func evidence(_ observed: TextEntryObservation?) -> TextEntryUnconfirmedEvidence? {
+      Self.unconfirmedTextEntryEvidence(requested: requested, baseline: baseline, observed: observed)
+    }
+    let moveDeadline = Date().addingTimeInterval(TextEntryTiming.unconfirmedSettleCeiling)
+    var latest = observe()
+    while let unmoved = latest, unmoved.isSettled(with: baseline), Date() < moveDeadline {
+      sleepFor(TextEntryTiming.pollInterval)
+      latest = observe()
+    }
+    guard evidence(latest) != nil else {
+      return nil
+    }
+    let ceiling = Date().addingTimeInterval(TextEntryTiming.unconfirmedSettleCeiling)
+    var stableSince = Date()
+    while Date().timeIntervalSince(stableSince) < TextEntryTiming.verificationStabilityWindow {
+      guard Date() < ceiling else {
+        return nil
+      }
+      sleepFor(TextEntryTiming.pollInterval)
+      let next = observe()
+      if let next, let settled = latest, next.isSettled(with: settled) {
+        latest = next
+        continue
+      }
+      latest = next
+      stableSince = Date()
+    }
+    guard let unconfirmed = evidence(latest) else {
+      return nil
+    }
+    NSLog(
+      "AGENT_DEVICE_RUNNER_TEXT_ENTRY_UNCONFIRMED expectedLength=%d observedLength=%d repaired=%d",
+      requested.count,
+      unconfirmed.after.count,
+      result.repaired ? 1 : 0
+    )
+    return TextEntryResult(
+      verified: nil,
+      repaired: result.repaired,
+      expectedText: requested,
+      observedText: unconfirmed.after,
+      unconfirmed: unconfirmed
+    )
+  }
+
   private func verifyTextEntry(
     app: XCUIApplication,
     target: TextEntryTarget,
@@ -503,18 +607,10 @@ extension RunnerTests {
     if observedText == expectedText {
       return true
     }
-    guard hasTextEntrySubmitSuffix(expectedText), element?.elementType != .textView else {
+    guard element?.elementType != .textView else {
       return false
     }
-    var submittedText = expectedText
-    while hasTextEntrySubmitSuffix(submittedText) {
-      submittedText.removeLast()
-    }
-    return observedText == submittedText
-  }
-
-  private func hasTextEntrySubmitSuffix(_ text: String) -> Bool {
-    text.hasSuffix("\n") || text.hasSuffix("\r")
+    return observedText == Self.textEntryRequestWithoutSubmitKeys(expectedText)
   }
 
   private func expectedTextEntryValue(
@@ -611,13 +707,6 @@ extension RunnerTests {
     guard missingCharacterCount <= max(2, expectedText.count / 4) else {
       return false
     }
-    var expectedIndex = expectedText.startIndex
-    for character in observedText {
-      guard let matchIndex = expectedText[expectedIndex...].firstIndex(of: character) else {
-        return false
-      }
-      expectedIndex = expectedText.index(after: matchIndex)
-    }
-    return true
+    return Self.isOrderedSubsequence(observedText, of: expectedText)
   }
 }

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift

@@ -27,6 +27,8 @@ extension RunnerTests {
       hasY: command.y != nil,
       xCTestChannelPenalized: xCTestChannelPenalized
     ), let x = command.x, let y = command.y {
+
+      let inputAtPoint = coordinateTapTextInputIdentityAt(app: activeApp, x: x, y: y)
       let policyKind = SynthesizedGesturePolicyKind.coordinateTap
       let context = synthesizedCoordinateContext(
         app: activeApp,
@@ -40,7 +42,8 @@ extension RunnerTests {
         resolvedCoordinateTarget = TextEntryTarget(
           element: nil,
           refreshPoint: CGPoint(x: x, y: y),
-          prefersFocusedElement: false
+          prefersFocusedElement: false,
+          inputAtRefreshPoint: inputAtPoint
         )
       case .xctestFallback:
         break
@@ -120,17 +123,27 @@ extension RunnerTests {
 
       frame = .zero
     }
-    return Response(
-      ok: true,
-      data: DataPayload(
-        message: textResult.repaired ? "typed after repair" : "typed",
-        x: point.map { Double($0.x) },
-        y: point.map { Double($0.y) },
-        referenceWidth: frame.isEmpty ? nil : Double(frame.width),
-        referenceHeight: frame.isEmpty ? nil : Double(frame.height),
-        maestroNonHittableCoordinateFallbackUsed: maestroNonHittableCoordinateFallbackUsed,
-        textEntryRoute: textResult.textEntryRoute
-      )
+    var data = DataPayload(
+      message: textResult.repaired ? "typed after repair" : "typed",
+      x: point.map { Double($0.x) },
+      y: point.map { Double($0.y) },
+      referenceWidth: frame.isEmpty ? nil : Double(frame.width),
+      referenceHeight: frame.isEmpty ? nil : Double(frame.height),
+      maestroNonHittableCoordinateFallbackUsed: maestroNonHittableCoordinateFallbackUsed,
+      textEntryRoute: textResult.textEntryRoute
     )
+    if let unconfirmed = textResult.unconfirmed {
+      data.verification = "unconfirmed"
+      data.requested = unconfirmed.requested
+      data.before = unconfirmed.before
+      data.after = unconfirmed.after
+      data.target = TextEntryVerificationTargetPayload(
+        resourceId: unconfirmed.target.identifier,
+        className: unconfirmed.target.elementType,
+        packageName: command.appBundleId,
+        rect: SnapshotRect(unconfirmed.target.frame)
+      )
+    }
+    return Response(ok: true, data: data)
   }
 }

package/dist/apple/runner/AgentDeviceRunner/RecordingScripts/RecordingExportSupport.swift

@@ -89,9 +89,14 @@ func makeRecordingExporter(
 }
 
 
+let defaultRecordingExportTimeoutSeconds: Double = 120
+
+
+
 
 func runRecordingExport(
   _ exporter: AVAssetExportSession,
+  timeoutSeconds: Double = defaultRecordingExportTimeoutSeconds,
   timeoutMessage: String,
   failureMessage: String
 ) throws {
@@ -99,7 +104,7 @@ func runRecordingExport(
   exporter.exportAsynchronously {
     semaphore.signal()
   }
-  if semaphore.wait(timeout: .now() + 120) == .timedOut {
+  if semaphore.wait(timeout: .now() + timeoutSeconds) == .timedOut {
     exporter.cancelExport()
     throw RecordingScriptError.exportFailed(timeoutMessage)
   }

package/dist/apple/runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift

@@ -88,7 +88,10 @@ func run() throws {
   let renderSize = resolvedRenderSize(for: sourceVideoTrack)
   let videoComposition = AVMutableVideoComposition()
   videoComposition.renderSize = renderSize
-  videoComposition.frameDuration = resolvedFrameDuration(for: sourceVideoTrack)
+  videoComposition.frameDuration = resolvedFrameDuration(
+    for: sourceVideoTrack,
+    maximumFrameRate: parsedArgs.maximumFrameRate
+  )
 
   let instruction = AVMutableVideoCompositionInstruction()
   instruction.timeRange = fullRange
@@ -145,6 +148,7 @@ func run() throws {
   )
   try runRecordingExport(
     exporter,
+    timeoutSeconds: parsedArgs.timeoutSeconds,
     timeoutMessage: "Touch overlay export timed out.",
     failureMessage: "Touch overlay export failed."
   )
@@ -157,10 +161,15 @@ func run() throws {
 
 func parseArguments(
   _ arguments: [String]
-) throws -> (inputPath: String, outputPath: String, eventsPath: String) {
+) throws -> (
+  inputPath: String, outputPath: String, eventsPath: String, timeoutSeconds: Double,
+  maximumFrameRate: Int32
+) {
   var inputPath: String?
   var outputPath: String?
   var eventsPath: String?
+  var timeoutSeconds = defaultRecordingExportTimeoutSeconds
+  var maximumFrameRate = maximumCompositedFrameRate
   var index = 0
 
   while index < arguments.count {
@@ -185,6 +194,20 @@ func parseArguments(
         throw RecordingScriptError.invalidArgs("--quality must be one of: medium, high")
       }
       index += 2
+    case "--timeout-ms":
+      let rawValue = try recordingOptionValue(arguments, nextIndex, "--timeout-ms")
+      guard let milliseconds = Double(rawValue), milliseconds > 0 else {
+        throw RecordingScriptError.invalidArgs("--timeout-ms must be a positive number")
+      }
+      timeoutSeconds = milliseconds / 1000
+      index += 2
+    case "--max-fps":
+      let rawValue = try recordingOptionValue(arguments, nextIndex, "--max-fps")
+      guard let framesPerSecond = Int32(rawValue), framesPerSecond > 0 else {
+        throw RecordingScriptError.invalidArgs("--max-fps must be a positive integer")
+      }
+      maximumFrameRate = min(framesPerSecond, maximumCompositedFrameRate)
+      index += 2
     default:
       throw RecordingScriptError.invalidArgs("Unknown argument: \(argument)")
     }
@@ -192,10 +215,10 @@ func parseArguments(
 
   guard let inputPath, let outputPath, let eventsPath else {
     throw RecordingScriptError.invalidArgs(
-      "Usage: recording-overlay.swift --input <video> --output <video> --events <json> [--quality <medium|high>]"
+      "Usage: recording-overlay.swift --input <video> --output <video> --events <json> [--quality <medium|high>] [--timeout-ms <ms>] [--max-fps <n>]"
     )
   }
-  return (inputPath, outputPath, eventsPath)
+  return (inputPath, outputPath, eventsPath, timeoutSeconds, maximumFrameRate)
 }
 
 
@@ -329,19 +352,26 @@ func frameMeanLuma(_ image: CGImage) -> Double? {
   return total / Double(count)
 }
 
-func resolvedFrameDuration(for track: AVAssetTrack) -> CMTime {
+
+
+let maximumCompositedFrameRate: Int32 = 30
+
+
+
+
+
+
+
+
+
+func resolvedFrameDuration(for track: AVAssetTrack, maximumFrameRate: Int32) -> CMTime {
+  let shortest = CMTime(value: 1, timescale: maximumFrameRate)
   let minFrameDuration = track.minFrameDuration
   if minFrameDuration.isValid && !minFrameDuration.isIndefinite && minFrameDuration.seconds > 0 {
-    return minFrameDuration
-  }
-
-  let nominalFrameRate = track.nominalFrameRate
-  if nominalFrameRate > 0 {
-    let timescale = Int32(max(1, round(nominalFrameRate)))
-    return CMTime(value: 1, timescale: timescale)
+    return CMTimeMaximum(minFrameDuration, shortest)
   }
 
-  return CMTime(value: 1, timescale: 60)
+  return shortest
 }
 
 func overlayPoint(event: GestureEvent, x: Double, y: Double, renderSize: CGSize) -> CGPoint {

package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift

@@ -29,7 +29,7 @@ public struct SnapshotRect: Codable, Equatable {
 }
 
 public struct RawAXNode: Equatable {
-  public let index: Int
+  public private(set) var index: Int
   public let type: String
   public let label: String?
   public let identifier: String?
@@ -42,10 +42,10 @@ public struct RawAXNode: Equatable {
   public let selected: Bool?
   /// Geometric actionability; `nil` when the capture has no viewport box to decide it against.
   public var hittable: Bool?
-  public let depth: Int
-  public let parentIndex: Int?
-  public let hiddenContentAbove: Bool?
-  public let hiddenContentBelow: Bool?
+  public private(set) var depth: Int
+  public private(set) var parentIndex: Int?
+  public private(set) var hiddenContentAbove: Bool?
+  public private(set) var hiddenContentBelow: Bool?
   public var actions: [String]?
 
   public init(
@@ -91,6 +91,21 @@ public struct RawAXNode: Equatable {
     return updated
   }
 
+  func reindexed(index: Int, depth: Int, parentIndex: Int?) -> RawAXNode {
+    var updated = self
+    updated.index = index
+    updated.depth = depth
+    updated.parentIndex = parentIndex
+    return updated
+  }
+
+  func addingHiddenContentHints(above: Bool, below: Bool) -> RawAXNode {
+    var updated = self
+    updated.hiddenContentAbove = hiddenContentAbove == true || above ? true : nil
+    updated.hiddenContentBelow = hiddenContentBelow == true || below ? true : nil
+    return updated
+  }
+
   public var hasSemanticContent: Bool {
     [label, identifier, value].contains {
       !($0?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ?? true)

package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentationProjection.swift

@@ -134,23 +134,10 @@ extension SnapshotPresentation {
     return nodes.enumerated().map { offset, node in
       let raw = node.raw
       return SnapshotPresentationNode(
-        raw: RawAXNode(
+        raw: raw.reindexed(
           index: offset,
-          type: raw.type,
-          label: raw.label,
-          identifier: raw.identifier,
-          value: raw.value,
-          placeholder: raw.placeholder,
-          rect: raw.rect,
-          enabled: raw.enabled,
-          focused: raw.focused,
-          selected: raw.selected,
-          hittable: raw.hittable,
           depth: max(0, raw.depth - depthOffset),
-          parentIndex: raw.parentIndex.flatMap { indexMap[$0] },
-          hiddenContentAbove: raw.hiddenContentAbove,
-          hiddenContentBelow: raw.hiddenContentBelow,
-          actions: raw.actions
+          parentIndex: raw.parentIndex.flatMap { indexMap[$0] }
         ),
         effectiveRect: node.effectiveRect
       )

package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotVisibilityFold.swift

@@ -191,33 +191,18 @@ public enum SnapshotVisibilityFold {
       if decision.isIncluded {
         let outIndex = kept.count
         let outDepth = keptDepth + 1
+        var projected = node.reindexed(index: outIndex, depth: outDepth, parentIndex: keptIndex)
+        projected.hittable = node.parentIndex == nil
+          ? false
+          : clippedHittability(
+            source: node.hittable,
+            enabled: node.enabled,
+            clippedFrame: decision.effectiveFrame,
+            viewport: viewport
+          )
         kept.append(
           SnapshotPresentationNode(
-            raw: RawAXNode(
-              index: outIndex,
-              type: node.type,
-              label: node.label,
-              identifier: node.identifier,
-              value: node.value,
-              placeholder: node.placeholder,
-              rect: node.rect,
-              enabled: node.enabled,
-              focused: node.focused,
-              selected: node.selected,
-              hittable: node.parentIndex == nil
-                ? false
-                : clippedHittability(
-                  source: node.hittable,
-                  enabled: node.enabled,
-                  clippedFrame: decision.effectiveFrame,
-                  viewport: viewport
-                ),
-              depth: outDepth,
-              parentIndex: keptIndex,
-              hiddenContentAbove: node.hiddenContentAbove,
-              hiddenContentBelow: node.hiddenContentBelow,
-              actions: node.actions
-            ),
+            raw: projected,
             effectiveRect: SnapshotGeometry.snapshotRect(
               from: decision.effectiveFrame,
               reportedFrame: rect

package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotVisibilityFoldProjection.swift

@@ -73,24 +73,7 @@ extension SnapshotVisibilityFold {
       let node = presentationNode.raw
       guard let hint = hints[node.index] else { return presentationNode }
       return SnapshotPresentationNode(
-        raw: RawAXNode(
-          index: node.index,
-          type: node.type,
-          label: node.label,
-          identifier: node.identifier,
-          value: node.value,
-          placeholder: node.placeholder,
-          rect: node.rect,
-          enabled: node.enabled,
-          focused: node.focused,
-          selected: node.selected,
-          hittable: node.hittable,
-          depth: node.depth,
-          parentIndex: node.parentIndex,
-          hiddenContentAbove: node.hiddenContentAbove == true || hint.above ? true : nil,
-          hiddenContentBelow: node.hiddenContentBelow == true || hint.below ? true : nil,
-          actions: node.actions
-        ),
+        raw: node.addingHiddenContentHints(above: hint.above, below: hint.below),
         effectiveRect: presentationNode.effectiveRect
       )
     }