agent-device 0.21.23 → 0.21.25

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift

@@ -114,6 +114,7 @@ extension RunnerTests {
     let app: XCUIApplication
 
     var systemSurface: SystemSurfaceHost? = nil
+    var observation: SnapshotObservationPayload? = nil
   }
 
   enum ActiveCommandPreparation {
@@ -494,6 +495,26 @@ extension RunnerTests {
     command: Command,
     routeToSpringboard: Bool = false
   ) -> ActiveCommandPreparation {
+    if command.command == .snapshot, command.observeOnly == true {
+#if os(iOS)
+      guard command.appBundleId?.trimmedNonEmpty != nil else {
+        return .response(observeOnlyUnavailableResponse())
+      }
+      let observedApp = resolveAppWithoutActivation(command: command)
+      let reportedState = observedApp.state
+      guard reportedState == .runningForeground else {
+        return .response(observeOnlyUnavailableResponse())
+      }
+      return .context(ActiveCommandContext(
+        app: observedApp,
+        observation: SnapshotObservationPayload(appState: Self.applicationStateName(reportedState))
+      ))
+#else
+      return .response(Response(ok: false, error: ErrorPayload(
+        code: "UNSUPPORTED_OPERATION", message: "observe-only snapshot is supported on iOS and iPadOS only"
+      )))
+#endif
+    }
     if routeToSpringboard {
       return .context(ActiveCommandContext(app: springboard))
     }
@@ -516,11 +537,26 @@ extension RunnerTests {
 
 
 
+
+
       return prepareActivatedTarget(command: command)
 #endif
     case .existingApp:
 
 
+#if os(macOS)
+      if let bundleId = command.appBundleId?.trimmedNonEmpty,
+        macReadMayBeServedInBackground(
+          command: command,
+          targetState: XCUIApplication(bundleIdentifier: bundleId).state
+        )
+      {
+
+
+        refreshCachedTargetIfProcessChanged(bundleId: bundleId)
+        return .context(ActiveCommandContext(app: resolveAppWithoutActivation(command: command)))
+      }
+#endif
       return prepareActivatedTarget(command: command)
     case .mayLaunch:
       if shouldSkipAppActivationPreflight(command) {
@@ -644,6 +680,14 @@ extension RunnerTests {
     return XCUIApplication(bundleIdentifier: host.bundleId).state
   }
 
+  func observeOnlyUnavailableResponse() -> Response {
+    Response(ok: false, error: ErrorPayload(
+      code: "OBSERVATION_UNAVAILABLE",
+      message: "The session app cannot be observed without activation.",
+      hint: "XCUIApplication must report runningForeground before and after capture. This state report does not prove screen ownership (#2696)."
+    ))
+  }
+
   func currentXCTestFailureCount() -> Int {
     return testRun?.failureCount ?? 0
   }
@@ -746,4 +790,16 @@ extension RunnerTests {
     }
     return XCUIApplication(bundleIdentifier: bundleId)
   }
+
+
+
+
+
+
+
+
+
+  func macReadMayBeServedInBackground(command: Command, targetState: XCUIApplication.State) -> Bool {
+    command.appBundleId?.trimmedNonEmpty != nil && targetState == .runningBackground
+  }
 }

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift

@@ -429,17 +429,26 @@ extension RunnerTests {
     case .screenshot:
 #if os(macOS)
 
-      if let bundleId = command.appBundleId, !bundleId.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
+
+
+
+      let screenshotBundleId = command.appBundleId?.trimmedNonEmpty
+      if let bundleId = screenshotBundleId {
         let targetApp = XCUIApplication(bundleIdentifier: bundleId)
-        targetApp.activate()
-        activeApp = targetApp
+        if macAppCaptureNeedsRaise(
+          fullscreen: command.fullscreen,
+          targetState: targetApp.state
+        ) {
+          targetApp.activate()
 
-        sleepFor(0.5)
+          sleepFor(0.5)
+        }
+        activeApp = targetApp
       }
       let screenshot: XCUIScreenshot
       if command.fullscreen == true {
         screenshot = XCUIScreen.main.screenshot()
-      } else if let bundleId = command.appBundleId, !bundleId.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
+      } else if screenshotBundleId != nil {
         screenshot = screenshotRoot(app: activeApp).screenshot()
       } else {
         screenshot = XCUIScreen.main.screenshot()

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Environment.swift

@@ -1,18 +1,25 @@
 import Foundation
+import Network
 
 
 
-enum RunnerEnv {
-  static func resolvePort() -> UInt16 {
-    if let env = ProcessInfo.processInfo.environment["AGENT_DEVICE_RUNNER_PORT"], let port = UInt16(env) {
-      return port
-    }
-    for arg in CommandLine.arguments {
-      if arg.hasPrefix("AGENT_DEVICE_RUNNER_PORT=") {
-        let value = arg.replacingOccurrences(of: "AGENT_DEVICE_RUNNER_PORT=", with: "")
-        if let port = UInt16(value) { return port }
-      }
+
+
+
+struct RunnerListenerConfiguration {
+  let port: NWEndpoint.Port
+  let token: RunnerListenerToken
+
+  static func fromEnvironment(
+    _ environment: [String: String] = ProcessInfo.processInfo.environment
+  ) -> RunnerListenerConfiguration? {
+    guard
+      let rawPort = environment["AGENT_DEVICE_RUNNER_PORT"].flatMap({ UInt16($0) }), rawPort > 0,
+      let port = NWEndpoint.Port(rawValue: rawPort),
+      let token = RunnerListenerToken(environment["AGENT_DEVICE_RUNNER_TOKEN"])
+    else {
+      return nil
     }
-    return 0
+    return RunnerListenerConfiguration(port: port, token: token)
   }
 }

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift

@@ -284,6 +284,14 @@ extension RunnerTests {
     return false
   }
 
+
+
+
+  func macAppCaptureNeedsRaise(fullscreen: Bool?, targetState: XCUIApplication.State) -> Bool {
+    guard targetState != .runningForeground else { return false }
+    return fullscreen != true || targetState == .notRunning
+  }
+
   @MainActor
   func canUseFastForegroundAppGuard(
     activeApp: XCUIApplication,

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift

@@ -66,6 +66,9 @@ enum CommandLaunchPolicy: Equatable {
 
 
 
+
+
+
   case existingApp
 
   case mayLaunch
@@ -239,6 +242,7 @@ struct Command: Codable {
   let interactiveOnly: Bool?
   let preferredBackend: String?
   let customActions: Bool?
+  let observeOnly: Bool?
   let depth: Int?
   let scope: String?
   let raw: Bool?
@@ -488,6 +492,7 @@ struct DataPayload: Codable {
   var failedStepIndex: Int?
   var sequenceResults: [SequenceStepResult]?
   var targetActivation: TargetActivationFactPayload?
+  var observation: SnapshotObservationPayload?
 
 
   var screenshotMetadata: ScreenshotMetadataPayload?
@@ -529,6 +534,13 @@ struct SystemSurfaceProvenancePayload: Codable {
   let kind: String
 }
 
+struct SnapshotObservationPayload: Codable {
+  var mode = "observe-only"
+  var activationPerformed = false
+  let appState: String
+  var appStateSource = "xcuiapplication-state"
+}
+
 struct SnapshotQualityPayload: Codable {
   let nodes: [PresentedNode]
   let truncated: Bool

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift

@@ -3,6 +3,8 @@ import XCTest
 
 extension RunnerTests {
   static let navigationBackKeywords = ["back", "close", "cancel"]
+  static let systemBackButtonIdentifier = "BackButton"
+  static let floatingBarBackControlMaxYFraction: CGFloat = 0.4
   static let navigationFallbackVerificationDelay: TimeInterval = 0.25
 
 
@@ -88,7 +90,11 @@ extension RunnerTests {
     let candidates = app.buttons.matching(Self.navigationBackPredicate()).allElementsBoundByIndex.compactMap {
       element -> (XCUIElement, Int)? in
       guard element.exists, element.isHittable else { return nil }
-      guard Self.isTopNavigationControlFrame(element.frame, in: frame) else { return nil }
+      guard Self.isTopNavigationBackCandidateFrame(
+        element.frame,
+        in: frame,
+        identifier: element.identifier
+      ) else { return nil }
       guard let rank = Self.navigationBackControlRank(
         label: element.label,
         identifier: element.identifier
@@ -132,6 +138,24 @@ extension RunnerTests {
     return candidate.midY >= window.minY && candidate.midY <= maxY
   }
 
+
+
+
+  static func isTopNavigationBackCandidateFrame(
+    _ candidate: CGRect,
+    in window: CGRect,
+    identifier: String
+  ) -> Bool {
+    if isTopNavigationControlFrame(candidate, in: window) { return true }
+    guard identifier == systemBackButtonIdentifier,
+      SnapshotGeometry.isPositiveFinite(candidate), SnapshotGeometry.isPositiveFinite(window)
+    else {
+      return false
+    }
+    let maxY = window.minY + window.height * floatingBarBackControlMaxYFraction
+    return candidate.midY >= window.minY && candidate.midY <= maxY
+  }
+
   static func topLeadingNavigationFallbackPoint(in frame: CGRect) -> CGPoint? {
     guard SnapshotGeometry.isPositiveFinite(frame) else {
       return nil

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift

@@ -17,7 +17,7 @@ struct SnapshotCaptureTarget {
 
 enum SnapshotCommandPreparation {
   case response(Response)
-  case capture(SnapshotCaptureTarget, systemSurface: SystemSurfaceHost?)
+  case capture(SnapshotCaptureTarget, systemSurface: SystemSurfaceHost?, observation: SnapshotObservationPayload?)
 }
 
 

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift

@@ -19,19 +19,27 @@ extension RunnerTests {
         return .response(response)
       case .context(let context):
         return .capture(
-          self.takeSnapshotCaptureTarget(app: context.app),
-          systemSurface: context.systemSurface
+          context.observation == nil
+            ? self.takeSnapshotCaptureTarget(app: context.app)
+            : SnapshotCaptureTarget(
+              app: context.app,
+              bundleId: command.appBundleId?.trimmedNonEmpty,
+              processIdentifier: Self.processIdentifier(of: context.app)
+            ),
+          systemSurface: context.systemSurface,
+          observation: context.observation
         )
       }
     }
     switch preparation {
     case .response(let response):
       return response
-    case .capture(let target, let systemSurface):
+    case .capture(let target, let systemSurface, let observation):
       return try executeSnapshotPrepared(
         command: command,
         target: target,
-        systemSurface: systemSurface
+        systemSurface: systemSurface,
+        observation: observation
       )
     }
   }
@@ -86,7 +94,8 @@ extension RunnerTests {
   private func executeSnapshotPrepared(
     command: Command,
     target: SnapshotCaptureTarget,
-    systemSurface: SystemSurfaceHost?
+    systemSurface: SystemSurfaceHost?,
+    observation: SnapshotObservationPayload?
   ) throws -> Response {
     let options = Self.presentationOptions(from: command)
     do {
@@ -102,6 +111,21 @@ extension RunnerTests {
           kind: systemSurface.kind.rawValue
         )
       }
+      if let observation {
+        guard let capturedProcessIdentifier = target.processIdentifier else {
+          return observeOnlyUnavailableResponse()
+        }
+        let unchanged = try runMainThreadWork(
+          "observe_only_state",
+          timeout: Self.mainThreadExecutionTimeout,
+          timeoutError: Self.mainThreadExecutionTimeoutError
+        ) {
+          target.app.state == .runningForeground
+            && Self.processIdentifier(of: target.app) == capturedProcessIdentifier
+        }
+        guard unchanged else { return observeOnlyUnavailableResponse() }
+        payload.observation = observation
+      }
       setNeedsPostSnapshotInteractionDelay()
       return Response(ok: true, data: payload)
     } catch let failure as SnapshotCaptureFailure {

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift

@@ -524,9 +524,6 @@ extension RunnerTests {
 
 
 
-
-
-
   static func awaitSynthesizedReplacementCommitOutcome(
     expectedText: String,
     placeholder: String?,

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift

@@ -64,9 +64,7 @@ extension RunnerTests {
     static let warmupValueTimeout: TimeInterval = 0.4
     static let verificationStabilityWindow: TimeInterval = 0.2
 
-
-
-    static let unconfirmedSettleCeiling: TimeInterval = 1.0
+    static let replacementSettleCeiling: TimeInterval = 1.0
 
 
 
@@ -243,7 +241,7 @@ extension RunnerTests {
     }
   }
 
-  func clearTextInput(_ element: XCUIElement) {
+  func clearTextInput(app: XCUIApplication, _ element: XCUIElement) {
 
 
 
@@ -257,7 +255,7 @@ extension RunnerTests {
       return
     }
 #if !os(tvOS)
-    moveCaretToEnd(element: element)
+    moveCaretToEnd(app: app, element: element)
 #endif
     let count = estimatedDeleteCount(for: element)
     let deletes = String(repeating: XCUIKeyboardKey.delete.rawValue, count: count)
@@ -358,6 +356,21 @@ extension RunnerTests {
   }
 
 
+
+
+
+
+
+
+
+
+  func textEntrySnapshotFrame(_ element: XCUIElement) -> CGRect? {
+
+
+    safely("TEXT_ENTRY_SNAPSHOT_FRAME", { element.exists ? try? element.snapshot() : nil })?.frame
+  }
+
+
   func probeTextEntryInput(_ element: XCUIElement) -> TextEntryInputProbe {
     var probe = TextEntryInputProbe.unavailable
     let (_, exception) = catchingObjCException(fallback: ()) {
@@ -395,20 +408,29 @@ extension RunnerTests {
     return nil
   }
 
-  private func moveCaretToEnd(element: XCUIElement) {
+  private func moveCaretToEnd(app: XCUIApplication, element: XCUIElement) {
 #if os(tvOS)
     return
 #else
-    let frame = element.frame
-    guard !frame.isEmpty else {
-      element.tap()
+
+
+
+
+
+
+
+
+
+
+
+    guard let frame = textEntrySnapshotFrame(element), !frame.isEmpty else {
       return
     }
-    let origin = element.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
-    let target = origin.withOffset(
-      CGVector(dx: max(2, frame.width - 4), dy: max(2, frame.height / 2))
+    _ = tapAt(
+      app: app,
+      x: frame.minX + max(2, frame.width - 4),
+      y: frame.minY + max(2, frame.height / 2)
     )
-    target.tap()
 #endif
   }
 

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryConfirmation.swift

@@ -39,41 +39,74 @@ extension RunnerTests {
     let target: TextEntryElementIdentity
   }
 
+  struct ReplacementTextEntryConfirmation {
+    private let requested: String
+    private let baseline: TextEntryObservation?
+    private var deadline: Date
+    private var latest: TextEntryObservation?
+    private var stableSince: Date
+
+    init(requested: String, baseline: TextEntryObservation?, startedAt: Date) {
+      self.requested = requested
+      self.baseline = baseline
+      deadline = startedAt.addingTimeInterval(TextEntryTiming.replacementSettleCeiling)
+      stableSince = startedAt
+    }
 
+    mutating func observe(_ observed: TextEntryObservation, at sampledAt: Date) -> TextEntryResult? {
 
-
-
-
-  static func textEntryValueEchoes(observed: String, expected: String, baseline: String) -> Bool {
-    let request = textEntryRequestWithoutSubmitKeys(expected)
-    let residualAndRequest = baseline + request
-    return isOrderedSubsequence(observed, of: residualAndRequest)
-      || (isOrderedSubsequence(request, of: observed) && !isOrderedSubsequence(request, of: baseline))
-      || isOrderedSubsequence(residualAndRequest, of: observed)
+      if latest == nil {
+        deadline = max(deadline, sampledAt.addingTimeInterval(TextEntryTiming.verificationStabilityWindow))
+      }
+      if latest.map({ observed.isSettled(with: $0) }) != true { stableSince = sampledAt }
+      latest = observed
+      let result = RunnerTests.replacementTextEntryResult(requested: requested, baseline: baseline, observed: observed)
+      let settled = sampledAt.timeIntervalSince(stableSince) >= TextEntryTiming.verificationStabilityWindow
+      let moved = baseline.map { !observed.isSettled(with: $0) } ?? true
+      if settled && (moved || result.verified != false || sampledAt >= deadline) {
+        return result
+      }
+      if sampledAt >= deadline {
+        return TextEntryResult(
+          verified: nil, repaired: false, expectedText: requested, observedText: observed.value,
+          failure: .commitNotObserved
+        )
+      }
+      return nil
+    }
   }
 
 
-
-
-  static func unconfirmedTextEntryEvidence(
+  static func replacementTextEntryResult(
     requested: String,
     baseline: TextEntryObservation?,
     observed: TextEntryObservation?
-  ) -> TextEntryUnconfirmedEvidence? {
-    guard !textEntryRequestWithoutSubmitKeys(requested).isEmpty,
-          let baseline,
-          let observed,
-          baseline.identity.isSameElement(as: observed.identity),
-          observed.value != baseline.value,
-          !textEntryValueEchoes(observed: observed.value, expected: requested, baseline: baseline.value)
-    else {
-      return nil
+  ) -> TextEntryResult {
+    func result(_ verified: Bool?) -> TextEntryResult {
+      TextEntryResult(verified: verified, repaired: false, expectedText: requested, observedText: observed?.value)
+    }
+    guard let observed else { return result(nil) }
+    if let baseline, !baseline.identity.isSameElement(as: observed.identity) {
+      return result(false)
     }
-    return TextEntryUnconfirmedEvidence(
-      requested: requested,
-      before: baseline.value,
-      after: observed.value,
-      target: observed.identity
+    let request = observed.identity.elementType == elementTypeNamesByRawValue[XCUIElement.ElementType.textView.rawValue]
+      ? requested : textEntryRequestWithoutSubmitKeys(requested)
+    if observed.value == requested || observed.value == request { return result(true) }
+
+    if request != requested { return result(nil) }
+    guard !request.isEmpty, let baseline else { return result(false) }
+    if isOrderedSubsequence(observed.value, of: baseline.value + request) { return result(false) }
+    return TextEntryResult(
+      verified: nil,
+      repaired: false,
+      expectedText: requested,
+      observedText: observed.value,
+      unconfirmed: TextEntryUnconfirmedEvidence(
+        requested: requested,
+        before: baseline.value,
+        after: observed.value,
+        target: observed.identity
+      )
     )
   }
 

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift

@@ -137,7 +137,7 @@ extension RunnerTests {
     if keyboardVisibleBeforeTap, let target {
       return TextEntryTarget(
         element: target,
-        refreshPoint: textEntryRefreshPoint(for: target) ?? requestedPoint,
+        refreshPoint: textEntryRefreshPoint(for: target, fallbackTo: requestedPoint),
         prefersFocusedElement: false
       )
     }
@@ -157,18 +157,32 @@ extension RunnerTests {
       : (waitForTextEntryReadiness(app: app, target: readyTarget) ?? stabilized.element ?? target)
     return TextEntryTarget(
       element: element,
-      refreshPoint: textEntryRefreshPoint(for: element) ?? requestedPoint,
+      refreshPoint: textEntryRefreshPoint(for: element, fallbackTo: requestedPoint),
       prefersFocusedElement: false
     )
   }
 
-  private func textEntryRefreshPoint(for element: XCUIElement?) -> CGPoint? {
+
+
+
+
+
+
+
+
+
+  private func textEntryRefreshPoint(
+    for element: XCUIElement?,
+    fallbackTo requestedPoint: CGPoint?
+  ) -> CGPoint? {
     guard let element else {
+      return requestedPoint
+    }
+    guard let frame = textEntrySnapshotFrame(element) else {
       return nil
     }
-    let frame = element.frame
     guard !frame.isEmpty else {
-      return nil
+      return requestedPoint
     }
     return CGPoint(x: frame.midX, y: frame.midY)
   }