agent-device 0.21.15 → 0.21.17

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift

@@ -42,8 +42,16 @@ extension RunnerTests {
 #if os(iOS)
       let postStartedAt = Date()
       let result = replacingExistingText
-        ? RunnerSynthesizedTextEntry.replaceText(withApplication: app, text: text)
-        : RunnerSynthesizedTextEntry.synthesizeText(withApplication: app, text: text)
+        ? RunnerSynthesizedTextEntry.replaceText(
+          withApplication: app,
+          text: text,
+          charactersPerSecond: TextEntryTiming.synthesizedCharactersPerSecond
+        )
+        : RunnerSynthesizedTextEntry.synthesizeText(
+          withApplication: app,
+          text: text,
+          charactersPerSecond: TextEntryTiming.synthesizedCharactersPerSecond
+        )
       NSLog(
         "[DEBUG-1874] synthesize posted %d chars status=%d tookMs=%.0f",
         text.count, result.status.rawValue, postStartedAt.timeIntervalSinceNow * -1000
@@ -73,90 +81,206 @@ extension RunnerTests {
 #endif
   }
 
-  struct SynthesizedReplacementStep: Equatable {
-    let text: String
-    let replacesExistingText: Bool
-  }
 
-  static func synthesizedReplacementSteps(
-    text: String,
-    delaySeconds: Double
-  ) -> [SynthesizedReplacementStep] {
-    guard synthesizedReplacementIsSpaced(characterCount: text.count, delaySeconds: delaySeconds)
-    else {
-      return [SynthesizedReplacementStep(text: text, replacesExistingText: true)]
+
+
+
+  struct SynthesizedTextPlan: Equatable {
+
+    struct Step: Equatable {
+      let characterCount: Int
+
+
+      var replacesExistingText = false
+
+
+
+
+      var pauseAfterSeconds: TimeInterval = 0
+
+      var warmsUpField = false
+
+
+      var synthesizeCallCount: Int {
+        replacesExistingText ? 2 : 1
+      }
     }
-    return Array(text).enumerated().map { index, character in
-      SynthesizedReplacementStep(
-        text: String(character),
-        replacesExistingText: index == 0
-      )
+
+    let steps: [Step]
+
+
+
+    static func spacedSteps(
+      characterCount: Int,
+      delaySeconds: Double,
+      replacesExistingTextOnFirstPost: Bool = false
+    ) -> [Step] {
+      (0..<characterCount).map { index in
+        Step(
+          characterCount: 1,
+          replacesExistingText: replacesExistingTextOnFirstPost && index == 0,
+          pauseAfterSeconds: index + 1 < characterCount ? delaySeconds : 0
+        )
+      }
+    }
+
+
+
+    var seconds: TimeInterval {
+      steps.reduce(0) { total, step in
+        total
+          + Double(step.characterCount) * TextEntryTiming.synthesizedCharacterInterval
+          + Double(step.synthesizeCallCount) * TextEntryTiming.synthesizeCallOverhead
+          + step.pauseAfterSeconds
+      }
+    }
+
+
+
+
+    var pacesEveryCharacter: Bool {
+      steps.count > 1 && !steps.contains(where: \.warmsUpField)
     }
   }
 
 
 
-  static func synthesizedReplacementIsSpaced(characterCount: Int, delaySeconds: Double) -> Bool {
+  static func synthesizedDeliveryIsSpaced(characterCount: Int, delaySeconds: Double) -> Bool {
     delaySeconds > 0 && characterCount > 1
   }
 
 
 
 
-  enum SynthesizedDeliveryBudget {
 
-    static var characterInterval: TimeInterval {
-      1.0 / Double(RunnerSynthesizedTextEntry.typingSpeedCharactersPerSecond())
-    }
 
 
 
 
+  static func synthesizedTextPlan(
+    characterCount: Int,
+    delaySeconds: Double,
+    selectsExistingText: Bool,
+    peelsWarmupCharacter: Bool = false
+  ) -> SynthesizedTextPlan {
+    if synthesizedDeliveryIsSpaced(characterCount: characterCount, delaySeconds: delaySeconds) {
+      return SynthesizedTextPlan(
+        steps: SynthesizedTextPlan.spacedSteps(
+          characterCount: characterCount,
+          delaySeconds: delaySeconds,
+          replacesExistingTextOnFirstPost: selectsExistingText
+        )
+      )
+    }
+    guard peelsWarmupCharacter && characterCount > 1 else {
+      return SynthesizedTextPlan(
+        steps: [
+          SynthesizedTextPlan.Step(
+            characterCount: characterCount,
+            replacesExistingText: selectsExistingText
+          ),
+        ]
+      )
+    }
+    return SynthesizedTextPlan(
+      steps: [
+        SynthesizedTextPlan.Step(
+          characterCount: 1,
+          pauseAfterSeconds: TextEntryTiming.pollInterval,
+          warmsUpField: true
+        ),
+        SynthesizedTextPlan.Step(characterCount: characterCount - 1),
+      ]
+    )
+  }
 
 
-    static func projectedSeconds(
-      textLength: Int,
-      delaySeconds: TimeInterval,
-      typeWarmup: Bool = false
-    ) -> TimeInterval {
-      let spaced = synthesizedReplacementIsSpaced(characterCount: textLength, delaySeconds: delaySeconds)
-      let warmupSplit = typeWarmup && textLength > 1 && !spaced
-      let calls = spaced ? textLength : (warmupSplit ? 2 : 1)
-      return Double(textLength) * characterInterval
-        + Double(calls) * TextEntryTiming.synthesizeCallOverhead
-        + Double(calls - 1) * delaySeconds
-        + (warmupSplit ? TextEntryTiming.pollInterval : 0)
-    }
 
-    static func exceeds(
-      textLength: Int,
-      delaySeconds: TimeInterval,
-      typeWarmup: Bool = false
-    ) -> Bool {
-      projectedSeconds(textLength: textLength, delaySeconds: delaySeconds, typeWarmup: typeWarmup)
-        > TextEntryTiming.synthesizedDeliveryCeiling
+  enum SynthesizedDeliveryBudget {
+    static func exceeds(_ plan: SynthesizedTextPlan) -> Bool {
+      plan.seconds > TextEntryTiming.synthesizedDeliveryCeiling
     }
 
 
+
+
     static func maxTextLength(delaySeconds: TimeInterval) -> Int {
       var length = 1
-      while !exceeds(textLength: length + 1, delaySeconds: delaySeconds) {
+      while !exceeds(
+        synthesizedTextPlan(
+          characterCount: length + 1,
+          delaySeconds: delaySeconds,
+          selectsExistingText: true
+        )
+      ) {
         length += 1
       }
       return length
     }
   }
 
+
+
+  enum SynthesizedStepDispatch {
+    case posted
+    case stop
+  }
+
+  struct SynthesizedPlanRun {
+    let postedCharacterCount: Int
+
+    let stoppedEarly: Bool
+  }
+
+
+
+
+
+
+
+
+
+
+
+  @MainActor
+  func runSynthesizedTextPlan(
+    _ plan: SynthesizedTextPlan,
+    text: String,
+    post: (_ characters: String, _ step: SynthesizedTextPlan.Step) -> SynthesizedStepDispatch,
+    waitAfterWarmupCharacter: (_ characters: String) -> Void,
+    didPostStep: (_ step: SynthesizedTextPlan.Step) -> Void = { _ in }
+  ) -> SynthesizedPlanRun {
+    let characters = Array(text)
+    var postedCount = 0
+    for step in plan.steps {
+      let nextCount = postedCount + step.characterCount
+      if post(String(characters[postedCount..<nextCount]), step) == .stop {
+
+        return SynthesizedPlanRun(postedCharacterCount: postedCount, stoppedEarly: true)
+      }
+      postedCount = nextCount
+      didPostStep(step)
+      if step.warmsUpField {
+        waitAfterWarmupCharacter(String(characters[postedCount - step.characterCount..<postedCount]))
+      } else if step.pauseAfterSeconds > 0 {
+        sleepFor(step.pauseAfterSeconds)
+      }
+    }
+    return SynthesizedPlanRun(postedCharacterCount: postedCount, stoppedEarly: false)
+  }
+
   @MainActor
   func runSynthesizedReplacementRoute(
     _ request: SynthesizedReplacementRequest
   ) -> SynthesizedReplacementRouteOutcome {
 #if os(iOS)
     NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=synthesized-first-responder-replacement")
-    if SynthesizedDeliveryBudget.exceeds(
-      textLength: request.text.count,
-      delaySeconds: request.delaySeconds
-    ) {
+    let plan = Self.synthesizedTextPlan(
+      characterCount: request.text.count,
+      delaySeconds: request.delaySeconds,
+      selectsExistingText: true
+    )
+    if SynthesizedDeliveryBudget.exceeds(plan) {
       NSLog(
         "AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=synthesized-first-responder-replacement "
           + "reason=delivery-budget-refused chars=%d budgetChars=%d",
@@ -174,33 +298,38 @@ extension RunnerTests {
         )
       )
     }
-    let steps = Self.synthesizedReplacementSteps(
+
+
+    let synthesisAvailable = runSynthesizedTextPlan(
+      plan,
       text: request.text,
-      delaySeconds: request.delaySeconds
+      post: { slice, step in
+        switch request.synthesizer.enterText(
+          app: request.app,
+          text: slice,
+          replacingExistingText: step.replacesExistingText
+        ) {
+        case .continueTyping:
+          return .posted
+        case .fallback:
+          return .stop
+        case .raise(let message):
+          NSException(
+            name: NSExceptionName.internalInconsistencyException,
+            reason: message ?? "private XCTest text synthesis failed"
+          ).raise()
+          return .stop
+        }
+      },
+
+      waitAfterWarmupCharacter: { _ in }
     )
-    for (index, step) in steps.enumerated() {
-      switch request.synthesizer.enterText(
-        app: request.app,
-        text: step.text,
-        replacingExistingText: step.replacesExistingText
-      ) {
-      case .fallback:
-        NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=verified-fallback reason=synthesis-unavailable")
-        guard let point = request.target.refreshPoint else { return .notApplicable }
-        return .fallback(
-          focusTextInputForTextEntry(app: request.app, x: point.x, y: point.y)
-        )
-      case .raise(let message):
-        NSException(
-          name: NSExceptionName.internalInconsistencyException,
-          reason: message ?? "private XCTest text synthesis failed"
-        ).raise()
-      case .continueTyping:
-        break
-      }
-      if index + 1 < steps.count {
-        sleepFor(request.delaySeconds)
-      }
+    if synthesisAvailable.stoppedEarly {
+      NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=verified-fallback reason=synthesis-unavailable")
+      guard let point = request.target.refreshPoint else { return .notApplicable }
+      return .fallback(
+        focusTextInputForTextEntry(app: request.app, x: point.x, y: point.y)
+      )
     }
 
 

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift

@@ -36,7 +36,11 @@ extension RunnerTests {
         let recoveryBudget = SynthesizedDeliveryBudget.maxTextLength(
           delaySeconds: Double(recoveryDelay) / 1000
         )
-        return "Fill at most \(SynthesizedDeliveryBudget.maxTextLength(delaySeconds: 0)) characters at a time without --delay-ms and append the rest with separate type commands, keeping each command inside that budget. --delay-ms lowers the budget, because each character then gets its own synthesize call and each gap between characters pays the delay: \(recoveryBudget) characters at --delay-ms \(recoveryDelay). A longer timeout does not help: this route is chosen when the accessibility channel is already degraded, and the pace is what makes the text long."
+
+
+
+
+        return "Fill at most \(SynthesizedDeliveryBudget.maxTextLength(delaySeconds: 0)) characters per command without --delay-ms and append the rest with separate type commands. --delay-ms lowers the limit: each character then gets its own synthesize call and each gap pays the delay, so \(recoveryDelay) ms fits \(recoveryBudget). This route is chosen when the accessibility channel is already degraded, so a longer timeout does not help."
       }
     }
   }
@@ -76,7 +80,18 @@ extension RunnerTests {
 
 
 
-    static let synthesizedAcknowledgeWindowSeconds: TimeInterval = 0.04
+
+
+
+
+
+
+
+
+
+    static let synthesizedCharactersPerSecond: UInt = 12
+
+    static let synthesizedCharacterInterval: TimeInterval = 1.0 / Double(synthesizedCharactersPerSecond)
 
 
 

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift

@@ -136,6 +136,39 @@ extension RunnerTests {
       }
     }
 
+
+
+
+
+
+    let synthesizedTypePlan = Self.synthesizedTextPlan(
+      characterCount: text.count,
+      delaySeconds: delaySeconds,
+      selectsExistingText: false,
+      peelsWarmupCharacter: repairMode != .none
+    )
+
+
+
+
+    let synthesizedTypeCarriesTheCommand = !SynthesizedDeliveryBudget.exceeds(synthesizedTypePlan)
+
+
+
+
+
+    func typeApplicationWide(_ value: String, reason: String) -> (element: XCUIElement?, failure: TextEntryFailure?) {
+      NSLog(
+        "AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=xctest-application-fallback reason=%@ chars=%d commandChars=%d",
+        reason,
+        value.count,
+        text.count
+      )
+      textEntryRoute = "xctest-application-fallback"
+      app.typeText(value)
+      return (resolveTextEntryElement(app: app, target: activeTarget), nil)
+    }
+
     func typeIntoCurrentTarget(_ value: String) -> (element: XCUIElement?, failure: TextEntryFailure?) {
 #if os(iOS)
       if shouldUseSynthesizedFirstResponderType {
@@ -167,35 +200,14 @@ extension RunnerTests {
 
 
 
-
-
-
-
-        if SynthesizedDeliveryBudget.exceeds(
-          textLength: text.count,
-          delaySeconds: delaySeconds,
-          typeWarmup: repairMode != .none
-        ) {
-          textEntryRoute = "xctest-application-fallback"
-          NSLog(
-            "AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=xctest-application-fallback "
-              + "reason=delivery-budget chars=%d",
-            value.count
-          )
-          app.typeText(value)
-          return (resolveTextEntryElement(app: app, target: activeTarget), nil)
+        guard synthesizedTypeCarriesTheCommand else {
+          return typeApplicationWide(value, reason: "delivery-budget")
         }
         textEntryRoute = "synthesized-first-responder"
         NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=synthesized-first-responder")
-        let action = synthesizer.enterText(
-          app: app,
-          text: value,
-          replacingExistingText: false
-        )
-        switch action {
+        switch synthesizer.enterText(app: app, text: value, replacingExistingText: false) {
         case .fallback:
-          textEntryRoute = "xctest-application-fallback"
-          app.typeText(value)
+          return typeApplicationWide(value, reason: "synthesis-unavailable")
         case .raise(let message):
           NSException(
             name: NSExceptionName.internalInconsistencyException,
@@ -223,131 +235,100 @@ extension RunnerTests {
       )
     }
 
-    func waitForWarmupValue(_ expectedValue: String?, target: TextEntryTarget) {
-      guard let expectedValue else {
+    func readBackWarmupCharacter(_ peeledCharacter: String) {
+
+
+
+      let warmupExpectedText = expectedTextEntryValue(
+        typedText: peeledCharacter,
+        mode: repairMode,
+        initialText: initialText
+      )
+      guard let warmupExpectedText else {
         sleepFor(TextEntryTiming.pollInterval)
         return
       }
       let deadline = Date().addingTimeInterval(TextEntryTiming.warmupValueTimeout)
       while Date() < deadline {
-        if editableTextValue(for: resolveTextEntryElement(app: app, target: target)) == expectedValue {
+        if editableTextValue(for: resolveTextEntryElement(app: app, target: activeTarget)) == warmupExpectedText {
           return
         }
         sleepFor(TextEntryTiming.pollInterval)
       }
     }
 
-    let characters = Array(text)
-    if delaySeconds > 0 && characters.count > 1 {
-      var typedTarget: XCUIElement?
-      let delayedTypeStartedAt = Date()
-      for (index, character) in characters.enumerated() {
-        let dispatch = typeIntoCurrentTarget(String(character))
+
+
+    var typedTarget: XCUIElement?
+    var sawWarmupPost = false
+    var planFailure: TextEntryResult?
+    var stepStartedAt = Date()
+    let pacedStartedAt = Date()
+    runSynthesizedTextPlan(
+      synthesizedTypePlan,
+      text: text,
+      post: { slice, step in
+        stepStartedAt = Date()
+        let dispatch = typeIntoCurrentTarget(slice)
         if let failure = dispatch.failure {
-          return dispatchFailureResult(failure)
+          planFailure = dispatchFailureResult(failure)
+          return .stop
         }
         typedTarget = dispatch.element ?? typedTarget
-        if index + 1 < characters.count {
-          sleepFor(delaySeconds)
+        if step.warmsUpField {
+
+          activeTarget = activeTarget.withElement(typedTarget)
         }
-      }
-      logTextEntryPhase(
-        commandId: commandId,
-        phase: "type-delayed",
-        startedAt: delayedTypeStartedAt,
-        chars: characters.count,
-        mode: repairMode
-      )
-      if repairMode == .none {
-        logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: text.count, mode: repairMode)
-        return TextEntryResult(
-          verified: nil,
-          repaired: false,
-          expectedText: nil,
-          observedText: nil,
-          textEntryRoute: textEntryRoute
+        return .posted
+      },
+      waitAfterWarmupCharacter: { peeledCharacter in
+        let warmupStartedAt = Date()
+        readBackWarmupCharacter(peeledCharacter)
+        logTextEntryPhase(
+          commandId: commandId,
+          phase: "warmup",
+          startedAt: warmupStartedAt,
+          chars: 1,
+          mode: repairMode
+        )
+      },
+      didPostStep: { step in
+        guard !synthesizedTypePlan.pacesEveryCharacter else {
+          return
+        }
+        let phase: String
+        if step.warmsUpField {
+          phase = "type-first"
+          sawWarmupPost = true
+        } else {
+          phase = sawWarmupPost ? "type-remaining" : "type-all"
+        }
+        logTextEntryPhase(
+          commandId: commandId,
+          phase: phase,
+          startedAt: stepStartedAt,
+          chars: step.characterCount,
+          mode: repairMode
         )
       }
-      let verifyStartedAt = Date()
-      var result = verifyTextEntryWithRepairIfNeeded(
-        app: app,
-        target: activeTarget.withElement(typedTarget),
-        expectedText: expectedText,
-        repairMode: repairMode
-      )
+    )
+    if synthesizedTypePlan.pacesEveryCharacter {
+
+
       logTextEntryPhase(
         commandId: commandId,
-        phase: "verify",
-        startedAt: verifyStartedAt,
-        chars: characters.count,
+        phase: "type-delayed",
+        startedAt: pacedStartedAt,
+        chars: text.count,
         mode: repairMode
       )
-      logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: text.count, mode: repairMode)
-      result.textEntryRoute = textEntryRoute
-      return result
     }
 
-    let typedTarget: XCUIElement?
-    if repairMode != .none && characters.count > 1 {
-      let firstCharacter = String(characters[0])
-      let firstStartedAt = Date()
-      let firstDispatch = typeIntoCurrentTarget(firstCharacter)
-      if let failure = firstDispatch.failure {
-        return dispatchFailureResult(failure)
-      }
-      var firstTypedTarget = firstDispatch.element
-      logTextEntryPhase(
-        commandId: commandId,
-        phase: "type-first",
-        startedAt: firstStartedAt,
-        chars: 1,
-        mode: repairMode
-      )
-      activeTarget = activeTarget.withElement(firstTypedTarget)
-      let warmupExpectedText = expectedTextEntryValue(
-        typedText: firstCharacter,
-        mode: repairMode,
-        initialText: initialText
-      )
-      let warmupStartedAt = Date()
-      waitForWarmupValue(warmupExpectedText, target: activeTarget)
-      logTextEntryPhase(
-        commandId: commandId,
-        phase: "warmup",
-        startedAt: warmupStartedAt,
-        chars: 1,
-        mode: repairMode
-      )
-      let remainingText = String(characters.dropFirst())
-      let remainingStartedAt = Date()
-      let remainingDispatch = typeIntoCurrentTarget(remainingText)
-      if let failure = remainingDispatch.failure {
-        return dispatchFailureResult(failure)
-      }
-      firstTypedTarget = remainingDispatch.element ?? firstTypedTarget
-      logTextEntryPhase(
-        commandId: commandId,
-        phase: "type-remaining",
-        startedAt: remainingStartedAt,
-        chars: characters.count - 1,
-        mode: repairMode
-      )
-      typedTarget = firstTypedTarget
-    } else {
-      let typeStartedAt = Date()
-      let dispatch = typeIntoCurrentTarget(text)
-      if let failure = dispatch.failure {
-        return dispatchFailureResult(failure)
-      }
-      typedTarget = dispatch.element
-      logTextEntryPhase(
-        commandId: commandId,
-        phase: "type-all",
-        startedAt: typeStartedAt,
-        chars: characters.count,
-        mode: repairMode
-      )
+
+    if let failure = planFailure {
+      return failure
     }
+    activeTarget = activeTarget.withElement(typedTarget)
     if repairMode == .none {
       logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: text.count, mode: repairMode)
       return TextEntryResult(
@@ -361,7 +342,7 @@ extension RunnerTests {
     let verifyStartedAt = Date()
     var result = verifyTextEntryWithRepairIfNeeded(
       app: app,
-      target: activeTarget.withElement(typedTarget),
+      target: activeTarget,
       expectedText: expectedText,
       repairMode: repairMode
     )
@@ -369,7 +350,7 @@ extension RunnerTests {
       commandId: commandId,
       phase: "verify",
       startedAt: verifyStartedAt,
-      chars: characters.count,
+      chars: text.count,
       mode: repairMode
     )
     logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: text.count, mode: repairMode)

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift

@@ -5,6 +5,7 @@
 
 
 
+import AgentDeviceSnapshotPresentation
 import XCTest
 import Network
 #if canImport(UIKit)
@@ -211,6 +212,9 @@ final class RunnerTests: XCTestCase {
 
 
 
+
+
+