agent-device 0.20.10 → 0.21.0

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift

@@ -0,0 +1,110 @@
+import Foundation
+import XCTest
+
+struct SnapshotCaptureTiming: Codable, Equatable {
+  let acquisitionMs: Double
+  let presentationMs: Double
+
+  init(acquisitionMs: Double, presentationMs: Double) {
+    self.acquisitionMs = max(0, acquisitionMs)
+    self.presentationMs = max(0, presentationMs)
+  }
+
+  init(acquisition: TimeInterval, presentation: TimeInterval) {
+    self.init(
+      acquisitionMs: max(0, acquisition) * 1_000,
+      presentationMs: max(0, presentation) * 1_000
+    )
+  }
+}
+
+enum SnapshotCapturePhase: Equatable {
+  case acquisition
+  case presentation
+}
+
+struct SnapshotPhaseTimer {
+  private let now: () -> Date
+  private var acquisitionSeconds: TimeInterval = 0
+  private var presentationSeconds: TimeInterval = 0
+
+  init(now: @escaping () -> Date = { Date() }) {
+    self.now = now
+  }
+
+  mutating func measure<T>(
+    _ phase: SnapshotCapturePhase,
+    _ operation: () throws -> T
+  ) rethrows -> T {
+    let startedAt = now()
+    defer {
+      let elapsed = max(0, now().timeIntervalSince(startedAt))
+      switch phase {
+      case .acquisition:
+        acquisitionSeconds += elapsed
+      case .presentation:
+        presentationSeconds += elapsed
+      }
+    }
+    return try operation()
+  }
+
+  var timing: SnapshotCaptureTiming {
+    SnapshotCaptureTiming(
+      acquisition: acquisitionSeconds,
+      presentation: presentationSeconds
+    )
+  }
+}
+
+extension RunnerTests {
+  struct SnapshotBackendAttempt {
+    enum Outcome {
+      case noCapture
+      case captured(SnapshotBackendCapture)
+      case failed(SnapshotCaptureFailure, phase: SnapshotCapturePhase)
+    }
+
+    /// The failure phase is part of the result, so penalty policy cannot infer it from duration
+    /// or error text.
+    let outcome: Outcome
+    let timing: SnapshotCaptureTiming
+  }
+
+  /// The penalty breaker observes only acquisition facts. Presentation is a separate phase and
+  /// cannot arm the breaker, even when it is slower than the acquisition that produced the tree.
+  private static func snapshotXCTestPenaltyReason(
+    kind: SnapshotBackendKind,
+    attempt: SnapshotBackendAttempt,
+    slowThresholdMs: Double
+  ) -> String? {
+    guard kind.usesXCTestAccessibilityChannel else { return nil }
+    if case let .failed(failure, phase: .acquisition) = attempt.outcome,
+      failure.code == Self.xCTestSnapshotTimeoutCode
+    {
+      return "\(kind.rawValue)_backend_timeout"
+    }
+    guard attempt.timing.acquisitionMs > slowThresholdMs else { return nil }
+    return "slow_\(kind.rawValue)_capture_\(Int(attempt.timing.acquisitionMs))ms"
+  }
+
+  func recordXCTestSnapshotBackendAttemptIfNeeded(
+    _ kind: SnapshotBackendKind,
+    attempt: SnapshotBackendAttempt,
+    penaltySuppressed: Bool
+  ) {
+    guard !penaltySuppressed else { return }
+    guard
+      let reason = Self.snapshotXCTestPenaltyReason(
+        kind: kind,
+        attempt: attempt,
+        slowThresholdMs: snapshotXCTestSlowCaptureThreshold * 1_000
+      )
+    else { return }
+    penalizeSnapshotXCTestChannel(
+      bundleId: currentBundleId,
+      reason: reason
+    )
+  }
+}
+

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift

@@ -0,0 +1,206 @@
+import XCTest
+
+// The synthesized text-entry commit wait, end to end: the deadline that bounds it, the two waits
+// that run it, the observation/pacing they poll through, and the value-free cadence line that path
+// is allowed to log. Split from RunnerTests+SynthesizedTextEntry.swift, which keeps the
+// private-XCTest synthesis boundary and the route policies; the pure outcome functions these wrap
+// stay there next to the rules they encode. Everything that touches the polled field value now
+// lives in this one file, which is the surface apple-runner-log-redaction.test.ts guards.
+extension RunnerTests {
+  /// One commit wait's running deadline.
+  ///
+  /// A synthesized burst can be *throttled* rather than dropped: on a loaded simulator the
+  /// characters keep landing, just slowly, and everything touching the input system slows with
+  /// them (#1874). A flat wall-clock budget cannot tell that apart from a wedged pipeline, so it
+  /// condemned both at the same instant — and the throttled case is a working command reported as
+  /// `TEXT_INPUT_COMMIT_NOT_OBSERVED`, which is what turned an environment episode into a red
+  /// lane on branches touching no iOS code.
+  ///
+  /// So time is granted against *progress* — the observed value's expected-prefix growing, the
+  /// same length-only evidence `logCommitCadence` already emits — with an absolute `ceiling`, so a
+  /// pipeline delivering one character per stall window cannot hold a command open forever. A
+  /// pipeline making no progress at all still expires at exactly the `stallBudget` the flat
+  /// deadline used, so nothing that fails today starts passing merely by waiting longer.
+  ///
+  /// Started by the poll loop from its own first `now()`, and held as a local `var` there. The
+  /// wait's setup reads the field's placeholder first, an AX round-trip that takes seconds on
+  /// exactly the loaded host this exists for; charging that to the deadline made the wait give up
+  /// sooner than the flat budget it replaced.
+  struct SynthesizedCommitDeadline {
+    private let startedAt: Date
+    private let stallBudget: TimeInterval
+    private let ceiling: TimeInterval
+    // Nothing has landed yet, and an observation of "still nothing" must not read as progress.
+    private var bestPrefixLength = 0
+    private var lastProgressAt: Date
+
+    init(startedAt: Date, stallBudget: TimeInterval, ceiling: TimeInterval) {
+      self.startedAt = startedAt
+      self.stallBudget = stallBudget
+      self.ceiling = ceiling
+      self.lastProgressAt = startedAt
+    }
+
+    /// Records one observation's expected-prefix length. Only forward movement counts: a shorter
+    /// read — the app clearing the field mid-flight, or a value that could not be read at all,
+    /// which measures as 0 — is not evidence the burst is still landing, so it neither buys time
+    /// nor takes any back.
+    mutating func record(expectedPrefixLength: Int, at now: Date) {
+      guard expectedPrefixLength > bestPrefixLength else { return }
+      bestPrefixLength = expectedPrefixLength
+      lastProgressAt = now
+    }
+
+    func isExpired(at now: Date) -> Bool {
+      now.timeIntervalSince(startedAt) >= ceiling
+        || now.timeIntervalSince(lastProgressAt) >= stallBudget
+    }
+  }
+
+  /// Blocks until the synthesized bare-type text is observable in the target field, so `type`
+  /// cannot report ok while trailing characters are still uncommitted on a slow simulator.
+  ///
+  /// Observation only. A stalled prefix cannot be told apart from a suffix still queued in the
+  /// event stream, so re-synthesizing the difference risks committing it twice after the command
+  /// already reported success (#1676 rejected exactly that repair). Reporting `.notObserved` is
+  /// what the caller does instead: the partial value is the agent's to resolve, and a named
+  /// failure beats a success that misdescribes the field. Text carrying a submit key is skipped
+  /// outright: the app may clear or rewrite the field on submit, so `textBefore + typedText` is
+  /// not the value to wait for.
+  func awaitSynthesizedFirstResponderCommit(
+    app: XCUIApplication,
+    target: TextEntryTarget,
+    textBefore: String?,
+    typedText: String
+  ) -> SynthesizedTextCommitOutcome {
+    guard let textBefore, !typedText.contains("\n"), !typedText.contains("\r") else {
+      return .unobservable
+    }
+    let expectedText = textBefore + typedText
+    let waitStartedAt = Date()
+    NSLog("[DEBUG-1874] wait start expectedLen=%ld route=append", expectedText.count)
+    let ingredients = synthesizedCommitPollingIngredients(app: app, target: target, expectedText: expectedText)
+    let outcome = Self.awaitSynthesizedCommitOutcome(
+      expectedText: expectedText,
+      placeholder: ingredients.placeholder,
+      now: { Date() },
+      observe: ingredients.observe,
+      waitForNextObservation: ingredients.waitForNextObservation
+    )
+    NSLog(
+      "[DEBUG-1874] wait outcome=%@ elapsedMs=%.0f route=append",
+      String(describing: outcome),
+      waitStartedAt.timeIntervalSinceNow * -1000
+    )
+    return outcome
+  }
+
+  /// Blocks until the synthesized replacement text (`fill`) is observable in the target field, so
+  /// `fill` cannot report ok while the select-all-and-retype it posted is still uncommitted — or
+  /// silently wrong — on a slow or channel-penalized simulator (this route runs only when the
+  /// XCTest channel is already penalized, and never resolves an `XCUIElement`, so it previously had
+  /// no verification at all).
+  ///
+  /// Unlike the append route, the expected value is the final text itself — replacement mode
+  /// clears the field first, so there is no `textBefore` prefix to account for — and unlike the
+  /// append route, a settled mismatch is always reported rather than trusted: see
+  /// `awaitSynthesizedReplacementCommitOutcome`'s doc comment.
+  func awaitSynthesizedReplacementCommit(
+    app: XCUIApplication,
+    target: TextEntryTarget,
+    expectedText: String
+  ) -> SynthesizedTextCommitOutcome {
+    guard !expectedText.contains("\n"), !expectedText.contains("\r") else {
+      return .unobservable
+    }
+    let waitStartedAt = Date()
+    NSLog("[DEBUG-1874] wait start expectedLen=%ld route=replacement", expectedText.count)
+    let ingredients = synthesizedCommitPollingIngredients(app: app, target: target, expectedText: expectedText)
+    let outcome = Self.awaitSynthesizedReplacementCommitOutcome(
+      expectedText: expectedText,
+      placeholder: ingredients.placeholder,
+      now: { Date() },
+      observe: ingredients.observe,
+      waitForNextObservation: ingredients.waitForNextObservation
+    )
+    NSLog(
+      "[DEBUG-1874] wait outcome=%@ elapsedMs=%.0f route=replacement",
+      String(describing: outcome),
+      waitStartedAt.timeIntervalSinceNow * -1000
+    )
+    return outcome
+  }
+
+  /// The emitted cadence line, as a pure function so its output is assertable. Only lengths and
+  /// a timestamp are representable here; there is no String parameter, so observed field
+  /// contents cannot reach runner.log through this boundary whatever they contain.
+  static func commitCadenceLogLine(
+    elapsedMs: Int,
+    observedLen: Int,
+    expectedPrefixLen: Int
+  ) -> String {
+    "[DEBUG-1874] poll t=\(elapsedMs)ms observedLen=\(observedLen) expectedPrefixLen=\(expectedPrefixLen)"
+  }
+
+  /// The typed boundary for commit-wait cadence evidence. The poll path must log through this
+  /// function and never through a raw NSLog: every parameter is an Int, so the polled value's
+  /// contents are unrepresentable at the call site.
+  static func logCommitCadence(
+    elapsedMs: Int,
+    observedLen: Int,
+    expectedPrefixLen: Int
+  ) {
+    NSLog(
+      "%@",
+      commitCadenceLogLine(
+        elapsedMs: elapsedMs,
+        observedLen: observedLen,
+        expectedPrefixLen: expectedPrefixLen
+      )
+    )
+  }
+
+  /// The placeholder/observe/pacing ingredients shared by the append route
+  /// (`awaitSynthesizedFirstResponderCommit`) and the replacement route
+  /// (`awaitSynthesizedReplacementCommit`). What must NOT be shared is which outcome function
+  /// consumes them: see `awaitSynthesizedReplacementCommitOutcome`'s doc comment for why append
+  /// mode's "trust a diverged value" rule is wrong for replacement mode. Each caller therefore
+  /// calls its own named outcome function directly, with real argument labels — deliberately not
+  /// a stored closure/function-value parameter here, which would erase those labels at the call
+  /// site and make the observe closure unrecognizable to the static content-redaction check in
+  /// `apple-runner-log-redaction.test.ts` (`extractObserveClosure` locates the labeled closure
+  /// literal by its text; a closure passed as a plain function value carries no such label).
+  private func synthesizedCommitPollingIngredients(
+    app: XCUIApplication,
+    target: TextEntryTarget,
+    expectedText: String
+  ) -> (
+    placeholder: String?,
+    observe: () -> String?,
+    waitForNextObservation: () -> Void
+  ) {
+    let placeholder = resolveTextEntryElement(app: app, target: target)?.placeholderValue
+    let waitStartedAt = Date()
+    return (
+      placeholder: placeholder,
+      observe: {
+        let observedText = self.editableTextValue(
+          for: self.resolveTextEntryElement(app: app, target: target),
+          treatingPlaceholderAsEmpty: true
+        )
+        // Cadence evidence stays value-free: the polled value is user content typed through
+        // `type`/`fill` and must never reach runner.log. Lengths and the expected-prefix walk
+        // are enough to distinguish throttling (prefix grows slowly) from a wedge (it freezes).
+        Self.logCommitCadence(
+          elapsedMs: Int(waitStartedAt.timeIntervalSinceNow * -1000),
+          observedLen: observedText?.count ?? -1,
+          expectedPrefixLen: observedText.map { Self.commonPrefixLength($0, expectedText) } ?? -1
+        )
+        return observedText
+      },
+      // XCUI resolution shares the automation channel with the in-flight synthesized event.
+      // Sparse reads let the target consume that event instead of continuously interrupting it.
+      waitForNextObservation: { self.sleepFor(TextEntryTiming.synthesizedCommitPollInterval) }
+    )
+  }
+}

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift

@@ -40,9 +40,14 @@ extension RunnerTests {
       replacingExistingText: Bool
     ) -> SynthesizedTextEntryAction {
 #if os(iOS)
+      let postStartedAt = Date()
       let result = replacingExistingText
         ? RunnerSynthesizedTextEntry.replaceText(withApplication: app, text: text)
         : RunnerSynthesizedTextEntry.synthesizeText(withApplication: app, text: text)
+      NSLog(
+        "[DEBUG-1874] synthesize posted %d chars status=%d tookMs=%.0f",
+        text.count, result.status.rawValue, postStartedAt.timeIntervalSinceNow * -1000
+      )
       return Self.action(status: result.status, message: result.message)
 #else
       return .fallback
@@ -122,6 +127,17 @@ extension RunnerTests {
         sleepFor(request.delaySeconds)
       }
     }
+    // The private synthesize call returns at post time, not commit time (same as bare `type`,
+    // see awaitSynthesizedFirstResponderCommit) — but this route never resolves an XCUIElement,
+    // so without this wait it had no way to notice a dropped or still-in-flight character at all
+    // and reported ok purely because the event posted. Wait here, on the same request.target
+    // (element nil, refreshPoint set) that gated this route, so each poll re-resolves via the
+    // refresh point rather than trusting a stale element handle.
+    let commit = awaitSynthesizedReplacementCommit(
+      app: request.app,
+      target: request.target,
+      expectedText: request.text
+    )
     logTextEntryPhase(
       commandId: request.commandId,
       phase: "total",
@@ -135,7 +151,8 @@ extension RunnerTests {
         repaired: false,
         expectedText: request.text,
         observedText: nil,
-        textEntryRoute: "synthesized-first-responder-replacement"
+        textEntryRoute: "synthesized-first-responder-replacement",
+        failure: Self.textEntryFailure(forCommitOutcome: commit)
       )
     )
 #else
@@ -184,38 +201,143 @@ extension RunnerTests {
     return expectedText.hasPrefix(observedText) ? .pending : .diverged
   }
 
-  /// Blocks until the synthesized bare-type text is observable in the target field, so `type`
-  /// cannot report ok while trailing characters are still uncommitted on a slow simulator.
+  /// Length of the shared prefix of two strings. Feeds value-free commit-wait logging: the
+  /// expected-prefix walk over time distinguishes throttled delivery (grows slowly) from a
+  /// wedged pipeline (freezes) without ever logging the field's contents.
+  static func commonPrefixLength(_ lhs: String, _ rhs: String) -> Int {
+    var length = 0
+    for (l, r) in zip(lhs, rhs) {
+      if l != r { break }
+      length += 1
+    }
+    return length
+  }
+
+  /// How the commit wait ended. Distinct from `SynthesizedTextCommitProgress`, which classifies a
+  /// single observation: this is the whole wait's verdict, and it exists so the deadline can be
+  /// told apart from success. The wait used to return `Void`, which made an expired deadline
+  /// indistinguishable from a committed one — `type` then reported ok with a partial value in the
+  /// field (#1874, #1844).
+  enum SynthesizedTextCommitOutcome: Equatable {
+    /// The wait's success case, but its meaning is route-specific: for append mode (bare `type`),
+    /// the expected text committed OR the app transformed the input in a way the runner must not
+    /// second-guess; for replacement mode (`fill`), it means only an exact match — see
+    /// `awaitSynthesizedReplacementCommitOutcome`'s doc comment for why replacement mode has no
+    /// "trust it" case.
+    case settled
+    /// There was nothing to wait for — no readable baseline, or the text carries a submit key.
+    case unobservable
+    /// The deadline expired with the expected text still not observed.
+    case notObserved
+  }
+
+  /// The commit wait's decision, with observation, pacing and the clock injected so both deadline
+  /// branches are exercisable without a simulator (the macOS host lane runs this; the member
+  /// wrapper below binds the real XCUI reads).
   ///
-  /// Observation only. A stalled prefix cannot be told apart from a suffix still queued in the
-  /// event stream, so re-synthesizing the difference risks committing it twice after the command
-  /// already reported success. Text carrying a submit key is skipped outright: the app may clear
-  /// or rewrite the field on submit, so `textBefore + typedText` is not the value to wait for.
-  func awaitSynthesizedFirstResponderCommit(
-    app: XCUIApplication,
-    target: TextEntryTarget,
-    textBefore: String?,
-    typedText: String
-  ) {
-    guard let textBefore, !typedText.contains("\n"), !typedText.contains("\r") else {
-      return
+  /// The deadline is a local `var`, started from this loop's own first `now()` and advanced from
+  /// the same observation the progress check reads, so "did the burst move" and "is time up" are
+  /// two statements in one loop. The budget defaults to the shipped one, so only a test that is
+  /// asking about time has to name it.
+  static func awaitSynthesizedCommitOutcome(
+    expectedText: String,
+    placeholder: String?,
+    stallBudget: TimeInterval = TextEntryTiming.synthesizedCommitStallTimeout,
+    ceiling: TimeInterval = TextEntryTiming.synthesizedCommitCeiling,
+    now: () -> Date = { Date() },
+    observe: () -> String?,
+    waitForNextObservation: () -> Void
+  ) -> SynthesizedTextCommitOutcome {
+    // A placeholder-equal AX value cannot prove a commit: an input handler may clear even a
+    // previously non-empty field after dispatch, making the empty field render the same value.
+    // Refuse before polling because no later read can distinguish those states.
+    if Self.textMatchesPlaceholder(expectedText, placeholder: placeholder) {
+      return .notObserved
     }
-    let expectedText = textBefore + typedText
-    let deadline = Date().addingTimeInterval(TextEntryTiming.synthesizedCommitTimeout)
-    while Date() < deadline {
-      let observedText = editableTextValue(
-        for: resolveTextEntryElement(app: app, target: target),
-        treatingPlaceholderAsEmpty: true
-      )
-      switch Self.synthesizedTextCommitProgress(observedText: observedText, expectedText: expectedText) {
+    var deadline = SynthesizedCommitDeadline(startedAt: now(), stallBudget: stallBudget, ceiling: ceiling)
+    // The deadline is checked AFTER an observation, never before one, so the last thing that
+    // happens before condemning a commit is a read. Checking first would condemn a commit that
+    // landed during the final poll sleep — the exact loaded-host timing this wait exists for.
+    while true {
+      let observedText = observe()
+      switch synthesizedTextCommitProgress(observedText: observedText, expectedText: expectedText) {
       case .committed, .diverged:
-        return
+        return .settled
       case .pending:
-        sleepFor(TextEntryTiming.pollInterval)
+        // One clock sample, so the instant an observation is recorded at is the instant it is
+        // judged against.
+        let sampledAt = now()
+        deadline.record(
+          expectedPrefixLength: Self.commonPrefixLength(observedText ?? "", expectedText),
+          at: sampledAt
+        )
+        if deadline.isExpired(at: sampledAt) { return .notObserved }
+        waitForNextObservation()
       }
     }
   }
 
+  /// The command-level consequence of a commit wait. `.unobservable` is not a failure: there was
+  /// no baseline to compare against, which is the pre-existing contract for submit-key text and
+  /// unreadable fields, not evidence that anything went wrong.
+  static func textEntryFailure(
+    forCommitOutcome outcome: SynthesizedTextCommitOutcome
+  ) -> TextEntryFailure? {
+    switch outcome {
+    case .settled, .unobservable:
+      return nil
+    case .notObserved:
+      return .commitNotObserved
+    }
+  }
+
+  /// The replacement-mode counterpart of `awaitSynthesizedCommitOutcome`. It must NOT reuse that
+  /// function's `synthesizedTextCommitProgress`: prefix-walk's `.diverged` case exists to trust an
+  /// app that transforms bare-`type` input (formatter, autocomplete) rather than second-guess it —
+  /// but that same rule silently accepts a dropped-character corruption too, because a value with a
+  /// hole in the middle ("ada@example" -> "aexample") is neither a matching prefix NOR the full
+  /// string, yet still gets classified `.diverged` -> `.settled` -> reported `ok: true`. This is not
+  /// a hypothetical: it is the exact shape of the corruption this wait exists to catch (`fill`
+  /// reporting success over "Avelace"/"aexample"-style drops), verified against both live examples
+  /// before writing this comment.
+  ///
+  /// `.replacement` mode does not need prefix tolerance for legitimate transforms either:
+  /// `isRepairableTextEntryMismatch` (RunnerTests+TextTyping.swift) already treats every mismatch in
+  /// `.replacement` mode as repairable unconditionally, with no formatter/autocomplete carve-out —
+  /// `fill` fully owns the field via select-all, so there is no legitimate reason for the settled
+  /// value to be anything other than exactly what was requested. A settled non-match is therefore
+  /// always the wait's failure case, never a `.settled` pass-through.
+  static func awaitSynthesizedReplacementCommitOutcome(
+    expectedText: String,
+    placeholder: String?,
+    stallBudget: TimeInterval = TextEntryTiming.synthesizedCommitStallTimeout,
+    ceiling: TimeInterval = TextEntryTiming.synthesizedCommitCeiling,
+    now: () -> Date = { Date() },
+    observe: () -> String?,
+    waitForNextObservation: () -> Void
+  ) -> SynthesizedTextCommitOutcome {
+    if Self.textMatchesPlaceholder(expectedText, placeholder: placeholder) {
+      return .notObserved
+    }
+    var deadline = SynthesizedCommitDeadline(startedAt: now(), stallBudget: stallBudget, ceiling: ceiling)
+    while true {
+      let observedText = observe()
+      if observedText == expectedText {
+        return .settled
+      }
+      // Prefix growth cannot settle this wait — a value with a hole in the middle is still a
+      // failure, see the doc comment above — but it is the same evidence that the burst is still
+      // landing, so it buys the same time here as it does in append mode.
+      let sampledAt = now()
+      deadline.record(
+        expectedPrefixLength: Self.commonPrefixLength(observedText ?? "", expectedText),
+        at: sampledAt
+      )
+      if deadline.isExpired(at: sampledAt) { return .notObserved }
+      waitForNextObservation()
+    }
+  }
+
   static func shouldUseResolvedCoordinateTextEntryRoute(
     repairMode: TextTypingRepairMode,
     hasX: Bool,

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift

@@ -1,4 +1,5 @@
 import XCTest
+import AgentDeviceSnapshotPresentation
 
 extension RunnerTests {
   // MARK: - Blocking System Modal Snapshot