agent-device 0.20.5 → 0.20.7

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.m

@@ -0,0 +1,212 @@
+#import "RunnerSynthesizedTextEntry.h"
+#import "RunnerXCTestEventBridge.h"
+
+#import <objc/message.h>
+
+static NSString *const RunnerTextSynthesisSurface = @"text";
+
+typedef id (*RunnerTextMsgSendInit)(id, SEL, NSString *);
+typedef id (*RunnerTextMsgSendInitPath)(id, SEL);
+typedef void (*RunnerTextMsgSendType)(id, SEL, NSString *, NSTimeInterval, NSUInteger, BOOL);
+typedef void (*RunnerTextMsgSendTypeKey)(id, SEL, NSString *, NSUInteger, NSTimeInterval);
+
+// Text-entry-specific extension of the shared bridge: the 1-arg `initWithName:`
+// record initializer and the text-input path selectors, none of which gesture
+// synthesis needs.
+typedef struct {
+  RunnerXCTestEventBridge core;
+  SEL initRecordSelector;
+  SEL initPathSelector;
+  SEL typeTextSelector;
+  SEL typeKeySelector;  // only required in `replace` mode
+} RunnerTextEventBridge;
+
+static NSString * _Nullable RunnerResolveTextEventBridge(
+  id application,
+  BOOL replace,
+  RunnerTextEventBridge *bridge
+);
+static RunnerSynthesizedTextEntryResult *RunnerTextEntryResult(
+  RunnerSynthesizedTextEntryStatus status,
+  NSString * _Nullable message
+);
+static RunnerSynthesizedTextEntryResult *RunnerTextEntryUnavailable(NSString *message);
+static RunnerSynthesizedTextEntryResult *RunnerTextEntryFailed(NSString *message);
+static RunnerSynthesizedTextEntryResult *RunnerSynthesizeTextWithMode(
+  id application,
+  NSString *text,
+  BOOL replace
+);
+
+@interface RunnerSynthesizedTextEntryResult ()
+
+@property(nonatomic, readwrite) RunnerSynthesizedTextEntryStatus status;
+@property(nonatomic, readwrite, nullable) NSString *message;
+
+@end
+
+
+@implementation RunnerSynthesizedTextEntryResult
+@end
+
+@implementation RunnerSynthesizedTextEntry
+
++ (RunnerSynthesizedTextEntryResult *)synthesizeTextWithApplication:(id)application
+                                                               text:(NSString *)text {
+  return RunnerSynthesizeTextWithMode(application, text, NO);
+}
+
++ (RunnerSynthesizedTextEntryResult *)replaceTextWithApplication:(id)application
+                                                           text:(NSString *)text {
+  return RunnerSynthesizeTextWithMode(application, text, YES);
+}
+
+@end
+
+static RunnerSynthesizedTextEntryResult *RunnerSynthesizeTextWithMode(
+  id application,
+  NSString *text,
+  BOOL replace
+) {
+  @try {
+    RunnerTextEventBridge bridge;
+    NSString *missing = RunnerResolveTextEventBridge(application, replace, &bridge);
+    if (missing != nil) return RunnerTextEntryUnavailable(missing);
+
+    NSInteger targetProcessID =
+      ((RunnerMsgSendInteger)objc_msgSend)(application, bridge.core.processIDSelector);
+    if (targetProcessID <= 0) {
+      return RunnerTextEntryUnavailable(
+        @"private XCTest text synthesis unavailable: could not resolve target process ID"
+      );
+    }
+
+    if (replace) {
+      id selectionRecord = ((RunnerTextMsgSendInit)objc_msgSend)(
+        [bridge.core.recordClass alloc], bridge.initRecordSelector, @"agent-device-fill-select-all"
+      );
+      id selectionPath =
+        ((RunnerTextMsgSendInitPath)objc_msgSend)([bridge.core.pathClass alloc], bridge.initPathSelector);
+      if (selectionRecord == nil || selectionPath == nil) {
+        return RunnerTextEntryFailed(
+          @"private XCTest text synthesis failed: could not create the selection event"
+        );
+      }
+      ((RunnerMsgSendSetInteger)objc_msgSend)(
+        selectionRecord, bridge.core.setTargetProcessIDSelector, targetProcessID
+      );
+      // XCUIKeyModifierCommand is public XCTest API (1 << 4). Keeping the value local
+      // avoids importing XCUIAutomation headers into the bridging helper. A text-input
+      // path only performs its first operation reliably, so selection and replacement
+      // use ordered records rather than one multi-operation path.
+      NSUInteger commandModifier = 1UL << 4;
+      ((RunnerTextMsgSendTypeKey)objc_msgSend)(
+        selectionPath, bridge.typeKeySelector, @"a", commandModifier, 0.0
+      );
+      ((RunnerMsgSendAddPath)objc_msgSend)(selectionRecord, bridge.core.addPathSelector, selectionPath);
+      NSError *selectionError = nil;
+      BOOL selected = ((RunnerMsgSendSynthesize)objc_msgSend)(
+        selectionRecord, bridge.core.synthesizeSelector, &selectionError
+      );
+      if (!selected) {
+        NSString *detail = selectionError.localizedDescription ?: @"synthesizeWithError returned false";
+        return RunnerTextEntryFailed(
+          [NSString stringWithFormat:@"private XCTest text selection failed: %@", detail]
+        );
+      }
+    }
+
+    id record = ((RunnerTextMsgSendInit)objc_msgSend)(
+      [bridge.core.recordClass alloc],
+      bridge.initRecordSelector,
+      replace ? @"agent-device-fill-text" : @"agent-device-type"
+    );
+    id path = ((RunnerTextMsgSendInitPath)objc_msgSend)([bridge.core.pathClass alloc], bridge.initPathSelector);
+    if (record == nil || path == nil) {
+      return RunnerTextEntryFailed(
+        @"private XCTest text synthesis failed: could not create the text event"
+      );
+    }
+    ((RunnerMsgSendSetInteger)objc_msgSend)(record, bridge.core.setTargetProcessIDSelector, targetProcessID);
+    ((RunnerTextMsgSendType)objc_msgSend)(path, bridge.typeTextSelector, text, 0.0, 60, YES);
+    ((RunnerMsgSendAddPath)objc_msgSend)(record, bridge.core.addPathSelector, path);
+
+    NSError *error = nil;
+    BOOL ok = ((RunnerMsgSendSynthesize)objc_msgSend)(record, bridge.core.synthesizeSelector, &error);
+    if (!ok) {
+      NSString *detail = error.localizedDescription ?: @"synthesizeWithError returned false";
+      return RunnerTextEntryFailed(
+        [NSString stringWithFormat:@"private XCTest text synthesis failed: %@", detail]
+      );
+    }
+    return RunnerTextEntryResult(RunnerSynthesizedTextEntryStatusSucceeded, nil);
+  } @catch (NSException *exception) {
+    return RunnerTextEntryFailed(
+      RunnerFormatXCTestException(exception, @"private XCTest text synthesis failed")
+    );
+  }
+}
+
+static NSString * _Nullable RunnerResolveTextEventBridge(
+  id application,
+  BOOL replace,
+  RunnerTextEventBridge *bridge
+) {
+  RunnerXCTestEventBridge core;
+  NSString *missing = RunnerResolveXCTestEventBridge(application, RunnerTextSynthesisSurface, &core);
+  if (missing != nil) return missing;
+
+  SEL initRecordSelector = NSSelectorFromString(@"initWithName:");
+  SEL initPathSelector = NSSelectorFromString(@"initForTextInput");
+  SEL typeTextSelector = NSSelectorFromString(@"typeText:atOffset:typingSpeed:shouldRedact:");
+  SEL typeKeySelector = NSSelectorFromString(@"typeKey:modifiers:atOffset:");
+
+  missing = RunnerRequireSelector(
+    core.recordClass, initRecordSelector, @"initWithName:", RunnerTextSynthesisSurface
+  );
+  if (missing != nil) return missing;
+  missing = RunnerRequireSelector(
+    core.pathClass, initPathSelector, @"initForTextInput", RunnerTextSynthesisSurface
+  );
+  if (missing != nil) return missing;
+  missing = RunnerRequireSelector(
+    core.pathClass,
+    typeTextSelector,
+    @"typeText:atOffset:typingSpeed:shouldRedact:",
+    RunnerTextSynthesisSurface
+  );
+  if (missing != nil) return missing;
+  if (replace) {
+    missing = RunnerRequireSelector(
+      core.pathClass, typeKeySelector, @"typeKey:modifiers:atOffset:", RunnerTextSynthesisSurface
+    );
+    if (missing != nil) return missing;
+  }
+
+  *bridge = (RunnerTextEventBridge){
+    .core = core,
+    .initRecordSelector = initRecordSelector,
+    .initPathSelector = initPathSelector,
+    .typeTextSelector = typeTextSelector,
+    .typeKeySelector = typeKeySelector,
+  };
+  return nil;
+}
+
+static RunnerSynthesizedTextEntryResult *RunnerTextEntryResult(
+  RunnerSynthesizedTextEntryStatus status,
+  NSString * _Nullable message
+) {
+  RunnerSynthesizedTextEntryResult *result = [RunnerSynthesizedTextEntryResult new];
+  result.status = status;
+  result.message = message;
+  return result;
+}
+
+static RunnerSynthesizedTextEntryResult *RunnerTextEntryUnavailable(NSString *message) {
+  return RunnerTextEntryResult(RunnerSynthesizedTextEntryStatusUnavailable, message);
+}
+
+static RunnerSynthesizedTextEntryResult *RunnerTextEntryFailed(NSString *message) {
+  return RunnerTextEntryResult(RunnerSynthesizedTextEntryStatusFailed, message);
+}

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift

@@ -2,6 +2,34 @@ import XCTest
 
 extension RunnerTests {
   private static let privateAXSnapshotMaxNodes = 5_000
+
+  /// Upper bound on element-rooted follow-up snapshot requests per capture. A
+  /// depth-capped Bluesky-class tree resolves in 1-3 chained requests (~100-300ms
+  /// each); the bound exists so a pathological tree cannot stack requests past
+  /// the capture-plan deadline, which is also enforced per call.
+  private static let privateAXDeepExtensionCallLimit = 8
+
+  /// Upper bound on per-element custom-action reads per capture. Each is its own
+  /// AX round trip (~100ms on an idle simulator), so this caps the opt-in cost
+  /// at roughly a second on top of the capture; the capture-plan deadline stops
+  /// it earlier under load. A screenful of merged cards is well under this.
+  private static let privateAXCustomActionLimit = 12
+
+  /// A capture is depth-limited unless its frontier extension resolved every
+  /// capped node: frontiers left pending (budget/deadline) or missed (element
+  /// vanished, re-rooted request failed) mean subtrees are still absent, and
+  /// presenting such a capture as complete would hide exactly the content the
+  /// extension exists to recover. nil extension counts = extension never ran.
+  static func privateAXDepthLimited(
+    effectiveDepth: Int,
+    requestedDepth: Int,
+    pendingFrontiers: Int?,
+    missedFrontiers: Int?
+  ) -> Bool {
+    guard effectiveDepth < requestedDepth else { return false }
+    guard let pendingFrontiers, let missedFrontiers else { return true }
+    return pendingFrontiers > 0 || missedFrontiers > 0
+  }
   /// Deep React Native trees make the AX server reject bulk snapshot requests outright with
   /// kAXErrorIllegalArgument once the requested depth crosses a tree-size-dependent limit
   /// (observed between depth 56 and 64 on the Bluesky Home feed; the limit moves with live
@@ -10,6 +38,71 @@ extension RunnerTests {
   /// apps where the AX surface is genuinely unavailable.
   static let privateAXSnapshotDepthLadder = [56, 40, 24, 12]
 
+  /// Ladder rungs for one capture. A remembered accepted depth (recorded when a prior capture's
+  /// deep request was rejected) drops the rungs above it, so the known-rejected deep request is
+  /// not re-paid on every capture of the same screen class.
+  static func privateAXAttemptDepths(requestedDepth: Int, rememberedDepth: Int?) -> [Int] {
+    var depths = [requestedDepth]
+    depths.append(contentsOf: privateAXSnapshotDepthLadder.filter { $0 < requestedDepth })
+    guard let remembered = rememberedDepth, remembered < requestedDepth else { return depths }
+    return depths.filter { $0 <= remembered }
+  }
+
+  /// The bridge omits the key entirely when the capture did not ask for custom
+  /// actions, which is the difference between "nothing to disclose" and "read
+  /// none of them" — so a missing key must stay nil, never (0, 0).
+  static func privateAXCustomActionCoverage(_ raw: Any?) -> SnapshotCustomActionCoverage? {
+    guard let coverage = raw as? [String: Any],
+      let read = (coverage[RunnerAXSnapshotCustomActionsReadKey] as? NSNumber)?.intValue,
+      let candidates = (coverage[RunnerAXSnapshotCustomActionsCandidatesKey] as? NSNumber)?.intValue
+    else { return nil }
+    // read/candidates are the ratio and must both be present; the truncation
+    // count is additive, so an older bridge that omits it reads as zero rather
+    // than voiding the whole coverage.
+    let truncated =
+      (coverage[RunnerAXSnapshotCustomActionsTruncatedKey] as? NSNumber)?.intValue ?? 0
+    let blocked = (coverage[RunnerAXSnapshotCustomActionsBlockedKey] as? NSNumber)?.boolValue ?? false
+    return SnapshotCustomActionCoverage(
+      read: read, candidates: candidates, truncated: truncated, blocked: blocked)
+  }
+
+  func rememberPrivateAXAcceptedDepth(bundleId: String?, processIdentifier: Int?, depth: Int) {
+    // No PID means no way to notice a relaunch later; record nothing rather than risk serving
+    // a stale shallow rung to a fresh process.
+    guard let processIdentifier else { return }
+    privateAXAcceptedDepthLock.lock()
+    privateAXAcceptedDepthBundleId = bundleId
+    privateAXAcceptedDepthProcessIdentifier = processIdentifier
+    privateAXAcceptedDepth = depth
+    privateAXAcceptedDepthUntil = Date().addingTimeInterval(snapshotXCTestChannelPenaltyDuration)
+    privateAXAcceptedDepthLock.unlock()
+    NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_DEPTH_REMEMBERED depth=%ld bundle=%@", depth, bundleId ?? "")
+  }
+
+  func rememberedPrivateAXAcceptedDepth(bundleId: String?, processIdentifier: Int?) -> Int? {
+    privateAXAcceptedDepthLock.lock()
+    defer { privateAXAcceptedDepthLock.unlock() }
+    guard Date() < privateAXAcceptedDepthUntil else { return nil }
+    guard privateAXAcceptedDepthBundleId == bundleId else { return nil }
+    guard let processIdentifier, privateAXAcceptedDepthProcessIdentifier == processIdentifier else {
+      return nil
+    }
+    return privateAXAcceptedDepth
+  }
+
+  func clearPrivateAXAcceptedDepth(reason: String) {
+    privateAXAcceptedDepthLock.lock()
+    let hadMemory = privateAXAcceptedDepth != nil && Date() < privateAXAcceptedDepthUntil
+    privateAXAcceptedDepthBundleId = nil
+    privateAXAcceptedDepthProcessIdentifier = nil
+    privateAXAcceptedDepth = nil
+    privateAXAcceptedDepthUntil = .distantPast
+    privateAXAcceptedDepthLock.unlock()
+    if hadMemory {
+      NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_DEPTH_MEMORY_CLEARED reason=%@", reason)
+    }
+  }
+
   func privateAXSnapshotCapture(
     app: XCUIApplication,
     options: SnapshotOptions,
@@ -17,9 +110,19 @@ extension RunnerTests {
   ) -> SnapshotBackendCapture? {
     #if os(iOS) && targetEnvironment(simulator)
       let requestedDepth = options.depth ?? 64
-      var attemptDepths = [requestedDepth]
-      attemptDepths.append(
-        contentsOf: Self.privateAXSnapshotDepthLadder.filter { $0 < requestedDepth }
+      // An explicit --depth request is honored as asked: no accepted-depth
+      // memory, no frontier extension past it.
+      let exactDepthRequested = options.depth != nil
+      let rememberedDepth =
+        exactDepthRequested
+        ? nil
+        : rememberedPrivateAXAcceptedDepth(
+          bundleId: currentBundleId,
+          processIdentifier: currentAppProcessIdentifier
+        )
+      let attemptDepths = Self.privateAXAttemptDepths(
+        requestedDepth: requestedDepth,
+        rememberedDepth: rememberedDepth
       )
       var response: [String: Any] = [:]
       var effectiveDepth = requestedDepth
@@ -35,7 +138,10 @@ extension RunnerTests {
         response = RunnerAXSnapshotBridge.snapshotTree(
           for: app,
           maxDepth: depth,
-          maxNodes: Self.privateAXSnapshotMaxNodes
+          maxNodes: Self.privateAXSnapshotMaxNodes,
+          deepExtensionCallLimit: exactDepthRequested ? 0 : Self.privateAXDeepExtensionCallLimit,
+          customActionLimit: options.customActions ? Self.privateAXCustomActionLimit : 0,
+          deadline: deadline
         )
         if response["ok"] as? Bool == true {
           effectiveDepth = depth
@@ -52,6 +158,16 @@ extension RunnerTests {
         NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_FAILED=%@", lastError)
         return nil
       }
+      // Only a capture that actually descended records memory: a first-rung success on a
+      // remembered depth deliberately does NOT refresh the TTL, so expiry re-probes the full
+      // requested depth once per window instead of capping this screen class forever.
+      if !exactDepthRequested, effectiveDepth != attemptDepths.first {
+        rememberPrivateAXAcceptedDepth(
+          bundleId: currentBundleId,
+          processIdentifier: currentAppProcessIdentifier,
+          depth: effectiveDepth
+        )
+      }
       guard let root = response["root"] as? [String: Any] else {
         NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_FAILED=missing root")
         return nil
@@ -74,24 +190,46 @@ extension RunnerTests {
         return nil
       }
 
-      let depthLimited = effectiveDepth < requestedDepth
+      // A capture whose frontier extension resolved every capped node is
+      // complete despite the per-request depth cap — reporting it as
+      // depth-limited would send agents chasing deeper content that is not
+      // there. Pending or missed frontiers keep the depth-limited verdict.
+      let deepExtension = response[RunnerAXSnapshotDeepExtensionKey] as? [String: Any]
+      let depthLimited = Self.privateAXDepthLimited(
+        effectiveDepth: effectiveDepth,
+        requestedDepth: requestedDepth,
+        pendingFrontiers: deepExtension?[RunnerAXSnapshotDeepExtensionPendingKey] as? Int,
+        missedFrontiers: deepExtension?[RunnerAXSnapshotDeepExtensionMissedKey] as? Int
+      )
       NSLog(
-        "AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_USED nodes=%ld depth=%ld",
+        "AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_USED nodes=%ld depth=%ld extended=%ld",
         nodes.count,
-        effectiveDepth
+        effectiveDepth,
+        deepExtension?[RunnerAXSnapshotDeepExtensionNodesAddedKey] as? Int ?? 0
       )
       return SnapshotBackendCapture(
         payload: DataPayload(nodes: nodes, truncated: (response["truncated"] as? Bool) == true),
-        effectiveDepth: depthLimited ? effectiveDepth : nil
+        effectiveDepth: depthLimited ? effectiveDepth : nil,
+        customActions: Self.privateAXCustomActionCoverage(
+          response[RunnerAXSnapshotCustomActionsKey]
+        )
       )
     #else
       return nil
     #endif
   }
 
+  /// The viewport read is XCTest main-thread work — the exact channel the penalty marks as
+  /// grinding on this screen class. Under penalty it reliably burns its full timeout and
+  /// falls back anyway (~1s added to every private AX capture on the Bluesky bench feed),
+  /// so honor the penalty here the same way capture plans do.
+  func shouldReadPrivateAXViewportViaXCTest() -> Bool {
+    !hasAbandonedTreeCapture() && !isSnapshotXCTestChannelPenalized(bundleId: currentBundleId)
+  }
+
   private func privateAXSnapshotViewport(app: XCUIApplication, rootFrame: CGRect) -> CGRect {
     let fallback = rootFrame.isEmpty ? CGRect.infinite : rootFrame
-    guard !hasAbandonedTreeCapture() else {
+    guard shouldReadPrivateAXViewportViaXCTest() else {
       return fallback
     }
     do {
@@ -164,7 +302,8 @@ extension RunnerTests {
           depth: depth,
           parentIndex: parentIndex,
           hiddenContentAbove: nil,
-          hiddenContentBelow: nil
+          hiddenContentBelow: nil,
+          actions: rawNode["actions"] as? [String]
         )
       )
     } else {