agent-device 0.20.9 → 0.21.0

package/dist/apple/runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift

@@ -30,30 +30,25 @@ struct GestureEvent: Decodable {
   let edge: String?
 }
 
-enum OverlayError: Error, CustomStringConvertible {
-  case invalidArgs(String)
-  case missingVideoTrack
-  case exportFailed(String)
+enum ExportQuality: String {
+  case medium
+  case high
+}
 
-  var description: String {
-    switch self {
-    case .invalidArgs(let message):
-      return message
-    case .missingVideoTrack:
-      return "Input video does not contain a video track."
-    case .exportFailed(let message):
-      return message
+/// Entry point: `@main` because multi-file swiftc compilation reserves top-level statements for
+/// `main.swift`, which cannot be shared per-script.
+@main
+enum RecordingOverlay {
+  static func main() {
+    do {
+      try run()
+    } catch {
+      fputs("recording-overlay: \(error)\n", stderr)
+      exit(1)
     }
   }
 }
 
-do {
-  try run()
-} catch {
-  fputs("recording-overlay: \(error)\n", stderr)
-  exit(1)
-}
-
 func run() throws {
   let arguments = Array(CommandLine.arguments.dropFirst())
   let parsedArgs = try parseArguments(arguments)
@@ -61,9 +56,7 @@ func run() throws {
   let outputURL = URL(fileURLWithPath: parsedArgs.outputPath)
   let eventsURL = URL(fileURLWithPath: parsedArgs.eventsPath)
 
-  if FileManager.default.fileExists(atPath: outputURL.path) {
-    try FileManager.default.removeItem(at: outputURL)
-  }
+  try removeStaleOutput(outputURL)
 
   let payload = try Data(contentsOf: eventsURL)
   let envelope = try JSONDecoder().decode(GestureEnvelope.self, from: payload)
@@ -74,27 +67,15 @@ func run() throws {
   }
 
   let asset = AVURLAsset(url: inputURL)
-  guard let sourceVideoTrack = asset.tracks(withMediaType: .video).first else {
-    throw OverlayError.missingVideoTrack
-  }
-
-  let composition = AVMutableComposition()
-  guard let compositionVideoTrack = composition.addMutableTrack(
-    withMediaType: .video,
-    preferredTrackID: kCMPersistentTrackID_Invalid
-  ) else {
-    throw OverlayError.exportFailed("Failed to create composition video track.")
-  }
-
+  let sourceVideoTrack = try sourceVideoTrack(of: asset)
   let fullRange = CMTimeRange(start: .zero, duration: asset.duration)
-  try compositionVideoTrack.insertTimeRange(fullRange, of: sourceVideoTrack, at: .zero)
-
-  if let sourceAudioTrack = asset.tracks(withMediaType: .audio).first,
-     let compositionAudioTrack = composition.addMutableTrack(
-       withMediaType: .audio,
-       preferredTrackID: kCMPersistentTrackID_Invalid
-     ) {
-    try? compositionAudioTrack.insertTimeRange(fullRange, of: sourceAudioTrack, at: .zero)
+  let composition = try makeRecordingComposition(
+    asset: asset,
+    videoTrack: sourceVideoTrack,
+    timeRange: fullRange
+  )
+  guard let compositionVideoTrack = composition.tracks(withMediaType: .video).first else {
+    throw RecordingScriptError.exportFailed("Failed to create composition video track.")
   }
 
   let renderSize = resolvedRenderSize(for: sourceVideoTrack)
@@ -149,32 +130,17 @@ func run() throws {
   // while avoiding very slow highest-quality exports. Pass --quality high to opt into
   // the slower highest-quality export.
   let presetName = exportPresetName(for: parsedArgs.exportQuality, compatibleWith: composition)
-  guard let exporter = AVAssetExportSession(asset: composition, presetName: presetName) else {
-    throw OverlayError.exportFailed("Failed to create export session.")
-  }
-
-  exporter.outputURL = outputURL
-  exporter.outputFileType = .mp4
-  exporter.videoComposition = videoComposition
-  exporter.shouldOptimizeForNetworkUse = true
-
-  let semaphore = DispatchSemaphore(value: 0)
-  exporter.exportAsynchronously {
-    semaphore.signal()
-  }
-  if semaphore.wait(timeout: .now() + 120) == .timedOut {
-    exporter.cancelExport()
-    throw OverlayError.exportFailed("Touch overlay export timed out.")
-  }
-
-  if exporter.status != .completed {
-    throw OverlayError.exportFailed(exporter.error?.localizedDescription ?? "Touch overlay export failed.")
-  }
-}
-
-enum ExportQuality: String {
-  case medium
-  case high
+  let exporter = try makeRecordingExporter(
+    composition,
+    presetName: presetName,
+    outputURL: outputURL,
+    videoComposition: videoComposition
+  )
+  try runRecordingExport(
+    exporter,
+    timeoutMessage: "Touch overlay export timed out.",
+    failureMessage: "Touch overlay export failed."
+  )
 }
 
 func parseArguments(
@@ -193,33 +159,28 @@ func parseArguments(
     let nextIndex = index + 1
     switch argument {
     case "--input":
-      guard nextIndex < arguments.count else { throw OverlayError.invalidArgs("--input requires a value") }
-      inputPath = arguments[nextIndex]
+      inputPath = try recordingOptionValue(arguments, nextIndex, "--input")
       index += 2
     case "--output":
-      guard nextIndex < arguments.count else { throw OverlayError.invalidArgs("--output requires a value") }
-      outputPath = arguments[nextIndex]
+      outputPath = try recordingOptionValue(arguments, nextIndex, "--output")
       index += 2
     case "--events":
-      guard nextIndex < arguments.count else { throw OverlayError.invalidArgs("--events requires a value") }
-      eventsPath = arguments[nextIndex]
+      eventsPath = try recordingOptionValue(arguments, nextIndex, "--events")
       index += 2
     case "--quality":
-      guard nextIndex < arguments.count else {
-        throw OverlayError.invalidArgs("--quality requires a value")
-      }
-      guard let parsed = ExportQuality(rawValue: arguments[nextIndex]) else {
-        throw OverlayError.invalidArgs("--quality must be one of: medium, high")
+      let rawValue = try recordingOptionValue(arguments, nextIndex, "--quality")
+      guard let parsed = ExportQuality(rawValue: rawValue) else {
+        throw RecordingScriptError.invalidArgs("--quality must be one of: medium, high")
       }
       exportQuality = parsed
       index += 2
     default:
-      throw OverlayError.invalidArgs("Unknown argument: \(argument)")
+      throw RecordingScriptError.invalidArgs("Unknown argument: \(argument)")
     }
   }
 
   guard let inputPath, let outputPath, let eventsPath else {
-    throw OverlayError.invalidArgs(
+    throw RecordingScriptError.invalidArgs(
       "Usage: recording-overlay.swift --input <video> --output <video> --events <json> [--quality <medium|high>]"
     )
   }

package/dist/apple/runner/AgentDeviceRunner/RecordingScripts/recording-trim.swift

@@ -1,102 +1,59 @@
 import AVFoundation
 import Foundation
 
-enum TrimError: Error, CustomStringConvertible {
-  case invalidArgs(String)
-  case invalidTrimRange
-  case missingVideoTrack
-  case exportFailed(String)
-
-  var description: String {
-    switch self {
-    case .invalidArgs(let message):
-      return message
-    case .invalidTrimRange:
-      return "Trim start must be before the end of the recording."
-    case .missingVideoTrack:
-      return "Input video does not contain a video track."
-    case .exportFailed(let message):
-      return message
+/// Entry point: `@main` because multi-file swiftc compilation reserves top-level statements for
+/// `main.swift`, which cannot be shared per-script.
+@main
+enum RecordingTrim {
+  static func main() {
+    do {
+      try run()
+    } catch {
+      fputs("recording-trim: \(error)\n", stderr)
+      exit(1)
     }
   }
 }
 
-do {
-  try run()
-} catch {
-  fputs("recording-trim: \(error)\n", stderr)
-  exit(1)
-}
-
 func run() throws {
   let arguments = Array(CommandLine.arguments.dropFirst())
   let parsedArgs = try parseArguments(arguments)
   let inputURL = URL(fileURLWithPath: parsedArgs.inputPath)
   let outputURL = URL(fileURLWithPath: parsedArgs.outputPath)
 
-  if FileManager.default.fileExists(atPath: outputURL.path) {
-    try FileManager.default.removeItem(at: outputURL)
-  }
+  try removeStaleOutput(outputURL)
 
   let asset = AVURLAsset(url: inputURL)
-  guard let sourceVideoTrack = asset.tracks(withMediaType: .video).first else {
-    throw TrimError.missingVideoTrack
-  }
-
+  let videoTrack = try sourceVideoTrack(of: asset)
   let trimStart = CMTime(seconds: parsedArgs.trimStartMs / 1000.0, preferredTimescale: 600)
   guard CMTimeCompare(trimStart, asset.duration) < 0 else {
-    throw TrimError.invalidTrimRange
+    throw RecordingScriptError.invalidTrimRange
   }
 
   let trimmedDuration = CMTimeSubtract(asset.duration, trimStart)
   guard CMTimeCompare(trimmedDuration, .zero) > 0 else {
-    throw TrimError.invalidTrimRange
-  }
-
-  let composition = AVMutableComposition()
-  let trimmedRange = CMTimeRange(start: trimStart, duration: trimmedDuration)
-
-  guard let compositionVideoTrack = composition.addMutableTrack(
-    withMediaType: .video,
-    preferredTrackID: kCMPersistentTrackID_Invalid
-  ) else {
-    throw TrimError.exportFailed("Failed to create composition video track.")
+    throw RecordingScriptError.invalidTrimRange
   }
-  try compositionVideoTrack.insertTimeRange(trimmedRange, of: sourceVideoTrack, at: .zero)
-  compositionVideoTrack.preferredTransform = sourceVideoTrack.preferredTransform
 
-  if let sourceAudioTrack = asset.tracks(withMediaType: .audio).first,
-     let compositionAudioTrack = composition.addMutableTrack(
-       withMediaType: .audio,
-       preferredTrackID: kCMPersistentTrackID_Invalid
-     ) {
-    try? compositionAudioTrack.insertTimeRange(trimmedRange, of: sourceAudioTrack, at: .zero)
-  }
+  let composition = try makeRecordingComposition(
+    asset: asset,
+    videoTrack: videoTrack,
+    timeRange: CMTimeRange(start: trimStart, duration: trimmedDuration)
+  )
+  composition.tracks(withMediaType: .video).first?.preferredTransform = videoTrack.preferredTransform
 
+  // Passthrough keeps the trim lossless when the composition supports it; otherwise re-encode at
+  // highest quality.
   let presetName = AVAssetExportSession.exportPresets(compatibleWith: composition)
     .contains(AVAssetExportPresetPassthrough)
     ? AVAssetExportPresetPassthrough
     : AVAssetExportPresetHighestQuality
-  guard let exporter = AVAssetExportSession(asset: composition, presetName: presetName) else {
-    throw TrimError.exportFailed("Failed to create export session.")
-  }
-
-  exporter.outputURL = outputURL
-  exporter.outputFileType = .mp4
-  exporter.shouldOptimizeForNetworkUse = true
-
-  let semaphore = DispatchSemaphore(value: 0)
-  exporter.exportAsynchronously {
-    semaphore.signal()
-  }
-  if semaphore.wait(timeout: .now() + 120) == .timedOut {
-    exporter.cancelExport()
-    throw TrimError.exportFailed("Trim export timed out.")
-  }
-
-  if exporter.status != .completed {
-    throw TrimError.exportFailed(exporter.error?.localizedDescription ?? "Trim export failed.")
-  }
+  let exporter = try makeRecordingExporter(composition, presetName: presetName, outputURL: outputURL)
+  try runRecordingExport(
+    exporter,
+    timeoutMessage: "Trim export timed out.",
+    failureMessage: "Trim export failed."
+  )
 }
 
 func parseArguments(_ arguments: [String]) throws -> (inputPath: String, outputPath: String, trimStartMs: Double) {
@@ -110,29 +67,25 @@ func parseArguments(_ arguments: [String]) throws -> (inputPath: String, outputP
     let nextIndex = index + 1
     switch argument {
     case "--input":
-      guard nextIndex < arguments.count else { throw TrimError.invalidArgs("--input requires a value") }
-      inputPath = arguments[nextIndex]
+      inputPath = try recordingOptionValue(arguments, nextIndex, "--input")
       index += 2
     case "--output":
-      guard nextIndex < arguments.count else { throw TrimError.invalidArgs("--output requires a value") }
-      outputPath = arguments[nextIndex]
+      outputPath = try recordingOptionValue(arguments, nextIndex, "--output")
       index += 2
     case "--trim-start-ms":
-      guard nextIndex < arguments.count else {
-        throw TrimError.invalidArgs("--trim-start-ms requires a value")
-      }
-      guard let parsed = Double(arguments[nextIndex]), parsed >= 0 else {
-        throw TrimError.invalidArgs("--trim-start-ms must be a non-negative number")
+      let rawValue = try recordingOptionValue(arguments, nextIndex, "--trim-start-ms")
+      guard let parsed = Double(rawValue), parsed >= 0 else {
+        throw RecordingScriptError.invalidArgs("--trim-start-ms must be a non-negative number")
       }
       trimStartMs = parsed
       index += 2
     default:
-      throw TrimError.invalidArgs("Unknown argument: \(argument)")
+      throw RecordingScriptError.invalidArgs("Unknown argument: \(argument)")
     }
   }
 
   guard let inputPath, let outputPath, let trimStartMs else {
-    throw TrimError.invalidArgs(
+    throw RecordingScriptError.invalidArgs(
       "Usage: recording-trim.swift --input <video> --output <video> --trim-start-ms <ms>"
     )
   }

package/dist/apple/snapshot-presentation/Package.swift

@@ -0,0 +1,21 @@
+// swift-tools-version: 5.9
+import PackageDescription
+
+let package = Package(
+  name: "agent-device-snapshot-presentation",
+  platforms: [
+    .iOS(.v15),
+    .macOS(.v13),
+    .tvOS(.v15),
+    .visionOS(.v1),
+  ],
+  products: [
+    .library(
+      name: "AgentDeviceSnapshotPresentation",
+      targets: ["AgentDeviceSnapshotPresentation"]
+    ),
+  ],
+  targets: [
+    .target(name: "AgentDeviceSnapshotPresentation"),
+  ]
+)

package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotGeometry.swift

@@ -0,0 +1,56 @@
+import Foundation
+import CoreGraphics
+
+public enum SnapshotGeometry {
+  public static func effectiveFrame(
+    reportedFrame: CGRect,
+    viewport: CGRect,
+    ancestorClip: CGRect?
+  ) -> CGRect {
+    var frame = reportedFrame
+    if !viewport.isInfinite {
+      frame = clipped(frame, to: viewport)
+    }
+    if let ancestorClip {
+      frame = clipped(frame, to: ancestorClip)
+    }
+    return frame
+  }
+
+  public static func snapshotRect(from frame: CGRect, reportedFrame: CGRect) -> SnapshotRect {
+    guard !frame.isNull, !frame.isEmpty else {
+      return SnapshotRect(
+        x: Double(reportedFrame.minX),
+        y: Double(reportedFrame.minY),
+        width: 0,
+        height: 0
+      )
+    }
+    return SnapshotRect(
+      x: Double(frame.origin.x),
+      y: Double(frame.origin.y),
+      width: Double(max(0, frame.size.width)),
+      height: Double(max(0, frame.size.height))
+    )
+  }
+
+  public static func isGeometricallyActionable(
+    enabled: Bool,
+    frame: CGRect,
+    viewport: CGRect
+  ) -> Bool {
+    guard enabled, !frame.isNull, !frame.isEmpty else { return false }
+    if viewport.isInfinite { return true }
+    let center = CGPoint(x: frame.midX, y: frame.midY)
+    return viewport.contains(center)
+  }
+
+  private static func clipped(_ frame: CGRect, to clip: CGRect) -> CGRect {
+    guard !frame.isNull, !frame.isEmpty else { return frame }
+    let intersection = frame.intersection(clip)
+    guard !intersection.isNull, !intersection.isEmpty else {
+      return CGRect(x: frame.minX, y: frame.minY, width: 0, height: 0)
+    }
+    return intersection
+  }
+}

package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift

@@ -0,0 +1,259 @@
+import Foundation
+import CoreGraphics
+
+public struct SnapshotRect: Codable, Equatable {
+  public let x: Double
+  public let y: Double
+  public let width: Double
+  public let height: Double
+
+  public init(x: Double, y: Double, width: Double, height: Double) {
+    self.x = x
+    self.y = y
+    self.width = width
+    self.height = height
+  }
+
+  public var cgRect: CGRect {
+    CGRect(x: x, y: y, width: width, height: height)
+  }
+}
+
+public struct RawAXNode: Equatable {
+  public let index: Int
+  public let type: String
+  public let label: String?
+  public let identifier: String?
+  public let value: String?
+  public let rect: SnapshotRect
+  public let enabled: Bool
+  public let focused: Bool?
+  public let selected: Bool?
+  public let hittable: Bool
+  public let depth: Int
+  public let parentIndex: Int?
+  public let hiddenContentAbove: Bool?
+  public let hiddenContentBelow: Bool?
+  public var actions: [String]?
+
+  public init(
+    index: Int,
+    type: String,
+    label: String?,
+    identifier: String?,
+    value: String?,
+    rect: SnapshotRect,
+    enabled: Bool,
+    focused: Bool?,
+    selected: Bool?,
+    hittable: Bool,
+    depth: Int,
+    parentIndex: Int?,
+    hiddenContentAbove: Bool?,
+    hiddenContentBelow: Bool?,
+    actions: [String]? = nil
+  ) {
+    self.index = index
+    self.type = type
+    self.label = label
+    self.identifier = identifier
+    self.value = value
+    self.rect = rect
+    self.enabled = enabled
+    self.focused = focused
+    self.selected = selected
+    self.hittable = hittable
+    self.depth = depth
+    self.parentIndex = parentIndex
+    self.hiddenContentAbove = hiddenContentAbove
+    self.hiddenContentBelow = hiddenContentBelow
+    self.actions = actions
+  }
+
+  public var hasSemanticContent: Bool {
+    [label, identifier, value].contains {
+      !($0?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ?? true)
+    }
+  }
+}
+
+public struct CaptureHint: Equatable {
+  public enum Projection: String {
+    case regular
+    case raw
+  }
+
+  public let projection: Projection
+  public let depth: Int?
+  public let regularPresentedDepth: Int?
+  public let interactiveOnly: Bool
+  public let customActions: Bool
+
+  public init(
+    projection: Projection,
+    depth: Int?,
+    regularPresentedDepth: Int?,
+    interactiveOnly: Bool,
+    customActions: Bool
+  ) {
+    self.projection = projection
+    self.depth = depth
+    self.regularPresentedDepth = regularPresentedDepth
+    self.interactiveOnly = interactiveOnly
+    self.customActions = customActions
+  }
+
+  public var rawTraversalDepth: Int? {
+    projection == .raw ? depth : nil
+  }
+
+  public var isRaw: Bool {
+    projection == .raw
+  }
+}
+
+public struct PresentationOptions: Equatable {
+  public let interactiveOnly: Bool
+  public let depth: Int?
+  public let scope: String?
+  public let raw: Bool
+  public var preferredBackend: String?
+  public var customActions: Bool
+
+  public init(
+    interactiveOnly: Bool,
+    depth: Int?,
+    scope: String?,
+    raw: Bool,
+    preferredBackend: String? = nil,
+    customActions: Bool = false
+  ) {
+    self.interactiveOnly = interactiveOnly
+    self.depth = depth
+    self.scope = scope
+    self.raw = raw
+    self.preferredBackend = preferredBackend
+    self.customActions = customActions
+  }
+}
+
+public struct SnapshotAcquisition {
+  public let hint: CaptureHint
+  public let nodes: [RawAXNode]
+  public let truncated: Bool
+  public let effectiveDepth: Int?
+  public var customActions: SnapshotCustomActionCoverage?
+  public let viewport: CGRect
+
+  public init(
+    hint: CaptureHint,
+    nodes: [RawAXNode],
+    truncated: Bool,
+    effectiveDepth: Int?,
+    customActions: SnapshotCustomActionCoverage? = nil,
+    viewport: CGRect
+  ) {
+    self.hint = hint
+    self.nodes = nodes
+    self.truncated = truncated
+    self.effectiveDepth = effectiveDepth
+    self.customActions = customActions
+    self.viewport = viewport
+  }
+}
+
+public struct SnapshotCustomActionCoverage: Codable, Equatable {
+  public let read: Int
+  public let candidates: Int
+  public let truncated: Int
+  public let blocked: Bool
+
+  public init(read: Int, candidates: Int, truncated: Int, blocked: Bool) {
+    self.read = read
+    self.candidates = candidates
+    self.truncated = truncated
+    self.blocked = blocked
+  }
+}
+
+public struct SnapshotPresentationNode {
+  public let raw: RawAXNode
+  public let effectiveRect: SnapshotRect
+
+  public init(raw: RawAXNode, effectiveRect: SnapshotRect) {
+    self.raw = raw
+    self.effectiveRect = effectiveRect
+  }
+
+  public static func reported(_ raw: RawAXNode) -> Self {
+    Self(raw: raw, effectiveRect: raw.rect)
+  }
+}
+
+public struct PresentedNode: Codable, Equatable {
+  public let index: Int
+  public let type: String
+  public let label: String?
+  public let identifier: String?
+  public let value: String?
+  public let rect: SnapshotRect
+  public let enabled: Bool
+  public let focused: Bool?
+  public let selected: Bool?
+  public let hittable: Bool
+  public let depth: Int
+  public let parentIndex: Int?
+  public let hiddenContentAbove: Bool?
+  public let hiddenContentBelow: Bool?
+  public let actions: [String]?
+
+  internal init(
+    presenting raw: RawAXNode,
+    rect: SnapshotRect? = nil,
+    index: Int? = nil,
+    depth: Int? = nil,
+    parentIndex: Int?? = nil
+  ) {
+    self.index = index ?? raw.index
+    self.type = raw.type
+    self.label = raw.label
+    self.identifier = raw.identifier
+    self.value = raw.value
+    self.rect = rect ?? raw.rect
+    self.enabled = raw.enabled
+    self.focused = raw.focused
+    self.selected = raw.selected
+    self.hittable = raw.hittable
+    self.depth = depth ?? raw.depth
+    self.parentIndex = parentIndex ?? raw.parentIndex
+    self.hiddenContentAbove = raw.hiddenContentAbove
+    self.hiddenContentBelow = raw.hiddenContentBelow
+    self.actions = raw.actions
+  }
+
+  internal init(presenting node: SnapshotPresentationNode) {
+    self.init(presenting: node.raw, rect: node.effectiveRect)
+  }
+}
+
+public struct SnapshotPresentationResult {
+  public let nodes: [PresentedNode]
+  public let truncated: Bool
+  public let effectiveDepth: Int?
+  public let customActions: SnapshotCustomActionCoverage?
+  public let qualityNodes: [PresentedNode]?
+
+  public init(
+    nodes: [PresentedNode],
+    truncated: Bool,
+    effectiveDepth: Int?,
+    customActions: SnapshotCustomActionCoverage?,
+    qualityNodes: [PresentedNode]?
+  ) {
+    self.nodes = nodes
+    self.truncated = truncated
+    self.effectiveDepth = effectiveDepth
+    self.customActions = customActions
+    self.qualityNodes = qualityNodes
+  }
+}