@kaeawc/auto-mobile 0.0.44 → 0.0.45

package/dist/ios/screen-capture/Sources/ScreenCaptureHelper/DeviceDiscovery.swift

@@ -0,0 +1,29 @@
+import AVFoundation
+import Foundation
+import ScreenCaptureCore
+
+/// Discovers USB-connected iOS devices visible to AVFoundation as external
+/// muxed (audio + video) capture devices.
+enum DeviceDiscovery {
+    static func discover() -> [AVCaptureDevice] {
+        let types: [AVCaptureDevice.DeviceType] = [.external]
+        return AVCaptureDevice.DiscoverySession(
+            deviceTypes: types,
+            mediaType: .muxed,
+            position: .unspecified
+        ).devices
+    }
+
+    static func toInfo(_ device: AVCaptureDevice) -> DeviceInfo {
+        DeviceInfo(
+            uniqueID: device.uniqueID,
+            localizedName: device.localizedName,
+            modelID: device.modelID,
+            manufacturer: device.manufacturer
+        )
+    }
+
+    static func find(uniqueID: String) -> AVCaptureDevice? {
+        discover().first { $0.uniqueID == uniqueID }
+    }
+}

package/dist/ios/screen-capture/Sources/ScreenCaptureHelper/FrameWriter+PixelBuffer.swift

@@ -0,0 +1,25 @@
+import CoreMedia
+import CoreVideo
+import Foundation
+import ScreenCaptureCore
+
+extension FrameWriter {
+    /// Writes one frame from a `CMSampleBuffer`, locking the underlying pixel
+    /// buffer for the duration of the synchronous sink write. Returns `false`
+    /// when the sample buffer has no image (e.g. dropped/idle frames).
+    @discardableResult
+    func write(sampleBuffer: CMSampleBuffer) -> Bool {
+        guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return false }
+        CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly)
+        defer { CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly) }
+        guard let base = CVPixelBufferGetBaseAddress(pixelBuffer) else { return false }
+
+        write(
+            width: CVPixelBufferGetWidth(pixelBuffer),
+            height: CVPixelBufferGetHeight(pixelBuffer),
+            bytesPerRow: CVPixelBufferGetBytesPerRow(pixelBuffer),
+            baseAddress: base
+        )
+        return true
+    }
+}

package/dist/ios/screen-capture/Sources/ScreenCaptureHelper/SimulatorCaptureSession.swift

@@ -0,0 +1,159 @@
+import CoreMedia
+import CoreVideo
+import Foundation
+import ScreenCaptureKit
+import ScreenCaptureCore
+
+/// Streams BGRA frames from a single iOS Simulator window via ScreenCaptureKit.
+/// The frame rate is configurable (5–60); 5 is the default for typical MCP
+/// automation workloads. Size changes (e.g. device rotation) trigger a stream
+/// reconfiguration so frames don't get cropped.
+final class SimulatorCaptureSession: NSObject, SCStreamOutput, SCStreamDelegate {
+    private let writer: FrameWriter
+    private let onFatalError: (Error) -> Void
+    private let queue = DispatchQueue(label: "automobile.simulator-capture.frames")
+    private var stream: SCStream?
+    private var configuredPixelWidth: Int = 0
+    private var configuredPixelHeight: Int = 0
+    private var hasReceivedFrame = false
+    private var fps: Int = CommandLineOptions.defaultSimulatorFPS
+    private var audioEnabled = false
+
+    init(writer: FrameWriter, onFatalError: @escaping (Error) -> Void) {
+        self.writer = writer
+        self.onFatalError = onFatalError
+    }
+
+    func start(window: SCWindow, fps: Int, audio: Bool) async throws {
+        self.fps = fps
+        audioEnabled = audio
+        let filter = SCContentFilter(desktopIndependentWindow: window)
+        let config = SimulatorCaptureSession.makeConfiguration(window: window, fps: fps, audio: audio)
+        configuredPixelWidth = config.width
+        configuredPixelHeight = config.height
+
+        let stream = SCStream(filter: filter, configuration: config, delegate: self)
+        try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: queue)
+        if audio {
+            try stream.addStreamOutput(self, type: .audio, sampleHandlerQueue: queue)
+        }
+        try await stream.startCapture()
+        self.stream = stream
+    }
+
+    func stop() async {
+        guard let stream = stream else { return }
+        // removeStreamOutput breaks the SCStream → self retain cycle so the
+        // session is collectable even before SCStream itself goes away.
+        try? stream.removeStreamOutput(self, type: .screen)
+        try? await stream.stopCapture()
+        self.stream = nil
+    }
+
+    /// Whether at least one frame has been delivered since `start()`.
+    /// Used by the CLI to detect a silent screen-recording permission denial.
+    var hasReceivedAnyFrame: Bool { hasReceivedFrame }
+
+    // MARK: - SCStreamOutput
+
+    func stream(
+        _ stream: SCStream,
+        didOutputSampleBuffer sampleBuffer: CMSampleBuffer,
+        of type: SCStreamOutputType
+    ) {
+        guard sampleBuffer.isValid else { return }
+        if type == .audio {
+            if let pcm16le = pcm16leAudio(sampleBuffer: sampleBuffer) {
+                writer.writeAudio(pcm16le: pcm16le)
+            }
+            return
+        }
+        guard type == .screen else { return }
+
+        // Drop non-complete statuses (idle, blank, suspended, stopped) so we
+        // don't re-emit identical pixels or partial buffers.
+        if let status = SimulatorCaptureSession.frameStatus(of: sampleBuffer),
+           status != .complete {
+            return
+        }
+
+        guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
+        let actualWidth = CVPixelBufferGetWidth(pixelBuffer)
+        let actualHeight = CVPixelBufferGetHeight(pixelBuffer)
+        if actualWidth != configuredPixelWidth || actualHeight != configuredPixelHeight {
+            reconfigure(width: actualWidth, height: actualHeight)
+        }
+
+        if writer.write(sampleBuffer: sampleBuffer) {
+            hasReceivedFrame = true
+        }
+    }
+
+    // MARK: - SCStreamDelegate
+
+    func stream(_ stream: SCStream, didStopWithError error: Error) {
+        onFatalError(error)
+    }
+
+    // MARK: - Internals
+
+    private func reconfigure(width: Int, height: Int) {
+        guard let stream = stream else { return }
+        configuredPixelWidth = width
+        configuredPixelHeight = height
+
+        let updated = SCStreamConfiguration()
+        updated.width = width
+        updated.height = height
+        updated.pixelFormat = kCVPixelFormatType_32BGRA
+        updated.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale(fps))
+        updated.showsCursor = false
+        updated.scalesToFit = false
+        updated.capturesAudio = audioEnabled
+        updated.sampleRate = 8_000
+        updated.channelCount = 1
+
+        // Fire-and-forget: if the update fails we keep using the old config and
+        // either resync on the next size change or stop with a delegate error.
+        Task {
+            do {
+                try await stream.updateConfiguration(updated)
+            } catch {
+                FileHandle.standardError.write(
+                    Data("warn: failed to update stream configuration: \(error)\n".utf8)
+                )
+            }
+        }
+    }
+
+    private static func makeConfiguration(window: SCWindow, fps: Int, audio: Bool) -> SCStreamConfiguration {
+        let config = SCStreamConfiguration()
+        // Logical (points) size; the delivered CVPixelBuffer is in native
+        // pixels (2x/3x for Retina), and downstream consumers must use
+        // CVPixelBufferGetWidth/Height — not these values — to allocate sinks.
+        config.width = Int(window.frame.width)
+        config.height = Int(window.frame.height)
+        config.pixelFormat = kCVPixelFormatType_32BGRA
+        config.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale(fps))
+        config.showsCursor = false
+        config.scalesToFit = false
+        config.capturesAudio = audio
+        config.sampleRate = 8_000
+        config.channelCount = 1
+        return config
+    }
+
+    private static func frameStatus(of sampleBuffer: CMSampleBuffer) -> SCFrameStatus? {
+        guard
+            let attachments = CMSampleBufferGetSampleAttachmentsArray(
+                sampleBuffer, createIfNecessary: false
+            ) as? [[SCStreamFrameInfo: Any]],
+            let info = attachments.first,
+            let rawStatus = info[.status] as? Int,
+            let status = SCFrameStatus(rawValue: rawStatus)
+        else {
+            return nil
+        }
+        return status
+    }
+}

package/dist/ios/screen-capture/Sources/ScreenCaptureHelper/SimulatorWindowDiscovery.swift

@@ -0,0 +1,37 @@
+import Foundation
+import ScreenCaptureKit
+import ScreenCaptureCore
+
+enum SimulatorWindowDiscovery {
+    /// Discovers visible iOS Simulator windows via ScreenCaptureKit.
+    /// Requires screen-recording permission; throws if the user has not granted it.
+    static func discover() async throws -> [SimulatorWindowInfo] {
+        let content = try await SCShareableContent.excludingDesktopWindows(
+            false,
+            onScreenWindowsOnly: true
+        )
+        return content.windows
+            .filter { $0.owningApplication?.bundleIdentifier == simulatorBundleIdentifier }
+            .map(toInfo)
+    }
+
+    static func find(windowID: UInt32) async throws -> SCWindow? {
+        let content = try await SCShareableContent.excludingDesktopWindows(
+            false,
+            onScreenWindowsOnly: true
+        )
+        return content.windows.first { $0.windowID == windowID }
+    }
+
+    static func toInfo(_ window: SCWindow) -> SimulatorWindowInfo {
+        SimulatorWindowInfo(
+            windowID: window.windowID,
+            title: window.title,
+            applicationName: window.owningApplication?.applicationName ?? "",
+            bundleIdentifier: window.owningApplication?.bundleIdentifier ?? "",
+            processID: window.owningApplication?.processID ?? 0,
+            width: Int(window.frame.width),
+            height: Int(window.frame.height)
+        )
+    }
+}

package/dist/ios/screen-capture/Sources/ScreenCaptureHelper/main.swift

@@ -0,0 +1,255 @@
+import AppKit
+import AVFoundation
+import Foundation
+import ScreenCaptureKit
+import ScreenCaptureCore
+
+// MARK: - Constants
+
+// ScreenCaptureKit cold-starts just beyond two seconds on hosted macOS runners.
+// Keep the permission hint behind the source's first-frame timeout so startup
+// latency is not misreported as a missing Screen Recording entitlement.
+let simulatorPermissionTimeoutSeconds: TimeInterval = 10.0
+
+// A command-line process has no AppKit application by default. ScreenCaptureKit
+// reaches CoreGraphics when creating an SCStream, and CoreGraphics aborts with
+// CGS_REQUIRE_INIT unless this connection is initialized first.
+_ = NSApplication.shared
+
+// MARK: - Logging
+
+func logError(_ message: String) {
+    FileHandle.standardError.write(Data("\(message)\n".utf8))
+}
+
+// ScreenCaptureKit failures can surface as Objective-C exceptions, which would
+// otherwise terminate this subprocess with only SIGABRT visible to its parent.
+// Emit the exception while stderr is still connected so the daemon artifact
+// identifies the failing capture stage.
+NSSetUncaughtExceptionHandler { exception in
+    logError(
+        "fatal: uncaught Objective-C exception \(exception.name.rawValue): "
+        + (exception.reason ?? "no reason provided")
+    )
+}
+
+func writeJSON<T: Encodable>(_ value: T) {
+    let encoder = JSONEncoder()
+    encoder.outputFormatting = [.prettyPrinted, .sortedKeys]
+    do {
+        let data = try encoder.encode(value)
+        FileHandle.standardOutput.write(data)
+        FileHandle.standardOutput.write(Data("\n".utf8))
+    } catch {
+        logError("error: failed to encode JSON: \(error)")
+        exit(1)
+    }
+}
+
+// MARK: - Argument parsing
+
+let options: CommandLineOptions
+do {
+    options = try CommandLineOptions.parse(CommandLine.arguments)
+} catch let CommandLineOptions.ParseError.missingValue(flag) {
+    logError("error: missing value for \(flag)")
+    logError(CommandLineOptions.helpText)
+    exit(2)
+} catch let CommandLineOptions.ParseError.unknownArgument(arg) {
+    logError("error: unknown argument \(arg)")
+    logError(CommandLineOptions.helpText)
+    exit(2)
+} catch let CommandLineOptions.ParseError.invalidValue(flag, value) {
+    logError("error: invalid value '\(value)' for \(flag)")
+    logError(CommandLineOptions.helpText)
+    exit(2)
+} catch let CommandLineOptions.ParseError.conflictingFlags(message) {
+    logError("error: conflicting flags — \(message)")
+    logError(CommandLineOptions.helpText)
+    exit(2)
+} catch {
+    logError("error: \(error)")
+    exit(2)
+}
+
+// MARK: - Dispatch
+
+switch options.mode {
+case .help:
+    print(CommandLineOptions.helpText)
+    exit(0)
+
+case .listDevices:
+    CMIOSystem.enableScreenCaptureDevices()
+    // Allow a brief moment for the system to register USB devices.
+    Thread.sleep(forTimeInterval: 0.5)
+    let infos = DeviceDiscovery.discover().map(DeviceDiscovery.toInfo)
+    writeJSON(DeviceListResponse(devices: infos))
+    exit(0)
+
+case .listSimulators:
+    switch runBlocking({ try await SimulatorWindowDiscovery.discover() }) {
+    case .success(let windows):
+        writeJSON(SimulatorWindowListResponse(windows: windows))
+        exit(0)
+    case .failure(let error):
+        logError("error: failed to query simulator windows: \(error)")
+        logError("hint: grant Screen Recording permission to your terminal/IDE.")
+        exit(1)
+    }
+
+case .captureSimulator(let windowID, let fps, let audio):
+    if audio {
+        switch runBlocking({ try await SimulatorWindowDiscovery.discover() }) {
+        case .success(let windows):
+            if let error = SimulatorAudioCaptureAvailability.errorMessage(for: windows) {
+                logError("error: \(error)")
+                exit(1)
+            }
+        case .failure(let error):
+            logError("error: failed to query simulator windows: \(error)")
+            exit(1)
+        }
+    }
+
+    let window: SCWindow
+    switch runBlocking({ try await SimulatorWindowDiscovery.find(windowID: windowID) }) {
+    case .success(.some(let resolved)):
+        window = resolved
+    case .success(.none):
+        logError("error: no window with CGWindowID \(windowID)")
+        exit(1)
+    case .failure(let error):
+        logError("error: failed to query simulator windows: \(error)")
+        exit(1)
+    }
+
+    let sink = FileHandleFrameSink(handle: .standardOutput)
+    let writer = FrameWriter(sink: sink)
+    let simSession = SimulatorCaptureSession(writer: writer) { error in
+        logError("error: ScreenCaptureKit stream stopped: \(error)")
+    }
+
+    if case .failure(let error) = runBlocking({
+        try await simSession.start(window: window, fps: fps, audio: audio)
+    }) {
+        logError("error: failed to start simulator capture: \(error)")
+        exit(1)
+    }
+
+    // ScreenCaptureKit silently emits no frames when the host process lacks
+    // Screen Recording permission — there's no error to surface. Emit a hint
+    // on stderr if nothing arrives within the deadline; this leaves stdout
+    // (the frame channel) untouched.
+    DispatchQueue.main.asyncAfter(deadline: .now() + simulatorPermissionTimeoutSeconds) {
+        if !simSession.hasReceivedAnyFrame {
+            logError(
+                "warn: no frames received within \(simulatorPermissionTimeoutSeconds)s. "
+                + "Grant 'Screen Recording' to your terminal/IDE in "
+                + "System Settings → Privacy & Security → Screen Recording."
+            )
+        }
+    }
+
+    installShutdownHandlers {
+        runBlocking { await simSession.stop() }
+        exit(0)
+    }
+    RunLoop.main.run()
+
+case .capture(let deviceID):
+    CMIOSystem.enableScreenCaptureDevices()
+    Thread.sleep(forTimeInterval: 0.5)
+
+    let resolved: AVCaptureDevice?
+    if let id = deviceID {
+        resolved = DeviceDiscovery.find(uniqueID: id)
+        if resolved == nil {
+            logError("error: no device with uniqueID '\(id)'")
+            exit(1)
+        }
+    } else {
+        resolved = DeviceDiscovery.discover().first
+        if resolved == nil {
+            logError("error: no muxed external capture devices found")
+            exit(1)
+        }
+    }
+    guard let device = resolved else { exit(1) }
+
+    let sink = FileHandleFrameSink(handle: .standardOutput)
+    let writer = FrameWriter(sink: sink)
+    let captureSession = DeviceCaptureSession(writer: writer) { error in
+        logError("error: iOS device capture failed: \(error)")
+    }
+
+    do {
+        try captureSession.start(device: device)
+    } catch {
+        logError("error: failed to start capture session: \(error)")
+        exit(1)
+    }
+
+    installShutdownHandlers {
+        captureSession.stop()
+        exit(0)
+    }
+    RunLoop.main.run()
+}
+
+// MARK: - Async/sync bridges
+
+// A reference box so the Task closure mutates a captured class instead of a
+// `var`. Swift 5.9 (Xcode 15) rejects `var` capture in `Task { … }` even when
+// the semaphore enforces a happens-before relationship; class capture is fine.
+private final class Box<T> {
+    var value: T
+    init(_ value: T) { self.value = value }
+}
+
+func runBlocking<T>(_ body: @escaping () async throws -> T) -> Result<T, Error> {
+    let semaphore = DispatchSemaphore(value: 0)
+    let box = Box<Result<T, Error>>(.failure(CancellationError()))
+    Task {
+        do {
+            box.value = .success(try await body())
+        } catch {
+            box.value = .failure(error)
+        }
+        semaphore.signal()
+    }
+    semaphore.wait()
+    return box.value
+}
+
+func runBlocking<T>(_ body: @escaping () async -> T) -> T {
+    let semaphore = DispatchSemaphore(value: 0)
+    let box = Box<T?>(nil)
+    Task {
+        box.value = await body()
+        semaphore.signal()
+    }
+    semaphore.wait()
+    // box.value is set by the async body before semaphore.signal(); wait() blocks until then.
+    return box.value!  // swiftlint:disable:this force_unwrapping
+}
+
+// MARK: - Signal handling
+
+// Retain signal sources beyond `installShutdownHandlers` to keep them alive
+// for the duration of the run loop.
+private var retainedSignalSources: [DispatchSourceSignal] = []
+
+func installShutdownHandlers(_ handler: @escaping () -> Void) {
+    let termSource = DispatchSource.makeSignalSource(signal: SIGTERM, queue: .main)
+    let intSource = DispatchSource.makeSignalSource(signal: SIGINT, queue: .main)
+    // Ignore the default disposition so the DispatchSources are the only
+    // handlers that run.
+    signal(SIGTERM, SIG_IGN)
+    signal(SIGINT, SIG_IGN)
+    termSource.setEventHandler(handler: handler)
+    intSource.setEventHandler(handler: handler)
+    termSource.resume()
+    intSource.resume()
+    retainedSignalSources = [termSource, intSource]
+}

package/dist/ios/screen-capture/Tests/ScreenCaptureCoreTests/AudioPcm16EncoderTests.swift

@@ -0,0 +1,17 @@
+import XCTest
+@testable import ScreenCaptureCore
+
+final class AudioPcm16EncoderTests: XCTestCase {
+    func testEncodesFloat32LittleEndianAsPCM16LE() {
+        let values: [Float] = [-1, 0, 1]
+        let input = values.withUnsafeBytes { Data($0) }
+
+        let output = AudioPcm16Encoder.encodeFloat32LE(input)
+
+        XCTAssertEqual(output, Data([0x01, 0x80, 0x00, 0x00, 0xFF, 0x7F]))
+    }
+
+    func testRejectsTruncatedFloat32Input() {
+        XCTAssertNil(AudioPcm16Encoder.encodeFloat32LE(Data([0, 0, 0])))
+    }
+}