@stim-cli/server 1.14.0 → 1.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1773 -92
- package/dist/protocol-BHSl7bQ_.mjs +2772 -0
- package/dist/protocol.d.mts +2 -496
- package/dist/protocol.mjs +3 -2
- package/dist/protocol.schema.json +5388 -684
- package/dist/scrcpy/LICENSE +204 -0
- package/dist/scrcpy/NOTICE +22 -0
- package/dist/scrcpy/scrcpy-server +0 -0
- package/dist/stats-read.d.mts +1 -0
- package/dist/stats-read.mjs +46 -0
- package/dist/stim-frames/AndroidDeviceStream.swift +284 -0
- package/dist/stim-frames/DevToolsClient.swift +180 -0
- package/dist/stim-frames/DeviceFrameGeometry.swift +41 -0
- package/dist/stim-frames/DeviceFrameView.swift +167 -0
- package/dist/stim-frames/DuoCaptureSource.swift +132 -0
- package/dist/stim-frames/DuoDeviceProfile.swift +37 -0
- package/dist/stim-frames/DuoFrameRenderer.swift +111 -0
- package/dist/stim-frames/DuoHingeMonitor.swift +89 -0
- package/dist/stim-frames/DuoMesh.swift +100 -0
- package/dist/stim-frames/DuoModelView.swift +275 -0
- package/dist/stim-frames/DuoPoseClip.swift +140 -0
- package/dist/stim-frames/DuoScreenHitMesh.swift +200 -0
- package/dist/stim-frames/EmulatorFrameArtwork.swift +113 -0
- package/dist/stim-frames/EmulatorInput.swift +13 -2
- package/dist/stim-frames/EmulatorPosture.swift +2 -2
- package/dist/stim-frames/FrameArtwork.swift +37 -0
- package/dist/stim-frames/H264Decoder.swift +109 -0
- package/dist/stim-frames/InputMessages.swift +27 -6
- package/dist/stim-frames/KeyboardLayoutMap.swift +70 -0
- package/dist/stim-frames/MacosSource.swift +677 -0
- package/dist/stim-frames/OwnedAppWindowReader.swift +87 -0
- package/dist/stim-frames/OwnedAppWindows.swift +108 -0
- package/dist/stim-frames/PhoneSource.swift +315 -0
- package/dist/stim-frames/ProcessStream.swift +155 -0
- package/dist/stim-frames/ScrcpyProtocol.swift +167 -0
- package/dist/stim-frames/SimulatorDevelopmentOptions.swift +81 -0
- package/dist/stim-frames/SimulatorFrameArtwork.swift +118 -0
- package/dist/stim-frames/SimulatorHID.swift +70 -29
- package/dist/stim-frames/SimulatorHingeAngle.swift +56 -0
- package/dist/stim-frames/SimulatorPosture.swift +277 -0
- package/dist/stim-frames/SimulatorRotation.swift +28 -8
- package/dist/stim-frames/Siniulator-LICENSE.txt +21 -0
- package/dist/stim-frames/StoragePaths.swift +42 -0
- package/dist/stim-frames/VideoEncoder.swift +8 -6
- package/dist/stim-frames/WebPage.swift +301 -0
- package/dist/stim-frames/main.swift +651 -22
- package/dist/stim-host/Info.plist +28 -0
- package/dist/stim-host/stim-host.c +153 -0
- package/dist/stim-server-CvqlgkBK.mjs +14887 -0
- package/dist/stim-server.mjs +6 -4421
- package/package.json +3 -3
- package/dist/protocol-IroIg8Td.mjs +0 -865
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import ApplicationServices
|
|
2
|
+
import CoreGraphics
|
|
3
|
+
|
|
4
|
+
/// Reads one process's windows for `OwnedAppWindows`, from CoreGraphics and Accessibility.
|
|
5
|
+
public enum OwnedAppWindowReader {
|
|
6
|
+
public struct Accessible {
|
|
7
|
+
public var windows: [OwnedAppWindows.Accessible]
|
|
8
|
+
public var elements: [AXUIElement]
|
|
9
|
+
/// Whether any window of the app is modal, or does not say.
|
|
10
|
+
public var modal: Bool
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/// The process's on-screen windows, front to back; nil when the inventory or a window's metadata is unavailable.
|
|
14
|
+
public static func screen(pid: pid_t) -> [OwnedAppWindows.Screen]? {
|
|
15
|
+
guard
|
|
16
|
+
let entries = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID)
|
|
17
|
+
as? [[String: Any]]
|
|
18
|
+
else { return nil }
|
|
19
|
+
var screen: [OwnedAppWindows.Screen] = []
|
|
20
|
+
for entry in entries where entry[kCGWindowOwnerPID as String] as? Int == Int(pid) {
|
|
21
|
+
guard let id = entry[kCGWindowNumber as String] as? UInt32, let layer = entry[kCGWindowLayer as String] as? Int,
|
|
22
|
+
let bounds = entry[kCGWindowBounds as String] as? [String: Any],
|
|
23
|
+
let frame = CGRect(dictionaryRepresentation: bounds as CFDictionary)
|
|
24
|
+
else { return nil }
|
|
25
|
+
screen.append(.init(id: id, layer: layer, frame: frame, title: entry[kCGWindowName as String] as? String ?? ""))
|
|
26
|
+
}
|
|
27
|
+
return screen
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/// The process's standard windows, front to back; nil when Accessibility does not list its windows.
|
|
31
|
+
public static func accessible(pid: pid_t) -> Accessible? {
|
|
32
|
+
let application = AXUIElementCreateApplication(pid)
|
|
33
|
+
var value: CFTypeRef?
|
|
34
|
+
guard AXUIElementCopyAttributeValue(application, kAXWindowsAttribute as CFString, &value) == .success,
|
|
35
|
+
let owned = value as? [AXUIElement]
|
|
36
|
+
else { return nil }
|
|
37
|
+
var main: CFTypeRef?
|
|
38
|
+
var focused: CFTypeRef?
|
|
39
|
+
_ = AXUIElementCopyAttributeValue(application, kAXMainWindowAttribute as CFString, &main)
|
|
40
|
+
_ = AXUIElementCopyAttributeValue(application, kAXFocusedWindowAttribute as CFString, &focused)
|
|
41
|
+
var result = Accessible(windows: [], elements: [], modal: false)
|
|
42
|
+
for own in owned {
|
|
43
|
+
var modal: CFTypeRef?
|
|
44
|
+
if AXUIElementCopyAttributeValue(own, kAXModalAttribute as CFString, &modal) != .success || modal as? Bool != false {
|
|
45
|
+
result.modal = true
|
|
46
|
+
}
|
|
47
|
+
var subrole: CFTypeRef?
|
|
48
|
+
guard AXUIElementCopyAttributeValue(own, kAXSubroleAttribute as CFString, &subrole) == .success,
|
|
49
|
+
subrole as? String == kAXStandardWindowSubrole, let frame = frame(own)
|
|
50
|
+
else { continue }
|
|
51
|
+
var title: CFTypeRef?
|
|
52
|
+
_ = AXUIElementCopyAttributeValue(own, kAXTitleAttribute as CFString, &title)
|
|
53
|
+
result.windows.append(
|
|
54
|
+
.init(
|
|
55
|
+
frame: frame, title: title as? String ?? "", main: main.map { CFEqual($0, own) } ?? false,
|
|
56
|
+
focused: focused.map { CFEqual($0, own) } ?? false))
|
|
57
|
+
result.elements.append(own)
|
|
58
|
+
}
|
|
59
|
+
return result
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/// The windows viewing shows and follows: by Accessibility when this process is trusted, else only an app whose one
|
|
63
|
+
/// window contains the others. Throws when an inventory is unavailable; nil when the app shows no window.
|
|
64
|
+
public static func selection(pid: pid_t) throws -> OwnedAppWindows.Selection? {
|
|
65
|
+
guard let screen = screen(pid: pid) else { throw Unavailable() }
|
|
66
|
+
guard AXIsProcessTrusted() else { return OwnedAppWindows.single(screen: screen) }
|
|
67
|
+
guard let accessible = accessible(pid: pid) else { throw Unavailable() }
|
|
68
|
+
return OwnedAppWindows.select(screen: screen, accessible: accessible.windows)
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
public struct Unavailable: Error {}
|
|
72
|
+
|
|
73
|
+
public static func frame(_ element: AXUIElement) -> CGRect? {
|
|
74
|
+
var positionValue: CFTypeRef?
|
|
75
|
+
var sizeValue: CFTypeRef?
|
|
76
|
+
var position = CGPoint.zero
|
|
77
|
+
var size = CGSize.zero
|
|
78
|
+
guard AXUIElementCopyAttributeValue(element, kAXPositionAttribute as CFString, &positionValue) == .success,
|
|
79
|
+
AXUIElementCopyAttributeValue(element, kAXSizeAttribute as CFString, &sizeValue) == .success,
|
|
80
|
+
let positionValue, let sizeValue, CFGetTypeID(positionValue) == AXValueGetTypeID(),
|
|
81
|
+
CFGetTypeID(sizeValue) == AXValueGetTypeID(),
|
|
82
|
+
AXValueGetValue(unsafeDowncast(positionValue, to: AXValue.self), .cgPoint, &position),
|
|
83
|
+
AXValueGetValue(unsafeDowncast(sizeValue, to: AXValue.self), .cgSize, &size)
|
|
84
|
+
else { return nil }
|
|
85
|
+
return CGRect(origin: position, size: size)
|
|
86
|
+
}
|
|
87
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import CoreGraphics
|
|
2
|
+
|
|
3
|
+
/// The windows of an owned macOS app that viewing and Control can show, and the one they follow.
|
|
4
|
+
public enum OwnedAppWindows {
|
|
5
|
+
/// An on-screen window of the app's process, from `CGWindowListCopyWindowInfo`, front to back.
|
|
6
|
+
public struct Screen: Equatable, Sendable {
|
|
7
|
+
public var id: UInt32
|
|
8
|
+
public var layer: Int
|
|
9
|
+
public var frame: CGRect
|
|
10
|
+
public var title: String
|
|
11
|
+
|
|
12
|
+
public init(id: UInt32, layer: Int, frame: CGRect, title: String) {
|
|
13
|
+
self.id = id
|
|
14
|
+
self.layer = layer
|
|
15
|
+
self.frame = frame
|
|
16
|
+
self.title = title
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/// A standard window of the app from Accessibility (`kAXWindowsAttribute`), front to back.
|
|
21
|
+
public struct Accessible: Equatable, Sendable {
|
|
22
|
+
public var frame: CGRect
|
|
23
|
+
public var title: String
|
|
24
|
+
public var main: Bool
|
|
25
|
+
public var focused: Bool
|
|
26
|
+
|
|
27
|
+
public init(frame: CGRect, title: String, main: Bool, focused: Bool) {
|
|
28
|
+
self.frame = frame
|
|
29
|
+
self.title = title
|
|
30
|
+
self.main = main
|
|
31
|
+
self.focused = focused
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/// `accessible` is the index of the window in the Accessibility list it was matched from, or nil without one.
|
|
36
|
+
public struct Window: Equatable, Sendable {
|
|
37
|
+
public var id: UInt32
|
|
38
|
+
public var title: String
|
|
39
|
+
public var frame: CGRect
|
|
40
|
+
public var accessible: Int?
|
|
41
|
+
|
|
42
|
+
public init(id: UInt32, title: String, frame: CGRect, accessible: Int?) {
|
|
43
|
+
self.id = id
|
|
44
|
+
self.title = title
|
|
45
|
+
self.frame = frame
|
|
46
|
+
self.accessible = accessible
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
public struct Selection: Equatable, Sendable {
|
|
51
|
+
public var current: Window
|
|
52
|
+
public var windows: [Window]
|
|
53
|
+
|
|
54
|
+
public init(current: Window, windows: [Window]) {
|
|
55
|
+
self.current = current
|
|
56
|
+
self.windows = windows
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/// Matches each standard window to the first unused on-screen layer-0 window with the same frame and title, so
|
|
61
|
+
/// identical windows pair in their front-to-back order, then pairs the rest by frame alone, as the two inventories
|
|
62
|
+
/// can disagree on a title that just changed. A standard window without an on-screen match (minimized, on another
|
|
63
|
+
/// Space) is left out. The current window is the app's main window, else its focused one, else the frontmost.
|
|
64
|
+
public static func select(screen: [Screen], accessible: [Accessible]) -> Selection? {
|
|
65
|
+
var unused = screen.filter { $0.layer == 0 }
|
|
66
|
+
var matched: [Int: Screen] = [:]
|
|
67
|
+
for sameTitle in [true, false] {
|
|
68
|
+
for (index, window) in accessible.enumerated() where matched[index] == nil {
|
|
69
|
+
guard
|
|
70
|
+
let match = unused.firstIndex(where: { $0.frame == window.frame && (!sameTitle || $0.title == window.title) })
|
|
71
|
+
else { continue }
|
|
72
|
+
matched[index] = unused.remove(at: match)
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
let windows = accessible.indices.compactMap { index in
|
|
76
|
+
matched[index].map { Window(id: $0.id, title: accessible[index].title, frame: $0.frame, accessible: index) }
|
|
77
|
+
}
|
|
78
|
+
let current =
|
|
79
|
+
windows.first { accessible[$0.accessible!].main } ?? windows.first { accessible[$0.accessible!].focused }
|
|
80
|
+
?? windows.first
|
|
81
|
+
return current.map { Selection(current: $0, windows: windows) }
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/// The selection with `pinned` as its current window while that window is still on screen; nil `pinned` in the
|
|
85
|
+
/// result means the pin is gone and capture follows the front window again. A pinned window that the two
|
|
86
|
+
/// inventories briefly disagree on stays pinned from its on-screen entry, without an Accessibility match.
|
|
87
|
+
public static func pin(_ selection: Selection, to pinned: UInt32?, screen: [Screen]) -> (
|
|
88
|
+
selection: Selection, pinned: UInt32?
|
|
89
|
+
) {
|
|
90
|
+
guard let pinned else { return (selection, nil) }
|
|
91
|
+
if let window = selection.windows.first(where: { $0.id == pinned }) {
|
|
92
|
+
return (Selection(current: window, windows: selection.windows), pinned)
|
|
93
|
+
}
|
|
94
|
+
guard let shown = screen.first(where: { $0.id == pinned && $0.layer == 0 }) else { return (selection, nil) }
|
|
95
|
+
let window = Window(id: shown.id, title: shown.title, frame: shown.frame, accessible: nil)
|
|
96
|
+
return (Selection(current: window, windows: selection.windows + [window]), pinned)
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/// Without Accessibility, the one layer-0 window that contains every other one, such as a main window with its
|
|
100
|
+
/// sheets; nil when the app shows disjoint windows.
|
|
101
|
+
public static func single(screen: [Screen]) -> Selection? {
|
|
102
|
+
let windows = screen.filter { $0.layer == 0 }
|
|
103
|
+
let containers = windows.filter { candidate in windows.allSatisfy { candidate.frame.contains($0.frame) } }
|
|
104
|
+
guard containers.count == 1, let only = containers.first else { return nil }
|
|
105
|
+
let window = Window(id: only.id, title: only.title, frame: only.frame, accessible: nil)
|
|
106
|
+
return Selection(current: window, windows: [window])
|
|
107
|
+
}
|
|
108
|
+
}
|
|
@@ -0,0 +1,315 @@
|
|
|
1
|
+
import AVFoundation
|
|
2
|
+
import CoreImage
|
|
3
|
+
import CoreMediaIO
|
|
4
|
+
import Foundation
|
|
5
|
+
import IOKit
|
|
6
|
+
|
|
7
|
+
/// The screen of a USB-cabled iPhone, view only. macOS lists a cabled iPhone as an external muxed capture device,
|
|
8
|
+
/// the one QuickTime's New Movie Recording shows, only after a process sets
|
|
9
|
+
/// `kCMIOHardwarePropertyAllowScreenCaptureDevices`, and the device appears some seconds later. Capture runs only
|
|
10
|
+
/// while a subscriber asks for frames.
|
|
11
|
+
///
|
|
12
|
+
/// The capture device's unique ID is a random UUID (macOS 27), and none of its CoreMediaIO properties names the
|
|
13
|
+
/// UDID. The USB device does: its serial number is the UDID without dashes. So the UDID must be cabled, and the
|
|
14
|
+
/// capture device is the iOS one named `name`, or the only one when there is no name and one iPhone is cabled.
|
|
15
|
+
/// Screens appear one by one, so with several iPhones cabled it waits for all of them; any other case refuses
|
|
16
|
+
/// rather than guess.
|
|
17
|
+
final class PhoneSource: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
|
|
18
|
+
static let searchSeconds = 15.0
|
|
19
|
+
static let retrySeconds = 2.0
|
|
20
|
+
static let starvedSeconds = 2.0
|
|
21
|
+
|
|
22
|
+
let udid: String
|
|
23
|
+
let name: String?
|
|
24
|
+
private var serial: String { udid.replacingOccurrences(of: "-", with: "").uppercased() }
|
|
25
|
+
let queue = DispatchQueue(label: "stim.frames.phone")
|
|
26
|
+
private let captureQueue = DispatchQueue(label: "stim.frames.phone-capture")
|
|
27
|
+
private var pacer: Pacer!
|
|
28
|
+
private let video = videoEncoder()
|
|
29
|
+
private let jpegGate = JpegGate()
|
|
30
|
+
private var config = Config(fps: 0)
|
|
31
|
+
private var device: AVCaptureDevice?
|
|
32
|
+
private var session: AVCaptureSession?
|
|
33
|
+
private var authorized = false
|
|
34
|
+
private var latest: CVPixelBuffer?
|
|
35
|
+
private var frameAt = Date.distantPast
|
|
36
|
+
private var watchdog: DispatchSourceTimer?
|
|
37
|
+
private var retrying = false
|
|
38
|
+
private var failure: String?
|
|
39
|
+
private var reportedStall: String??
|
|
40
|
+
private var observers: [NSObjectProtocol] = []
|
|
41
|
+
private var watches: [NSKeyValueObservation] = []
|
|
42
|
+
|
|
43
|
+
init(udid: String, name: String?) {
|
|
44
|
+
self.udid = udid
|
|
45
|
+
self.name = name
|
|
46
|
+
super.init()
|
|
47
|
+
pacer = Pacer { [unowned self] config in
|
|
48
|
+
guard let pixels = self.queue.sync(execute: { self.latest }) else { return }
|
|
49
|
+
self.render(pixels, config: config)
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/// The serial numbers of the iOS devices on USB, which are their UDIDs without dashes. The registry is walked
|
|
54
|
+
/// instead of matched, because macOS leaves a cabled iPhone's USB device unregistered.
|
|
55
|
+
static func cabledSerials() -> Set<String> {
|
|
56
|
+
var iterator: io_iterator_t = 0
|
|
57
|
+
guard
|
|
58
|
+
IORegistryEntryCreateIterator(
|
|
59
|
+
IORegistryGetRootEntry(kIOMainPortDefault), "IOUSB", IOOptionBits(kIORegistryIterateRecursively), &iterator)
|
|
60
|
+
== KERN_SUCCESS
|
|
61
|
+
else { return [] }
|
|
62
|
+
defer { IOObjectRelease(iterator) }
|
|
63
|
+
var serials: Set<String> = []
|
|
64
|
+
while case let entry = IOIteratorNext(iterator), entry != 0 {
|
|
65
|
+
defer { IOObjectRelease(entry) }
|
|
66
|
+
let property = { (key: String) in
|
|
67
|
+
IORegistryEntryCreateCFProperty(entry, key as CFString, kCFAllocatorDefault, 0)?.takeRetainedValue()
|
|
68
|
+
}
|
|
69
|
+
guard property("UsbAppleDeviceECID") != nil, let serial = property("USB Serial Number") as? String else { continue }
|
|
70
|
+
serials.insert(serial.uppercased())
|
|
71
|
+
}
|
|
72
|
+
return serials
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
func start() {
|
|
76
|
+
var address = CMIOObjectPropertyAddress(
|
|
77
|
+
mSelector: CMIOObjectPropertySelector(kCMIOHardwarePropertyAllowScreenCaptureDevices),
|
|
78
|
+
mScope: CMIOObjectPropertyScope(kCMIOObjectPropertyScopeGlobal),
|
|
79
|
+
mElement: CMIOObjectPropertyElement(kCMIOObjectPropertyElementMain))
|
|
80
|
+
var allow: UInt32 = 1
|
|
81
|
+
let status = CMIOObjectSetPropertyData(
|
|
82
|
+
CMIOObjectID(kCMIOObjectSystemObject), &address, 0, nil, UInt32(MemoryLayout<UInt32>.size), &allow)
|
|
83
|
+
guard status == 0 else { fail("macOS refused to list iPhone screens as capture devices (CoreMediaIO \(status)).") }
|
|
84
|
+
let center = NotificationCenter.default
|
|
85
|
+
observers.append(
|
|
86
|
+
center.addObserver(forName: AVCaptureDevice.wasConnectedNotification, object: nil, queue: nil) { [weak self] _ in
|
|
87
|
+
self?.queue.async { self?.find(until: nil) }
|
|
88
|
+
})
|
|
89
|
+
observers.append(
|
|
90
|
+
center.addObserver(forName: AVCaptureSession.runtimeErrorNotification, object: nil, queue: nil) {
|
|
91
|
+
[weak self] note in
|
|
92
|
+
let error = note.userInfo?[AVCaptureSessionErrorKey] as? Error
|
|
93
|
+
self?.queue.async {
|
|
94
|
+
guard let self, let session = self.session, session === note.object as? AVCaptureSession else { return }
|
|
95
|
+
self.failure = "Capture of the iPhone \(self.udid) stopped (\(error?.localizedDescription ?? "unknown error"))."
|
|
96
|
+
self.close()
|
|
97
|
+
self.retry()
|
|
98
|
+
}
|
|
99
|
+
})
|
|
100
|
+
observers.append(
|
|
101
|
+
center.addObserver(forName: AVCaptureDevice.wasDisconnectedNotification, object: nil, queue: nil) {
|
|
102
|
+
[weak self] note in
|
|
103
|
+
guard let gone = note.object as? AVCaptureDevice else { return }
|
|
104
|
+
self?.queue.async {
|
|
105
|
+
guard let self, let device = self.device, device.uniqueID == gone.uniqueID else { return }
|
|
106
|
+
fail("The iPhone \(self.udid) was disconnected from this Mac.")
|
|
107
|
+
}
|
|
108
|
+
})
|
|
109
|
+
find(until: Date().addingTimeInterval(Self.searchSeconds))
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
private func find(until deadline: Date?) {
|
|
113
|
+
guard device == nil else { return }
|
|
114
|
+
let cabled = Self.cabledSerials()
|
|
115
|
+
guard cabled.contains(serial) else {
|
|
116
|
+
fail(
|
|
117
|
+
"The iPhone \(udid) is not cabled to this Mac. Stim shows a physical iPhone's screen only over a USB cable.")
|
|
118
|
+
}
|
|
119
|
+
let screens = AVCaptureDevice.DiscoverySession(deviceTypes: [.external], mediaType: .muxed, position: .unspecified)
|
|
120
|
+
.devices.filter { $0.modelID == "iOS Device" }
|
|
121
|
+
if cabled.count > 1 && name == nil {
|
|
122
|
+
fail("Several iPhones are cabled to this Mac, and Stim has no name to tell \(udid) apart from them.")
|
|
123
|
+
}
|
|
124
|
+
let named = name.map { name in screens.filter { $0.localizedName == name } } ?? screens
|
|
125
|
+
let listed = screens.count >= cabled.count
|
|
126
|
+
if listed && named.count > 1 {
|
|
127
|
+
fail(
|
|
128
|
+
"Several cabled iPhones are named \(name ?? "alike"), so Stim cannot tell which one is \(udid). Rename one in "
|
|
129
|
+
+ "Settings > General > About > Name.")
|
|
130
|
+
}
|
|
131
|
+
guard listed, named.count == 1, let found = named.first else {
|
|
132
|
+
guard let deadline else { return }
|
|
133
|
+
if Date() > deadline {
|
|
134
|
+
if screens.isEmpty {
|
|
135
|
+
fail("macOS shows no screen for the iPhone \(udid). Unlock it, and tap Trust if it asks to trust this Mac.")
|
|
136
|
+
}
|
|
137
|
+
if !listed {
|
|
138
|
+
fail(
|
|
139
|
+
"Not every cabled iPhone shows its screen yet, so Stim cannot tell \(udid) apart from them. Unlock each "
|
|
140
|
+
+ "one, and tap Trust if it asks to trust this Mac.")
|
|
141
|
+
}
|
|
142
|
+
fail(
|
|
143
|
+
"No cabled iPhone's screen is named \(name ?? "") as the lease of \(udid) records. If it was renamed, run "
|
|
144
|
+
+ "stim device lock ios \(udid) again.")
|
|
145
|
+
}
|
|
146
|
+
queue.asyncAfter(deadline: .now() + 1) { self.find(until: deadline) }
|
|
147
|
+
return
|
|
148
|
+
}
|
|
149
|
+
device = found
|
|
150
|
+
for key in [\AVCaptureDevice.isSuspended, \AVCaptureDevice.isInUseByAnotherApplication] {
|
|
151
|
+
watches.append(found.observe(key) { [weak self] _, _ in self?.queue.async { self?.reportStall() } })
|
|
152
|
+
}
|
|
153
|
+
authorize()
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/// The first capture asks the user for Camera access, which macOS attributes to the app that started stim-server.
|
|
157
|
+
private func authorize() {
|
|
158
|
+
switch AVCaptureDevice.authorizationStatus(for: .video) {
|
|
159
|
+
case .authorized:
|
|
160
|
+
authorized = true
|
|
161
|
+
failure = nil
|
|
162
|
+
reportStall()
|
|
163
|
+
apply()
|
|
164
|
+
case .notDetermined:
|
|
165
|
+
failure = "macOS is asking on the Mac whether the app that runs stim-server may use the camera."
|
|
166
|
+
reportStall()
|
|
167
|
+
AVCaptureDevice.requestAccess(for: .video) { granted in
|
|
168
|
+
self.queue.async {
|
|
169
|
+
if !granted { self.denied() }
|
|
170
|
+
self.authorize()
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
default:
|
|
174
|
+
denied()
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/// macOS denies without a prompt when the responsible app has no camera usage description, or runs with the
|
|
179
|
+
/// hardened runtime and lacks the camera entitlement, such as a bare `node`.
|
|
180
|
+
private func denied() -> Never {
|
|
181
|
+
fail(
|
|
182
|
+
"macOS denied Camera access, which capturing an iPhone screen needs. Run stim-server from Stim, and allow Stim "
|
|
183
|
+
+ "in System Settings > Privacy & Security > Camera.")
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
func configure(_ config: Config) {
|
|
187
|
+
video.configure(enabled: config.video, maxEdge: config.maxEdge, fps: Int(config.fps), bitrate: config.bitrate)
|
|
188
|
+
queue.async {
|
|
189
|
+
self.config = config
|
|
190
|
+
self.pacer.queue.async {
|
|
191
|
+
self.pacer.config = config
|
|
192
|
+
self.pacer.changed()
|
|
193
|
+
}
|
|
194
|
+
self.apply()
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
func keyframe() {
|
|
199
|
+
video.requestKeyframe()
|
|
200
|
+
pacer.changed()
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
private func apply() {
|
|
204
|
+
let wanted = config.fps > 0 && (config.jpeg || config.video)
|
|
205
|
+
if wanted, session == nil, authorized, let device { open(device) }
|
|
206
|
+
if !wanted, session != nil {
|
|
207
|
+
close()
|
|
208
|
+
latest = nil
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
private func close() {
|
|
213
|
+
guard let session else { return }
|
|
214
|
+
self.session = nil
|
|
215
|
+
watchdog?.cancel()
|
|
216
|
+
watchdog = nil
|
|
217
|
+
captureQueue.async { session.stopRunning() }
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
private func open(_ device: AVCaptureDevice) {
|
|
221
|
+
let session = AVCaptureSession()
|
|
222
|
+
do {
|
|
223
|
+
let input = try AVCaptureDeviceInput(device: device)
|
|
224
|
+
for port in input.ports where port.mediaType != .video { port.isEnabled = false }
|
|
225
|
+
guard session.canAddInput(input) else { throw CaptureError.refused }
|
|
226
|
+
session.addInput(input)
|
|
227
|
+
} catch {
|
|
228
|
+
failure = "The iPhone \(udid) could not be opened for capture (\(error.localizedDescription))."
|
|
229
|
+
return retry()
|
|
230
|
+
}
|
|
231
|
+
let output = AVCaptureVideoDataOutput()
|
|
232
|
+
output.alwaysDiscardsLateVideoFrames = true
|
|
233
|
+
output.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA]
|
|
234
|
+
output.setSampleBufferDelegate(self, queue: captureQueue)
|
|
235
|
+
guard session.canAddOutput(output) else {
|
|
236
|
+
failure = "The iPhone \(udid) offers no video output macOS can read."
|
|
237
|
+
return retry()
|
|
238
|
+
}
|
|
239
|
+
session.addOutput(output)
|
|
240
|
+
self.session = session
|
|
241
|
+
failure = nil
|
|
242
|
+
frameAt = Date()
|
|
243
|
+
reportStall()
|
|
244
|
+
let watchdog = DispatchSource.makeTimerSource(queue: queue)
|
|
245
|
+
watchdog.schedule(deadline: .now() + 1, repeating: 1)
|
|
246
|
+
watchdog.setEventHandler { [weak self] in
|
|
247
|
+
guard let self else { return }
|
|
248
|
+
if !Self.cabledSerials().contains(self.serial) { fail("The iPhone \(self.udid) was disconnected from this Mac.") }
|
|
249
|
+
self.reportStall()
|
|
250
|
+
}
|
|
251
|
+
watchdog.resume()
|
|
252
|
+
self.watchdog = watchdog
|
|
253
|
+
captureQueue.async { session.startRunning() }
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
private func retry() {
|
|
257
|
+
if !Self.cabledSerials().contains(serial) { fail("The iPhone \(udid) was disconnected from this Mac.") }
|
|
258
|
+
reportStall()
|
|
259
|
+
guard !retrying else { return }
|
|
260
|
+
retrying = true
|
|
261
|
+
queue.asyncAfter(deadline: .now() + Self.retrySeconds) {
|
|
262
|
+
self.retrying = false
|
|
263
|
+
self.apply()
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/// A failed open, or frames that stopped, are stalls: the last frame stays on screen with the reason, and frames
|
|
268
|
+
/// resume when the reason clears. A cabled iPhone sends frames continuously, so a gap names its cause when macOS
|
|
269
|
+
/// reports one. Another app capturing the iPhone alone is no stall, because several processes can capture it.
|
|
270
|
+
private func reportStall() {
|
|
271
|
+
let starved = session != nil && Date().timeIntervalSince(frameAt) > Self.starvedSeconds
|
|
272
|
+
let stall: String?
|
|
273
|
+
if starved, let device, device.isSuspended {
|
|
274
|
+
stall = "The iPhone is locked. Unlock it to see its screen."
|
|
275
|
+
} else if starved, let device, device.isInUseByAnotherApplication {
|
|
276
|
+
stall = "Another app, such as QuickTime Player, is capturing this iPhone."
|
|
277
|
+
} else if starved {
|
|
278
|
+
stall = "The iPhone stopped sending its screen."
|
|
279
|
+
} else {
|
|
280
|
+
stall = failure
|
|
281
|
+
}
|
|
282
|
+
guard reportedStall != .some(stall) else { return }
|
|
283
|
+
reportedStall = .some(stall)
|
|
284
|
+
Output.notice(["stalled": stall.map { $0 as Any } ?? NSNull()])
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
func captureOutput(_ output: AVCaptureOutput, didOutput sample: CMSampleBuffer, from connection: AVCaptureConnection) {
|
|
288
|
+
guard let pixels = CMSampleBufferGetImageBuffer(sample) else { return }
|
|
289
|
+
queue.async {
|
|
290
|
+
self.latest = pixels
|
|
291
|
+
self.frameAt = Date()
|
|
292
|
+
if self.reportedStall != .some(self.failure) { self.reportStall() }
|
|
293
|
+
}
|
|
294
|
+
pacer.changed()
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
private func render(_ pixels: CVPixelBuffer, config: Config) {
|
|
298
|
+
if config.video { video.encode(pixels, quarterTurns: 0, capturedAt: now()) }
|
|
299
|
+
guard config.jpeg, jpegGate.admit(config, pacer: pacer),
|
|
300
|
+
let (data, width, height) = jpeg(CIImage(cvPixelBuffer: pixels), config: config)
|
|
301
|
+
else { return }
|
|
302
|
+
Output.frame(jpeg: data, width: width, height: height)
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
private enum CaptureError: Error { case refused }
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
/// stim-server records only the simulators and emulators Stim owns, so a physical iPhone runs no recording encoder.
|
|
309
|
+
extension PhoneSource: Source {
|
|
310
|
+
func recordKeyframe() {}
|
|
311
|
+
|
|
312
|
+
func input(_ command: Command) {
|
|
313
|
+
Output.notice(["inputError": "Stim shows a physical iPhone's screen but does not send it input."])
|
|
314
|
+
}
|
|
315
|
+
}
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
|
|
3
|
+
public enum OutputChannel: Sendable {
|
|
4
|
+
case stdout
|
|
5
|
+
case stderr
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
public struct OutputLine: Hashable, Sendable {
|
|
9
|
+
public var channel: OutputChannel
|
|
10
|
+
public var text: String
|
|
11
|
+
|
|
12
|
+
public init(_ channel: OutputChannel, _ text: String) {
|
|
13
|
+
self.channel = channel
|
|
14
|
+
self.text = text
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
extension String {
|
|
19
|
+
/// The text without ANSI CSI sequences (colors, cursor moves) and OSC sequences (titles, links).
|
|
20
|
+
public var strippingANSI: String {
|
|
21
|
+
replacingOccurrences(
|
|
22
|
+
of: "\u{1B}\\[[0-?]*[ -/]*[@-~]|\u{1B}\\][^\u{07}\u{1B}]*(\u{07}|\u{1B}\\\\)", with: "",
|
|
23
|
+
options: .regularExpression)
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/// Splits a byte stream into lines. A chunk can end inside a line or inside a
|
|
28
|
+
/// UTF-8 sequence, so bytes after the last newline wait for the next chunk.
|
|
29
|
+
public struct LineBuffer: Sendable {
|
|
30
|
+
private var pending = Data()
|
|
31
|
+
|
|
32
|
+
public init() {}
|
|
33
|
+
|
|
34
|
+
public mutating func append(_ data: Data) -> [String] {
|
|
35
|
+
pending.append(data)
|
|
36
|
+
guard let last = pending.lastIndex(of: UInt8(ascii: "\n")) else { return [] }
|
|
37
|
+
let complete = pending[pending.startIndex..<last]
|
|
38
|
+
pending = Data(pending[(last + 1)...])
|
|
39
|
+
return complete.split(separator: UInt8(ascii: "\n"), omittingEmptySubsequences: false).map(Self.decode)
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
public mutating func finish() -> [String] {
|
|
43
|
+
defer { pending = Data() }
|
|
44
|
+
return pending.isEmpty ? [] : [Self.decode(pending)]
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
private static func decode<Bytes: Collection<UInt8>>(_ bytes: Bytes) -> String {
|
|
48
|
+
var text = String(decoding: bytes, as: UTF8.self)
|
|
49
|
+
if text.hasSuffix("\r") { text.removeLast() }
|
|
50
|
+
return text
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/// Runs an executable with an argument list, never through a shell, and
|
|
55
|
+
/// reports its output line by line and then its exit status.
|
|
56
|
+
public enum ProcessStream {
|
|
57
|
+
/// A child the command leaves running can hold the pipes open, so the exit
|
|
58
|
+
/// is reported this long after termination even without end of file.
|
|
59
|
+
static let drainGrace: TimeInterval = 2
|
|
60
|
+
|
|
61
|
+
@discardableResult
|
|
62
|
+
public static func start(
|
|
63
|
+
executable: String,
|
|
64
|
+
arguments: [String],
|
|
65
|
+
cwd: String,
|
|
66
|
+
environment: [String: String]? = nil,
|
|
67
|
+
onLine: @escaping @Sendable (OutputLine) -> Void,
|
|
68
|
+
onExit: @escaping @Sendable (Int32) -> Void
|
|
69
|
+
) throws -> Process {
|
|
70
|
+
let process = Process()
|
|
71
|
+
process.executableURL = URL(fileURLWithPath: executable)
|
|
72
|
+
process.arguments = arguments
|
|
73
|
+
process.currentDirectoryURL = URL(fileURLWithPath: cwd)
|
|
74
|
+
if let environment { process.environment = environment }
|
|
75
|
+
process.standardInput = FileHandle.nullDevice
|
|
76
|
+
let out = Pipe()
|
|
77
|
+
let err = Pipe()
|
|
78
|
+
process.standardOutput = out
|
|
79
|
+
process.standardError = err
|
|
80
|
+
|
|
81
|
+
let state = StreamState(onLine: onLine, onExit: onExit)
|
|
82
|
+
for (pipe, channel) in [(out, OutputChannel.stdout), (err, .stderr)] {
|
|
83
|
+
pipe.fileHandleForReading.readabilityHandler = { handle in
|
|
84
|
+
let data = handle.availableData
|
|
85
|
+
if data.isEmpty {
|
|
86
|
+
handle.readabilityHandler = nil
|
|
87
|
+
state.close(channel)
|
|
88
|
+
} else {
|
|
89
|
+
state.receive(data, on: channel)
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
process.terminationHandler = { process in
|
|
94
|
+
let status = process.terminationStatus
|
|
95
|
+
state.terminated(status)
|
|
96
|
+
DispatchQueue.global().asyncAfter(deadline: .now() + drainGrace) {
|
|
97
|
+
out.fileHandleForReading.readabilityHandler = nil
|
|
98
|
+
err.fileHandleForReading.readabilityHandler = nil
|
|
99
|
+
state.close(.stdout)
|
|
100
|
+
state.close(.stderr)
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
do {
|
|
104
|
+
try process.run()
|
|
105
|
+
} catch {
|
|
106
|
+
out.fileHandleForReading.readabilityHandler = nil
|
|
107
|
+
err.fileHandleForReading.readabilityHandler = nil
|
|
108
|
+
throw error
|
|
109
|
+
}
|
|
110
|
+
return process
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
private final class StreamState: @unchecked Sendable {
|
|
115
|
+
private let lock = NSLock()
|
|
116
|
+
private var buffers: [OutputChannel: LineBuffer] = [.stdout: LineBuffer(), .stderr: LineBuffer()]
|
|
117
|
+
private var open: Set<OutputChannel> = [.stdout, .stderr]
|
|
118
|
+
private var status: Int32?
|
|
119
|
+
private var reported = false
|
|
120
|
+
private let onLine: @Sendable (OutputLine) -> Void
|
|
121
|
+
private let onExit: @Sendable (Int32) -> Void
|
|
122
|
+
|
|
123
|
+
init(onLine: @escaping @Sendable (OutputLine) -> Void, onExit: @escaping @Sendable (Int32) -> Void) {
|
|
124
|
+
self.onLine = onLine
|
|
125
|
+
self.onExit = onExit
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
func receive(_ data: Data, on channel: OutputChannel) {
|
|
129
|
+
lock.lock()
|
|
130
|
+
defer { lock.unlock() }
|
|
131
|
+
guard open.contains(channel) else { return }
|
|
132
|
+
for line in buffers[channel]!.append(data) { onLine(OutputLine(channel, line)) }
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
func close(_ channel: OutputChannel) {
|
|
136
|
+
lock.lock()
|
|
137
|
+
defer { lock.unlock() }
|
|
138
|
+
guard open.remove(channel) != nil else { return }
|
|
139
|
+
for line in buffers[channel]!.finish() { onLine(OutputLine(channel, line)) }
|
|
140
|
+
reportIfDone()
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
func terminated(_ status: Int32) {
|
|
144
|
+
lock.lock()
|
|
145
|
+
defer { lock.unlock() }
|
|
146
|
+
self.status = status
|
|
147
|
+
reportIfDone()
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
private func reportIfDone() {
|
|
151
|
+
guard !reported, open.isEmpty, let status else { return }
|
|
152
|
+
reported = true
|
|
153
|
+
onExit(status)
|
|
154
|
+
}
|
|
155
|
+
}
|