@stim-cli/server 1.14.0 → 1.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +1773 -92
  2. package/dist/protocol-BHSl7bQ_.mjs +2772 -0
  3. package/dist/protocol.d.mts +2 -496
  4. package/dist/protocol.mjs +3 -2
  5. package/dist/protocol.schema.json +5388 -684
  6. package/dist/scrcpy/LICENSE +204 -0
  7. package/dist/scrcpy/NOTICE +22 -0
  8. package/dist/scrcpy/scrcpy-server +0 -0
  9. package/dist/stats-read.d.mts +1 -0
  10. package/dist/stats-read.mjs +46 -0
  11. package/dist/stim-frames/AndroidDeviceStream.swift +284 -0
  12. package/dist/stim-frames/DevToolsClient.swift +180 -0
  13. package/dist/stim-frames/DeviceFrameGeometry.swift +41 -0
  14. package/dist/stim-frames/DeviceFrameView.swift +167 -0
  15. package/dist/stim-frames/DuoCaptureSource.swift +132 -0
  16. package/dist/stim-frames/DuoDeviceProfile.swift +37 -0
  17. package/dist/stim-frames/DuoFrameRenderer.swift +111 -0
  18. package/dist/stim-frames/DuoHingeMonitor.swift +89 -0
  19. package/dist/stim-frames/DuoMesh.swift +100 -0
  20. package/dist/stim-frames/DuoModelView.swift +275 -0
  21. package/dist/stim-frames/DuoPoseClip.swift +140 -0
  22. package/dist/stim-frames/DuoScreenHitMesh.swift +200 -0
  23. package/dist/stim-frames/EmulatorFrameArtwork.swift +113 -0
  24. package/dist/stim-frames/EmulatorInput.swift +13 -2
  25. package/dist/stim-frames/EmulatorPosture.swift +2 -2
  26. package/dist/stim-frames/FrameArtwork.swift +37 -0
  27. package/dist/stim-frames/H264Decoder.swift +109 -0
  28. package/dist/stim-frames/InputMessages.swift +27 -6
  29. package/dist/stim-frames/KeyboardLayoutMap.swift +70 -0
  30. package/dist/stim-frames/MacosSource.swift +677 -0
  31. package/dist/stim-frames/OwnedAppWindowReader.swift +87 -0
  32. package/dist/stim-frames/OwnedAppWindows.swift +108 -0
  33. package/dist/stim-frames/PhoneSource.swift +315 -0
  34. package/dist/stim-frames/ProcessStream.swift +155 -0
  35. package/dist/stim-frames/ScrcpyProtocol.swift +167 -0
  36. package/dist/stim-frames/SimulatorDevelopmentOptions.swift +81 -0
  37. package/dist/stim-frames/SimulatorFrameArtwork.swift +118 -0
  38. package/dist/stim-frames/SimulatorHID.swift +70 -29
  39. package/dist/stim-frames/SimulatorHingeAngle.swift +56 -0
  40. package/dist/stim-frames/SimulatorPosture.swift +277 -0
  41. package/dist/stim-frames/SimulatorRotation.swift +28 -8
  42. package/dist/stim-frames/Siniulator-LICENSE.txt +21 -0
  43. package/dist/stim-frames/StoragePaths.swift +42 -0
  44. package/dist/stim-frames/VideoEncoder.swift +8 -6
  45. package/dist/stim-frames/WebPage.swift +301 -0
  46. package/dist/stim-frames/main.swift +651 -22
  47. package/dist/stim-host/Info.plist +28 -0
  48. package/dist/stim-host/stim-host.c +153 -0
  49. package/dist/stim-server-CvqlgkBK.mjs +14887 -0
  50. package/dist/stim-server.mjs +6 -4421
  51. package/package.json +3 -3
  52. package/dist/protocol-IroIg8Td.mjs +0 -865
@@ -0,0 +1,87 @@
1
+ import ApplicationServices
2
+ import CoreGraphics
3
+
4
+ /// Reads one process's windows for `OwnedAppWindows`, from CoreGraphics and Accessibility.
5
+ public enum OwnedAppWindowReader {
6
+ public struct Accessible {
7
+ public var windows: [OwnedAppWindows.Accessible]
8
+ public var elements: [AXUIElement]
9
+ /// Whether any window of the app is modal, or does not say.
10
+ public var modal: Bool
11
+ }
12
+
13
+ /// The process's on-screen windows, front to back; nil when the inventory or a window's metadata is unavailable.
14
+ public static func screen(pid: pid_t) -> [OwnedAppWindows.Screen]? {
15
+ guard
16
+ let entries = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID)
17
+ as? [[String: Any]]
18
+ else { return nil }
19
+ var screen: [OwnedAppWindows.Screen] = []
20
+ for entry in entries where entry[kCGWindowOwnerPID as String] as? Int == Int(pid) {
21
+ guard let id = entry[kCGWindowNumber as String] as? UInt32, let layer = entry[kCGWindowLayer as String] as? Int,
22
+ let bounds = entry[kCGWindowBounds as String] as? [String: Any],
23
+ let frame = CGRect(dictionaryRepresentation: bounds as CFDictionary)
24
+ else { return nil }
25
+ screen.append(.init(id: id, layer: layer, frame: frame, title: entry[kCGWindowName as String] as? String ?? ""))
26
+ }
27
+ return screen
28
+ }
29
+
30
+ /// The process's standard windows, front to back; nil when Accessibility does not list its windows.
31
+ public static func accessible(pid: pid_t) -> Accessible? {
32
+ let application = AXUIElementCreateApplication(pid)
33
+ var value: CFTypeRef?
34
+ guard AXUIElementCopyAttributeValue(application, kAXWindowsAttribute as CFString, &value) == .success,
35
+ let owned = value as? [AXUIElement]
36
+ else { return nil }
37
+ var main: CFTypeRef?
38
+ var focused: CFTypeRef?
39
+ _ = AXUIElementCopyAttributeValue(application, kAXMainWindowAttribute as CFString, &main)
40
+ _ = AXUIElementCopyAttributeValue(application, kAXFocusedWindowAttribute as CFString, &focused)
41
+ var result = Accessible(windows: [], elements: [], modal: false)
42
+ for own in owned {
43
+ var modal: CFTypeRef?
44
+ if AXUIElementCopyAttributeValue(own, kAXModalAttribute as CFString, &modal) != .success || modal as? Bool != false {
45
+ result.modal = true
46
+ }
47
+ var subrole: CFTypeRef?
48
+ guard AXUIElementCopyAttributeValue(own, kAXSubroleAttribute as CFString, &subrole) == .success,
49
+ subrole as? String == kAXStandardWindowSubrole, let frame = frame(own)
50
+ else { continue }
51
+ var title: CFTypeRef?
52
+ _ = AXUIElementCopyAttributeValue(own, kAXTitleAttribute as CFString, &title)
53
+ result.windows.append(
54
+ .init(
55
+ frame: frame, title: title as? String ?? "", main: main.map { CFEqual($0, own) } ?? false,
56
+ focused: focused.map { CFEqual($0, own) } ?? false))
57
+ result.elements.append(own)
58
+ }
59
+ return result
60
+ }
61
+
62
+ /// The windows viewing shows and follows: by Accessibility when this process is trusted, else only an app whose one
63
+ /// window contains the others. Throws when an inventory is unavailable; nil when the app shows no window.
64
+ public static func selection(pid: pid_t) throws -> OwnedAppWindows.Selection? {
65
+ guard let screen = screen(pid: pid) else { throw Unavailable() }
66
+ guard AXIsProcessTrusted() else { return OwnedAppWindows.single(screen: screen) }
67
+ guard let accessible = accessible(pid: pid) else { throw Unavailable() }
68
+ return OwnedAppWindows.select(screen: screen, accessible: accessible.windows)
69
+ }
70
+
71
+ public struct Unavailable: Error {}
72
+
73
+ public static func frame(_ element: AXUIElement) -> CGRect? {
74
+ var positionValue: CFTypeRef?
75
+ var sizeValue: CFTypeRef?
76
+ var position = CGPoint.zero
77
+ var size = CGSize.zero
78
+ guard AXUIElementCopyAttributeValue(element, kAXPositionAttribute as CFString, &positionValue) == .success,
79
+ AXUIElementCopyAttributeValue(element, kAXSizeAttribute as CFString, &sizeValue) == .success,
80
+ let positionValue, let sizeValue, CFGetTypeID(positionValue) == AXValueGetTypeID(),
81
+ CFGetTypeID(sizeValue) == AXValueGetTypeID(),
82
+ AXValueGetValue(unsafeDowncast(positionValue, to: AXValue.self), .cgPoint, &position),
83
+ AXValueGetValue(unsafeDowncast(sizeValue, to: AXValue.self), .cgSize, &size)
84
+ else { return nil }
85
+ return CGRect(origin: position, size: size)
86
+ }
87
+ }
@@ -0,0 +1,108 @@
1
+ import CoreGraphics
2
+
3
+ /// The windows of an owned macOS app that viewing and Control can show, and the one they follow.
4
+ public enum OwnedAppWindows {
5
+ /// An on-screen window of the app's process, from `CGWindowListCopyWindowInfo`, front to back.
6
+ public struct Screen: Equatable, Sendable {
7
+ public var id: UInt32
8
+ public var layer: Int
9
+ public var frame: CGRect
10
+ public var title: String
11
+
12
+ public init(id: UInt32, layer: Int, frame: CGRect, title: String) {
13
+ self.id = id
14
+ self.layer = layer
15
+ self.frame = frame
16
+ self.title = title
17
+ }
18
+ }
19
+
20
+ /// A standard window of the app from Accessibility (`kAXWindowsAttribute`), front to back.
21
+ public struct Accessible: Equatable, Sendable {
22
+ public var frame: CGRect
23
+ public var title: String
24
+ public var main: Bool
25
+ public var focused: Bool
26
+
27
+ public init(frame: CGRect, title: String, main: Bool, focused: Bool) {
28
+ self.frame = frame
29
+ self.title = title
30
+ self.main = main
31
+ self.focused = focused
32
+ }
33
+ }
34
+
35
+ /// `accessible` is the index of the window in the Accessibility list it was matched from, or nil without one.
36
+ public struct Window: Equatable, Sendable {
37
+ public var id: UInt32
38
+ public var title: String
39
+ public var frame: CGRect
40
+ public var accessible: Int?
41
+
42
+ public init(id: UInt32, title: String, frame: CGRect, accessible: Int?) {
43
+ self.id = id
44
+ self.title = title
45
+ self.frame = frame
46
+ self.accessible = accessible
47
+ }
48
+ }
49
+
50
+ public struct Selection: Equatable, Sendable {
51
+ public var current: Window
52
+ public var windows: [Window]
53
+
54
+ public init(current: Window, windows: [Window]) {
55
+ self.current = current
56
+ self.windows = windows
57
+ }
58
+ }
59
+
60
+ /// Matches each standard window to the first unused on-screen layer-0 window with the same frame and title, so
61
+ /// identical windows pair in their front-to-back order, then pairs the rest by frame alone, as the two inventories
62
+ /// can disagree on a title that just changed. A standard window without an on-screen match (minimized, on another
63
+ /// Space) is left out. The current window is the app's main window, else its focused one, else the frontmost.
64
+ public static func select(screen: [Screen], accessible: [Accessible]) -> Selection? {
65
+ var unused = screen.filter { $0.layer == 0 }
66
+ var matched: [Int: Screen] = [:]
67
+ for sameTitle in [true, false] {
68
+ for (index, window) in accessible.enumerated() where matched[index] == nil {
69
+ guard
70
+ let match = unused.firstIndex(where: { $0.frame == window.frame && (!sameTitle || $0.title == window.title) })
71
+ else { continue }
72
+ matched[index] = unused.remove(at: match)
73
+ }
74
+ }
75
+ let windows = accessible.indices.compactMap { index in
76
+ matched[index].map { Window(id: $0.id, title: accessible[index].title, frame: $0.frame, accessible: index) }
77
+ }
78
+ let current =
79
+ windows.first { accessible[$0.accessible!].main } ?? windows.first { accessible[$0.accessible!].focused }
80
+ ?? windows.first
81
+ return current.map { Selection(current: $0, windows: windows) }
82
+ }
83
+
84
+ /// The selection with `pinned` as its current window while that window is still on screen; nil `pinned` in the
85
+ /// result means the pin is gone and capture follows the front window again. A pinned window that the two
86
+ /// inventories briefly disagree on stays pinned from its on-screen entry, without an Accessibility match.
87
+ public static func pin(_ selection: Selection, to pinned: UInt32?, screen: [Screen]) -> (
88
+ selection: Selection, pinned: UInt32?
89
+ ) {
90
+ guard let pinned else { return (selection, nil) }
91
+ if let window = selection.windows.first(where: { $0.id == pinned }) {
92
+ return (Selection(current: window, windows: selection.windows), pinned)
93
+ }
94
+ guard let shown = screen.first(where: { $0.id == pinned && $0.layer == 0 }) else { return (selection, nil) }
95
+ let window = Window(id: shown.id, title: shown.title, frame: shown.frame, accessible: nil)
96
+ return (Selection(current: window, windows: selection.windows + [window]), pinned)
97
+ }
98
+
99
+ /// Without Accessibility, the one layer-0 window that contains every other one, such as a main window with its
100
+ /// sheets; nil when the app shows disjoint windows.
101
+ public static func single(screen: [Screen]) -> Selection? {
102
+ let windows = screen.filter { $0.layer == 0 }
103
+ let containers = windows.filter { candidate in windows.allSatisfy { candidate.frame.contains($0.frame) } }
104
+ guard containers.count == 1, let only = containers.first else { return nil }
105
+ let window = Window(id: only.id, title: only.title, frame: only.frame, accessible: nil)
106
+ return Selection(current: window, windows: [window])
107
+ }
108
+ }
@@ -0,0 +1,315 @@
1
+ import AVFoundation
2
+ import CoreImage
3
+ import CoreMediaIO
4
+ import Foundation
5
+ import IOKit
6
+
7
+ /// The screen of a USB-cabled iPhone, view only. macOS lists a cabled iPhone as an external muxed capture device,
8
+ /// the one QuickTime's New Movie Recording shows, only after a process sets
9
+ /// `kCMIOHardwarePropertyAllowScreenCaptureDevices`, and the device appears some seconds later. Capture runs only
10
+ /// while a subscriber asks for frames.
11
+ ///
12
+ /// The capture device's unique ID is a random UUID (macOS 27), and none of its CoreMediaIO properties names the
13
+ /// UDID. The USB device does: its serial number is the UDID without dashes. So the UDID must be cabled, and the
14
+ /// capture device is the iOS one named `name`, or the only one when there is no name and one iPhone is cabled.
15
+ /// Screens appear one by one, so with several iPhones cabled it waits for all of them; any other case refuses
16
+ /// rather than guess.
17
+ final class PhoneSource: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
18
+ static let searchSeconds = 15.0
19
+ static let retrySeconds = 2.0
20
+ static let starvedSeconds = 2.0
21
+
22
+ let udid: String
23
+ let name: String?
24
+ private var serial: String { udid.replacingOccurrences(of: "-", with: "").uppercased() }
25
+ let queue = DispatchQueue(label: "stim.frames.phone")
26
+ private let captureQueue = DispatchQueue(label: "stim.frames.phone-capture")
27
+ private var pacer: Pacer!
28
+ private let video = videoEncoder()
29
+ private let jpegGate = JpegGate()
30
+ private var config = Config(fps: 0)
31
+ private var device: AVCaptureDevice?
32
+ private var session: AVCaptureSession?
33
+ private var authorized = false
34
+ private var latest: CVPixelBuffer?
35
+ private var frameAt = Date.distantPast
36
+ private var watchdog: DispatchSourceTimer?
37
+ private var retrying = false
38
+ private var failure: String?
39
+ private var reportedStall: String??
40
+ private var observers: [NSObjectProtocol] = []
41
+ private var watches: [NSKeyValueObservation] = []
42
+
43
+ init(udid: String, name: String?) {
44
+ self.udid = udid
45
+ self.name = name
46
+ super.init()
47
+ pacer = Pacer { [unowned self] config in
48
+ guard let pixels = self.queue.sync(execute: { self.latest }) else { return }
49
+ self.render(pixels, config: config)
50
+ }
51
+ }
52
+
53
+ /// The serial numbers of the iOS devices on USB, which are their UDIDs without dashes. The registry is walked
54
+ /// instead of matched, because macOS leaves a cabled iPhone's USB device unregistered.
55
+ static func cabledSerials() -> Set<String> {
56
+ var iterator: io_iterator_t = 0
57
+ guard
58
+ IORegistryEntryCreateIterator(
59
+ IORegistryGetRootEntry(kIOMainPortDefault), "IOUSB", IOOptionBits(kIORegistryIterateRecursively), &iterator)
60
+ == KERN_SUCCESS
61
+ else { return [] }
62
+ defer { IOObjectRelease(iterator) }
63
+ var serials: Set<String> = []
64
+ while case let entry = IOIteratorNext(iterator), entry != 0 {
65
+ defer { IOObjectRelease(entry) }
66
+ let property = { (key: String) in
67
+ IORegistryEntryCreateCFProperty(entry, key as CFString, kCFAllocatorDefault, 0)?.takeRetainedValue()
68
+ }
69
+ guard property("UsbAppleDeviceECID") != nil, let serial = property("USB Serial Number") as? String else { continue }
70
+ serials.insert(serial.uppercased())
71
+ }
72
+ return serials
73
+ }
74
+
75
+ func start() {
76
+ var address = CMIOObjectPropertyAddress(
77
+ mSelector: CMIOObjectPropertySelector(kCMIOHardwarePropertyAllowScreenCaptureDevices),
78
+ mScope: CMIOObjectPropertyScope(kCMIOObjectPropertyScopeGlobal),
79
+ mElement: CMIOObjectPropertyElement(kCMIOObjectPropertyElementMain))
80
+ var allow: UInt32 = 1
81
+ let status = CMIOObjectSetPropertyData(
82
+ CMIOObjectID(kCMIOObjectSystemObject), &address, 0, nil, UInt32(MemoryLayout<UInt32>.size), &allow)
83
+ guard status == 0 else { fail("macOS refused to list iPhone screens as capture devices (CoreMediaIO \(status)).") }
84
+ let center = NotificationCenter.default
85
+ observers.append(
86
+ center.addObserver(forName: AVCaptureDevice.wasConnectedNotification, object: nil, queue: nil) { [weak self] _ in
87
+ self?.queue.async { self?.find(until: nil) }
88
+ })
89
+ observers.append(
90
+ center.addObserver(forName: AVCaptureSession.runtimeErrorNotification, object: nil, queue: nil) {
91
+ [weak self] note in
92
+ let error = note.userInfo?[AVCaptureSessionErrorKey] as? Error
93
+ self?.queue.async {
94
+ guard let self, let session = self.session, session === note.object as? AVCaptureSession else { return }
95
+ self.failure = "Capture of the iPhone \(self.udid) stopped (\(error?.localizedDescription ?? "unknown error"))."
96
+ self.close()
97
+ self.retry()
98
+ }
99
+ })
100
+ observers.append(
101
+ center.addObserver(forName: AVCaptureDevice.wasDisconnectedNotification, object: nil, queue: nil) {
102
+ [weak self] note in
103
+ guard let gone = note.object as? AVCaptureDevice else { return }
104
+ self?.queue.async {
105
+ guard let self, let device = self.device, device.uniqueID == gone.uniqueID else { return }
106
+ fail("The iPhone \(self.udid) was disconnected from this Mac.")
107
+ }
108
+ })
109
+ find(until: Date().addingTimeInterval(Self.searchSeconds))
110
+ }
111
+
112
+ private func find(until deadline: Date?) {
113
+ guard device == nil else { return }
114
+ let cabled = Self.cabledSerials()
115
+ guard cabled.contains(serial) else {
116
+ fail(
117
+ "The iPhone \(udid) is not cabled to this Mac. Stim shows a physical iPhone's screen only over a USB cable.")
118
+ }
119
+ let screens = AVCaptureDevice.DiscoverySession(deviceTypes: [.external], mediaType: .muxed, position: .unspecified)
120
+ .devices.filter { $0.modelID == "iOS Device" }
121
+ if cabled.count > 1 && name == nil {
122
+ fail("Several iPhones are cabled to this Mac, and Stim has no name to tell \(udid) apart from them.")
123
+ }
124
+ let named = name.map { name in screens.filter { $0.localizedName == name } } ?? screens
125
+ let listed = screens.count >= cabled.count
126
+ if listed && named.count > 1 {
127
+ fail(
128
+ "Several cabled iPhones are named \(name ?? "alike"), so Stim cannot tell which one is \(udid). Rename one in "
129
+ + "Settings > General > About > Name.")
130
+ }
131
+ guard listed, named.count == 1, let found = named.first else {
132
+ guard let deadline else { return }
133
+ if Date() > deadline {
134
+ if screens.isEmpty {
135
+ fail("macOS shows no screen for the iPhone \(udid). Unlock it, and tap Trust if it asks to trust this Mac.")
136
+ }
137
+ if !listed {
138
+ fail(
139
+ "Not every cabled iPhone shows its screen yet, so Stim cannot tell \(udid) apart from them. Unlock each "
140
+ + "one, and tap Trust if it asks to trust this Mac.")
141
+ }
142
+ fail(
143
+ "No cabled iPhone's screen is named \(name ?? "") as the lease of \(udid) records. If it was renamed, run "
144
+ + "stim device lock ios \(udid) again.")
145
+ }
146
+ queue.asyncAfter(deadline: .now() + 1) { self.find(until: deadline) }
147
+ return
148
+ }
149
+ device = found
150
+ for key in [\AVCaptureDevice.isSuspended, \AVCaptureDevice.isInUseByAnotherApplication] {
151
+ watches.append(found.observe(key) { [weak self] _, _ in self?.queue.async { self?.reportStall() } })
152
+ }
153
+ authorize()
154
+ }
155
+
156
+ /// The first capture asks the user for Camera access, which macOS attributes to the app that started stim-server.
157
+ private func authorize() {
158
+ switch AVCaptureDevice.authorizationStatus(for: .video) {
159
+ case .authorized:
160
+ authorized = true
161
+ failure = nil
162
+ reportStall()
163
+ apply()
164
+ case .notDetermined:
165
+ failure = "macOS is asking on the Mac whether the app that runs stim-server may use the camera."
166
+ reportStall()
167
+ AVCaptureDevice.requestAccess(for: .video) { granted in
168
+ self.queue.async {
169
+ if !granted { self.denied() }
170
+ self.authorize()
171
+ }
172
+ }
173
+ default:
174
+ denied()
175
+ }
176
+ }
177
+
178
+ /// macOS denies without a prompt when the responsible app has no camera usage description, or runs with the
179
+ /// hardened runtime and lacks the camera entitlement, such as a bare `node`.
180
+ private func denied() -> Never {
181
+ fail(
182
+ "macOS denied Camera access, which capturing an iPhone screen needs. Run stim-server from Stim, and allow Stim "
183
+ + "in System Settings > Privacy & Security > Camera.")
184
+ }
185
+
186
+ func configure(_ config: Config) {
187
+ video.configure(enabled: config.video, maxEdge: config.maxEdge, fps: Int(config.fps), bitrate: config.bitrate)
188
+ queue.async {
189
+ self.config = config
190
+ self.pacer.queue.async {
191
+ self.pacer.config = config
192
+ self.pacer.changed()
193
+ }
194
+ self.apply()
195
+ }
196
+ }
197
+
198
+ func keyframe() {
199
+ video.requestKeyframe()
200
+ pacer.changed()
201
+ }
202
+
203
+ private func apply() {
204
+ let wanted = config.fps > 0 && (config.jpeg || config.video)
205
+ if wanted, session == nil, authorized, let device { open(device) }
206
+ if !wanted, session != nil {
207
+ close()
208
+ latest = nil
209
+ }
210
+ }
211
+
212
+ private func close() {
213
+ guard let session else { return }
214
+ self.session = nil
215
+ watchdog?.cancel()
216
+ watchdog = nil
217
+ captureQueue.async { session.stopRunning() }
218
+ }
219
+
220
+ private func open(_ device: AVCaptureDevice) {
221
+ let session = AVCaptureSession()
222
+ do {
223
+ let input = try AVCaptureDeviceInput(device: device)
224
+ for port in input.ports where port.mediaType != .video { port.isEnabled = false }
225
+ guard session.canAddInput(input) else { throw CaptureError.refused }
226
+ session.addInput(input)
227
+ } catch {
228
+ failure = "The iPhone \(udid) could not be opened for capture (\(error.localizedDescription))."
229
+ return retry()
230
+ }
231
+ let output = AVCaptureVideoDataOutput()
232
+ output.alwaysDiscardsLateVideoFrames = true
233
+ output.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA]
234
+ output.setSampleBufferDelegate(self, queue: captureQueue)
235
+ guard session.canAddOutput(output) else {
236
+ failure = "The iPhone \(udid) offers no video output macOS can read."
237
+ return retry()
238
+ }
239
+ session.addOutput(output)
240
+ self.session = session
241
+ failure = nil
242
+ frameAt = Date()
243
+ reportStall()
244
+ let watchdog = DispatchSource.makeTimerSource(queue: queue)
245
+ watchdog.schedule(deadline: .now() + 1, repeating: 1)
246
+ watchdog.setEventHandler { [weak self] in
247
+ guard let self else { return }
248
+ if !Self.cabledSerials().contains(self.serial) { fail("The iPhone \(self.udid) was disconnected from this Mac.") }
249
+ self.reportStall()
250
+ }
251
+ watchdog.resume()
252
+ self.watchdog = watchdog
253
+ captureQueue.async { session.startRunning() }
254
+ }
255
+
256
+ private func retry() {
257
+ if !Self.cabledSerials().contains(serial) { fail("The iPhone \(udid) was disconnected from this Mac.") }
258
+ reportStall()
259
+ guard !retrying else { return }
260
+ retrying = true
261
+ queue.asyncAfter(deadline: .now() + Self.retrySeconds) {
262
+ self.retrying = false
263
+ self.apply()
264
+ }
265
+ }
266
+
267
+ /// A failed open, or frames that stopped, are stalls: the last frame stays on screen with the reason, and frames
268
+ /// resume when the reason clears. A cabled iPhone sends frames continuously, so a gap names its cause when macOS
269
+ /// reports one. Another app capturing the iPhone alone is no stall, because several processes can capture it.
270
+ private func reportStall() {
271
+ let starved = session != nil && Date().timeIntervalSince(frameAt) > Self.starvedSeconds
272
+ let stall: String?
273
+ if starved, let device, device.isSuspended {
274
+ stall = "The iPhone is locked. Unlock it to see its screen."
275
+ } else if starved, let device, device.isInUseByAnotherApplication {
276
+ stall = "Another app, such as QuickTime Player, is capturing this iPhone."
277
+ } else if starved {
278
+ stall = "The iPhone stopped sending its screen."
279
+ } else {
280
+ stall = failure
281
+ }
282
+ guard reportedStall != .some(stall) else { return }
283
+ reportedStall = .some(stall)
284
+ Output.notice(["stalled": stall.map { $0 as Any } ?? NSNull()])
285
+ }
286
+
287
+ func captureOutput(_ output: AVCaptureOutput, didOutput sample: CMSampleBuffer, from connection: AVCaptureConnection) {
288
+ guard let pixels = CMSampleBufferGetImageBuffer(sample) else { return }
289
+ queue.async {
290
+ self.latest = pixels
291
+ self.frameAt = Date()
292
+ if self.reportedStall != .some(self.failure) { self.reportStall() }
293
+ }
294
+ pacer.changed()
295
+ }
296
+
297
+ private func render(_ pixels: CVPixelBuffer, config: Config) {
298
+ if config.video { video.encode(pixels, quarterTurns: 0, capturedAt: now()) }
299
+ guard config.jpeg, jpegGate.admit(config, pacer: pacer),
300
+ let (data, width, height) = jpeg(CIImage(cvPixelBuffer: pixels), config: config)
301
+ else { return }
302
+ Output.frame(jpeg: data, width: width, height: height)
303
+ }
304
+
305
+ private enum CaptureError: Error { case refused }
306
+ }
307
+
308
+ /// stim-server records only the simulators and emulators Stim owns, so a physical iPhone runs no recording encoder.
309
+ extension PhoneSource: Source {
310
+ func recordKeyframe() {}
311
+
312
+ func input(_ command: Command) {
313
+ Output.notice(["inputError": "Stim shows a physical iPhone's screen but does not send it input."])
314
+ }
315
+ }
@@ -0,0 +1,155 @@
1
+ import Foundation
2
+
3
+ public enum OutputChannel: Sendable {
4
+ case stdout
5
+ case stderr
6
+ }
7
+
8
+ public struct OutputLine: Hashable, Sendable {
9
+ public var channel: OutputChannel
10
+ public var text: String
11
+
12
+ public init(_ channel: OutputChannel, _ text: String) {
13
+ self.channel = channel
14
+ self.text = text
15
+ }
16
+ }
17
+
18
+ extension String {
19
+ /// The text without ANSI CSI sequences (colors, cursor moves) and OSC sequences (titles, links).
20
+ public var strippingANSI: String {
21
+ replacingOccurrences(
22
+ of: "\u{1B}\\[[0-?]*[ -/]*[@-~]|\u{1B}\\][^\u{07}\u{1B}]*(\u{07}|\u{1B}\\\\)", with: "",
23
+ options: .regularExpression)
24
+ }
25
+ }
26
+
27
+ /// Splits a byte stream into lines. A chunk can end inside a line or inside a
28
+ /// UTF-8 sequence, so bytes after the last newline wait for the next chunk.
29
+ public struct LineBuffer: Sendable {
30
+ private var pending = Data()
31
+
32
+ public init() {}
33
+
34
+ public mutating func append(_ data: Data) -> [String] {
35
+ pending.append(data)
36
+ guard let last = pending.lastIndex(of: UInt8(ascii: "\n")) else { return [] }
37
+ let complete = pending[pending.startIndex..<last]
38
+ pending = Data(pending[(last + 1)...])
39
+ return complete.split(separator: UInt8(ascii: "\n"), omittingEmptySubsequences: false).map(Self.decode)
40
+ }
41
+
42
+ public mutating func finish() -> [String] {
43
+ defer { pending = Data() }
44
+ return pending.isEmpty ? [] : [Self.decode(pending)]
45
+ }
46
+
47
+ private static func decode<Bytes: Collection<UInt8>>(_ bytes: Bytes) -> String {
48
+ var text = String(decoding: bytes, as: UTF8.self)
49
+ if text.hasSuffix("\r") { text.removeLast() }
50
+ return text
51
+ }
52
+ }
53
+
54
+ /// Runs an executable with an argument list, never through a shell, and
55
+ /// reports its output line by line and then its exit status.
56
+ public enum ProcessStream {
57
+ /// A child the command leaves running can hold the pipes open, so the exit
58
+ /// is reported this long after termination even without end of file.
59
+ static let drainGrace: TimeInterval = 2
60
+
61
+ @discardableResult
62
+ public static func start(
63
+ executable: String,
64
+ arguments: [String],
65
+ cwd: String,
66
+ environment: [String: String]? = nil,
67
+ onLine: @escaping @Sendable (OutputLine) -> Void,
68
+ onExit: @escaping @Sendable (Int32) -> Void
69
+ ) throws -> Process {
70
+ let process = Process()
71
+ process.executableURL = URL(fileURLWithPath: executable)
72
+ process.arguments = arguments
73
+ process.currentDirectoryURL = URL(fileURLWithPath: cwd)
74
+ if let environment { process.environment = environment }
75
+ process.standardInput = FileHandle.nullDevice
76
+ let out = Pipe()
77
+ let err = Pipe()
78
+ process.standardOutput = out
79
+ process.standardError = err
80
+
81
+ let state = StreamState(onLine: onLine, onExit: onExit)
82
+ for (pipe, channel) in [(out, OutputChannel.stdout), (err, .stderr)] {
83
+ pipe.fileHandleForReading.readabilityHandler = { handle in
84
+ let data = handle.availableData
85
+ if data.isEmpty {
86
+ handle.readabilityHandler = nil
87
+ state.close(channel)
88
+ } else {
89
+ state.receive(data, on: channel)
90
+ }
91
+ }
92
+ }
93
+ process.terminationHandler = { process in
94
+ let status = process.terminationStatus
95
+ state.terminated(status)
96
+ DispatchQueue.global().asyncAfter(deadline: .now() + drainGrace) {
97
+ out.fileHandleForReading.readabilityHandler = nil
98
+ err.fileHandleForReading.readabilityHandler = nil
99
+ state.close(.stdout)
100
+ state.close(.stderr)
101
+ }
102
+ }
103
+ do {
104
+ try process.run()
105
+ } catch {
106
+ out.fileHandleForReading.readabilityHandler = nil
107
+ err.fileHandleForReading.readabilityHandler = nil
108
+ throw error
109
+ }
110
+ return process
111
+ }
112
+ }
113
+
114
+ private final class StreamState: @unchecked Sendable {
115
+ private let lock = NSLock()
116
+ private var buffers: [OutputChannel: LineBuffer] = [.stdout: LineBuffer(), .stderr: LineBuffer()]
117
+ private var open: Set<OutputChannel> = [.stdout, .stderr]
118
+ private var status: Int32?
119
+ private var reported = false
120
+ private let onLine: @Sendable (OutputLine) -> Void
121
+ private let onExit: @Sendable (Int32) -> Void
122
+
123
+ init(onLine: @escaping @Sendable (OutputLine) -> Void, onExit: @escaping @Sendable (Int32) -> Void) {
124
+ self.onLine = onLine
125
+ self.onExit = onExit
126
+ }
127
+
128
+ func receive(_ data: Data, on channel: OutputChannel) {
129
+ lock.lock()
130
+ defer { lock.unlock() }
131
+ guard open.contains(channel) else { return }
132
+ for line in buffers[channel]!.append(data) { onLine(OutputLine(channel, line)) }
133
+ }
134
+
135
+ func close(_ channel: OutputChannel) {
136
+ lock.lock()
137
+ defer { lock.unlock() }
138
+ guard open.remove(channel) != nil else { return }
139
+ for line in buffers[channel]!.finish() { onLine(OutputLine(channel, line)) }
140
+ reportIfDone()
141
+ }
142
+
143
+ func terminated(_ status: Int32) {
144
+ lock.lock()
145
+ defer { lock.unlock() }
146
+ self.status = status
147
+ reportIfDone()
148
+ }
149
+
150
+ private func reportIfDone() {
151
+ guard !reported, open.isEmpty, let status else { return }
152
+ reported = true
153
+ onExit(status)
154
+ }
155
+ }