@kaeawc/auto-mobile 0.0.44 → 0.0.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/README.md.backup +3 -3
- package/dist/ios/screen-capture/Package.swift +35 -0
- package/dist/ios/screen-capture/Sources/ScreenCaptureCore/AudioPcm16Encoder.swift +22 -0
- package/dist/ios/screen-capture/Sources/ScreenCaptureCore/CommandLineOptions.swift +152 -0
- package/dist/ios/screen-capture/Sources/ScreenCaptureCore/DeviceInfo.swift +25 -0
- package/dist/ios/screen-capture/Sources/ScreenCaptureCore/FrameProtocol.swift +113 -0
- package/dist/ios/screen-capture/Sources/ScreenCaptureCore/FrameWriter.swift +62 -0
- package/dist/ios/screen-capture/Sources/ScreenCaptureCore/SimulatorWindowInfo.swift +51 -0
- package/dist/ios/screen-capture/Sources/ScreenCaptureHelper/AudioSampleBuffer.swift +41 -0
- package/dist/ios/screen-capture/Sources/ScreenCaptureHelper/CMIOSystem.swift +26 -0
- package/dist/ios/screen-capture/Sources/ScreenCaptureHelper/DeviceCaptureSession.swift +117 -0
- package/dist/ios/screen-capture/Sources/ScreenCaptureHelper/DeviceDiscovery.swift +29 -0
- package/dist/ios/screen-capture/Sources/ScreenCaptureHelper/FrameWriter+PixelBuffer.swift +25 -0
- package/dist/ios/screen-capture/Sources/ScreenCaptureHelper/SimulatorCaptureSession.swift +159 -0
- package/dist/ios/screen-capture/Sources/ScreenCaptureHelper/SimulatorWindowDiscovery.swift +37 -0
- package/dist/ios/screen-capture/Sources/ScreenCaptureHelper/main.swift +255 -0
- package/dist/ios/screen-capture/Tests/ScreenCaptureCoreTests/AudioPcm16EncoderTests.swift +17 -0
- package/dist/ios/screen-capture/Tests/ScreenCaptureCoreTests/CommandLineOptionsTests.swift +181 -0
- package/dist/ios/screen-capture/Tests/ScreenCaptureCoreTests/DeviceInfoTests.swift +35 -0
- package/dist/ios/screen-capture/Tests/ScreenCaptureCoreTests/FrameProtocolTests.swift +123 -0
- package/dist/ios/screen-capture/Tests/ScreenCaptureCoreTests/FrameWriterTests.swift +62 -0
- package/dist/ios/screen-capture/Tests/ScreenCaptureCoreTests/SimulatorAudioCaptureAvailabilityTests.swift +27 -0
- package/dist/ios/screen-capture/Tests/ScreenCaptureCoreTests/SimulatorWindowInfoTests.swift +40 -0
- package/dist/schemas/tool-definitions.json +1274 -849
- package/dist/src/db/migrations/2026_07_03_000_repair_datetime_now_defaults.ts +11 -2
- package/dist/src/index.js +670 -407
- package/dist/src/index.js.map +1 -1
- package/package.json +25 -6
- package/schemas/tool-definitions.json +1274 -849
package/README.md
CHANGED
|
@@ -8,9 +8,9 @@
|
|
|
8
8
|

|
|
9
9
|

|
|
10
10
|
|
|
11
|
-

|
|
12
|
+

|
|
13
|
+

|
|
14
14
|

|
|
15
15
|

|
|
16
16
|
|
package/README.md.backup
CHANGED
|
@@ -8,9 +8,9 @@
|
|
|
8
8
|

|
|
9
9
|

|
|
10
10
|
|
|
11
|
-

|
|
12
|
+

|
|
13
|
+

|
|
14
14
|

|
|
15
15
|

|
|
16
16
|
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
// swift-tools-version: 5.9
|
|
2
|
+
import PackageDescription
|
|
3
|
+
|
|
4
|
+
let package = Package(
|
|
5
|
+
name: "ScreenCaptureHelper",
|
|
6
|
+
platforms: [
|
|
7
|
+
.macOS(.v14),
|
|
8
|
+
],
|
|
9
|
+
products: [
|
|
10
|
+
.executable(
|
|
11
|
+
name: "screen-capture-helper",
|
|
12
|
+
targets: ["ScreenCaptureHelper"]
|
|
13
|
+
),
|
|
14
|
+
.library(
|
|
15
|
+
name: "ScreenCaptureCore",
|
|
16
|
+
targets: ["ScreenCaptureCore"]
|
|
17
|
+
),
|
|
18
|
+
],
|
|
19
|
+
targets: [
|
|
20
|
+
.target(
|
|
21
|
+
name: "ScreenCaptureCore",
|
|
22
|
+
path: "Sources/ScreenCaptureCore"
|
|
23
|
+
),
|
|
24
|
+
.executableTarget(
|
|
25
|
+
name: "ScreenCaptureHelper",
|
|
26
|
+
dependencies: ["ScreenCaptureCore"],
|
|
27
|
+
path: "Sources/ScreenCaptureHelper"
|
|
28
|
+
),
|
|
29
|
+
.testTarget(
|
|
30
|
+
name: "ScreenCaptureCoreTests",
|
|
31
|
+
dependencies: ["ScreenCaptureCore"],
|
|
32
|
+
path: "Tests/ScreenCaptureCoreTests"
|
|
33
|
+
),
|
|
34
|
+
]
|
|
35
|
+
)
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
|
|
3
|
+
/// Converts the Float32 little-endian PCM emitted by ScreenCaptureKit into
|
|
4
|
+
/// signed little-endian PCM16 for the shared WebRTC PCMU packetizer.
|
|
5
|
+
public enum AudioPcm16Encoder {
|
|
6
|
+
public static func encodeFloat32LE(_ input: Data) -> Data? {
|
|
7
|
+
guard input.count.isMultiple(of: MemoryLayout<Float>.size) else { return nil }
|
|
8
|
+
|
|
9
|
+
var output = Data(count: input.count / 2)
|
|
10
|
+
input.withUnsafeBytes { source in
|
|
11
|
+
output.withUnsafeMutableBytes { destination in
|
|
12
|
+
guard let sourceBase = source.baseAddress, let destinationBase = destination.baseAddress else { return }
|
|
13
|
+
for index in 0..<(input.count / MemoryLayout<Float>.size) {
|
|
14
|
+
let float = sourceBase.advanced(by: index * 4).loadUnaligned(as: Float.self)
|
|
15
|
+
let sample = Int16(max(-1, min(1, float)) * Float(Int16.max))
|
|
16
|
+
destinationBase.advanced(by: index * 2).storeBytes(of: sample.littleEndian, as: Int16.self)
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
return output
|
|
21
|
+
}
|
|
22
|
+
}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
|
|
3
|
+
/// Parsed CLI arguments for the screen-capture helper.
|
|
4
|
+
public struct CommandLineOptions: Equatable {
|
|
5
|
+
public enum Mode: Equatable {
|
|
6
|
+
case listDevices
|
|
7
|
+
case capture(deviceID: String?)
|
|
8
|
+
case listSimulators
|
|
9
|
+
case captureSimulator(windowID: UInt32, fps: Int, audio: Bool)
|
|
10
|
+
case help
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
public let mode: Mode
|
|
14
|
+
|
|
15
|
+
public init(mode: Mode) {
|
|
16
|
+
self.mode = mode
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
public enum ParseError: Error, Equatable {
|
|
20
|
+
case missingValue(flag: String)
|
|
21
|
+
case unknownArgument(String)
|
|
22
|
+
case invalidValue(flag: String, value: String)
|
|
23
|
+
case conflictingFlags(String)
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
public static let defaultSimulatorFPS = 5
|
|
27
|
+
public static let minSimulatorFPS = 5
|
|
28
|
+
public static let maxSimulatorFPS = 60
|
|
29
|
+
|
|
30
|
+
public static func parse(_ arguments: [String]) throws -> CommandLineOptions {
|
|
31
|
+
var iterator = arguments.makeIterator()
|
|
32
|
+
_ = iterator.next()
|
|
33
|
+
|
|
34
|
+
var listDevices = false
|
|
35
|
+
var deviceID: String?
|
|
36
|
+
var listSimulators = false
|
|
37
|
+
var simulatorWindowID: UInt32?
|
|
38
|
+
var simulatorFPS: Int?
|
|
39
|
+
var audio = false
|
|
40
|
+
var help = false
|
|
41
|
+
|
|
42
|
+
while let arg = iterator.next() {
|
|
43
|
+
switch arg {
|
|
44
|
+
case "--list-devices":
|
|
45
|
+
listDevices = true
|
|
46
|
+
case "--device-id":
|
|
47
|
+
guard let value = iterator.next() else {
|
|
48
|
+
throw ParseError.missingValue(flag: arg)
|
|
49
|
+
}
|
|
50
|
+
deviceID = value
|
|
51
|
+
case "--list-simulators":
|
|
52
|
+
listSimulators = true
|
|
53
|
+
case "--simulator-window":
|
|
54
|
+
guard let value = iterator.next() else {
|
|
55
|
+
throw ParseError.missingValue(flag: arg)
|
|
56
|
+
}
|
|
57
|
+
guard let parsed = UInt32(value) else {
|
|
58
|
+
throw ParseError.invalidValue(flag: arg, value: value)
|
|
59
|
+
}
|
|
60
|
+
simulatorWindowID = parsed
|
|
61
|
+
case "--simulator-fps":
|
|
62
|
+
guard let value = iterator.next() else {
|
|
63
|
+
throw ParseError.missingValue(flag: arg)
|
|
64
|
+
}
|
|
65
|
+
guard
|
|
66
|
+
let parsed = Int(value),
|
|
67
|
+
parsed >= minSimulatorFPS,
|
|
68
|
+
parsed <= maxSimulatorFPS
|
|
69
|
+
else {
|
|
70
|
+
throw ParseError.invalidValue(flag: arg, value: value)
|
|
71
|
+
}
|
|
72
|
+
simulatorFPS = parsed
|
|
73
|
+
case "--audio":
|
|
74
|
+
audio = true
|
|
75
|
+
case "-h", "--help":
|
|
76
|
+
help = true
|
|
77
|
+
default:
|
|
78
|
+
throw ParseError.unknownArgument(arg)
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
if help {
|
|
83
|
+
return CommandLineOptions(mode: .help)
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
let modeFlagCount = [
|
|
87
|
+
listDevices,
|
|
88
|
+
listSimulators,
|
|
89
|
+
simulatorWindowID != nil,
|
|
90
|
+
deviceID != nil,
|
|
91
|
+
].filter { $0 }.count
|
|
92
|
+
if modeFlagCount > 1 {
|
|
93
|
+
throw ParseError.conflictingFlags(
|
|
94
|
+
"choose one of --list-devices, --device-id, --list-simulators, --simulator-window"
|
|
95
|
+
)
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
if simulatorFPS != nil && simulatorWindowID == nil {
|
|
99
|
+
throw ParseError.conflictingFlags(
|
|
100
|
+
"--simulator-fps requires --simulator-window"
|
|
101
|
+
)
|
|
102
|
+
}
|
|
103
|
+
if audio && simulatorWindowID == nil {
|
|
104
|
+
throw ParseError.conflictingFlags("--audio requires --simulator-window")
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
if listDevices {
|
|
108
|
+
return CommandLineOptions(mode: .listDevices)
|
|
109
|
+
}
|
|
110
|
+
if listSimulators {
|
|
111
|
+
return CommandLineOptions(mode: .listSimulators)
|
|
112
|
+
}
|
|
113
|
+
if let windowID = simulatorWindowID {
|
|
114
|
+
return CommandLineOptions(
|
|
115
|
+
mode: .captureSimulator(
|
|
116
|
+
windowID: windowID,
|
|
117
|
+
fps: simulatorFPS ?? defaultSimulatorFPS,
|
|
118
|
+
audio: audio
|
|
119
|
+
)
|
|
120
|
+
)
|
|
121
|
+
}
|
|
122
|
+
return CommandLineOptions(mode: .capture(deviceID: deviceID))
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
public static let helpText = """
|
|
126
|
+
screen-capture-helper — AutoMobile iOS screen-capture helper
|
|
127
|
+
|
|
128
|
+
USAGE:
|
|
129
|
+
screen-capture-helper [--device-id <id>]
|
|
130
|
+
screen-capture-helper --list-devices
|
|
131
|
+
screen-capture-helper --simulator-window <windowID> [--simulator-fps <n>] [--audio]
|
|
132
|
+
screen-capture-helper --list-simulators
|
|
133
|
+
|
|
134
|
+
DEVICE OPTIONS (USB-connected iOS devices via AVFoundation):
|
|
135
|
+
--list-devices Emit discovered devices as JSON to stdout and exit.
|
|
136
|
+
--device-id <id> uniqueID of the device to capture. If omitted,
|
|
137
|
+
the first muxed external device is used.
|
|
138
|
+
|
|
139
|
+
SIMULATOR OPTIONS (macOS iOS Simulator windows via ScreenCaptureKit):
|
|
140
|
+
--list-simulators Emit discovered simulator windows as JSON.
|
|
141
|
+
--simulator-window <n> CGWindowID of the simulator window to capture.
|
|
142
|
+
--simulator-fps <n> Target frame rate (5-60, default 5). Higher
|
|
143
|
+
values waste CPU for typical MCP workloads.
|
|
144
|
+
--audio Capture Simulator window audio as 8 kHz mono PCM16LE.
|
|
145
|
+
|
|
146
|
+
-h, --help Show this help.
|
|
147
|
+
|
|
148
|
+
Frames are written to stdout: 16-byte little-endian header
|
|
149
|
+
(width, height, bytesPerRow, timestampMs) followed by
|
|
150
|
+
height * bytesPerRow bytes of BGRA pixel data.
|
|
151
|
+
"""
|
|
152
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
|
|
3
|
+
/// Serializable description of a discovered capture device. Emitted to stdout
|
|
4
|
+
/// in JSON form when `screen-capture-helper --list-devices` is invoked.
|
|
5
|
+
public struct DeviceInfo: Codable, Equatable {
|
|
6
|
+
public let uniqueID: String
|
|
7
|
+
public let localizedName: String
|
|
8
|
+
public let modelID: String
|
|
9
|
+
public let manufacturer: String
|
|
10
|
+
|
|
11
|
+
public init(uniqueID: String, localizedName: String, modelID: String, manufacturer: String) {
|
|
12
|
+
self.uniqueID = uniqueID
|
|
13
|
+
self.localizedName = localizedName
|
|
14
|
+
self.modelID = modelID
|
|
15
|
+
self.manufacturer = manufacturer
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
public struct DeviceListResponse: Codable, Equatable {
|
|
20
|
+
public let devices: [DeviceInfo]
|
|
21
|
+
|
|
22
|
+
public init(devices: [DeviceInfo]) {
|
|
23
|
+
self.devices = devices
|
|
24
|
+
}
|
|
25
|
+
}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
|
|
3
|
+
/// Binary frame protocol matching docs/design-docs/plat/ios/screen-streaming.md.
|
|
4
|
+
///
|
|
5
|
+
/// Wire format (header = 24 bytes, little-endian UInt32):
|
|
6
|
+
///
|
|
7
|
+
/// ┌─────────┬─────────────┬──────────┬───────────┬─────────────┬─────────────┐
|
|
8
|
+
/// │ magic(4)│ checksum(4) │ width(4) │ height(4) │ bytesPerRow │ timestampMs │
|
|
9
|
+
/// └─────────┴─────────────┴──────────┴───────────┴─────────────┴─────────────┘
|
|
10
|
+
///
|
|
11
|
+
/// Followed by `height * bytesPerRow` bytes of BGRA pixel data. `magic` is a
|
|
12
|
+
/// fixed sync marker ("AMF1" on the wire) and `checksum` is a CRC-32 (IEEE) over
|
|
13
|
+
/// the 16 field bytes that follow it, so the decoder can recover frame
|
|
14
|
+
/// boundaries deterministically after corruption (issue #4270). The TypeScript
|
|
15
|
+
/// decoder (`src/features/screen-stream/frameProtocol.ts`) must agree byte for
|
|
16
|
+
/// byte; the shared CRC-32 check vector pins that on both sides.
|
|
17
|
+
public enum FrameProtocol {
|
|
18
|
+
public static let headerSize = 24
|
|
19
|
+
/// Sync marker; on the wire the bytes read "AMF1".
|
|
20
|
+
public static let magic: UInt32 = 0x3146_4D41
|
|
21
|
+
/// Offset where the checksummed field bytes begin.
|
|
22
|
+
static let fieldsOffset = 8
|
|
23
|
+
public static let audioSampleRate: UInt32 = 8_000
|
|
24
|
+
public static let audioChannelCount: UInt32 = 1
|
|
25
|
+
|
|
26
|
+
public struct Header: Equatable {
|
|
27
|
+
public let width: UInt32
|
|
28
|
+
public let height: UInt32
|
|
29
|
+
public let bytesPerRow: UInt32
|
|
30
|
+
public let timestampMs: UInt32
|
|
31
|
+
|
|
32
|
+
public init(width: UInt32, height: UInt32, bytesPerRow: UInt32, timestampMs: UInt32) {
|
|
33
|
+
self.width = width
|
|
34
|
+
self.height = height
|
|
35
|
+
self.bytesPerRow = bytesPerRow
|
|
36
|
+
self.timestampMs = timestampMs
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
public static func encodeHeader(_ header: Header) -> Data {
|
|
41
|
+
var data = Data(count: headerSize)
|
|
42
|
+
data.withUnsafeMutableBytes { ptr in
|
|
43
|
+
// `data` was allocated with `Data(count: headerSize)` (headerSize > 0), so
|
|
44
|
+
// the buffer is non-empty and baseAddress is non-nil. All offsets are
|
|
45
|
+
// multiples of 4, so the aligned stores are valid.
|
|
46
|
+
let base = ptr.baseAddress! // swiftlint:disable:this force_unwrapping
|
|
47
|
+
base.storeBytes(of: magic.littleEndian, as: UInt32.self)
|
|
48
|
+
base.advanced(by: 8).storeBytes(of: header.width.littleEndian, as: UInt32.self)
|
|
49
|
+
base.advanced(by: 12).storeBytes(of: header.height.littleEndian, as: UInt32.self)
|
|
50
|
+
base.advanced(by: 16).storeBytes(of: header.bytesPerRow.littleEndian, as: UInt32.self)
|
|
51
|
+
base.advanced(by: 20).storeBytes(of: header.timestampMs.littleEndian, as: UInt32.self)
|
|
52
|
+
}
|
|
53
|
+
let checksum = crc32(data.subdata(in: fieldsOffset..<headerSize))
|
|
54
|
+
data.withUnsafeMutableBytes { ptr in
|
|
55
|
+
let base = ptr.baseAddress! // swiftlint:disable:this force_unwrapping
|
|
56
|
+
base.advanced(by: 4).storeBytes(of: checksum.littleEndian, as: UInt32.self)
|
|
57
|
+
}
|
|
58
|
+
return data
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
public static func encodeAudioHeader(payloadLength: Int) -> Data {
|
|
62
|
+
encodeHeader(Header(
|
|
63
|
+
width: 0,
|
|
64
|
+
height: audioSampleRate,
|
|
65
|
+
bytesPerRow: audioChannelCount,
|
|
66
|
+
timestampMs: UInt32(payloadLength)
|
|
67
|
+
))
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/// Decode + validate a header. Returns nil when the marker or checksum does
|
|
71
|
+
/// not match — i.e. these bytes are not a real frame boundary.
|
|
72
|
+
public static func decodeHeader(_ data: Data) -> Header? {
|
|
73
|
+
guard data.count >= headerSize else { return nil }
|
|
74
|
+
return data.withUnsafeBytes { ptr -> Header? in
|
|
75
|
+
// Guarded above by `data.count >= headerSize` (headerSize > 0), so the
|
|
76
|
+
// buffer is non-empty and baseAddress is non-nil. `loadUnaligned` has no
|
|
77
|
+
// alignment requirement, so a header at an odd address is safe (#3627).
|
|
78
|
+
let base = ptr.baseAddress! // swiftlint:disable:this force_unwrapping
|
|
79
|
+
let marker = UInt32(littleEndian: base.loadUnaligned(as: UInt32.self))
|
|
80
|
+
guard marker == magic else { return nil }
|
|
81
|
+
let stored = UInt32(littleEndian: base.advanced(by: 4).loadUnaligned(as: UInt32.self))
|
|
82
|
+
let fields = Data(bytes: base.advanced(by: fieldsOffset), count: headerSize - fieldsOffset)
|
|
83
|
+
guard stored == crc32(fields) else { return nil }
|
|
84
|
+
return Header(
|
|
85
|
+
width: UInt32(littleEndian: base.advanced(by: 8).loadUnaligned(as: UInt32.self)),
|
|
86
|
+
height: UInt32(littleEndian: base.advanced(by: 12).loadUnaligned(as: UInt32.self)),
|
|
87
|
+
bytesPerRow: UInt32(littleEndian: base.advanced(by: 16).loadUnaligned(as: UInt32.self)),
|
|
88
|
+
timestampMs: UInt32(littleEndian: base.advanced(by: 20).loadUnaligned(as: UInt32.self))
|
|
89
|
+
)
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
private static let crc32Table: [UInt32] = {
|
|
94
|
+
(0..<256).map { index -> UInt32 in
|
|
95
|
+
var remainder = UInt32(index)
|
|
96
|
+
for _ in 0..<8 {
|
|
97
|
+
remainder = (remainder & 1) != 0 ? 0xEDB8_8320 ^ (remainder >> 1) : remainder >> 1
|
|
98
|
+
}
|
|
99
|
+
return remainder
|
|
100
|
+
}
|
|
101
|
+
}()
|
|
102
|
+
|
|
103
|
+
/// CRC-32 (IEEE 802.3, reflected, polynomial 0xEDB88320) — the standard CRC
|
|
104
|
+
/// used by zip/gzip/PNG. Implemented here because Foundation ships no CRC-32
|
|
105
|
+
/// primitive and the TypeScript decoder must produce an identical checksum.
|
|
106
|
+
public static func crc32(_ data: Data) -> UInt32 {
|
|
107
|
+
var crc: UInt32 = 0xFFFF_FFFF
|
|
108
|
+
for byte in data {
|
|
109
|
+
crc = (crc >> 8) ^ crc32Table[Int((crc ^ UInt32(byte)) & 0xFF)]
|
|
110
|
+
}
|
|
111
|
+
return crc ^ 0xFFFF_FFFF
|
|
112
|
+
}
|
|
113
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
|
|
3
|
+
/// Sink that receives encoded frames. Implementations write to stdout, a file,
|
|
4
|
+
/// a Unix socket, or an in-memory buffer for tests.
|
|
5
|
+
public protocol FrameSink: AnyObject {
|
|
6
|
+
func write(_ data: Data)
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
/// Writes encoded frame header + BGRA payload to a `FrameSink`.
|
|
10
|
+
public final class FrameWriter {
|
|
11
|
+
private let sink: FrameSink
|
|
12
|
+
private let startTime: Date
|
|
13
|
+
|
|
14
|
+
public init(sink: FrameSink, startTime: Date = Date()) {
|
|
15
|
+
self.sink = sink
|
|
16
|
+
self.startTime = startTime
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
public func write(
|
|
20
|
+
width: Int,
|
|
21
|
+
height: Int,
|
|
22
|
+
bytesPerRow: Int,
|
|
23
|
+
baseAddress: UnsafeRawPointer,
|
|
24
|
+
timestamp: Date = Date()
|
|
25
|
+
) {
|
|
26
|
+
let elapsedMs = max(0, timestamp.timeIntervalSince(startTime) * 1000)
|
|
27
|
+
let header = FrameProtocol.Header(
|
|
28
|
+
width: UInt32(width),
|
|
29
|
+
height: UInt32(height),
|
|
30
|
+
bytesPerRow: UInt32(bytesPerRow),
|
|
31
|
+
timestampMs: UInt32(truncatingIfNeeded: UInt64(elapsedMs))
|
|
32
|
+
)
|
|
33
|
+
sink.write(FrameProtocol.encodeHeader(header))
|
|
34
|
+
// bytesNoCopy avoids copying the pixel buffer into a transient Data.
|
|
35
|
+
// The caller (DeviceCaptureSession.captureOutput) holds the underlying
|
|
36
|
+
// CVPixelBuffer locked for the duration of this synchronous write.
|
|
37
|
+
let payload = Data(
|
|
38
|
+
bytesNoCopy: UnsafeMutableRawPointer(mutating: baseAddress),
|
|
39
|
+
count: bytesPerRow * height,
|
|
40
|
+
deallocator: .none
|
|
41
|
+
)
|
|
42
|
+
sink.write(payload)
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
public func writeAudio(pcm16le: Data) {
|
|
46
|
+
sink.write(FrameProtocol.encodeAudioHeader(payloadLength: pcm16le.count))
|
|
47
|
+
sink.write(pcm16le)
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/// Sink that writes to a `FileHandle` (e.g. `FileHandle.standardOutput`).
|
|
52
|
+
public final class FileHandleFrameSink: FrameSink {
|
|
53
|
+
private let handle: FileHandle
|
|
54
|
+
|
|
55
|
+
public init(handle: FileHandle) {
|
|
56
|
+
self.handle = handle
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
public func write(_ data: Data) {
|
|
60
|
+
handle.write(data)
|
|
61
|
+
}
|
|
62
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
|
|
3
|
+
/// Serializable description of an iOS Simulator window discovered via
|
|
4
|
+
/// ScreenCaptureKit. Emitted as JSON by `screen-capture-helper --list-simulators`.
|
|
5
|
+
public struct SimulatorWindowInfo: Codable, Equatable {
|
|
6
|
+
public let windowID: UInt32
|
|
7
|
+
public let title: String?
|
|
8
|
+
public let applicationName: String
|
|
9
|
+
public let bundleIdentifier: String
|
|
10
|
+
public let processID: Int32
|
|
11
|
+
public let width: Int
|
|
12
|
+
public let height: Int
|
|
13
|
+
|
|
14
|
+
public init(
|
|
15
|
+
windowID: UInt32,
|
|
16
|
+
title: String?,
|
|
17
|
+
applicationName: String,
|
|
18
|
+
bundleIdentifier: String,
|
|
19
|
+
processID: Int32,
|
|
20
|
+
width: Int,
|
|
21
|
+
height: Int
|
|
22
|
+
) {
|
|
23
|
+
self.windowID = windowID
|
|
24
|
+
self.title = title
|
|
25
|
+
self.applicationName = applicationName
|
|
26
|
+
self.bundleIdentifier = bundleIdentifier
|
|
27
|
+
self.processID = processID
|
|
28
|
+
self.width = width
|
|
29
|
+
self.height = height
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
public struct SimulatorWindowListResponse: Codable, Equatable {
|
|
34
|
+
public let windows: [SimulatorWindowInfo]
|
|
35
|
+
|
|
36
|
+
public init(windows: [SimulatorWindowInfo]) {
|
|
37
|
+
self.windows = windows
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/// Bundle identifier of the iOS Simulator host application on macOS.
|
|
42
|
+
public let simulatorBundleIdentifier = "com.apple.iphonesimulator"
|
|
43
|
+
|
|
44
|
+
/// ScreenCaptureKit cannot isolate a single Simulator window's audio from the
|
|
45
|
+
/// Simulator host application, so audio capture is safe only with one window.
|
|
46
|
+
public enum SimulatorAudioCaptureAvailability {
|
|
47
|
+
public static func errorMessage(for visibleSimulatorWindows: [SimulatorWindowInfo]) -> String? {
|
|
48
|
+
guard visibleSimulatorWindows.count > 1 else { return nil }
|
|
49
|
+
return "iOS Simulator audio capture requires exactly one visible Simulator window because ScreenCaptureKit cannot isolate audio to a selected Simulator window. Close other Simulator windows and try again."
|
|
50
|
+
}
|
|
51
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import AudioToolbox
|
|
2
|
+
import CoreMedia
|
|
3
|
+
import Foundation
|
|
4
|
+
import ScreenCaptureCore
|
|
5
|
+
|
|
6
|
+
/// ScreenCaptureKit is configured for 8 kHz mono output. Convert its common
|
|
7
|
+
/// Float32 or signed-16-bit linear PCM representation to signed PCM16LE.
|
|
8
|
+
func pcm16leAudio(sampleBuffer: CMSampleBuffer) -> Data? {
|
|
9
|
+
guard let format = CMSampleBufferGetFormatDescription(sampleBuffer),
|
|
10
|
+
let asbd = CMAudioFormatDescriptionGetStreamBasicDescription(format)?.pointee,
|
|
11
|
+
asbd.mSampleRate == 8_000,
|
|
12
|
+
asbd.mChannelsPerFrame == 1,
|
|
13
|
+
asbd.mFormatID == kAudioFormatLinearPCM
|
|
14
|
+
else { return nil }
|
|
15
|
+
|
|
16
|
+
var audioBufferList = AudioBufferList()
|
|
17
|
+
var blockBuffer: CMBlockBuffer?
|
|
18
|
+
let result = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(
|
|
19
|
+
sampleBuffer,
|
|
20
|
+
bufferListSizeNeededOut: nil,
|
|
21
|
+
bufferListOut: &audioBufferList,
|
|
22
|
+
bufferListSize: MemoryLayout<AudioBufferList>.size,
|
|
23
|
+
blockBufferAllocator: nil,
|
|
24
|
+
blockBufferMemoryAllocator: nil,
|
|
25
|
+
flags: 0,
|
|
26
|
+
blockBufferOut: &blockBuffer
|
|
27
|
+
)
|
|
28
|
+
guard result == noErr, let input = audioBufferList.mBuffers.mData else { return nil }
|
|
29
|
+
|
|
30
|
+
let sampleCount = CMSampleBufferGetNumSamples(sampleBuffer)
|
|
31
|
+
if asbd.mBitsPerChannel == 16 {
|
|
32
|
+
return Data(bytes: input, count: sampleCount * MemoryLayout<Int16>.size)
|
|
33
|
+
}
|
|
34
|
+
if asbd.mBitsPerChannel == 32,
|
|
35
|
+
(asbd.mFormatFlags & kAudioFormatFlagIsFloat) != 0 {
|
|
36
|
+
return AudioPcm16Encoder.encodeFloat32LE(
|
|
37
|
+
Data(bytes: input, count: sampleCount * MemoryLayout<Float>.size)
|
|
38
|
+
)
|
|
39
|
+
}
|
|
40
|
+
return nil
|
|
41
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import CoreMediaIO
|
|
2
|
+
import Foundation
|
|
3
|
+
|
|
4
|
+
/// Enables iOS device screen-capture devices to appear in AVFoundation. This
|
|
5
|
+
/// is the same toggle QuickTime Player flips when you select an iOS device.
|
|
6
|
+
///
|
|
7
|
+
/// Per design doc: rapidly toggling this property can stall device discovery
|
|
8
|
+
/// for up to 60 seconds, so we only flip it to `true` and leave it.
|
|
9
|
+
enum CMIOSystem {
|
|
10
|
+
static func enableScreenCaptureDevices() {
|
|
11
|
+
var prop = CMIOObjectPropertyAddress(
|
|
12
|
+
mSelector: CMIOObjectPropertySelector(kCMIOHardwarePropertyAllowScreenCaptureDevices),
|
|
13
|
+
mScope: CMIOObjectPropertyScope(kCMIOObjectPropertyScopeGlobal),
|
|
14
|
+
mElement: CMIOObjectPropertyElement(kCMIOObjectPropertyElementMain)
|
|
15
|
+
)
|
|
16
|
+
var allow: UInt32 = 1
|
|
17
|
+
CMIOObjectSetPropertyData(
|
|
18
|
+
CMIOObjectID(kCMIOObjectSystemObject),
|
|
19
|
+
&prop,
|
|
20
|
+
0,
|
|
21
|
+
nil,
|
|
22
|
+
UInt32(MemoryLayout<UInt32>.size),
|
|
23
|
+
&allow
|
|
24
|
+
)
|
|
25
|
+
}
|
|
26
|
+
}
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import AVFoundation
|
|
2
|
+
import CoreVideo
|
|
3
|
+
import Foundation
|
|
4
|
+
import ScreenCaptureCore
|
|
5
|
+
|
|
6
|
+
/// Wraps an `AVCaptureSession` configured to deliver BGRA frames from an iOS
|
|
7
|
+
/// device to a `FrameWriter`.
|
|
8
|
+
final class DeviceCaptureSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
|
|
9
|
+
private let session = AVCaptureSession()
|
|
10
|
+
private let output = AVCaptureVideoDataOutput()
|
|
11
|
+
private let writer: FrameWriter
|
|
12
|
+
private let onFatalError: (Error) -> Void
|
|
13
|
+
private let queue = DispatchQueue(label: "automobile.screen-capture.frames")
|
|
14
|
+
private var observers: [NSObjectProtocol] = []
|
|
15
|
+
private var stopping = false
|
|
16
|
+
|
|
17
|
+
init(writer: FrameWriter, onFatalError: @escaping (Error) -> Void) {
|
|
18
|
+
self.writer = writer
|
|
19
|
+
self.onFatalError = onFatalError
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
deinit {
|
|
23
|
+
removeObservers()
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
func start(device: AVCaptureDevice) throws {
|
|
27
|
+
let input = try AVCaptureDeviceInput(device: device)
|
|
28
|
+
|
|
29
|
+
session.beginConfiguration()
|
|
30
|
+
guard session.canAddInput(input) else {
|
|
31
|
+
session.commitConfiguration()
|
|
32
|
+
throw CaptureError.couldNotAddInput
|
|
33
|
+
}
|
|
34
|
+
session.addInput(input)
|
|
35
|
+
output.alwaysDiscardsLateVideoFrames = true
|
|
36
|
+
output.videoSettings = [
|
|
37
|
+
kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA
|
|
38
|
+
]
|
|
39
|
+
output.setSampleBufferDelegate(self, queue: queue)
|
|
40
|
+
guard session.canAddOutput(output) else {
|
|
41
|
+
session.commitConfiguration()
|
|
42
|
+
throw CaptureError.couldNotAddOutput
|
|
43
|
+
}
|
|
44
|
+
session.addOutput(output)
|
|
45
|
+
session.commitConfiguration()
|
|
46
|
+
installObservers()
|
|
47
|
+
session.startRunning()
|
|
48
|
+
guard session.isRunning else {
|
|
49
|
+
throw CaptureError.didNotStart
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
func stop() {
|
|
54
|
+
stopping = true
|
|
55
|
+
removeObservers()
|
|
56
|
+
session.stopRunning()
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
private func installObservers() {
|
|
60
|
+
let center = NotificationCenter.default
|
|
61
|
+
observers = [
|
|
62
|
+
center.addObserver(
|
|
63
|
+
forName: AVCaptureSession.runtimeErrorNotification,
|
|
64
|
+
object: session,
|
|
65
|
+
queue: .main
|
|
66
|
+
) { [weak self] notification in
|
|
67
|
+
let error = notification.userInfo?[AVCaptureSessionErrorKey] as? Error
|
|
68
|
+
?? CaptureError.runtimeFailure
|
|
69
|
+
self?.reportFatal(error)
|
|
70
|
+
},
|
|
71
|
+
center.addObserver(
|
|
72
|
+
forName: AVCaptureSession.wasInterruptedNotification,
|
|
73
|
+
object: session,
|
|
74
|
+
queue: .main
|
|
75
|
+
) { [weak self] _ in
|
|
76
|
+
self?.reportFatal(CaptureError.interrupted)
|
|
77
|
+
},
|
|
78
|
+
]
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
private func removeObservers() {
|
|
82
|
+
let center = NotificationCenter.default
|
|
83
|
+
observers.forEach(center.removeObserver)
|
|
84
|
+
observers = []
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
private func reportFatal(_ error: Error) {
|
|
88
|
+
guard !stopping else { return }
|
|
89
|
+
onFatalError(error)
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
private enum CaptureError: LocalizedError {
|
|
93
|
+
case couldNotAddInput
|
|
94
|
+
case couldNotAddOutput
|
|
95
|
+
case didNotStart
|
|
96
|
+
case runtimeFailure
|
|
97
|
+
case interrupted
|
|
98
|
+
|
|
99
|
+
var errorDescription: String? {
|
|
100
|
+
switch self {
|
|
101
|
+
case .couldNotAddInput: return "Unable to add the iOS capture input."
|
|
102
|
+
case .couldNotAddOutput: return "Unable to add the iOS capture output."
|
|
103
|
+
case .didNotStart: return "The iOS capture session did not start."
|
|
104
|
+
case .runtimeFailure: return "The iOS capture session reported a runtime failure."
|
|
105
|
+
case .interrupted: return "The iOS capture session was interrupted."
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
func captureOutput(
|
|
111
|
+
_ output: AVCaptureOutput,
|
|
112
|
+
didOutput sampleBuffer: CMSampleBuffer,
|
|
113
|
+
from connection: AVCaptureConnection
|
|
114
|
+
) {
|
|
115
|
+
writer.write(sampleBuffer: sampleBuffer)
|
|
116
|
+
}
|
|
117
|
+
}
|