@mindexec/cli 0.2.441 → 0.2.443
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/remote-fast/osx-arm64/MacScreenCaptureKitH264.swift +339 -0
- package/remote-fast/osx-arm64/MacSystemAudioCapture.swift +214 -0
- package/remote-fast/osx-arm64/UnixRawCaptureHelper.mjs +80 -0
- package/remote-fast/osx-arm64/livedesk-raw-capture-resampler.mjs +73 -0
- package/remote-fast/osx-arm64/mindexec-remote-fast.dll +0 -0
- package/remote-fast/osx-x64/MacScreenCaptureKitH264.swift +339 -0
- package/remote-fast/osx-x64/MacSystemAudioCapture.swift +214 -0
- package/remote-fast/osx-x64/UnixRawCaptureHelper.mjs +80 -0
- package/remote-fast/osx-x64/livedesk-raw-capture-resampler.mjs +73 -0
- package/remote-fast/osx-x64/mindexec-remote-fast.dll +0 -0
- package/remote-fast/win-x64/NAudio.Asio.dll +0 -0
- package/remote-fast/win-x64/NAudio.Core.dll +0 -0
- package/remote-fast/win-x64/NAudio.Midi.dll +0 -0
- package/remote-fast/win-x64/NAudio.Wasapi.dll +0 -0
- package/remote-fast/win-x64/NAudio.WinForms.dll +0 -0
- package/remote-fast/win-x64/NAudio.WinMM.dll +0 -0
- package/remote-fast/win-x64/NAudio.dll +0 -0
- package/remote-fast/win-x64/SharpGen.Runtime.COM.dll +0 -0
- package/remote-fast/win-x64/SharpGen.Runtime.dll +0 -0
- package/remote-fast/win-x64/Vortice.DXGI.dll +0 -0
- package/remote-fast/win-x64/Vortice.Direct3D11.dll +0 -0
- package/remote-fast/win-x64/Vortice.DirectX.dll +0 -0
- package/remote-fast/win-x64/Vortice.Mathematics.dll +0 -0
- package/remote-fast/win-x64/mindexec-remote-fast.deps.json +238 -0
- package/remote-fast/win-x64/mindexec-remote-fast.dll +0 -0
- package/remote-fast/win-x64/mindexec-remote-fast.exe +0 -0
- package/remote-hub.js +75 -5
- package/server.js +32 -7
- package/wwwroot/_content/MindExecution.Shared/js/mind-map-css3d-manager.js +55 -9
- package/wwwroot/_content/MindExecution.Shared/js/mind-map-remote-codecs.js +199 -0
- package/wwwroot/_framework/{MindExecution.Core.808jj176e1.dll → MindExecution.Core.4moacskf64.dll} +0 -0
- package/wwwroot/_framework/{MindExecution.Kernel.75hxk08tt0.dll → MindExecution.Kernel.pa251cfsz4.dll} +0 -0
- package/wwwroot/_framework/{MindExecution.Plugins.Admin.0zkmk8wahq.dll → MindExecution.Plugins.Admin.dxyuwmxv1s.dll} +0 -0
- package/wwwroot/_framework/{MindExecution.Plugins.Business.vyw3769hrw.dll → MindExecution.Plugins.Business.7uj5erqhhn.dll} +0 -0
- package/wwwroot/_framework/{MindExecution.Plugins.Concept.cvmglrxsjb.dll → MindExecution.Plugins.Concept.108rmsxc96.dll} +0 -0
- package/wwwroot/_framework/{MindExecution.Plugins.Directory.jtw9q1dm9n.dll → MindExecution.Plugins.Directory.ccsy3g0fe5.dll} +0 -0
- package/wwwroot/_framework/{MindExecution.Plugins.PlanMaster.8hmc6ps1ux.dll → MindExecution.Plugins.PlanMaster.1vjpuoe9ff.dll} +0 -0
- package/wwwroot/_framework/{MindExecution.Plugins.YouTube.xb9sou9ox0.dll → MindExecution.Plugins.YouTube.q8ml98phj9.dll} +0 -0
- package/wwwroot/_framework/{MindExecution.Shared.pqab76cjb6.dll → MindExecution.Shared.2pywq19zjy.dll} +0 -0
- package/wwwroot/_framework/{MindExecution.Web.mc24u4bzol.dll → MindExecution.Web.qucbls1wfr.dll} +0 -0
- package/wwwroot/_framework/blazor.boot.json +21 -21
- package/wwwroot/index.html +2 -1
- package/wwwroot/service-worker-assets.js +28 -24
- package/wwwroot/service-worker.js +1 -1
package/package.json
CHANGED
|
@@ -0,0 +1,339 @@
|
|
|
1
|
+
import CoreFoundation
|
|
2
|
+
import CoreGraphics
|
|
3
|
+
import CoreMedia
|
|
4
|
+
import CoreVideo
|
|
5
|
+
import Foundation
|
|
6
|
+
import IOSurface
|
|
7
|
+
import ScreenCaptureKit
|
|
8
|
+
import VideoToolbox
|
|
9
|
+
|
|
10
|
+
@available(macOS 12.3, *)
|
|
11
|
+
final class LiveDeskScreenCaptureKitEncoder: NSObject, SCStreamOutput, SCStreamDelegate {
|
|
12
|
+
private let displayIndex: Int
|
|
13
|
+
private let width: Int
|
|
14
|
+
private let height: Int
|
|
15
|
+
private let fps: Int
|
|
16
|
+
private let bitrateKbps: Int
|
|
17
|
+
private let queue = DispatchQueue(label: "dev.livedesk.screencapturekit.frames", qos: .userInteractive)
|
|
18
|
+
private let writeLock = NSLock()
|
|
19
|
+
private var stream: SCStream?
|
|
20
|
+
private var compressionSession: VTCompressionSession?
|
|
21
|
+
private var sampleCount: Int = 0
|
|
22
|
+
private var frameCount: Int = 0
|
|
23
|
+
private var skippedCount: Int = 0
|
|
24
|
+
private var encodedCount: Int = 0
|
|
25
|
+
private var encodedKeyCount: Int = 0
|
|
26
|
+
private var encodedDeltaCount: Int = 0
|
|
27
|
+
private var loggedInputSurface = false
|
|
28
|
+
|
|
29
|
+
init(displayIndex: Int, width: Int, height: Int, fps: Int, bitrateKbps: Int) {
|
|
30
|
+
self.displayIndex = max(0, displayIndex)
|
|
31
|
+
self.width = max(2, width)
|
|
32
|
+
self.height = max(2, height)
|
|
33
|
+
self.fps = max(1, fps)
|
|
34
|
+
self.bitrateKbps = max(500, bitrateKbps)
|
|
35
|
+
super.init()
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
func start() async throws {
|
|
39
|
+
try createCompressionSession()
|
|
40
|
+
|
|
41
|
+
let shareableContent = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true)
|
|
42
|
+
guard !shareableContent.displays.isEmpty else {
|
|
43
|
+
throw LiveDeskSCKError.noDisplays
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
let mainDisplayID = CGMainDisplayID()
|
|
47
|
+
let orderedDisplays = shareableContent.displays.sorted { left, right in
|
|
48
|
+
let leftIsMain = left.displayID == mainDisplayID
|
|
49
|
+
let rightIsMain = right.displayID == mainDisplayID
|
|
50
|
+
if leftIsMain != rightIsMain {
|
|
51
|
+
return leftIsMain
|
|
52
|
+
}
|
|
53
|
+
return left.displayID < right.displayID
|
|
54
|
+
}
|
|
55
|
+
let display = orderedDisplays[min(displayIndex, orderedDisplays.count - 1)]
|
|
56
|
+
let filter = SCContentFilter(display: display, excludingWindows: [])
|
|
57
|
+
let configuration = SCStreamConfiguration()
|
|
58
|
+
configuration.width = width
|
|
59
|
+
configuration.height = height
|
|
60
|
+
configuration.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale(fps))
|
|
61
|
+
configuration.queueDepth = 3
|
|
62
|
+
configuration.showsCursor = false
|
|
63
|
+
configuration.pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
|
|
64
|
+
|
|
65
|
+
let stream = SCStream(filter: filter, configuration: configuration, delegate: self)
|
|
66
|
+
try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: queue)
|
|
67
|
+
self.stream = stream
|
|
68
|
+
try await stream.startCapture()
|
|
69
|
+
|
|
70
|
+
fputs("LiveDeskSCK started displayIndex=\(displayIndex) displayID=\(display.displayID) size=\(width)x\(height) fps=\(fps) bitrateKbps=\(bitrateKbps)\n", stderr)
|
|
71
|
+
while !Task.isCancelled {
|
|
72
|
+
try await Task.sleep(nanoseconds: 1_000_000_000)
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
private func createCompressionSession() throws {
|
|
77
|
+
var session: VTCompressionSession?
|
|
78
|
+
let encoderSpecification: [CFString: Any] = [
|
|
79
|
+
kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder: true
|
|
80
|
+
]
|
|
81
|
+
let imageBufferAttributes: [CFString: Any] = [
|
|
82
|
+
kCVPixelBufferPixelFormatTypeKey: NSNumber(value: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange),
|
|
83
|
+
kCVPixelBufferWidthKey: NSNumber(value: width),
|
|
84
|
+
kCVPixelBufferHeightKey: NSNumber(value: height),
|
|
85
|
+
kCVPixelBufferMetalCompatibilityKey: true,
|
|
86
|
+
kCVPixelBufferIOSurfacePropertiesKey: [:] as CFDictionary
|
|
87
|
+
]
|
|
88
|
+
let status = VTCompressionSessionCreate(
|
|
89
|
+
allocator: kCFAllocatorDefault,
|
|
90
|
+
width: Int32(width),
|
|
91
|
+
height: Int32(height),
|
|
92
|
+
codecType: kCMVideoCodecType_H264,
|
|
93
|
+
encoderSpecification: encoderSpecification as CFDictionary,
|
|
94
|
+
imageBufferAttributes: imageBufferAttributes as CFDictionary,
|
|
95
|
+
compressedDataAllocator: nil,
|
|
96
|
+
outputCallback: Self.compressionCallback,
|
|
97
|
+
refcon: UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque()),
|
|
98
|
+
compressionSessionOut: &session)
|
|
99
|
+
|
|
100
|
+
guard status == noErr, let session else {
|
|
101
|
+
throw LiveDeskSCKError.encoderCreate(status)
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_RealTime, value: kCFBooleanTrue)
|
|
105
|
+
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_PrioritizeEncodingSpeedOverQuality, value: kCFBooleanTrue)
|
|
106
|
+
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AllowTemporalCompression, value: kCFBooleanTrue)
|
|
107
|
+
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AllowFrameReordering, value: kCFBooleanFalse)
|
|
108
|
+
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxFrameDelayCount, value: NSNumber(value: 0))
|
|
109
|
+
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ProfileLevel, value: kVTProfileLevel_H264_Baseline_AutoLevel)
|
|
110
|
+
let recoveryKeyInterval = max(1, fps / 2)
|
|
111
|
+
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameInterval, value: NSNumber(value: recoveryKeyInterval))
|
|
112
|
+
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, value: NSNumber(value: 0.5))
|
|
113
|
+
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ExpectedFrameRate, value: NSNumber(value: fps))
|
|
114
|
+
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AverageBitRate, value: NSNumber(value: bitrateKbps * 1000))
|
|
115
|
+
VTCompressionSessionPrepareToEncodeFrames(session)
|
|
116
|
+
compressionSession = session
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
private static let compressionCallback: VTCompressionOutputCallback = { refCon, _, status, _, sampleBuffer in
|
|
120
|
+
guard status == noErr,
|
|
121
|
+
let refCon,
|
|
122
|
+
let sampleBuffer,
|
|
123
|
+
CMSampleBufferDataIsReady(sampleBuffer) else {
|
|
124
|
+
return
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
let encoder = Unmanaged<LiveDeskScreenCaptureKitEncoder>.fromOpaque(refCon).takeUnretainedValue()
|
|
128
|
+
encoder.writeEncodedSampleBuffer(sampleBuffer)
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {
|
|
132
|
+
guard type == .screen,
|
|
133
|
+
CMSampleBufferDataIsReady(sampleBuffer),
|
|
134
|
+
let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer),
|
|
135
|
+
let session = compressionSession else {
|
|
136
|
+
return
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
sampleCount += 1
|
|
140
|
+
guard isEncodableFrame(sampleBuffer) else {
|
|
141
|
+
skippedCount += 1
|
|
142
|
+
logCaptureProgressIfNeeded()
|
|
143
|
+
return
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
frameCount += 1
|
|
147
|
+
if !loggedInputSurface {
|
|
148
|
+
loggedInputSurface = true
|
|
149
|
+
let pixelFormat = CVPixelBufferGetPixelFormatType(imageBuffer)
|
|
150
|
+
let hasIOSurface = CVPixelBufferGetIOSurface(imageBuffer) != nil
|
|
151
|
+
fputs("LiveDeskSCK input pixelFormat=\(pixelFormat) iosurface=\(hasIOSurface ? 1 : 0) queueDepth=3 zeroCopyCandidate=\(hasIOSurface ? 1 : 0)\n", stderr)
|
|
152
|
+
}
|
|
153
|
+
let pts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
|
|
154
|
+
let duration = CMTime(value: 1, timescale: CMTimeScale(fps))
|
|
155
|
+
var frameProperties: CFDictionary?
|
|
156
|
+
if frameCount == 1 {
|
|
157
|
+
let properties = NSMutableDictionary()
|
|
158
|
+
properties[kVTEncodeFrameOptionKey_ForceKeyFrame] = kCFBooleanTrue
|
|
159
|
+
frameProperties = properties
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
let status = VTCompressionSessionEncodeFrame(
|
|
163
|
+
session,
|
|
164
|
+
imageBuffer: imageBuffer,
|
|
165
|
+
presentationTimeStamp: pts,
|
|
166
|
+
duration: duration,
|
|
167
|
+
frameProperties: frameProperties,
|
|
168
|
+
sourceFrameRefcon: nil,
|
|
169
|
+
infoFlagsOut: nil)
|
|
170
|
+
|
|
171
|
+
if status != noErr && (frameCount == 1 || frameCount % 120 == 0) {
|
|
172
|
+
fputs("LiveDeskSCK encode status=\(status) frames=\(frameCount)\n", stderr)
|
|
173
|
+
}
|
|
174
|
+
logCaptureProgressIfNeeded()
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
func stream(_ stream: SCStream, didStopWithError error: Error) {
|
|
178
|
+
fputs("LiveDeskSCK stream stopped: \(error)\n", stderr)
|
|
179
|
+
exit(3)
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
private func isEncodableFrame(_ sampleBuffer: CMSampleBuffer) -> Bool {
|
|
183
|
+
guard let attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, createIfNecessary: false) as NSArray?,
|
|
184
|
+
let first = attachments.firstObject as? NSDictionary,
|
|
185
|
+
let rawStatus = first[SCStreamFrameInfo.status] as? Int else {
|
|
186
|
+
return true
|
|
187
|
+
}
|
|
188
|
+
return rawStatus == SCFrameStatus.complete.rawValue
|
|
189
|
+
|| rawStatus == SCFrameStatus.idle.rawValue
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
private func logCaptureProgressIfNeeded() {
|
|
193
|
+
let interval = max(1, fps * 5)
|
|
194
|
+
if sampleCount == 1 || sampleCount % interval == 0 {
|
|
195
|
+
fputs("LiveDeskSCK capture samples=\(sampleCount) complete=\(frameCount) skipped=\(skippedCount)\n", stderr)
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
private func writeEncodedSampleBuffer(_ sampleBuffer: CMSampleBuffer) {
|
|
200
|
+
guard let blockBuffer = CMSampleBufferGetDataBuffer(sampleBuffer) else {
|
|
201
|
+
return
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
let keyFrame = isKeyFrame(sampleBuffer)
|
|
205
|
+
var output = Data()
|
|
206
|
+
appendAccessUnitDelimiter(to: &output)
|
|
207
|
+
if keyFrame, let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer) {
|
|
208
|
+
appendParameterSet(formatDescription, index: 0, to: &output)
|
|
209
|
+
appendParameterSet(formatDescription, index: 1, to: &output)
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
var lengthAtOffset = 0
|
|
213
|
+
var totalLength = 0
|
|
214
|
+
var dataPointer: UnsafeMutablePointer<Int8>?
|
|
215
|
+
let status = CMBlockBufferGetDataPointer(
|
|
216
|
+
blockBuffer,
|
|
217
|
+
atOffset: 0,
|
|
218
|
+
lengthAtOffsetOut: &lengthAtOffset,
|
|
219
|
+
totalLengthOut: &totalLength,
|
|
220
|
+
dataPointerOut: &dataPointer)
|
|
221
|
+
|
|
222
|
+
guard status == noErr, let dataPointer, totalLength > 4 else {
|
|
223
|
+
return
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
var offset = 0
|
|
227
|
+
while offset + 4 <= totalLength {
|
|
228
|
+
var nalLength: UInt32 = 0
|
|
229
|
+
memcpy(&nalLength, dataPointer.advanced(by: offset), 4)
|
|
230
|
+
nalLength = CFSwapInt32BigToHost(nalLength)
|
|
231
|
+
let length = Int(nalLength)
|
|
232
|
+
if length <= 0 || offset + 4 + length > totalLength {
|
|
233
|
+
break
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
appendStartCode(to: &output)
|
|
237
|
+
output.append(UnsafeRawPointer(dataPointer.advanced(by: offset + 4)).assumingMemoryBound(to: UInt8.self), count: length)
|
|
238
|
+
offset += 4 + length
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
guard !output.isEmpty else {
|
|
242
|
+
return
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
writeLock.lock()
|
|
246
|
+
FileHandle.standardOutput.write(output)
|
|
247
|
+
writeLock.unlock()
|
|
248
|
+
|
|
249
|
+
encodedCount += 1
|
|
250
|
+
if keyFrame {
|
|
251
|
+
encodedKeyCount += 1
|
|
252
|
+
} else {
|
|
253
|
+
encodedDeltaCount += 1
|
|
254
|
+
}
|
|
255
|
+
if encodedCount == 1 || encodedCount % max(1, fps * 5) == 0 {
|
|
256
|
+
fputs("LiveDeskSCK encoded frames=\(encodedCount) key=\(encodedKeyCount) delta=\(encodedDeltaCount) bytes=\(output.count)\n", stderr)
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
private func isKeyFrame(_ sampleBuffer: CMSampleBuffer) -> Bool {
|
|
261
|
+
guard let attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, createIfNecessary: false) as NSArray?,
|
|
262
|
+
let first = attachments.firstObject as? NSDictionary else {
|
|
263
|
+
return true
|
|
264
|
+
}
|
|
265
|
+
return !(first[kCMSampleAttachmentKey_NotSync] as? Bool ?? false)
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
private func appendParameterSet(_ formatDescription: CMFormatDescription, index: Int, to output: inout Data) {
|
|
269
|
+
var pointer: UnsafePointer<UInt8>?
|
|
270
|
+
var size = 0
|
|
271
|
+
var count = 0
|
|
272
|
+
var nalHeaderLength: Int32 = 0
|
|
273
|
+
let status = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(
|
|
274
|
+
formatDescription,
|
|
275
|
+
parameterSetIndex: index,
|
|
276
|
+
parameterSetPointerOut: &pointer,
|
|
277
|
+
parameterSetSizeOut: &size,
|
|
278
|
+
parameterSetCountOut: &count,
|
|
279
|
+
nalUnitHeaderLengthOut: &nalHeaderLength)
|
|
280
|
+
|
|
281
|
+
guard status == noErr, let pointer, size > 0 else {
|
|
282
|
+
return
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
appendStartCode(to: &output)
|
|
286
|
+
output.append(pointer, count: size)
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
private func appendStartCode(to output: inout Data) {
|
|
290
|
+
output.append(contentsOf: [0, 0, 0, 1])
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
private func appendAccessUnitDelimiter(to output: inout Data) {
|
|
294
|
+
appendStartCode(to: &output)
|
|
295
|
+
output.append(contentsOf: [0x09, 0xF0])
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
enum LiveDeskSCKError: Error {
|
|
300
|
+
case noDisplays
|
|
301
|
+
case encoderCreate(OSStatus)
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
func intArg(_ name: String, default defaultValue: Int) -> Int {
|
|
305
|
+
let args = CommandLine.arguments
|
|
306
|
+
guard let index = args.firstIndex(of: name), index + 1 < args.count else {
|
|
307
|
+
return defaultValue
|
|
308
|
+
}
|
|
309
|
+
return Int(args[index + 1]) ?? defaultValue
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
@main
|
|
313
|
+
struct LiveDeskSCKMain {
|
|
314
|
+
static func main() async {
|
|
315
|
+
guard #available(macOS 12.3, *) else {
|
|
316
|
+
fputs("LiveDeskSCK requires macOS 12.3 or later.\n", stderr)
|
|
317
|
+
exit(2)
|
|
318
|
+
}
|
|
319
|
+
guard CGPreflightScreenCaptureAccess() else {
|
|
320
|
+
fputs("LiveDeskSCK screen recording permission is not granted.\n", stderr)
|
|
321
|
+
exit(10)
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
let displayIndex = intArg("--display-index", default: 0)
|
|
325
|
+
let width = intArg("--width", default: 960)
|
|
326
|
+
let height = intArg("--height", default: 540)
|
|
327
|
+
let fps = intArg("--fps", default: 30)
|
|
328
|
+
let bitrateKbps = intArg("--bitrate-kbps", default: max(3500, width * height * fps / 4000))
|
|
329
|
+
|
|
330
|
+
do {
|
|
331
|
+
let encoder = LiveDeskScreenCaptureKitEncoder(displayIndex: displayIndex, width: width, height: height, fps: fps, bitrateKbps: bitrateKbps)
|
|
332
|
+
try await encoder.start()
|
|
333
|
+
} catch {
|
|
334
|
+
fputs("LiveDeskSCK failed: \(error)\n", stderr)
|
|
335
|
+
exit(1)
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
import AudioToolbox
|
|
2
|
+
import AVFAudio
|
|
3
|
+
import CoreGraphics
|
|
4
|
+
import CoreMedia
|
|
5
|
+
import Foundation
|
|
6
|
+
import ScreenCaptureKit
|
|
7
|
+
|
|
8
|
+
@available(macOS 13.0, *)
|
|
9
|
+
final class LiveDeskSystemAudioCapture: NSObject, SCStreamOutput, SCStreamDelegate {
|
|
10
|
+
private let sampleRate: Int
|
|
11
|
+
private let channelCount: Int
|
|
12
|
+
private let queue = DispatchQueue(label: "dev.livedesk.screencapturekit.audio", qos: .userInteractive)
|
|
13
|
+
private let writeLock = NSLock()
|
|
14
|
+
private var stream: SCStream?
|
|
15
|
+
private var audioFrameCount = 0
|
|
16
|
+
private var loggedFormat = false
|
|
17
|
+
|
|
18
|
+
init(sampleRate: Int, channelCount: Int) {
|
|
19
|
+
self.sampleRate = max(8_000, min(192_000, sampleRate))
|
|
20
|
+
self.channelCount = max(1, min(2, channelCount))
|
|
21
|
+
super.init()
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
func start() async throws {
|
|
25
|
+
let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true)
|
|
26
|
+
guard !content.displays.isEmpty else {
|
|
27
|
+
throw LiveDeskSystemAudioError.noDisplays
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
let mainDisplayID = CGMainDisplayID()
|
|
31
|
+
let display = content.displays.sorted { left, right in
|
|
32
|
+
let leftIsMain = left.displayID == mainDisplayID
|
|
33
|
+
let rightIsMain = right.displayID == mainDisplayID
|
|
34
|
+
if leftIsMain != rightIsMain {
|
|
35
|
+
return leftIsMain
|
|
36
|
+
}
|
|
37
|
+
return left.displayID < right.displayID
|
|
38
|
+
}[0]
|
|
39
|
+
|
|
40
|
+
let configuration = SCStreamConfiguration()
|
|
41
|
+
configuration.width = 2
|
|
42
|
+
configuration.height = 2
|
|
43
|
+
configuration.minimumFrameInterval = CMTime(value: 1, timescale: 1)
|
|
44
|
+
configuration.queueDepth = 8
|
|
45
|
+
configuration.capturesAudio = true
|
|
46
|
+
configuration.sampleRate = sampleRate
|
|
47
|
+
configuration.channelCount = channelCount
|
|
48
|
+
configuration.excludesCurrentProcessAudio = true
|
|
49
|
+
|
|
50
|
+
let stream = SCStream(filter: SCContentFilter(display: display, excludingWindows: []), configuration: configuration, delegate: self)
|
|
51
|
+
try stream.addStreamOutput(self, type: .audio, sampleHandlerQueue: queue)
|
|
52
|
+
self.stream = stream
|
|
53
|
+
try await stream.startCapture()
|
|
54
|
+
fputs("LiveDeskSCKAudio started displayID=\(display.displayID) sampleRate=\(sampleRate) channels=\(channelCount)\n", stderr)
|
|
55
|
+
|
|
56
|
+
while !Task.isCancelled {
|
|
57
|
+
try await Task.sleep(nanoseconds: 1_000_000_000)
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {
|
|
62
|
+
guard type == .audio, sampleBuffer.isValid, CMSampleBufferDataIsReady(sampleBuffer) else {
|
|
63
|
+
return
|
|
64
|
+
}
|
|
65
|
+
writePcm(sampleBuffer)
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
func stream(_ stream: SCStream, didStopWithError error: Error) {
|
|
69
|
+
fputs("LiveDeskSCKAudio stream stopped: \(error)\n", stderr)
|
|
70
|
+
exit(3)
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
private func writePcm(_ sampleBuffer: CMSampleBuffer) {
|
|
74
|
+
guard let description = sampleBuffer.formatDescription?.audioStreamBasicDescription else {
|
|
75
|
+
return
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
let channels = Int(description.mChannelsPerFrame)
|
|
79
|
+
let frames = Int(CMSampleBufferGetNumSamples(sampleBuffer))
|
|
80
|
+
let isFloat32 = description.mFormatID == kAudioFormatLinearPCM
|
|
81
|
+
&& description.mBitsPerChannel == 32
|
|
82
|
+
&& (description.mFormatFlags & kAudioFormatFlagIsFloat) != 0
|
|
83
|
+
let isInterleaved = (description.mFormatFlags & kAudioFormatFlagIsNonInterleaved) == 0
|
|
84
|
+
guard isFloat32, frames > 0, channels > 0, channels <= 8 else {
|
|
85
|
+
if !loggedFormat {
|
|
86
|
+
loggedFormat = true
|
|
87
|
+
fputs("LiveDeskSCKAudio unsupported PCM format id=\(description.mFormatID) flags=\(description.mFormatFlags) bits=\(description.mBitsPerChannel) channels=\(channels)\n", stderr)
|
|
88
|
+
}
|
|
89
|
+
return
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
do {
|
|
93
|
+
try sampleBuffer.withAudioBufferList { audioBufferList, _ in
|
|
94
|
+
let bytesPerSample = MemoryLayout<Float>.size
|
|
95
|
+
let bytesPerChannel = frames * bytesPerSample
|
|
96
|
+
let outputBytes = bytesPerChannel * channels
|
|
97
|
+
guard audioBufferList.count > 0, outputBytes > 0 else {
|
|
98
|
+
return
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
let output: Data
|
|
102
|
+
if isInterleaved {
|
|
103
|
+
let sourceBuffer = audioBufferList[0]
|
|
104
|
+
guard let source = sourceBuffer.mData, Int(sourceBuffer.mDataByteSize) >= outputBytes else {
|
|
105
|
+
return
|
|
106
|
+
}
|
|
107
|
+
output = Data(bytes: source, count: outputBytes)
|
|
108
|
+
} else {
|
|
109
|
+
guard audioBufferList.count >= channels else {
|
|
110
|
+
return
|
|
111
|
+
}
|
|
112
|
+
var interleaved = Data(count: outputBytes)
|
|
113
|
+
var copied = true
|
|
114
|
+
interleaved.withUnsafeMutableBytes { rawBuffer in
|
|
115
|
+
guard let destination = rawBuffer.baseAddress?.assumingMemoryBound(to: Float.self) else {
|
|
116
|
+
copied = false
|
|
117
|
+
return
|
|
118
|
+
}
|
|
119
|
+
for channelIndex in 0..<channels {
|
|
120
|
+
let sourceBuffer = audioBufferList[channelIndex]
|
|
121
|
+
guard let source = sourceBuffer.mData,
|
|
122
|
+
Int(sourceBuffer.mDataByteSize) >= bytesPerChannel else {
|
|
123
|
+
copied = false
|
|
124
|
+
return
|
|
125
|
+
}
|
|
126
|
+
let sourceSamples = source.assumingMemoryBound(to: Float.self)
|
|
127
|
+
for frameIndex in 0..<frames {
|
|
128
|
+
destination[frameIndex * channels + channelIndex] = sourceSamples[frameIndex]
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
guard copied else {
|
|
133
|
+
return
|
|
134
|
+
}
|
|
135
|
+
output = interleaved
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
if !loggedFormat {
|
|
139
|
+
loggedFormat = true
|
|
140
|
+
fputs("LiveDeskSCKAudio format sampleRate=\(Int(description.mSampleRate.rounded())) channels=\(channels) frames=\(frames) interleaved=\(isInterleaved ? 1 : 0)\n", stderr)
|
|
141
|
+
}
|
|
142
|
+
writePacket(output, sampleRate: Int(description.mSampleRate.rounded()), channels: channels)
|
|
143
|
+
audioFrameCount += 1
|
|
144
|
+
if audioFrameCount == 1 || audioFrameCount % 250 == 0 {
|
|
145
|
+
fputs("LiveDeskSCKAudio frames=\(audioFrameCount) bytes=\(output.count)\n", stderr)
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
} catch {
|
|
149
|
+
fputs("LiveDeskSCKAudio audio buffer error: \(error)\n", stderr)
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
private func writePacket(_ pcm: Data, sampleRate: Int, channels: Int) {
|
|
154
|
+
guard sampleRate > 0, channels > 0, pcm.count <= Int(UInt32.max) else {
|
|
155
|
+
return
|
|
156
|
+
}
|
|
157
|
+
var packet = Data()
|
|
158
|
+
packet.reserveCapacity(16 + pcm.count)
|
|
159
|
+
packet.append(contentsOf: [0x4c, 0x44, 0x41, 0x55])
|
|
160
|
+
appendLittleEndian(UInt16(1), to: &packet)
|
|
161
|
+
appendLittleEndian(UInt16(min(channels, Int(UInt16.max))), to: &packet)
|
|
162
|
+
appendLittleEndian(UInt32(min(sampleRate, Int(UInt32.max))), to: &packet)
|
|
163
|
+
appendLittleEndian(UInt32(pcm.count), to: &packet)
|
|
164
|
+
packet.append(pcm)
|
|
165
|
+
|
|
166
|
+
writeLock.lock()
|
|
167
|
+
FileHandle.standardOutput.write(packet)
|
|
168
|
+
writeLock.unlock()
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
private func appendLittleEndian<T: FixedWidthInteger>(_ value: T, to data: inout Data) {
|
|
172
|
+
var littleEndian = value.littleEndian
|
|
173
|
+
withUnsafeBytes(of: &littleEndian) { bytes in
|
|
174
|
+
data.append(contentsOf: bytes)
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
enum LiveDeskSystemAudioError: Error {
|
|
180
|
+
case noDisplays
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
func intArg(_ name: String, default defaultValue: Int) -> Int {
|
|
184
|
+
let args = CommandLine.arguments
|
|
185
|
+
guard let index = args.firstIndex(of: name), index + 1 < args.count else {
|
|
186
|
+
return defaultValue
|
|
187
|
+
}
|
|
188
|
+
return Int(args[index + 1]) ?? defaultValue
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
@main
|
|
192
|
+
struct LiveDeskSystemAudioMain {
|
|
193
|
+
static func main() async {
|
|
194
|
+
guard #available(macOS 13.0, *) else {
|
|
195
|
+
fputs("LiveDeskSCKAudio requires macOS 13 or later.\n", stderr)
|
|
196
|
+
exit(2)
|
|
197
|
+
}
|
|
198
|
+
guard CGPreflightScreenCaptureAccess() else {
|
|
199
|
+
fputs("LiveDeskSCKAudio Screen & System Audio Recording permission is not granted.\n", stderr)
|
|
200
|
+
exit(10)
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
let sampleRate = intArg("--sample-rate", default: 48_000)
|
|
204
|
+
let channels = intArg("--channels", default: 2)
|
|
205
|
+
do {
|
|
206
|
+
let capture = LiveDeskSystemAudioCapture(sampleRate: sampleRate, channelCount: channels)
|
|
207
|
+
try await capture.start()
|
|
208
|
+
} catch {
|
|
209
|
+
fputs("LiveDeskSCKAudio failed: \(error)\n", stderr)
|
|
210
|
+
exit(1)
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import readline from 'node:readline';
|
|
4
|
+
import { convertRgbaToRgb565 } from './livedesk-raw-capture-resampler.mjs';
|
|
5
|
+
|
|
6
|
+
const screenshotModule = await import('node-screenshots');
|
|
7
|
+
const Monitor = screenshotModule.Monitor || screenshotModule.default?.Monitor;
|
|
8
|
+
if (!Monitor?.all) {
|
|
9
|
+
throw new Error('node-screenshots Monitor API is unavailable.');
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
function clampInteger(value, min, max, fallback) {
|
|
13
|
+
const number = Number(value);
|
|
14
|
+
if (!Number.isFinite(number)) return fallback;
|
|
15
|
+
return Math.max(min, Math.min(max, Math.round(number)));
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function pickMonitor(index) {
|
|
19
|
+
const monitors = Monitor.all();
|
|
20
|
+
if (!monitors.length) {
|
|
21
|
+
throw new Error('No capturable desktop monitor was found.');
|
|
22
|
+
}
|
|
23
|
+
const selectedIndex = clampInteger(index, 0, monitors.length - 1, 0);
|
|
24
|
+
const monitor = monitors[selectedIndex]
|
|
25
|
+
|| monitors.find(item => item?.isPrimary?.() === true)
|
|
26
|
+
|| monitors[0];
|
|
27
|
+
return { monitor, selectedIndex, monitorCount: monitors.length };
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function writeResponse(metadata, payload = Buffer.alloc(0)) {
|
|
31
|
+
const header = Buffer.from(JSON.stringify(metadata), 'utf8');
|
|
32
|
+
const prefix = Buffer.allocUnsafe(8);
|
|
33
|
+
prefix.writeUInt32BE(header.length, 0);
|
|
34
|
+
prefix.writeUInt32BE(payload.length, 4);
|
|
35
|
+
process.stdout.write(prefix);
|
|
36
|
+
process.stdout.write(header);
|
|
37
|
+
if (payload.length > 0) process.stdout.write(payload);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const input = readline.createInterface({ input: process.stdin, crlfDelay: Infinity });
|
|
41
|
+
for await (const line of input) {
|
|
42
|
+
if (!line.trim()) continue;
|
|
43
|
+
let request;
|
|
44
|
+
try {
|
|
45
|
+
request = JSON.parse(line);
|
|
46
|
+
const id = Number.isSafeInteger(Number(request.id)) ? Number(request.id) : 0;
|
|
47
|
+
const maxWidth = clampInteger(request.maxWidth, 1, 480, 320);
|
|
48
|
+
const maxHeight = clampInteger(request.maxHeight, 1, 270, 180);
|
|
49
|
+
const { monitor, selectedIndex, monitorCount } = pickMonitor(request.monitorIndex);
|
|
50
|
+
const captureStartedAt = performance.now();
|
|
51
|
+
const image = await monitor.captureImage();
|
|
52
|
+
const raw = await image.toRaw(false);
|
|
53
|
+
const captureMs = Math.max(0, performance.now() - captureStartedAt);
|
|
54
|
+
const sourceWidth = Number(image.width || monitor.width?.() || 0);
|
|
55
|
+
const sourceHeight = Number(image.height || monitor.height?.() || 0);
|
|
56
|
+
const convertStartedAt = performance.now();
|
|
57
|
+
const converted = convertRgbaToRgb565(raw, sourceWidth, sourceHeight, maxWidth, maxHeight);
|
|
58
|
+
const convertMs = Math.max(0, performance.now() - convertStartedAt);
|
|
59
|
+
writeResponse({
|
|
60
|
+
ok: true,
|
|
61
|
+
id,
|
|
62
|
+
width: converted.width,
|
|
63
|
+
height: converted.height,
|
|
64
|
+
sourceWidth,
|
|
65
|
+
sourceHeight,
|
|
66
|
+
monitorIndex: selectedIndex,
|
|
67
|
+
monitorCount,
|
|
68
|
+
captureMs: Math.round(captureMs),
|
|
69
|
+
convertMs: Math.round(convertMs),
|
|
70
|
+
resizeFilter: converted.filter
|
|
71
|
+
}, converted.output);
|
|
72
|
+
} catch (error) {
|
|
73
|
+
writeResponse({
|
|
74
|
+
ok: false,
|
|
75
|
+
id: Number.isSafeInteger(Number(request?.id)) ? Number(request.id) : 0,
|
|
76
|
+
error: String(error?.message || error || 'capture failed').slice(0, 1000)
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|