@livedesk/client 0.1.71 → 0.1.73

package/fast/osx-arm64/MindExec.RemoteAgent.Fast/MacScreenCaptureKitH264.swift

@@ -0,0 +1,260 @@
+import CoreFoundation
+import CoreGraphics
+import CoreMedia
+import CoreVideo
+import Foundation
+import ScreenCaptureKit
+import VideoToolbox
+
+@available(macOS 12.3, *)
+final class LiveDeskScreenCaptureKitEncoder: NSObject, SCStreamOutput, SCStreamDelegate {
+    private let displayIndex: Int
+    private let width: Int
+    private let height: Int
+    private let fps: Int
+    private let queue = DispatchQueue(label: "dev.livedesk.screencapturekit.frames")
+    private let writeLock = NSLock()
+    private var stream: SCStream?
+    private var compressionSession: VTCompressionSession?
+    private var frameCount: Int = 0
+    private var encodedCount: Int = 0
+
+    init(displayIndex: Int, width: Int, height: Int, fps: Int) {
+        self.displayIndex = max(0, displayIndex)
+        self.width = max(2, width)
+        self.height = max(2, height)
+        self.fps = max(1, fps)
+        super.init()
+    }
+
+    func start() async throws {
+        try createCompressionSession()
+
+        let shareableContent = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true)
+        guard !shareableContent.displays.isEmpty else {
+            throw LiveDeskSCKError.noDisplays
+        }
+
+        let display = shareableContent.displays[min(displayIndex, shareableContent.displays.count - 1)]
+        let filter = SCContentFilter(display: display, excludingWindows: [])
+        let configuration = SCStreamConfiguration()
+        configuration.width = width
+        configuration.height = height
+        configuration.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale(fps))
+        configuration.queueDepth = 3
+        configuration.showsCursor = false
+        configuration.pixelFormat = kCVPixelFormatType_32BGRA
+
+        let stream = SCStream(filter: filter, configuration: configuration, delegate: self)
+        try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: queue)
+        self.stream = stream
+        try await stream.startCapture()
+
+        fputs("LiveDeskSCK started displayIndex=\(displayIndex) displayID=\(display.displayID) size=\(width)x\(height) fps=\(fps)\n", stderr)
+        RunLoop.main.run()
+    }
+
+    private func createCompressionSession() throws {
+        var session: VTCompressionSession?
+        let status = VTCompressionSessionCreate(
+            allocator: kCFAllocatorDefault,
+            width: Int32(width),
+            height: Int32(height),
+            codecType: kCMVideoCodecType_H264,
+            encoderSpecification: nil,
+            imageBufferAttributes: nil,
+            compressedDataAllocator: nil,
+            outputCallback: Self.compressionCallback,
+            refcon: UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque()),
+            compressionSessionOut: &session)
+
+        guard status == noErr, let session else {
+            throw LiveDeskSCKError.encoderCreate(status)
+        }
+
+        VTSessionSetProperty(session, key: kVTCompressionPropertyKey_RealTime, value: kCFBooleanTrue)
+        VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AllowFrameReordering, value: kCFBooleanFalse)
+        VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ProfileLevel, value: kVTProfileLevel_H264_Baseline_AutoLevel)
+        VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameInterval, value: NSNumber(value: fps))
+        VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, value: NSNumber(value: 1))
+        VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ExpectedFrameRate, value: NSNumber(value: fps))
+        VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AverageBitRate, value: NSNumber(value: max(2_500_000, width * height * fps / 6)))
+        VTCompressionSessionPrepareToEncodeFrames(session)
+        compressionSession = session
+    }
+
+    private static let compressionCallback: VTCompressionOutputCallback = { refCon, _, status, _, sampleBuffer in
+        guard status == noErr,
+              let refCon,
+              let sampleBuffer,
+              CMSampleBufferDataIsReady(sampleBuffer) else {
+            return
+        }
+
+        let encoder = Unmanaged<LiveDeskScreenCaptureKitEncoder>.fromOpaque(refCon).takeUnretainedValue()
+        encoder.writeEncodedSampleBuffer(sampleBuffer)
+    }
+
+    func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {
+        guard type == .screen,
+              CMSampleBufferDataIsReady(sampleBuffer),
+              isCompleteFrame(sampleBuffer),
+              let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer),
+              let session = compressionSession else {
+            return
+        }
+
+        frameCount += 1
+        let pts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
+        let duration = CMTime(value: 1, timescale: CMTimeScale(fps))
+        let status = VTCompressionSessionEncodeFrame(
+            session,
+            imageBuffer: imageBuffer,
+            presentationTimeStamp: pts,
+            duration: duration,
+            frameProperties: nil,
+            sourceFrameRefcon: nil,
+            infoFlagsOut: nil)
+
+        if status != noErr && (frameCount == 1 || frameCount % 120 == 0) {
+            fputs("LiveDeskSCK encode status=\(status) frames=\(frameCount)\n", stderr)
+        }
+    }
+
+    func stream(_ stream: SCStream, didStopWithError error: Error) {
+        fputs("LiveDeskSCK stream stopped: \(error)\n", stderr)
+        exit(3)
+    }
+
+    private func isCompleteFrame(_ sampleBuffer: CMSampleBuffer) -> Bool {
+        guard let attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, createIfNecessary: false) as NSArray?,
+              let first = attachments.firstObject as? NSDictionary,
+              let rawStatus = first[SCStreamFrameInfo.status] as? Int else {
+            return true
+        }
+        return rawStatus == SCFrameStatus.complete.rawValue
+    }
+
+    private func writeEncodedSampleBuffer(_ sampleBuffer: CMSampleBuffer) {
+        guard let blockBuffer = CMSampleBufferGetDataBuffer(sampleBuffer) else {
+            return
+        }
+
+        var output = Data()
+        if isKeyFrame(sampleBuffer), let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer) {
+            appendParameterSet(formatDescription, index: 0, to: &output)
+            appendParameterSet(formatDescription, index: 1, to: &output)
+        }
+
+        var lengthAtOffset = 0
+        var totalLength = 0
+        var dataPointer: UnsafeMutablePointer<Int8>?
+        let status = CMBlockBufferGetDataPointer(
+            blockBuffer,
+            atOffset: 0,
+            lengthAtOffsetOut: &lengthAtOffset,
+            totalLengthOut: &totalLength,
+            dataPointerOut: &dataPointer)
+
+        guard status == noErr, let dataPointer, totalLength > 4 else {
+            return
+        }
+
+        var offset = 0
+        while offset + 4 <= totalLength {
+            var nalLength: UInt32 = 0
+            memcpy(&nalLength, dataPointer.advanced(by: offset), 4)
+            nalLength = CFSwapInt32BigToHost(nalLength)
+            let length = Int(nalLength)
+            if length <= 0 || offset + 4 + length > totalLength {
+                break
+            }
+
+            appendStartCode(to: &output)
+            output.append(UnsafeRawPointer(dataPointer.advanced(by: offset + 4)).assumingMemoryBound(to: UInt8.self), count: length)
+            offset += 4 + length
+        }
+
+        guard !output.isEmpty else {
+            return
+        }
+
+        writeLock.lock()
+        FileHandle.standardOutput.write(output)
+        writeLock.unlock()
+
+        encodedCount += 1
+        if encodedCount == 1 || encodedCount % max(1, fps * 5) == 0 {
+            fputs("LiveDeskSCK encoded frames=\(encodedCount) bytes=\(output.count)\n", stderr)
+        }
+    }
+
+    private func isKeyFrame(_ sampleBuffer: CMSampleBuffer) -> Bool {
+        guard let attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, createIfNecessary: false) as NSArray?,
+              let first = attachments.firstObject as? NSDictionary else {
+            return true
+        }
+        return !(first[kCMSampleAttachmentKey_NotSync] as? Bool ?? false)
+    }
+
+    private func appendParameterSet(_ formatDescription: CMFormatDescription, index: Int, to output: inout Data) {
+        var pointer: UnsafePointer<UInt8>?
+        var size = 0
+        var count = 0
+        var nalHeaderLength: Int32 = 0
+        let status = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(
+            formatDescription,
+            parameterSetIndex: index,
+            parameterSetPointerOut: &pointer,
+            parameterSetSizeOut: &size,
+            parameterSetCountOut: &count,
+            nalUnitHeaderLengthOut: &nalHeaderLength)
+
+        guard status == noErr, let pointer, size > 0 else {
+            return
+        }
+
+        appendStartCode(to: &output)
+        output.append(pointer, count: size)
+    }
+
+    private func appendStartCode(to output: inout Data) {
+        output.append(contentsOf: [0, 0, 0, 1])
+    }
+}
+
+enum LiveDeskSCKError: Error {
+    case noDisplays
+    case encoderCreate(OSStatus)
+}
+
+func intArg(_ name: String, default defaultValue: Int) -> Int {
+    let args = CommandLine.arguments
+    guard let index = args.firstIndex(of: name), index + 1 < args.count else {
+        return defaultValue
+    }
+    return Int(args[index + 1]) ?? defaultValue
+}
+
+@main
+struct LiveDeskSCKMain {
+    static func main() async {
+        guard #available(macOS 12.3, *) else {
+            fputs("LiveDeskSCK requires macOS 12.3 or later.\n", stderr)
+            exit(2)
+        }
+
+        let displayIndex = intArg("--display-index", default: 0)
+        let width = intArg("--width", default: 960)
+        let height = intArg("--height", default: 540)
+        let fps = intArg("--fps", default: 30)
+
+        do {
+            let encoder = LiveDeskScreenCaptureKitEncoder(displayIndex: displayIndex, width: width, height: height, fps: fps)
+            try await encoder.start()
+        } catch {
+            fputs("LiveDeskSCK failed: \(error)\n", stderr)
+            exit(1)
+        }
+    }
+}

package/fast/osx-x64/MindExec.RemoteAgent.Fast/MacScreenCaptureKitH264.swift

@@ -0,0 +1,260 @@
+import CoreFoundation
+import CoreGraphics
+import CoreMedia
+import CoreVideo
+import Foundation
+import ScreenCaptureKit
+import VideoToolbox
+
+@available(macOS 12.3, *)
+final class LiveDeskScreenCaptureKitEncoder: NSObject, SCStreamOutput, SCStreamDelegate {
+    private let displayIndex: Int
+    private let width: Int
+    private let height: Int
+    private let fps: Int
+    private let queue = DispatchQueue(label: "dev.livedesk.screencapturekit.frames")
+    private let writeLock = NSLock()
+    private var stream: SCStream?
+    private var compressionSession: VTCompressionSession?
+    private var frameCount: Int = 0
+    private var encodedCount: Int = 0
+
+    init(displayIndex: Int, width: Int, height: Int, fps: Int) {
+        self.displayIndex = max(0, displayIndex)
+        self.width = max(2, width)
+        self.height = max(2, height)
+        self.fps = max(1, fps)
+        super.init()
+    }
+
+    func start() async throws {
+        try createCompressionSession()
+
+        let shareableContent = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true)
+        guard !shareableContent.displays.isEmpty else {
+            throw LiveDeskSCKError.noDisplays
+        }
+
+        let display = shareableContent.displays[min(displayIndex, shareableContent.displays.count - 1)]
+        let filter = SCContentFilter(display: display, excludingWindows: [])
+        let configuration = SCStreamConfiguration()
+        configuration.width = width
+        configuration.height = height
+        configuration.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale(fps))
+        configuration.queueDepth = 3
+        configuration.showsCursor = false
+        configuration.pixelFormat = kCVPixelFormatType_32BGRA
+
+        let stream = SCStream(filter: filter, configuration: configuration, delegate: self)
+        try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: queue)
+        self.stream = stream
+        try await stream.startCapture()
+
+        fputs("LiveDeskSCK started displayIndex=\(displayIndex) displayID=\(display.displayID) size=\(width)x\(height) fps=\(fps)\n", stderr)
+        RunLoop.main.run()
+    }
+
+    private func createCompressionSession() throws {
+        var session: VTCompressionSession?
+        let status = VTCompressionSessionCreate(
+            allocator: kCFAllocatorDefault,
+            width: Int32(width),
+            height: Int32(height),
+            codecType: kCMVideoCodecType_H264,
+            encoderSpecification: nil,
+            imageBufferAttributes: nil,
+            compressedDataAllocator: nil,
+            outputCallback: Self.compressionCallback,
+            refcon: UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque()),
+            compressionSessionOut: &session)
+
+        guard status == noErr, let session else {
+            throw LiveDeskSCKError.encoderCreate(status)
+        }
+
+        VTSessionSetProperty(session, key: kVTCompressionPropertyKey_RealTime, value: kCFBooleanTrue)
+        VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AllowFrameReordering, value: kCFBooleanFalse)
+        VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ProfileLevel, value: kVTProfileLevel_H264_Baseline_AutoLevel)
+        VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameInterval, value: NSNumber(value: fps))
+        VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, value: NSNumber(value: 1))
+        VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ExpectedFrameRate, value: NSNumber(value: fps))
+        VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AverageBitRate, value: NSNumber(value: max(2_500_000, width * height * fps / 6)))
+        VTCompressionSessionPrepareToEncodeFrames(session)
+        compressionSession = session
+    }
+
+    private static let compressionCallback: VTCompressionOutputCallback = { refCon, _, status, _, sampleBuffer in
+        guard status == noErr,
+              let refCon,
+              let sampleBuffer,
+              CMSampleBufferDataIsReady(sampleBuffer) else {
+            return
+        }
+
+        let encoder = Unmanaged<LiveDeskScreenCaptureKitEncoder>.fromOpaque(refCon).takeUnretainedValue()
+        encoder.writeEncodedSampleBuffer(sampleBuffer)
+    }
+
+    func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {
+        guard type == .screen,
+              CMSampleBufferDataIsReady(sampleBuffer),
+              isCompleteFrame(sampleBuffer),
+              let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer),
+              let session = compressionSession else {
+            return
+        }
+
+        frameCount += 1
+        let pts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
+        let duration = CMTime(value: 1, timescale: CMTimeScale(fps))
+        let status = VTCompressionSessionEncodeFrame(
+            session,
+            imageBuffer: imageBuffer,
+            presentationTimeStamp: pts,
+            duration: duration,
+            frameProperties: nil,
+            sourceFrameRefcon: nil,
+            infoFlagsOut: nil)
+
+        if status != noErr && (frameCount == 1 || frameCount % 120 == 0) {
+            fputs("LiveDeskSCK encode status=\(status) frames=\(frameCount)\n", stderr)
+        }
+    }
+
+    func stream(_ stream: SCStream, didStopWithError error: Error) {
+        fputs("LiveDeskSCK stream stopped: \(error)\n", stderr)
+        exit(3)
+    }
+
+    private func isCompleteFrame(_ sampleBuffer: CMSampleBuffer) -> Bool {
+        guard let attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, createIfNecessary: false) as NSArray?,
+              let first = attachments.firstObject as? NSDictionary,
+              let rawStatus = first[SCStreamFrameInfo.status] as? Int else {
+            return true
+        }
+        return rawStatus == SCFrameStatus.complete.rawValue
+    }
+
+    private func writeEncodedSampleBuffer(_ sampleBuffer: CMSampleBuffer) {
+        guard let blockBuffer = CMSampleBufferGetDataBuffer(sampleBuffer) else {
+            return
+        }
+
+        var output = Data()
+        if isKeyFrame(sampleBuffer), let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer) {
+            appendParameterSet(formatDescription, index: 0, to: &output)
+            appendParameterSet(formatDescription, index: 1, to: &output)
+        }
+
+        var lengthAtOffset = 0
+        var totalLength = 0
+        var dataPointer: UnsafeMutablePointer<Int8>?
+        let status = CMBlockBufferGetDataPointer(
+            blockBuffer,
+            atOffset: 0,
+            lengthAtOffsetOut: &lengthAtOffset,
+            totalLengthOut: &totalLength,
+            dataPointerOut: &dataPointer)
+
+        guard status == noErr, let dataPointer, totalLength > 4 else {
+            return
+        }
+
+        var offset = 0
+        while offset + 4 <= totalLength {
+            var nalLength: UInt32 = 0
+            memcpy(&nalLength, dataPointer.advanced(by: offset), 4)
+            nalLength = CFSwapInt32BigToHost(nalLength)
+            let length = Int(nalLength)
+            if length <= 0 || offset + 4 + length > totalLength {
+                break
+            }
+
+            appendStartCode(to: &output)
+            output.append(UnsafeRawPointer(dataPointer.advanced(by: offset + 4)).assumingMemoryBound(to: UInt8.self), count: length)
+            offset += 4 + length
+        }
+
+        guard !output.isEmpty else {
+            return
+        }
+
+        writeLock.lock()
+        FileHandle.standardOutput.write(output)
+        writeLock.unlock()
+
+        encodedCount += 1
+        if encodedCount == 1 || encodedCount % max(1, fps * 5) == 0 {
+            fputs("LiveDeskSCK encoded frames=\(encodedCount) bytes=\(output.count)\n", stderr)
+        }
+    }
+
+    private func isKeyFrame(_ sampleBuffer: CMSampleBuffer) -> Bool {
+        guard let attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, createIfNecessary: false) as NSArray?,
+              let first = attachments.firstObject as? NSDictionary else {
+            return true
+        }
+        return !(first[kCMSampleAttachmentKey_NotSync] as? Bool ?? false)
+    }
+
+    private func appendParameterSet(_ formatDescription: CMFormatDescription, index: Int, to output: inout Data) {
+        var pointer: UnsafePointer<UInt8>?
+        var size = 0
+        var count = 0
+        var nalHeaderLength: Int32 = 0
+        let status = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(
+            formatDescription,
+            parameterSetIndex: index,
+            parameterSetPointerOut: &pointer,
+            parameterSetSizeOut: &size,
+            parameterSetCountOut: &count,
+            nalUnitHeaderLengthOut: &nalHeaderLength)
+
+        guard status == noErr, let pointer, size > 0 else {
+            return
+        }
+
+        appendStartCode(to: &output)
+        output.append(pointer, count: size)
+    }
+
+    private func appendStartCode(to output: inout Data) {
+        output.append(contentsOf: [0, 0, 0, 1])
+    }
+}
+
+enum LiveDeskSCKError: Error {
+    case noDisplays
+    case encoderCreate(OSStatus)
+}
+
+func intArg(_ name: String, default defaultValue: Int) -> Int {
+    let args = CommandLine.arguments
+    guard let index = args.firstIndex(of: name), index + 1 < args.count else {
+        return defaultValue
+    }
+    return Int(args[index + 1]) ?? defaultValue
+}
+
+@main
+struct LiveDeskSCKMain {
+    static func main() async {
+        guard #available(macOS 12.3, *) else {
+            fputs("LiveDeskSCK requires macOS 12.3 or later.\n", stderr)
+            exit(2)
+        }
+
+        let displayIndex = intArg("--display-index", default: 0)
+        let width = intArg("--width", default: 960)
+        let height = intArg("--height", default: 540)
+        let fps = intArg("--fps", default: 30)
+
+        do {
+            let encoder = LiveDeskScreenCaptureKitEncoder(displayIndex: displayIndex, width: width, height: height, fps: fps)
+            try await encoder.start()
+        } catch {
+            fputs("LiveDeskSCK failed: \(error)\n", stderr)
+            exit(1)
+        }
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livedesk/client",
-  "version": "0.1.71",
+  "version": "0.1.73",
   "description": "LiveDesk local remote client",
   "type": "module",
   "bin": {