@livedesk/client 0.1.94 → 0.1.96

package/fast/osx-arm64/MindExec.RemoteAgent.Fast/MacScreenCaptureKitH264.swift

@@ -17,8 +17,12 @@ final class LiveDeskScreenCaptureKitEncoder: NSObject, SCStreamOutput, SCStreamD
     private let writeLock = NSLock()
     private var stream: SCStream?
     private var compressionSession: VTCompressionSession?
+    private var sampleCount: Int = 0
     private var frameCount: Int = 0
+    private var skippedCount: Int = 0
     private var encodedCount: Int = 0
+    private var encodedKeyCount: Int = 0
+    private var encodedDeltaCount: Int = 0
 
     init(displayIndex: Int, width: Int, height: Int, fps: Int, bitrateKbps: Int) {
         self.displayIndex = max(0, displayIndex)
@@ -52,7 +56,7 @@ final class LiveDeskScreenCaptureKitEncoder: NSObject, SCStreamOutput, SCStreamD
         configuration.width = width
         configuration.height = height
         configuration.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale(fps))
-        configuration.queueDepth = 1
+        configuration.queueDepth = 3
         configuration.showsCursor = false
         configuration.pixelFormat = kCVPixelFormatType_32BGRA
 
@@ -69,12 +73,15 @@ final class LiveDeskScreenCaptureKitEncoder: NSObject, SCStreamOutput, SCStreamD
 
     private func createCompressionSession() throws {
         var session: VTCompressionSession?
+        let encoderSpecification: [CFString: Any] = [
+            kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder: true
+        ]
         let status = VTCompressionSessionCreate(
             allocator: kCFAllocatorDefault,
             width: Int32(width),
             height: Int32(height),
             codecType: kCMVideoCodecType_H264,
-            encoderSpecification: nil,
+            encoderSpecification: encoderSpecification as CFDictionary,
             imageBufferAttributes: nil,
             compressedDataAllocator: nil,
             outputCallback: Self.compressionCallback,
@@ -86,7 +93,9 @@ final class LiveDeskScreenCaptureKitEncoder: NSObject, SCStreamOutput, SCStreamD
         }
 
         VTSessionSetProperty(session, key: kVTCompressionPropertyKey_RealTime, value: kCFBooleanTrue)
+        VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AllowTemporalCompression, value: kCFBooleanTrue)
         VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AllowFrameReordering, value: kCFBooleanFalse)
+        VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxFrameDelayCount, value: NSNumber(value: 1))
         VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ProfileLevel, value: kVTProfileLevel_H264_Baseline_AutoLevel)
         VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameInterval, value: NSNumber(value: fps))
         VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, value: NSNumber(value: 1.0))
@@ -111,12 +120,18 @@ final class LiveDeskScreenCaptureKitEncoder: NSObject, SCStreamOutput, SCStreamD
     func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {
         guard type == .screen,
               CMSampleBufferDataIsReady(sampleBuffer),
-              isCompleteFrame(sampleBuffer),
               let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer),
               let session = compressionSession else {
             return
         }
 
+        sampleCount += 1
+        guard isCompleteFrame(sampleBuffer) else {
+            skippedCount += 1
+            logCaptureProgressIfNeeded()
+            return
+        }
+
         frameCount += 1
         let pts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
         let duration = CMTime(value: 1, timescale: CMTimeScale(fps))
@@ -139,6 +154,7 @@ final class LiveDeskScreenCaptureKitEncoder: NSObject, SCStreamOutput, SCStreamD
         if status != noErr && (frameCount == 1 || frameCount % 120 == 0) {
             fputs("LiveDeskSCK encode status=\(status) frames=\(frameCount)\n", stderr)
         }
+        logCaptureProgressIfNeeded()
     }
 
     func stream(_ stream: SCStream, didStopWithError error: Error) {
@@ -155,14 +171,22 @@ final class LiveDeskScreenCaptureKitEncoder: NSObject, SCStreamOutput, SCStreamD
         return rawStatus == SCFrameStatus.complete.rawValue
     }
 
+    private func logCaptureProgressIfNeeded() {
+        let interval = max(1, fps * 5)
+        if sampleCount == 1 || sampleCount % interval == 0 {
+            fputs("LiveDeskSCK capture samples=\(sampleCount) complete=\(frameCount) skipped=\(skippedCount)\n", stderr)
+        }
+    }
+
     private func writeEncodedSampleBuffer(_ sampleBuffer: CMSampleBuffer) {
         guard let blockBuffer = CMSampleBufferGetDataBuffer(sampleBuffer) else {
             return
         }
 
+        let keyFrame = isKeyFrame(sampleBuffer)
         var output = Data()
         appendAccessUnitDelimiter(to: &output)
-        if isKeyFrame(sampleBuffer), let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer) {
+        if keyFrame, let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer) {
             appendParameterSet(formatDescription, index: 0, to: &output)
             appendParameterSet(formatDescription, index: 1, to: &output)
         }
@@ -205,8 +229,13 @@ final class LiveDeskScreenCaptureKitEncoder: NSObject, SCStreamOutput, SCStreamD
         writeLock.unlock()
 
         encodedCount += 1
+        if keyFrame {
+            encodedKeyCount += 1
+        } else {
+            encodedDeltaCount += 1
+        }
         if encodedCount == 1 || encodedCount % max(1, fps * 5) == 0 {
-            fputs("LiveDeskSCK encoded frames=\(encodedCount) bytes=\(output.count)\n", stderr)
+            fputs("LiveDeskSCK encoded frames=\(encodedCount) key=\(encodedKeyCount) delta=\(encodedDeltaCount) bytes=\(output.count)\n", stderr)
         }
     }
 

package/fast/osx-x64/MindExec.RemoteAgent.Fast/MacScreenCaptureKitH264.swift

@@ -17,8 +17,12 @@ final class LiveDeskScreenCaptureKitEncoder: NSObject, SCStreamOutput, SCStreamD
     private let writeLock = NSLock()
     private var stream: SCStream?
     private var compressionSession: VTCompressionSession?
+    private var sampleCount: Int = 0
     private var frameCount: Int = 0
+    private var skippedCount: Int = 0
     private var encodedCount: Int = 0
+    private var encodedKeyCount: Int = 0
+    private var encodedDeltaCount: Int = 0
 
     init(displayIndex: Int, width: Int, height: Int, fps: Int, bitrateKbps: Int) {
         self.displayIndex = max(0, displayIndex)
@@ -52,7 +56,7 @@ final class LiveDeskScreenCaptureKitEncoder: NSObject, SCStreamOutput, SCStreamD
         configuration.width = width
         configuration.height = height
         configuration.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale(fps))
-        configuration.queueDepth = 1
+        configuration.queueDepth = 3
         configuration.showsCursor = false
         configuration.pixelFormat = kCVPixelFormatType_32BGRA
 
@@ -69,12 +73,15 @@ final class LiveDeskScreenCaptureKitEncoder: NSObject, SCStreamOutput, SCStreamD
 
     private func createCompressionSession() throws {
         var session: VTCompressionSession?
+        let encoderSpecification: [CFString: Any] = [
+            kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder: true
+        ]
         let status = VTCompressionSessionCreate(
             allocator: kCFAllocatorDefault,
             width: Int32(width),
             height: Int32(height),
             codecType: kCMVideoCodecType_H264,
-            encoderSpecification: nil,
+            encoderSpecification: encoderSpecification as CFDictionary,
             imageBufferAttributes: nil,
             compressedDataAllocator: nil,
             outputCallback: Self.compressionCallback,
@@ -86,7 +93,9 @@ final class LiveDeskScreenCaptureKitEncoder: NSObject, SCStreamOutput, SCStreamD
         }
 
         VTSessionSetProperty(session, key: kVTCompressionPropertyKey_RealTime, value: kCFBooleanTrue)
+        VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AllowTemporalCompression, value: kCFBooleanTrue)
         VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AllowFrameReordering, value: kCFBooleanFalse)
+        VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxFrameDelayCount, value: NSNumber(value: 1))
         VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ProfileLevel, value: kVTProfileLevel_H264_Baseline_AutoLevel)
         VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameInterval, value: NSNumber(value: fps))
         VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, value: NSNumber(value: 1.0))
@@ -111,12 +120,18 @@ final class LiveDeskScreenCaptureKitEncoder: NSObject, SCStreamOutput, SCStreamD
     func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {
         guard type == .screen,
               CMSampleBufferDataIsReady(sampleBuffer),
-              isCompleteFrame(sampleBuffer),
               let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer),
               let session = compressionSession else {
             return
         }
 
+        sampleCount += 1
+        guard isCompleteFrame(sampleBuffer) else {
+            skippedCount += 1
+            logCaptureProgressIfNeeded()
+            return
+        }
+
         frameCount += 1
         let pts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
         let duration = CMTime(value: 1, timescale: CMTimeScale(fps))
@@ -139,6 +154,7 @@ final class LiveDeskScreenCaptureKitEncoder: NSObject, SCStreamOutput, SCStreamD
         if status != noErr && (frameCount == 1 || frameCount % 120 == 0) {
             fputs("LiveDeskSCK encode status=\(status) frames=\(frameCount)\n", stderr)
         }
+        logCaptureProgressIfNeeded()
     }
 
     func stream(_ stream: SCStream, didStopWithError error: Error) {
@@ -155,14 +171,22 @@ final class LiveDeskScreenCaptureKitEncoder: NSObject, SCStreamOutput, SCStreamD
         return rawStatus == SCFrameStatus.complete.rawValue
     }
 
+    private func logCaptureProgressIfNeeded() {
+        let interval = max(1, fps * 5)
+        if sampleCount == 1 || sampleCount % interval == 0 {
+            fputs("LiveDeskSCK capture samples=\(sampleCount) complete=\(frameCount) skipped=\(skippedCount)\n", stderr)
+        }
+    }
+
     private func writeEncodedSampleBuffer(_ sampleBuffer: CMSampleBuffer) {
         guard let blockBuffer = CMSampleBufferGetDataBuffer(sampleBuffer) else {
             return
         }
 
+        let keyFrame = isKeyFrame(sampleBuffer)
         var output = Data()
         appendAccessUnitDelimiter(to: &output)
-        if isKeyFrame(sampleBuffer), let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer) {
+        if keyFrame, let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer) {
             appendParameterSet(formatDescription, index: 0, to: &output)
             appendParameterSet(formatDescription, index: 1, to: &output)
         }
@@ -205,8 +229,13 @@ final class LiveDeskScreenCaptureKitEncoder: NSObject, SCStreamOutput, SCStreamD
         writeLock.unlock()
 
         encodedCount += 1
+        if keyFrame {
+            encodedKeyCount += 1
+        } else {
+            encodedDeltaCount += 1
+        }
         if encodedCount == 1 || encodedCount % max(1, fps * 5) == 0 {
-            fputs("LiveDeskSCK encoded frames=\(encodedCount) bytes=\(output.count)\n", stderr)
+            fputs("LiveDeskSCK encoded frames=\(encodedCount) key=\(encodedKeyCount) delta=\(encodedDeltaCount) bytes=\(output.count)\n", stderr)
         }
     }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livedesk/client",
-  "version": "0.1.94",
+  "version": "0.1.96",
   "description": "LiveDesk local remote client",
   "type": "module",
   "bin": {