@capgo/camera-preview 8.11.9 → 8.11.10

package/ios/Sources/CapgoCameraPreviewPlugin/AspectRatioLayout.swift

@@ -0,0 +1,107 @@
+import CoreGraphics
+import Foundation
+
+/// Viewport sizing for numeric aspect ratios (`4:3`, `16:9`) vs full-frame `fill`.
+enum AspectRatioLayout {
+    static func isFillMode(_ aspectRatio: String) -> Bool {
+        aspectRatio.lowercased() == "fill"
+    }
+
+    /// Width-to-height ratio for the preview view in the current interface orientation.
+    /// Returns `nil` for `fill` and other non-numeric values.
+    static func parseViewportAspectRatio(_ ratio: String, isPortrait: Bool) -> CGFloat? {
+        if isFillMode(ratio) {
+            return nil
+        }
+
+        let parts = ratio.split(separator: ":")
+        guard parts.count == 2,
+              let width = Double(parts[0]), let height = Double(parts[1]),
+              width.isFinite, height.isFinite,
+              width > 0, height > 0 else {
+            return nil
+        }
+
+        let viewportRatio = CGFloat(isPortrait ? height / width : width / height)
+        guard viewportRatio.isFinite, viewportRatio > 0 else {
+            return nil
+        }
+
+        return viewportRatio
+    }
+
+    static func dimensionsForAspectRatio(
+        _ aspectRatio: String,
+        availableWidth: CGFloat,
+        availableHeight: CGFloat,
+        isPortrait: Bool
+    ) -> (width: CGFloat, height: CGFloat) {
+        guard let ratio = parseViewportAspectRatio(aspectRatio, isPortrait: isPortrait) else {
+            return (width: availableWidth, height: availableHeight)
+        }
+
+        let maxWidthByHeight = availableHeight * ratio
+        let maxHeightByWidth = availableWidth / ratio
+
+        if maxWidthByHeight <= availableWidth {
+            return (width: maxWidthByHeight, height: availableHeight)
+        }
+
+        return (width: availableWidth, height: maxHeightByWidth)
+    }
+
+    /// Length left in `available` after an explicit `origin`, clamped at zero.
+    /// Used to size an explicitly positioned `fill` axis from the web view.
+    static func remainingLength(from origin: CGFloat, in available: CGFloat) -> CGFloat {
+        max(0, available - origin)
+    }
+
+    /// Size of a `fill` preview with explicit `x` and `y` and no width or height:
+    /// the web-view space remaining to the right of `x` and below `y`.
+    static func positionedFillSize(
+        x: CGFloat,
+        y: CGFloat,
+        viewportWidth: CGFloat,
+        viewportHeight: CGFloat
+    ) -> (width: CGFloat, height: CGFloat) {
+        (width: remainingLength(from: x, in: viewportWidth), height: remainingLength(from: y, in: viewportHeight))
+    }
+
+    /// Width-to-height ratio of the area visible in a `fill` preview using `cover`
+    /// (`resizeAspectFill`), expressed in the orientation of the captured photo.
+    /// Returns `nil` when the preview size is unusable.
+    static func visibleFillCaptureAspectRatio(
+        previewSize: CGSize,
+        interfaceIsPortrait: Bool,
+        captureIsPortrait: Bool
+    ) -> CGFloat? {
+        guard previewSize.width.isFinite, previewSize.height.isFinite,
+              previewSize.width > 0, previewSize.height > 0 else {
+            return nil
+        }
+
+        let ratio = previewSize.width / previewSize.height
+        return interfaceIsPortrait == captureIsPortrait ? ratio : 1 / ratio
+    }
+
+    /// Centered crop rect with `targetAspectRatio` (width / height) inside `imageSize`,
+    /// matching how `resizeAspectFill` centers the visible area.
+    static func centerCropRect(imageSize: CGSize, targetAspectRatio: CGFloat) -> CGRect {
+        let bounds = CGRect(origin: .zero, size: imageSize)
+        guard targetAspectRatio.isFinite, targetAspectRatio > 0,
+              imageSize.width > 0, imageSize.height > 0 else {
+            return bounds
+        }
+
+        let imageAspectRatio = imageSize.width / imageSize.height
+        let rect: CGRect
+        if imageAspectRatio > targetAspectRatio {
+            let width = imageSize.height * targetAspectRatio
+            rect = CGRect(x: (imageSize.width - width) / 2, y: 0, width: width, height: imageSize.height)
+        } else {
+            let height = imageSize.width / targetAspectRatio
+            rect = CGRect(x: 0, y: (imageSize.height - height) / 2, width: imageSize.width, height: height)
+        }
+        return rect.intersection(bounds).integral.intersection(bounds)
+    }
+}

package/ios/Sources/CapgoCameraPreviewPlugin/CameraController.swift

@@ -4,8 +4,24 @@ import UIKit
 import CoreLocation
 import UniformTypeIdentifiers
 import CoreMotion
+import CallKit
+
+class CameraController: NSObject, CXCallObserverDelegate {
+    override init() {
+        super.init()
+        self.callObserver.setDelegate(self, queue: nil)
+    }
+
+    func callObserver(_ callObserver: CXCallObserver, callChanged call: CXCall) {
+        guard call.hasEnded,
+              callObserver.calls.allSatisfy({ $0.hasEnded }),
+              let session = self.captureSession else { return }
+        self.sessionRecoveryQueue.async { [weak self] in
+            guard let self = self, self.captureSession === session else { return }
+            self.restoreAudioInputIfNeeded(for: session)
+        }
+    }
 
-class CameraController: NSObject {
     private func getVideoOrientation() -> AVCaptureVideoOrientation {
         var orientation: AVCaptureVideoOrientation = .portrait
         if Thread.isMainThread {
@@ -151,6 +167,16 @@ class CameraController: NSObject {
     var firstFrameReadyCallback: (() -> Void)?
     var hasReceivedFirstFrame = false
 
+    var onCameraInterrupted: ((String, Bool) -> Void)?
+    var onCameraInterruptionEnded: (() -> Void)?
+    var onStartFailure: ((Error) -> Void)?
+
+    private let callObserver = CXCallObserver()
+    private weak var observedCaptureSession: AVCaptureSession?
+    private let sessionRecoveryQueue = DispatchQueue(label: "com.capgo.camera.sessionRecovery")
+    private var activePrepareToken: UInt = 0
+    private var audioInputEnabledByUser = false
+
     var audioDevice: AVCaptureDevice?
     var audioInput: AVCaptureDeviceInput?
 
@@ -658,9 +684,19 @@ extension CameraController {
         self.outputsPrepared = true
     }
 
-    func prepare(cameraPosition: String, deviceId: String? = nil, disableAudio: Bool, cameraMode: Bool, aspectRatio: String? = nil, aspectMode: String = "contain", initialZoomLevel: Float?, disableFocusIndicator: Bool = false, videoQuality: String = "high", completionHandler: @escaping (Error?) -> Void) {
+    func invalidateActivePrepare() {
+        self.activePrepareToken = 0
+    }
+
+    private func isPrepareTokenValid(_ token: UInt) -> Bool {
+        return token != 0 && token == self.activePrepareToken
+    }
+
+    func prepare(cameraPosition: String, deviceId: String? = nil, disableAudio: Bool, cameraMode: Bool, aspectRatio: String? = nil, aspectMode: String = "contain", initialZoomLevel: Float?, disableFocusIndicator: Bool = false, videoQuality: String = "high", startToken: UInt, completionHandler: @escaping (Error?) -> Void) {
         print("[CameraPreview] 🎬 Starting prepare - position: \(cameraPosition), deviceId: \(deviceId ?? "nil"), disableAudio: \(disableAudio), cameraMode: \(cameraMode), aspectRatio: \(aspectRatio ?? "nil"), aspectMode: \(aspectMode), zoom: \(initialZoomLevel ?? 1)")
 
+        self.activePrepareToken = startToken
+
         DispatchQueue.global(qos: .userInitiated).async { [weak self] in
             guard let self = self else {
                 DispatchQueue.main.async {
@@ -669,6 +705,10 @@ extension CameraController {
                 return
             }
 
+            guard self.isPrepareTokenValid(startToken) else {
+                return
+            }
+
             // Start accelerometer
             var startedAccelerometer = false
             if self.motionManager.isAccelerometerAvailable {
@@ -758,12 +798,19 @@ extension CameraController {
                 NotificationCenter.default.removeObserver(self, name: .AVCaptureDeviceSubjectAreaDidChange, object: nil)
                 NotificationCenter.default.addObserver(self, selector: #selector(self.subjectAreaDidChange), name: .AVCaptureDeviceSubjectAreaDidChange, object: nil)
 
+                self.registerCaptureSessionObservers(for: captureSession)
+
+                guard self.isPrepareTokenValid(startToken) else {
+                    return
+                }
+
                 // Start the session - all outputs are already configured
                 captureSession.startRunning()
 
                 // Bring to full opacity after a tiny moment to smooth any visual artifacts
                 DispatchQueue.main.asyncAfter(deadline: .now() + 0.05) { [weak self] in
-                    if let layer = self?.previewLayer {
+                    guard let self = self, self.isPrepareTokenValid(startToken) else { return }
+                    if let layer = self.previewLayer {
                         CATransaction.begin()
                         CATransaction.setAnimationDuration(0.1)
                         layer.opacity = 1.0
@@ -773,6 +820,7 @@ extension CameraController {
 
                 // Success callback
                 DispatchQueue.main.async {
+                    guard self.isPrepareTokenValid(startToken) else { return }
                     completionHandler(nil)
                 }
             } catch {
@@ -780,6 +828,7 @@ extension CameraController {
                     self.motionManager.stopAccelerometerUpdates()
                 }
                 DispatchQueue.main.async {
+                    guard self.isPrepareTokenValid(startToken) else { return }
                     completionHandler(error)
                 }
             }
@@ -951,6 +1000,8 @@ extension CameraController {
     private func configureDeviceInputs(cameraPosition: String, deviceId: String?, disableAudio: Bool) throws {
         guard let captureSession = self.captureSession else { throw CameraControllerError.captureSessionIsMissing }
 
+        self.audioInputEnabledByUser = !disableAudio
+
         // Ensure cameras are discovered before configuring inputs
         ensureCamerasDiscovered()
 
@@ -991,8 +1042,8 @@ extension CameraController {
             throw CameraControllerError.inputsAreInvalid
         }
 
-        // Add audio input if needed
-        if !disableAudio {
+        // Add audio input if needed (skip while a phone call owns the audio session)
+        if self.shouldAttachAudioInput(disableAudio: disableAudio) {
             if self.audioDevice == nil {
                 self.audioDevice = AVCaptureDevice.default(for: AVMediaType.audio)
             }
@@ -1344,6 +1395,9 @@ extension CameraController {
             self.lastCaptureOrientation = captureOrientation
             self.setVideoOrientation(captureOrientation, on: connection)
         }
+        // For fill + cover the preview shows a centered crop of the frame. Snapshot its
+        // shape now so the photo can be cropped to the same visible area.
+        let visibleFillRatio = self.visibleFillCaptureAspectRatio()
         let settings = AVCapturePhotoSettings()
         // Configure photo capture settings optimized for speed
         // Only use high res if explicitly requesting large dimensions
@@ -1415,7 +1469,7 @@ extension CameraController {
                 // When max dimensions are specified, we used high-res capture
                 // First crop to aspect ratio if needed, then resize to max dimensions
                 if let aspectRatio = self.requestedAspectRatio {
-                    finalImage = self.cropImageToAspectRatio(image: image, aspectRatio: aspectRatio) ?? image
+                    finalImage = self.cropCapturedImage(image, aspectRatio: aspectRatio, visibleFillRatio: visibleFillRatio)
                     print("[CameraPreview] Cropped high-res image to aspect ratio \(aspectRatio)")
                 }
                 // Then resize to fit within maximum dimensions while maintaining aspect ratio
@@ -1424,7 +1478,7 @@ extension CameraController {
             } else if let aspectRatio = self.requestedAspectRatio {
                 // No max dimensions specified, but aspect ratio is specified
                 // Always apply aspect ratio cropping to ensure correct orientation
-                finalImage = self.cropImageToAspectRatio(image: image, aspectRatio: aspectRatio) ?? image
+                finalImage = self.cropCapturedImage(image, aspectRatio: aspectRatio, visibleFillRatio: visibleFillRatio)
                 print("[CameraPreview] Applied aspect ratio cropping for \(aspectRatio): \(finalImage.size.width)x\(finalImage.size.height)")
             }
 
@@ -1741,6 +1795,55 @@ extension CameraController {
         return resizeImage(image: image, to: targetSize)
     }
 
+    /// Ratio of the visible preview area for `fill` + `cover`, in capture orientation.
+    /// Returns `nil` for numeric ratios and for `contain`, where the full frame is visible.
+    func visibleFillCaptureAspectRatio() -> CGFloat? {
+        guard let aspectRatio = self.requestedAspectRatio,
+              AspectRatioLayout.isFillMode(aspectRatio),
+              self.requestedAspectMode == "cover",
+              let previewLayer = self.previewLayer else {
+            return nil
+        }
+
+        let orientation = self.lastCaptureOrientation ?? self.getPhysicalOrientation()
+        let captureIsPortrait = orientation == .portrait || orientation == .portraitUpsideDown
+        let screenBounds = UIScreen.main.bounds
+        return AspectRatioLayout.visibleFillCaptureAspectRatio(
+            previewSize: previewLayer.bounds.size,
+            interfaceIsPortrait: screenBounds.height >= screenBounds.width,
+            captureIsPortrait: captureIsPortrait
+        )
+    }
+
+    /// Crops a captured photo so it matches what the preview shows.
+    func cropCapturedImage(_ image: UIImage, aspectRatio: String, visibleFillRatio: CGFloat?) -> UIImage {
+        if AspectRatioLayout.isFillMode(aspectRatio) {
+            guard let ratio = visibleFillRatio else {
+                // fill + contain shows the whole frame, so keep the full photo.
+                return image
+            }
+            return cropImage(image, toAspectRatio: ratio) ?? image
+        }
+        return cropImageToAspectRatio(image: image, aspectRatio: aspectRatio) ?? image
+    }
+
+    /// Center-crops `image` (after normalizing its orientation) to a width / height ratio.
+    func cropImage(_ image: UIImage, toAspectRatio targetAspectRatio: CGFloat) -> UIImage? {
+        let normalizedImage = image.imageOrientation == .up ? image : (image.fixedOrientation() ?? image)
+        guard let cgImage = normalizedImage.cgImage else {
+            return nil
+        }
+
+        let pixelSize = CGSize(width: cgImage.width, height: cgImage.height)
+        let cropRect = AspectRatioLayout.centerCropRect(imageSize: pixelSize, targetAspectRatio: targetAspectRatio)
+        guard let croppedCGImage = cgImage.cropping(to: cropRect) else {
+            return nil
+        }
+
+        print("[CameraPreview] cropImage - Cropped \(pixelSize.width)x\(pixelSize.height) to visible fill area \(cropRect)")
+        return UIImage(cgImage: croppedCGImage, scale: normalizedImage.scale, orientation: .up)
+    }
+
     func cropImageToAspectRatio(image: UIImage, aspectRatio: String) -> UIImage? {
         let components = aspectRatio.split(separator: ":").compactMap {Float(String($0))}
         guard components.count == 2 else {
@@ -2536,10 +2639,168 @@ extension CameraController {
         self.updateVideoOrientation()
     }
 
+    private func isPhoneCallActive() -> Bool {
+        self.callObserver.calls.contains { !$0.hasEnded }
+    }
+
+    private func shouldAttachAudioInput(disableAudio: Bool) -> Bool {
+        return !disableAudio && !self.isPhoneCallActive()
+    }
+
+    private static let insufficientAudioPriorityOSStatus = 561017449
+
+    private func isAudioPriorityError(_ error: NSError) -> Bool {
+        var current: NSError? = error
+        while let err = current {
+            if err.domain == NSOSStatusErrorDomain && err.code == CameraController.insufficientAudioPriorityOSStatus {
+                return true
+            }
+            current = err.userInfo[NSUnderlyingErrorKey] as? NSError
+        }
+        return false
+    }
+
+    private func registerCaptureSessionObservers(for session: AVCaptureSession) {
+        self.unregisterCaptureSessionObservers()
+        self.observedCaptureSession = session
+        NotificationCenter.default.addObserver(
+            self,
+            selector: #selector(self.handleCaptureSessionRuntimeError(_:)),
+            name: .AVCaptureSessionRuntimeError,
+            object: session
+        )
+        NotificationCenter.default.addObserver(
+            self,
+            selector: #selector(self.handleCaptureSessionWasInterrupted(_:)),
+            name: .AVCaptureSessionWasInterrupted,
+            object: session
+        )
+        NotificationCenter.default.addObserver(
+            self,
+            selector: #selector(self.handleCaptureSessionInterruptionEnded(_:)),
+            name: .AVCaptureSessionInterruptionEnded,
+            object: session
+        )
+    }
+
+    private func unregisterCaptureSessionObservers() {
+        guard let session = self.observedCaptureSession else { return }
+        NotificationCenter.default.removeObserver(self, name: .AVCaptureSessionRuntimeError, object: session)
+        NotificationCenter.default.removeObserver(self, name: .AVCaptureSessionWasInterrupted, object: session)
+        NotificationCenter.default.removeObserver(self, name: .AVCaptureSessionInterruptionEnded, object: session)
+        self.observedCaptureSession = nil
+    }
+
+    private func restoreAudioInputIfNeeded(for session: AVCaptureSession) {
+        guard self.audioInputEnabledByUser, self.audioInput == nil, !self.isPhoneCallActive() else { return }
+        guard self.captureSession === session else { return }
+
+        do {
+            session.beginConfiguration()
+            defer { session.commitConfiguration() }
+            if self.audioDevice == nil {
+                self.audioDevice = AVCaptureDevice.default(for: AVMediaType.audio)
+            }
+            guard let audioDevice = self.audioDevice else {
+                return
+            }
+            let input = try AVCaptureDeviceInput(device: audioDevice)
+            if session.canAddInput(input) {
+                session.addInput(input)
+                self.audioInput = input
+            }
+        } catch {
+            print("[CameraPreview] Failed to restore audio input after interruption: \(error)")
+        }
+    }
+
+    private func dropAudioInputAndRestartSession(for session: AVCaptureSession) {
+        guard self.captureSession === session else { return }
+
+        session.beginConfiguration()
+        for input in session.inputs {
+            guard let deviceInput = input as? AVCaptureDeviceInput else { continue }
+            if deviceInput.device.hasMediaType(.audio) {
+                session.removeInput(deviceInput)
+            }
+        }
+        self.audioInput = nil
+        session.commitConfiguration()
+
+        if session.isRunning {
+            session.stopRunning()
+        }
+        session.startRunning()
+    }
+
+    @objc private func handleCaptureSessionRuntimeError(_ notification: Notification) {
+        guard let error = notification.userInfo?[AVCaptureSessionErrorKey] as? NSError else { return }
+        print("[CameraPreview] AVCaptureSession runtime error: \(error)")
+
+        let audioConflict = self.isAudioPriorityError(error)
+        guard let session = notification.object as? AVCaptureSession else { return }
+        if audioConflict {
+            self.sessionRecoveryQueue.async { [weak self] in
+                self?.dropAudioInputAndRestartSession(for: session)
+            }
+        }
+
+        DispatchQueue.main.async { [weak self] in
+            guard let self = self else { return }
+            self.onCameraInterrupted?("runtimeError", audioConflict)
+            if !audioConflict && !self.hasReceivedFirstFrame {
+                self.sessionRecoveryQueue.async { [weak self] in
+                    guard let self = self, self.captureSession === session else { return }
+                    session.stopRunning()
+                    DispatchQueue.main.async {
+                        self.onStartFailure?(error)
+                    }
+                }
+            }
+        }
+    }
+
+    @objc private func handleCaptureSessionWasInterrupted(_ notification: Notification) {
+        guard let session = notification.object as? AVCaptureSession else { return }
+        var audioRelated = false
+        if let reasonValue = notification.userInfo?[AVCaptureSessionInterruptionReasonKey] as? Int,
+           let reason = AVCaptureSession.InterruptionReason(rawValue: reasonValue),
+           reason == .audioDeviceInUseByAnotherClient {
+            audioRelated = true
+            self.sessionRecoveryQueue.async { [weak self] in
+                self?.dropAudioInputAndRestartSession(for: session)
+            }
+        }
+
+        DispatchQueue.main.async { [weak self] in
+            self?.onCameraInterrupted?("interrupted", audioRelated)
+        }
+    }
+
+    @objc private func handleCaptureSessionInterruptionEnded(_ notification: Notification) {
+        guard let session = notification.object as? AVCaptureSession else { return }
+        self.sessionRecoveryQueue.async { [weak self] in
+            guard let self = self, self.captureSession === session else { return }
+            self.restoreAudioInputIfNeeded(for: session)
+            if !session.isRunning {
+                session.startRunning()
+            }
+            DispatchQueue.main.async {
+                guard self.captureSession === session else { return }
+                self.onCameraInterruptionEnded?()
+            }
+        }
+    }
+
     func cleanup() {
+        self.invalidateActivePrepare()
         self.cancelPendingFocusExposureRestore()
         configuredVideoFrameRate = nil
         stopBarcodeScanner()
+        self.unregisterCaptureSessionObservers()
+        self.onCameraInterrupted = nil
+        self.onCameraInterruptionEnded = nil
+        self.onStartFailure = nil
         if let captureSession = self.captureSession {
             captureSession.stopRunning()
             captureSession.inputs.forEach { captureSession.removeInput($0) }
@@ -2572,6 +2833,7 @@ extension CameraController {
         self.captureSession = nil
         self.currentCameraPosition = nil
         self.preferredExposureMode = "CONTINUOUS"
+        self.audioInputEnabledByUser = false
 
         // Reset output preparation status
         self.outputsPrepared = false