@capacitor-mlkit/selfie-segmentation 5.2.0

package/ios/Plugin/SelfieSegmentation.swift

@@ -0,0 +1,199 @@
+import Foundation
+import MLKitVision
+import MLKitSegmentationSelfie
+
+@objc public class SelfieSegmentation: NSObject {
+    public let plugin: SelfieSegmentationPlugin
+
+    init(plugin: SelfieSegmentationPlugin) {
+        self.plugin = plugin
+    }
+
+    @objc func createImageFromFilePath(_ path: String) -> UIImage? {
+        guard let url = URL.init(string: path) else {
+            return nil
+        }
+        if FileManager.default.fileExists(atPath: url.path) {
+            return UIImage.init(contentsOfFile: url.path)
+        } else {
+            return nil
+        }
+    }
+
+    enum ProcessError: Error {
+        case createImageBuffer
+    }
+
+    private var segmenter: Segmenter?
+
+    @objc func processImage(_ options: ProcessImageOptions, completion: @escaping (ProcessImageResult?, Error?) -> Void) {
+        let image = options.getImage()
+        let threshold = options.getConfidence()
+
+        let visionImage = VisionImage.init(image: image)
+        visionImage.orientation = image.imageOrientation
+
+        let selfieSegmenterOptions: SelfieSegmenterOptions = SelfieSegmenterOptions()
+        selfieSegmenterOptions.segmenterMode = .singleImage
+
+        segmenter = Segmenter.segmenter(
+            options: selfieSegmenterOptions
+        )
+
+        segmenter?.process(visionImage) { mask, error in
+            self.segmenter = nil
+
+            guard error == nil, let mask = mask else {
+                return completion(nil, error)
+            }
+
+            do {
+                guard let imageBuffer = self.createImageBuffer(from: image) else {
+                    throw ProcessError.createImageBuffer
+                }
+
+                self.applySegmentationMask(
+                    mask: mask, to: imageBuffer, threshold: threshold
+                )
+
+                let image = self.createImage(from: imageBuffer)
+                let result = ProcessImageResult(image: image)
+
+                completion(result, nil)
+            } catch {
+                completion(nil, error)
+            }
+        }
+    }
+
+    func createImageBuffer(from image: UIImage) -> CVImageBuffer? {
+        guard let cgImage = image.cgImage else { return nil }
+        let width = cgImage.width
+        let height = cgImage.height
+
+        var buffer: CVPixelBuffer?
+        CVPixelBufferCreate(
+            kCFAllocatorDefault,
+            width,
+            height,
+            kCVPixelFormatType_32BGRA,
+            nil,
+            &buffer)
+        guard let imageBuffer = buffer else { return nil }
+
+        let flags = CVPixelBufferLockFlags(rawValue: 0)
+        CVPixelBufferLockBaseAddress(imageBuffer, flags)
+        let baseAddress = CVPixelBufferGetBaseAddress(imageBuffer)
+        let colorSpace = CGColorSpaceCreateDeviceRGB()
+        let bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer)
+        let context = CGContext(
+            data: baseAddress,
+            width: width,
+            height: height,
+            bitsPerComponent: 8,
+            bytesPerRow: bytesPerRow,
+            space: colorSpace,
+            bitmapInfo: (CGImageAlphaInfo.premultipliedFirst.rawValue
+                            | CGBitmapInfo.byteOrder32Little.rawValue))
+
+        if let context = context {
+            let rect = CGRect.init(x: 0, y: 0, width: width, height: height)
+            context.draw(cgImage, in: rect)
+            CVPixelBufferUnlockBaseAddress(imageBuffer, flags)
+            return imageBuffer
+        } else {
+            CVPixelBufferUnlockBaseAddress(imageBuffer, flags)
+            return nil
+        }
+    }
+
+    //    func createSampleBuffer(with imageBuffer: CVImageBuffer) -> CMSampleBuffer? {
+    //        var timingInfo = CMSampleTimingInfo()
+    ////        guard CMSampleBufferGetSampleTimingInfo(sampleBuffer, at: 0, timingInfoOut: &timingInfo) == 0 else {
+    ////            return nil
+    ////        }
+    //        var outputSampleBuffer: CMSampleBuffer?
+    //        var newFormatDescription: CMFormatDescription?
+    //        CMVideoFormatDescriptionCreateForImageBuffer(allocator: nil, imageBuffer: imageBuffer, formatDescriptionOut: &newFormatDescription)
+    //        guard let formatDescription = newFormatDescription else {
+    //            return nil
+    //        }
+    //        CMSampleBufferCreateReadyWithImageBuffer(allocator: nil, imageBuffer: imageBuffer, formatDescription: formatDescription, sampleTiming: &timingInfo, sampleBufferOut: &outputSampleBuffer)
+    //        guard let buffer = outputSampleBuffer else {
+    //            return nil
+    //        }
+    //        return buffer
+    //    }
+
+    func createImage(
+        from imageBuffer: CVImageBuffer
+    ) -> UIImage {
+        let ciImage = CIImage(cvPixelBuffer: imageBuffer)
+        let context = CIContext(options: nil)
+        let cgImage = context.createCGImage(ciImage, from: ciImage.extent)!
+        return UIImage(cgImage: cgImage)
+    }
+
+    func applySegmentationMask(
+        mask: SegmentationMask, to imageBuffer: CVImageBuffer, threshold: CGFloat
+    ) {
+        let bgraBytesPerPixel = 4
+
+        assert(
+            CVPixelBufferGetPixelFormatType(imageBuffer) == kCVPixelFormatType_32BGRA,
+            "Image buffer must have 32BGRA pixel format type")
+
+        let width = CVPixelBufferGetWidth(mask.buffer)
+        let height = CVPixelBufferGetHeight(mask.buffer)
+        assert(CVPixelBufferGetWidth(imageBuffer) == width, "Width must match")
+        assert(CVPixelBufferGetHeight(imageBuffer) == height, "Height must match")
+
+        let writeFlags = CVPixelBufferLockFlags(rawValue: 0)
+        CVPixelBufferLockBaseAddress(imageBuffer, writeFlags)
+        CVPixelBufferLockBaseAddress(mask.buffer, CVPixelBufferLockFlags.readOnly)
+
+        let maskBytesPerRow = CVPixelBufferGetBytesPerRow(mask.buffer)
+        var maskAddress =
+            CVPixelBufferGetBaseAddress(mask.buffer)!.bindMemory(
+                to: Float32.self, capacity: maskBytesPerRow * height)
+
+        let imageBytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer)
+        var imageAddress = CVPixelBufferGetBaseAddress(imageBuffer)!.bindMemory(
+            to: UInt8.self, capacity: imageBytesPerRow * height)
+
+        for _ in 0...(height - 1) {
+            for col in 0...(width - 1) {
+                let pixelOffset = col * bgraBytesPerPixel
+                let blueOffset = pixelOffset
+                let greenOffset = pixelOffset + 1
+                let redOffset = pixelOffset + 2
+                let alphaOffset = pixelOffset + 3
+
+                let confidence: CGFloat = CGFloat(maskAddress[col])
+
+                if confidence >= threshold {
+                    let red = CGFloat(imageAddress[redOffset])
+                    let green = CGFloat(imageAddress[greenOffset])
+                    let blue = CGFloat(imageAddress[blueOffset])
+                    // let alpha = CGFloat(imageAddress[alphaOffset])
+
+                    imageAddress[redOffset] = UInt8(red * confidence)
+                    imageAddress[greenOffset] = UInt8(green * confidence)
+                    imageAddress[blueOffset] = UInt8(blue * confidence)
+                    imageAddress[alphaOffset] = UInt8(0xff)
+                } else {
+                    imageAddress[redOffset] = UInt8(0x00)
+                    imageAddress[greenOffset] = UInt8(0x00)
+                    imageAddress[blueOffset] = UInt8(0x00)
+                    imageAddress[alphaOffset] = UInt8(0x00)
+                }
+            }
+
+            imageAddress += imageBytesPerRow / MemoryLayout<UInt8>.size
+            maskAddress += maskBytesPerRow / MemoryLayout<Float32>.size
+        }
+
+        CVPixelBufferUnlockBaseAddress(imageBuffer, writeFlags)
+        CVPixelBufferUnlockBaseAddress(mask.buffer, CVPixelBufferLockFlags.readOnly)
+    }
+}

package/ios/Plugin/SelfieSegmentationPlugin.h

@@ -0,0 +1,10 @@
+#import <UIKit/UIKit.h>
+
+//! Project version number for Plugin.
+FOUNDATION_EXPORT double PluginVersionNumber;
+
+//! Project version string for Plugin.
+FOUNDATION_EXPORT const unsigned char PluginVersionString[];
+
+// In this header, you should import all the public headers of your framework using statements like #import <Plugin/PublicHeader.h>
+

package/ios/Plugin/SelfieSegmentationPlugin.m

@@ -0,0 +1,8 @@
+#import <Foundation/Foundation.h>
+#import <Capacitor/Capacitor.h>
+
+// Define the plugin using the CAP_PLUGIN Macro, and
+// each method the plugin supports using the CAP_PLUGIN_METHOD macro.
+CAP_PLUGIN(SelfieSegmentationPlugin, "SelfieSegmentation",
+           CAP_PLUGIN_METHOD(processImage, CAPPluginReturnPromise);
+)

package/ios/Plugin/SelfieSegmentationPlugin.swift

@@ -0,0 +1,61 @@
+import Foundation
+import Capacitor
+
+/**
+ * Please read the Capacitor iOS Plugin Development Guide
+ * here: https://capacitorjs.com/docs/plugins/ios
+ */
+@objc(SelfieSegmentationPlugin)
+public class SelfieSegmentationPlugin: CAPPlugin {
+    public let tag = "SelfieSegmentation"
+
+    public let errorPathMissing = "path must be provided."
+    public let errorLoadImageFailed = "The image could not be loaded."
+    public let errorWriteFileFailed = "The result could not be created."
+
+    public let defaultConfidence: Float = 0.9
+
+    private var implementation: SelfieSegmentation?
+
+    override public func load() {
+        implementation = SelfieSegmentation(plugin: self)
+    }
+
+    @objc func processImage(_ call: CAPPluginCall) {
+        guard let path = call.getString("path") else {
+            call.reject(errorPathMissing)
+            return
+        }
+
+        let width = call.getInt("width")
+        let height = call.getInt("height")
+
+        let confidence = call.getFloat("confidence", defaultConfidence)
+
+        guard let image = implementation?.createImageFromFilePath(path) else {
+            call.reject(errorLoadImageFailed)
+            return
+        }
+
+        let options = ProcessImageOptions(image: image,
+                                          width: width,
+                                          height: height,
+                                          confidence: CGFloat(confidence))
+
+        implementation?.processImage(options, completion: { result, error in
+            if let error = error {
+                CAPLog.print("[", self.tag, "] ", error)
+                call.reject(error.localizedDescription, nil, error)
+                return
+            }
+
+            if let result = result {
+                do {
+                    call.resolve(try result.toJSObject())
+                } catch {
+                    call.reject(self.errorWriteFileFailed)
+                }
+            }
+        })
+    }
+}

package/package.json

@@ -0,0 +1,91 @@
+{
+  "name": "@capacitor-mlkit/selfie-segmentation",
+  "version": "5.2.0",
+  "description": "Capacitor plugin for ML Kit Selfie Segmentation.",
+  "main": "dist/plugin.cjs.js",
+  "module": "dist/esm/index.js",
+  "types": "dist/esm/index.d.ts",
+  "unpkg": "dist/plugin.js",
+  "files": [
+    "android/src/main/",
+    "android/build.gradle",
+    "dist/",
+    "ios/Plugin/",
+    "CapacitorMlkitSelfieSegmentation.podspec"
+  ],
+  "author": "Robin Genz <mail@robingenz.dev>",
+  "license": "Apache-2.0",
+  "homepage": "https://capawesome.io/",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/capawesome-team/capacitor-mlkit.git"
+  },
+  "bugs": {
+    "url": "https://github.com/capawesome-team/capacitor-mlkit/issues"
+  },
+  "funding": [
+    {
+      "type": "github",
+      "url": "https://github.com/sponsors/capawesome-team/"
+    },
+    {
+      "type": "opencollective",
+      "url": "https://opencollective.com/capawesome"
+    }
+  ],
+  "keywords": [
+    "capacitor",
+    "plugin",
+    "native"
+  ],
+  "scripts": {
+    "verify": "npm run verify:ios && npm run verify:android && npm run verify:web",
+    "verify:ios": "cd ios && pod install && xcodebuild -workspace Plugin.xcworkspace -scheme Plugin -destination generic/platform=iOS && cd ..",
+    "verify:android": "cd android && ./gradlew clean build test && cd ..",
+    "verify:web": "npm run build",
+    "lint": "npm run eslint && npm run prettier -- --check && npm run swiftlint -- lint",
+    "fmt": "npm run eslint -- --fix && npm run prettier -- --write && npm run swiftlint -- --fix --format",
+    "eslint": "eslint . --ext ts",
+    "prettier": "prettier \"**/*.{css,html,ts,js,java}\"",
+    "swiftlint": "node-swiftlint",
+    "docgen": "docgen --api SelfieSegmentationPlugin --output-readme README.md --output-json dist/docs.json",
+    "build": "npm run clean && npm run docgen && tsc && rollup -c rollup.config.js",
+    "clean": "rimraf ./dist",
+    "watch": "tsc --watch",
+    "prepublishOnly": "npm run build"
+  },
+  "devDependencies": {
+    "@capacitor/android": "5.0.0",
+    "@capacitor/cli": "5.0.0",
+    "@capacitor/core": "5.0.0",
+    "@capacitor/docgen": "0.2.1",
+    "@capacitor/ios": "5.0.0",
+    "@ionic/eslint-config": "0.3.0",
+    "@ionic/swiftlint-config": "1.1.2",
+    "eslint": "7.32.0",
+    "prettier": "2.3.2",
+    "prettier-plugin-java": "1.0.2",
+    "rimraf": "3.0.2",
+    "rollup": "2.77.2",
+    "swiftlint": "1.0.1",
+    "typescript": "4.1.5"
+  },
+  "peerDependencies": {
+    "@capacitor/core": "^5.0.0"
+  },
+  "swiftlint": "@ionic/swiftlint-config",
+  "eslintConfig": {
+    "extends": "@ionic/eslint-config/recommended"
+  },
+  "capacitor": {
+    "ios": {
+      "src": "ios"
+    },
+    "android": {
+      "src": "android"
+    }
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}