@imagowave/vision-ocr 0.0.1

package/NitroVisionOcr.podspec

@@ -0,0 +1,32 @@
+require "json"
+
+package = JSON.parse(File.read(File.join(__dir__, "package.json")))
+
+Pod::Spec.new do |s|
+  s.name         = "NitroVisionOcr"
+  s.version      = package["version"]
+  s.summary      = package["description"]
+  s.homepage     = package["homepage"]
+  s.license      = package["license"]
+  s.authors      = package["author"]
+
+  s.platforms    = { :ios => min_ios_version_supported, :visionos => 1.0 }
+  s.source       = { :git => "https://github.com/mrousavy/nitro.git", :tag => "#{s.version}" }
+
+  s.source_files = [
+    # Implementation (Swift)
+    "ios/**/*.{swift}",
+    # Autolinking/Registration (Objective-C++)
+    "ios/**/*.{m,mm}",
+    # Implementation (C++ objects)
+    "cpp/**/*.{hpp,cpp}",
+  ]
+
+  load 'nitrogen/generated/ios/NitroVisionOcr+autolinking.rb'
+  add_nitrogen_files(s)
+
+  s.dependency 'React-jsi'
+  s.dependency 'React-callinvoker'
+  s.dependency 'VisionCamera'
+  install_modules_dependencies(s)
+end

package/README.md

@@ -0,0 +1,38 @@
+# react-native-nitro-template
+
+This is a template for Nitro Modules.
+
+## Usage
+
+Clone this repo, and change all `$$*$$` names according to your `nitro.json` file.
+
+## Contributing
+
+Contribute a change to this template to update it for newer React Native versions.
+
+## Structure
+
+- [`android/`](android): All your `android`-specific implementations.
+  - [`build.gradle`](android/build.gradle): The gradle build file. This contains four important pieces:
+    1. Standard react-native library boilerplate code
+    2. Configures Kotlin (`apply plugin: 'org.jetbrains.kotlin.android'`)
+    3. Adds all Nitrogen files (`apply from: '.../NitroVisionOcr+autolinking.gradle'`)
+    4. Triggers the native C++ build (via CMake/`externalNativeBuild`)
+  - [`CMakeLists.txt`](android/CMakeLists.txt): The CMake build file to build C++ code. This contains four important pieces:
+    1. Creates a library called `NitroVisionOcr` (same as in `nitro.json`)
+    2. Adds all Nitrogen files (`include(.../NitroVisionOcr+autolinking.cmake)`)
+    3. Adds all custom C++ files (only `HybridTestObjectCpp.cpp`)
+    4. Adds a `cpp-adapter.cpp` file, which autolinks all C++ HybridObjects (only `HybridTestObjectCpp`)
+  - [`src/main/java/com/margelo/nitro/visionocr/`](android/src/main/java/com/margelo/nitro/visionocr/): All Kotlin implementations.
+    - [`NitroVisionOcrPackage.kt`](android/src/main/java/com/margelo/nitro/visionocr/NitroVisionOcrPackage.kt): The react-native package. You need this because the react-native CLI only adds libraries if they have a `*Package.kt` file. In here, you can autolink all Kotlin HybridObjects.
+- [`cpp/`](cpp): All your cross-platform implementations. (only `HybridTestObjectCpp.cpp`)
+- [`ios/`](ios): All your iOS-specific implementations.
+- [`nitrogen/`](nitrogen): All files generated by nitrogen. You should commit this folder to git.
+- [`src/`](src): The TypeScript codebase. This defines all HybridObjects and loads them at runtime.
+  - [`specs/`](src/specs): All HybridObject types. Nitrogen will run on all `*.nitro.ts` files.
+- [`nitro.json`](nitro.json): The configuration file for nitrogen. This will define all native namespaces, as well as the library name.
+- [`NitroVisionOcr.podspec`](NitroVisionOcr.podspec): The iOS podspec build file to build the iOS code. This contains three important pieces:
+  1. Specifies the Pod's name. This must be identical to the name specified in `nitro.json`.
+  2. Adds all of your `.swift` or `.cpp` files (implementations).
+  3. Adds all Nitrogen files (`add_nitrogen_files(s)`)
+- [`package.json`](package.json): The npm package.json file. `react-native-nitro-modules` should be a `peerDependency`.

package/android/CMakeLists.txt

@@ -0,0 +1,31 @@
+project(NitroVisionOcr)
+cmake_minimum_required(VERSION 3.9.0)
+
+set (PACKAGE_NAME NitroVisionOcr)
+set (CMAKE_VERBOSE_MAKEFILE ON)
+set (CMAKE_CXX_STANDARD 20)
+
+# Define C++ library and add all sources
+add_library(${PACKAGE_NAME} SHARED
+        src/main/cpp/cpp-adapter.cpp
+)
+
+# Add Nitrogen specs :)
+include(${CMAKE_SOURCE_DIR}/../nitrogen/generated/android/NitroVisionOcr+autolinking.cmake)
+
+# Set up local includes
+include_directories(
+        "src/main/cpp"
+        "../cpp"
+)
+
+find_package(react-native-vision-camera REQUIRED)
+find_library(LOG_LIB log)
+
+# Link all libraries together
+target_link_libraries(
+        ${PACKAGE_NAME}
+        ${LOG_LIB}
+        android                                   # <-- Android core
+        react-native-vision-camera::VisionCamera
+)

package/android/build.gradle

@@ -0,0 +1,143 @@
+buildscript {
+  repositories {
+    google()
+    mavenCentral()
+  }
+
+  dependencies {
+    classpath "com.android.tools.build:gradle:9.2.1"
+  }
+}
+
+def reactNativeArchitectures() {
+  def value = rootProject.getProperties().get("reactNativeArchitectures")
+  return value ? value.split(",") : ["armeabi-v7a", "x86", "x86_64", "arm64-v8a"]
+}
+
+def isNewArchitectureEnabled() {
+  return rootProject.hasProperty("newArchEnabled") && rootProject.getProperty("newArchEnabled") == "true"
+}
+
+apply plugin: "com.android.library"
+apply plugin: 'org.jetbrains.kotlin.android'
+apply from: '../nitrogen/generated/android/NitroVisionOcr+autolinking.gradle'
+apply from: "./fix-prefab.gradle"
+
+if (isNewArchitectureEnabled()) {
+  apply plugin: "com.facebook.react"
+}
+
+def getExtOrDefault(name) {
+  return rootProject.ext.has(name) ? rootProject.ext.get(name) : project.properties["NitroVisionOcr_" + name]
+}
+
+def getExtOrIntegerDefault(name) {
+  return rootProject.ext.has(name) ? rootProject.ext.get(name) : (project.properties["NitroVisionOcr_" + name]).toInteger()
+}
+
+android {
+  namespace "com.margelo.nitro.visionocr"
+
+  ndkVersion getExtOrDefault("ndkVersion")
+  compileSdkVersion getExtOrIntegerDefault("compileSdkVersion")
+
+  defaultConfig {
+    minSdkVersion getExtOrIntegerDefault("minSdkVersion")
+    targetSdkVersion getExtOrIntegerDefault("targetSdkVersion")
+    buildConfigField "boolean", "IS_NEW_ARCHITECTURE_ENABLED", isNewArchitectureEnabled().toString()
+
+    externalNativeBuild {
+      cmake {
+        cppFlags "-frtti -fexceptions -Wall -Wextra -fstack-protector-all"
+        arguments "-DANDROID_STL=c++_shared", "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON"
+        abiFilters (*reactNativeArchitectures())
+
+        buildTypes {
+          debug {
+            cppFlags "-O1 -g"
+          }
+          release {
+            cppFlags "-O2"
+          }
+        }
+      }
+    }
+  }
+
+  externalNativeBuild {
+    cmake {
+      path "CMakeLists.txt"
+    }
+  }
+
+  packagingOptions {
+    excludes = [
+            "META-INF",
+            "META-INF/**",
+            "**/libc++_shared.so",
+            "**/libNitroModules.so",
+            "**/libfbjni.so",
+            "**/libjsi.so",
+            "**/libfolly_json.so",
+            "**/libfolly_runtime.so",
+            "**/libglog.so",
+            "**/libhermes.so",
+            "**/libhermes-executor-debug.so",
+            "**/libhermes_executor.so",
+            "**/libreactnative.so",
+            "**/libreactnativejni.so",
+            "**/libturbomodulejsijni.so",
+            "**/libreact_nativemodule_core.so",
+            "**/libjscexecutor.so"
+    ]
+  }
+
+  buildFeatures {
+    buildConfig true
+    prefab true
+  }
+
+  buildTypes {
+    release {
+      minifyEnabled false
+    }
+  }
+
+  lintOptions {
+    disable "GradleCompatible"
+  }
+
+  compileOptions {
+    sourceCompatibility JavaVersion.VERSION_1_8
+    targetCompatibility JavaVersion.VERSION_1_8
+  }
+
+  sourceSets {
+    main {
+      if (isNewArchitectureEnabled()) {
+        java.srcDirs += [
+          // React Codegen files
+          "${project.buildDir}/generated/source/codegen/java"
+        ]
+      }
+    }
+  }
+}
+
+repositories {
+  mavenCentral()
+  google()
+}
+
+
+dependencies {
+  // For < 0.71, this will be from the local maven repo
+  // For > 0.71, this will be replaced by `com.facebook.react:react-android:$version` by react gradle plugin
+  //noinspection GradleDynamicVersion
+  implementation "com.facebook.react:react-native:+"
+
+  // Add a dependency on NitroModules
+  implementation project(":react-native-nitro-modules")
+  implementation project(":react-native-vision-camera")
+}
+

package/android/fix-prefab.gradle

@@ -0,0 +1,51 @@
+tasks.configureEach { task ->
+  // Make sure that we generate our prefab publication file only after having built the native library
+  // so that not a header publication file, but a full configuration publication will be generated, which
+  // will include the .so file
+
+  def prefabConfigurePattern = ~/^prefab(.+)ConfigurePackage$/
+  def matcher = task.name =~ prefabConfigurePattern
+  if (matcher.matches()) {
+    def variantName = matcher[0][1]
+    task.outputs.upToDateWhen { false }
+    task.dependsOn("externalNativeBuild${variantName}")
+  }
+}
+
+afterEvaluate {
+  def abis = reactNativeArchitectures()
+  rootProject.allprojects.each { proj ->
+    if (proj === rootProject) return
+
+    def dependsOnThisLib = proj.configurations.findAll { it.canBeResolved }.any { config ->
+      config.dependencies.any { dep ->
+        dep.group == project.group && dep.name == project.name
+      }
+    }
+    if (!dependsOnThisLib && proj != project) return
+
+    if (!proj.plugins.hasPlugin('com.android.application') && !proj.plugins.hasPlugin('com.android.library')) {
+      return
+    }
+
+    def variants = proj.android.hasProperty('applicationVariants') ? proj.android.applicationVariants : proj.android.libraryVariants
+    // Touch the prefab_config.json files to ensure that in ExternalNativeJsonGenerator.kt we will re-trigger the prefab CLI to
+    // generate a libnameConfig.cmake file that will contain our native library (.so).
+    // See this condition: https://cs.android.com/android-studio/platform/tools/base/+/mirror-goog-studio-main:build-system/gradle-core/src/main/java/com/android/build/gradle/tasks/ExternalNativeJsonGenerator.kt;l=207-219?q=createPrefabBuildSystemGlue
+    variants.all { variant ->
+      def variantName = variant.name
+      abis.each { abi ->
+        def searchDir = new File(proj.projectDir, ".cxx/${variantName}")
+        if (!searchDir.exists()) return
+        def matches = []
+        searchDir.eachDir { randomDir ->
+          def prefabFile = new File(randomDir, "${abi}/prefab_config.json")
+          if (prefabFile.exists()) matches << prefabFile
+        }
+        matches.each { prefabConfig ->
+          prefabConfig.setLastModified(System.currentTimeMillis())
+        }
+      }
+    }
+  }
+}

package/android/gradle.properties

@@ -0,0 +1,5 @@
+NitroVisionOcr_kotlinVersion=2.1.20
+NitroVisionOcr_minSdkVersion=23
+NitroVisionOcr_targetSdkVersion=36
+NitroVisionOcr_compileSdkVersion=36
+NitroVisionOcr_ndkVersion=27.1.12297006

package/android/src/main/AndroidManifest.xml

@@ -0,0 +1,2 @@
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+</manifest>

package/android/src/main/cpp/cpp-adapter.cpp

@@ -0,0 +1,9 @@
+#include <jni.h>
+#include <fbjni/fbjni.h>
+#include "NitroVisionOcrOnLoad.hpp"
+
+JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void*) {
+  return facebook::jni::initialize(vm, []() {
+    margelo::nitro::visionocr::registerAllNatives();
+  });
+}

package/android/src/main/java/com/margelo/nitro/visionocr/NitroVisionOcrPackage.kt

@@ -0,0 +1,18 @@
+package com.margelo.nitro.visionocr
+
+import com.facebook.react.bridge.NativeModule
+import com.facebook.react.bridge.ReactApplicationContext
+import com.facebook.react.module.model.ReactModuleInfoProvider
+import com.facebook.react.BaseReactPackage
+
+class NitroVisionOcrPackage : BaseReactPackage() {
+    override fun getModule(name: String, reactContext: ReactApplicationContext): NativeModule? = null
+
+    override fun getReactModuleInfoProvider(): ReactModuleInfoProvider = ReactModuleInfoProvider { HashMap() }
+
+    companion object {
+        init {
+            NitroVisionOcrOnLoad.initializeNative()
+        }
+    }
+}

package/android/src/main/java/com/margelo/nitro/visionocr/VisionOcr.kt

@@ -0,0 +1,120 @@
+package com.margelo.nitro.math
+import com.margelo.nitro.core.*
+import android.graphics.Point
+import android.graphics.Rect
+import android.media.Image
+import android.util.Log
+import com.facebook.react.bridge.WritableNativeArray
+import com.facebook.react.bridge.WritableNativeMap
+import com.google.android.gms.tasks.Tasks
+import com.google.mlkit.vision.common.InputImage
+import com.google.mlkit.vision.text.Text
+import com.google.mlkit.vision.text.TextRecognition
+import com.google.mlkit.vision.text.latin.TextRecognizerOptions
+import com.mrousavy.camera.frameprocessors.Frame
+import com.mrousavy.camera.frameprocessors.FrameProcessorPlugin
+import com.mrousavy.camera.frameprocessors.VisionCameraProxy
+import java.util.HashMap
+
+class VisionOCR : HybridVisionOCRSpec() {
+  private val recognizer = TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS)
+
+  override fun call(frame: Frame, arguments: Map<String, Any>?): HashMap<String, Any>? {
+    val data = WritableNativeMap()
+
+    // Log the options for debugging
+    if (options != null) {
+      Log.d("OcrDetector", "Plugin options: $options")
+      val model = options["model"] as? String
+      if (model != null) {
+        Log.d("OcrDetector", "Using model: $model")
+        // TODO: Implement different model options based on 'model' parameter
+        // Currently ML Kit only supports DEFAULT_OPTIONS for text recognition
+        // Future versions might support different accuracy/speed trade-offs
+      }
+    }
+
+    // Read call-time arguments
+    val includeBoxes = (arguments?.get("includeBoxes") as? Boolean) ?: false
+    val includeConfidence = (arguments?.get("includeConfidence") as? Boolean) ?: false
+    Log.d("OcrDetector", "Args includeBoxes=$includeBoxes includeConfidence=$includeConfidence")
+
+    val mediaImage: Image = frame.image
+    val image = InputImage.fromMediaImage(mediaImage, frame.imageProxy.imageInfo.rotationDegrees)
+
+    return try {
+      val visionText: Text = Tasks.await(recognizer.process(image))
+
+      if (visionText.text.isEmpty()) {
+        return null
+      }
+
+      data.putString("text", visionText.text)
+
+      if (includeBoxes) {
+        val blocksArray = WritableNativeArray()
+        for (block in visionText.textBlocks) {
+          val blockMap = WritableNativeMap()
+          blockMap.putString("text", block.text)
+
+          val blockRect: Rect? = block.boundingBox
+          if (blockRect != null) {
+            val boxMap = WritableNativeMap()
+            boxMap.putDouble("x", blockRect.left.toDouble())
+            boxMap.putDouble("y", blockRect.top.toDouble())
+            boxMap.putDouble("width", (blockRect.right - blockRect.left).toDouble())
+            boxMap.putDouble("height", (blockRect.bottom - blockRect.top).toDouble())
+            blockMap.putMap("box", boxMap)
+          }
+
+          val linesArray = WritableNativeArray()
+          for (line in block.lines) {
+            val lineMap = WritableNativeMap()
+            lineMap.putString("text", line.text)
+
+            val lineRect: Rect? = line.boundingBox
+            if (lineRect != null) {
+              val lbox = WritableNativeMap()
+              lbox.putDouble("x", lineRect.left.toDouble())
+              lbox.putDouble("y", lineRect.top.toDouble())
+              lbox.putDouble("width", (lineRect.right - lineRect.left).toDouble())
+              lbox.putDouble("height", (lineRect.bottom - lineRect.top).toDouble())
+              lineMap.putMap("box", lbox)
+            }
+
+            val wordsArray = WritableNativeArray()
+            for (element in line.elements) {
+              val wordMap = WritableNativeMap()
+              wordMap.putString("text", element.text)
+              val wRect: Rect? = element.boundingBox
+              if (wRect != null) {
+                val wbox = WritableNativeMap()
+                wbox.putDouble("x", wRect.left.toDouble())
+                wbox.putDouble("y", wRect.top.toDouble())
+                wbox.putDouble("width", (wRect.right - wRect.left).toDouble())
+                wbox.putDouble("height", (wRect.bottom - wRect.top).toDouble())
+                wordMap.putMap("box", wbox)
+              }
+              wordsArray.pushMap(wordMap)
+            }
+            if (wordsArray.size() > 0) {
+              lineMap.putArray("words", wordsArray)
+            }
+            linesArray.pushMap(lineMap)
+          }
+          if (linesArray.size() > 0) {
+            blockMap.putArray("lines", linesArray)
+          }
+          blocksArray.pushMap(blockMap)
+        }
+        data.putArray("blocks", blocksArray)
+      }
+
+      @Suppress("UNCHECKED_CAST")
+      data.toHashMap() as HashMap<String, Any>
+    } catch (e: Exception) {
+      Log.e("OcrDetector", "OCR recognition error: ${e.localizedMessage}")
+      null
+    }
+  }
+}

package/ios/Bridge.h

@@ -0,0 +1,8 @@
+//
+//  Bridge.h
+//  NitroVisionOcr
+//
+//  Created by Marc Rousavy on 22.07.24.
+//
+
+#pragma once

package/ios/VisionOCR.swift

@@ -0,0 +1,137 @@
+import CoreImage
+import Foundation
+import NitroModules
+import ImageIO
+import Vision
+import VisionCamera
+
+class VisionOCR : HybridVisionOCRSpec {
+  func call(frame: (any HybridFrameSpec), options: OcrOptions?) throws -> OcrResult? {
+    let includeBoxes = options?.includeBoxes ?? false
+    let includeConfidence = options?.includeConfidence ?? false
+    NSLog(
+      "Args includeBoxes=%@ includeConfidence=%@",
+      includeBoxes ? "YES" : "NO",
+      includeConfidence ? "YES" : "NO"
+    )
+
+  	let sampleBuffer = try sampleBuffer(from: frame)
+    let pixelBuffer = try pixelBuffer(from: sampleBuffer)
+
+    let ciImage = CIImage(cvPixelBuffer: pixelBuffer)
+    let orientation = cgImageOrientation(from: frame.orientation, isMirrored: frame.isMirrored)
+    let handler = VNImageRequestHandler(ciImage: ciImage, orientation: orientation, options: [:])
+
+    var lineResults: [OcrLine] = []
+
+    let request = VNRecognizeTextRequest { request, error in
+      if let error {
+        NSLog("OCR recognition failed: %@", error.localizedDescription)
+        return
+      }
+
+      guard let results = request.results as? [VNRecognizedTextObservation] else {
+        return
+      }
+
+      for observation in results {
+        guard let topCandidate = observation.topCandidates(1).first else {
+          continue
+        }
+
+        let rect = observation.boundingBox
+
+        var line: OcrLine = OcrLine(
+          text: topCandidate.string, 
+          box: includeBoxes ? OcrBox(
+            x: rect.origin.x,
+            y: rect.origin.y,
+            width: rect.size.width,
+            height: rect.size.height
+          ) : nil,
+          words: nil,
+          confidence: includeConfidence ? Double(topCandidate.confidence) : nil, 
+        )
+        lineResults.append(line)
+      }
+    }
+
+      if let level = options?.recognitionLevel {
+      if level == .accurate {
+        request.recognitionLevel = VNRequestTextRecognitionLevel.accurate
+      } else if level == .fast {
+        request.recognitionLevel = VNRequestTextRecognitionLevel.fast
+      }
+    }
+
+    if let languages = options?.recognitionLanguages {
+      request.recognitionLanguages = languages
+    }
+
+    if let useCorrection = options?.usesLanguageCorrection {
+      request.usesLanguageCorrection = useCorrection
+    }
+
+    let semaphore = DispatchSemaphore(value: 0)
+
+    DispatchQueue.global(qos: .userInitiated).async {
+      do {
+        try handler.perform([request])
+      } catch {
+        NSLog("Failed to perform OCR recognition: %@", error.localizedDescription)
+      }
+      semaphore.signal()
+    }
+
+    semaphore.wait()
+
+    if lineResults.isEmpty {
+      return nil
+    }
+
+    let joinedText = lineResults.compactMap { $0.text }.joined(separator: " ")
+
+    return OcrResult(
+      text: joinedText, 
+      blocks: includeBoxes ? [
+        OcrBlock(
+          text: joinedText, 
+          box: nil, 
+          lines: lineResults
+        )
+      ] : nil)
+  }
+
+  private func cgImageOrientation(from orientation: CameraOrientation, isMirrored: Bool) -> CGImagePropertyOrientation {
+    switch orientation {
+    case .up:
+      return isMirrored ? .upMirrored : .up
+    case .down:
+      return isMirrored ? .downMirrored : .down
+    case .left:
+      return isMirrored ? .leftMirrored : .left
+    case .right:
+      return isMirrored ? .rightMirrored : .right
+    @unknown default:
+      return .up
+    }
+  }
+
+  private func sampleBuffer(from frame: any HybridFrameSpec) throws -> CMSampleBuffer {
+    guard let nativeFrame = frame as? any NativeFrame else {
+      throw RuntimeError.error(withMessage: "The given Frame is not of type `NativeFrame`!")
+    }
+    guard let sampleBuffer = nativeFrame.sampleBuffer else {
+      throw RuntimeError.error(withMessage: "The given Frame's `sampleBuffer` is no longer valid!")
+    }
+    return sampleBuffer
+  }
+
+  private func pixelBuffer(from sampleBuffer: CMSampleBuffer) throws -> CVPixelBuffer {
+    guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
+      throw RuntimeError.error(
+        withMessage: "The given Frame does not contain a valid image buffer!")
+    }
+    return pixelBuffer
+  }
+}

package/nitro.json

@@ -0,0 +1,19 @@
+{
+  "$schema": "https://nitro.margelo.com/nitro.schema.json",
+  "cxxNamespace": [
+    "visionocr"
+  ],
+  "ios": {
+    "iosModuleName": "NitroVisionOcr"
+  },
+  "android": {
+    "androidNamespace": [
+      "visionocr"
+    ],
+    "androidCxxLibName": "NitroVisionOcr"
+  },
+  "autolinking": {},
+  "ignorePaths": [
+    "**/node_modules"
+  ]
+}

package/nitrogen/generated/.gitattributes

@@ -0,0 +1 @@
+** linguist-generated=true