@nitro-mlkit/digital-ink 0.1.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/NitroMLKitDigitalInk.podspec +34 -0
- package/README.md +67 -0
- package/android/CMakeLists.txt +12 -0
- package/android/build.gradle +77 -0
- package/android/src/main/AndroidManifest.xml +3 -0
- package/android/src/main/cpp/cpp-adapter.cpp +9 -0
- package/android/src/main/kotlin/com/margelo/nitro/nitromlkit/digitalink/HybridDigitalInkRecognizer.kt +84 -0
- package/android/src/main/kotlin/com/nitromlkit/digitalink/NitroMLKitDigitalInkPackage.kt +19 -0
- package/expo-module.config.json +9 -0
- package/ios/HybridDigitalInkRecognizer.swift +156 -0
- package/nitro.json +16 -0
- package/nitrogen/generated/.gitattributes +1 -0
- package/nitrogen/generated/android/NitroMLKitDigitalInk+autolinking.cmake +81 -0
- package/nitrogen/generated/android/NitroMLKitDigitalInk+autolinking.gradle +27 -0
- package/nitrogen/generated/android/NitroMLKitDigitalInkOnLoad.cpp +54 -0
- package/nitrogen/generated/android/NitroMLKitDigitalInkOnLoad.hpp +34 -0
- package/nitrogen/generated/android/c++/JHybridDigitalInkRecognizerSpec.cpp +149 -0
- package/nitrogen/generated/android/c++/JHybridDigitalInkRecognizerSpec.hpp +67 -0
- package/nitrogen/generated/android/c++/JInkPoint.hpp +65 -0
- package/nitrogen/generated/android/c++/JInkStroke.hpp +78 -0
- package/nitrogen/generated/android/c++/JRecognitionCandidate.hpp +62 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/nitromlkit/digitalink/HybridDigitalInkRecognizerSpec.kt +71 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/nitromlkit/digitalink/InkPoint.kt +61 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/nitromlkit/digitalink/InkStroke.kt +51 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/nitromlkit/digitalink/NitroMLKitDigitalInkOnLoad.kt +35 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/nitromlkit/digitalink/RecognitionCandidate.kt +56 -0
- package/nitrogen/generated/ios/NitroMLKitDigitalInk+autolinking.rb +62 -0
- package/nitrogen/generated/ios/NitroMLKitDigitalInk-Swift-Cxx-Bridge.cpp +65 -0
- package/nitrogen/generated/ios/NitroMLKitDigitalInk-Swift-Cxx-Bridge.hpp +265 -0
- package/nitrogen/generated/ios/NitroMLKitDigitalInk-Swift-Cxx-Umbrella.hpp +56 -0
- package/nitrogen/generated/ios/NitroMLKitDigitalInkAutolinking.mm +33 -0
- package/nitrogen/generated/ios/NitroMLKitDigitalInkAutolinking.swift +26 -0
- package/nitrogen/generated/ios/c++/HybridDigitalInkRecognizerSpecSwift.cpp +11 -0
- package/nitrogen/generated/ios/c++/HybridDigitalInkRecognizerSpecSwift.hpp +125 -0
- package/nitrogen/generated/ios/swift/Func_void.swift +46 -0
- package/nitrogen/generated/ios/swift/Func_void_bool.swift +46 -0
- package/nitrogen/generated/ios/swift/Func_void_std__exception_ptr.swift +46 -0
- package/nitrogen/generated/ios/swift/Func_void_std__vector_RecognitionCandidate_.swift +46 -0
- package/nitrogen/generated/ios/swift/HybridDigitalInkRecognizerSpec.swift +59 -0
- package/nitrogen/generated/ios/swift/HybridDigitalInkRecognizerSpec_cxx.swift +220 -0
- package/nitrogen/generated/ios/swift/InkPoint.swift +52 -0
- package/nitrogen/generated/ios/swift/InkStroke.swift +35 -0
- package/nitrogen/generated/ios/swift/RecognitionCandidate.swift +47 -0
- package/nitrogen/generated/shared/c++/HybridDigitalInkRecognizerSpec.cpp +25 -0
- package/nitrogen/generated/shared/c++/HybridDigitalInkRecognizerSpec.hpp +73 -0
- package/nitrogen/generated/shared/c++/InkPoint.hpp +91 -0
- package/nitrogen/generated/shared/c++/InkStroke.hpp +85 -0
- package/nitrogen/generated/shared/c++/RecognitionCandidate.hpp +88 -0
- package/package.json +54 -0
- package/src/index.ts +19 -0
- package/src/specs/DigitalInkRecognizer.nitro.ts +51 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Gonzalo Polo
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
|
|
3
|
+
package = JSON.parse(File.read(File.join(__dir__, "package.json")))
|
|
4
|
+
|
|
5
|
+
Pod::Spec.new do |s|
|
|
6
|
+
s.name = "NitroMLKitDigitalInk"
|
|
7
|
+
# Must match nitro.json's `ios.iosModuleName` — Nitrogen's generated bridge
|
|
8
|
+
# imports "NitroMLKitDigitalInk-Swift.h", so the compiled Swift module name has to be NitroMLKitDigitalInk.
|
|
9
|
+
s.module_name = "NitroMLKitDigitalInk"
|
|
10
|
+
s.version = package["version"]
|
|
11
|
+
s.summary = package["description"]
|
|
12
|
+
s.homepage = package["repository"]["url"]
|
|
13
|
+
s.license = package["license"]
|
|
14
|
+
s.authors = package["author"]
|
|
15
|
+
s.source = { :git => "https://github.com/pologonzalo/react-native-nitro-mlkit.git", :tag => s.version }
|
|
16
|
+
|
|
17
|
+
s.platforms = { :ios => "15.5" }
|
|
18
|
+
|
|
19
|
+
s.source_files = [
|
|
20
|
+
"ios/**/*.{swift,h,m,mm,cpp}"
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
s.dependency "GoogleMLKit/DigitalInkRecognition", "~> 7.0"
|
|
24
|
+
|
|
25
|
+
s.pod_target_xcconfig = {
|
|
26
|
+
"SWIFT_VERSION" => "5.9"
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
# Add all files generated by Nitrogen (generated sources + NitroModules dep +
|
|
30
|
+
# the C++/Swift-interop build flags — same setup as the working
|
|
31
|
+
# NitroMLKitFaceDetection podspec).
|
|
32
|
+
load 'nitrogen/generated/ios/NitroMLKitDigitalInk+autolinking.rb'
|
|
33
|
+
add_nitrogen_files(s)
|
|
34
|
+
end
|
package/README.md
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# React Native ML Kit — Digital Ink Recognition
|
|
2
|
+
|
|
3
|
+
**`@nitro-mlkit/digital-ink`** · on-device Google ML Kit via [Nitro Modules](https://github.com/mrousavy/nitro) — JSI, no bridge.
|
|
4
|
+
|
|
5
|
+
> ⚠️ **Beta (`0.1.0-beta.x`).** Android verified on-device. iOS builds & links
|
|
6
|
+
> but device runtime is pending — see [Platform status](#platform-status).
|
|
7
|
+
|
|
8
|
+
High-performance, on-device **handwriting / digital-ink recognition** for React
|
|
9
|
+
Native, built with [Nitro Modules](https://github.com/mrousavy/nitro) (JSI, no bridge).
|
|
10
|
+
|
|
11
|
+
Powered by **Google ML Kit**. Feed it the **strokes** the user drew (points with
|
|
12
|
+
optional timestamps) and a language tag; get back candidate transcriptions.
|
|
13
|
+
300+ languages, plus autodraw/shapes and emoji. The per-language model
|
|
14
|
+
**downloads at runtime** on first use. **All on-device.**
|
|
15
|
+
|
|
16
|
+
## Installation
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
npm install @nitro-mlkit/digital-ink@beta react-native-nitro-modules
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
No config plugin (autolinked Expo module). Install and `npx expo prebuild`.
|
|
23
|
+
Not available in Expo Go.
|
|
24
|
+
|
|
25
|
+
## Usage
|
|
26
|
+
|
|
27
|
+
```ts
|
|
28
|
+
import { NitroDigitalInk, type InkStroke } from "@nitro-mlkit/digital-ink";
|
|
29
|
+
|
|
30
|
+
// Collect strokes from your canvas (e.g. via PanResponder). Each stroke is the
|
|
31
|
+
// list of points between pen-down and pen-up. Timestamps (t) are optional but
|
|
32
|
+
// improve accuracy.
|
|
33
|
+
const strokes: InkStroke[] = [
|
|
34
|
+
{ points: [ { x: 10, y: 20, t: 0 }, { x: 12, y: 22, t: 16 }, /* … */ ] },
|
|
35
|
+
];
|
|
36
|
+
|
|
37
|
+
const candidates = await NitroDigitalInk.recognize(strokes, "en-US");
|
|
38
|
+
// → [{ text: "hello", score: … }, { text: "helio", … }, …] (best first)
|
|
39
|
+
|
|
40
|
+
// Model management
|
|
41
|
+
await NitroDigitalInk.downloadModel("es-ES");
|
|
42
|
+
await NitroDigitalInk.isModelDownloaded("es-ES"); // boolean
|
|
43
|
+
await NitroDigitalInk.deleteModel("es-ES");
|
|
44
|
+
|
|
45
|
+
NitroDigitalInk.isAvailable(); // boolean
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Language tags follow ML Kit's identifiers (e.g. `"en-US"`, `"es-ES"`, `"fr-FR"`,
|
|
49
|
+
`"zh-Hani"`, `"emoji"`).
|
|
50
|
+
|
|
51
|
+
## Platform status
|
|
52
|
+
|
|
53
|
+
| Platform | Min | Status |
|
|
54
|
+
| -------- | --- | ------ |
|
|
55
|
+
| Android | API 26+ | ✅ Verified on-device (Pixel 9, API 36) |
|
|
56
|
+
| iOS | 15.5+ | ⚠️ Swift impl written; on-device build & run pending¹ |
|
|
57
|
+
| tvOS/macOS | — | 🔜 Planned |
|
|
58
|
+
|
|
59
|
+
¹ ML Kit's iOS pods ship no `arm64` Simulator slice; validate on a physical device.
|
|
60
|
+
|
|
61
|
+
## Part of `nitro-mlkit`
|
|
62
|
+
|
|
63
|
+
The full ML Kit suite on Nitro — see the other `@nitro-mlkit/*` packages.
|
|
64
|
+
|
|
65
|
+
## License
|
|
66
|
+
|
|
67
|
+
MIT © Gonzalo Polo
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
project(NitroMLKitDigitalInk)
|
|
2
|
+
cmake_minimum_required(VERSION 3.9.0)
|
|
3
|
+
|
|
4
|
+
set(CMAKE_CXX_STANDARD 20)
|
|
5
|
+
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
|
6
|
+
|
|
7
|
+
add_library(NitroMLKitDigitalInk SHARED
|
|
8
|
+
src/main/cpp/cpp-adapter.cpp
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
# Add all files generated by Nitrogen
|
|
12
|
+
include(${CMAKE_SOURCE_DIR}/../nitrogen/generated/android/NitroMLKitDigitalInk+autolinking.cmake)
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
buildscript {
|
|
2
|
+
ext.safeExtGet = { prop, fallback ->
|
|
3
|
+
rootProject.ext.has(prop) ? rootProject.ext.get(prop) : fallback
|
|
4
|
+
}
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
def reactNativeArchitectures() {
|
|
8
|
+
def value = rootProject.getProperties().get("reactNativeArchitectures")
|
|
9
|
+
return value ? value.split(",") : ["armeabi-v7a", "x86", "x86_64", "arm64-v8a"]
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
apply plugin: "com.android.library"
|
|
13
|
+
apply plugin: "kotlin-android"
|
|
14
|
+
|
|
15
|
+
android {
|
|
16
|
+
namespace "com.nitromlkit.digitalink"
|
|
17
|
+
compileSdkVersion safeExtGet("compileSdkVersion", 35)
|
|
18
|
+
ndkVersion safeExtGet("ndkVersion", "27.1.12297006")
|
|
19
|
+
|
|
20
|
+
defaultConfig {
|
|
21
|
+
minSdkVersion safeExtGet("minSdkVersion", 26)
|
|
22
|
+
targetSdkVersion safeExtGet("targetSdkVersion", 35)
|
|
23
|
+
|
|
24
|
+
externalNativeBuild {
|
|
25
|
+
cmake {
|
|
26
|
+
arguments "-DANDROID_STL=c++_shared"
|
|
27
|
+
abiFilters (*reactNativeArchitectures())
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
externalNativeBuild {
|
|
33
|
+
cmake {
|
|
34
|
+
path "CMakeLists.txt"
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
buildFeatures {
|
|
39
|
+
prefab true
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
compileOptions {
|
|
43
|
+
sourceCompatibility JavaVersion.VERSION_17
|
|
44
|
+
targetCompatibility JavaVersion.VERSION_17
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
kotlinOptions {
|
|
48
|
+
jvmTarget = "17"
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
sourceSets {
|
|
52
|
+
main {
|
|
53
|
+
java.srcDirs += [
|
|
54
|
+
"src/main/kotlin",
|
|
55
|
+
"../nitrogen/generated/android/kotlin"
|
|
56
|
+
]
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
dependencies {
|
|
62
|
+
// MLKit
|
|
63
|
+
implementation "com.google.mlkit:digital-ink-recognition:19.0.0"
|
|
64
|
+
|
|
65
|
+
// Nitro Modules
|
|
66
|
+
implementation project(":react-native-nitro-modules")
|
|
67
|
+
|
|
68
|
+
// Expo Modules (only used so autolinking calls our OnLoad at startup)
|
|
69
|
+
implementation project(":expo-modules-core")
|
|
70
|
+
|
|
71
|
+
// Kotlin coroutines
|
|
72
|
+
implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:1.9.0"
|
|
73
|
+
implementation "org.jetbrains.kotlinx:kotlinx-coroutines-play-services:1.9.0"
|
|
74
|
+
|
|
75
|
+
// React Native
|
|
76
|
+
implementation "com.facebook.react:react-android"
|
|
77
|
+
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
package com.margelo.nitro.nitromlkit.digitalink
|
|
2
|
+
|
|
3
|
+
import com.google.mlkit.common.model.DownloadConditions
|
|
4
|
+
import com.google.mlkit.common.model.RemoteModelManager
|
|
5
|
+
import com.google.mlkit.vision.digitalink.recognition.DigitalInkRecognition
|
|
6
|
+
import com.google.mlkit.vision.digitalink.recognition.DigitalInkRecognitionModel
|
|
7
|
+
import com.google.mlkit.vision.digitalink.recognition.DigitalInkRecognitionModelIdentifier
|
|
8
|
+
import com.google.mlkit.vision.digitalink.recognition.DigitalInkRecognizerOptions
|
|
9
|
+
import com.google.mlkit.vision.digitalink.recognition.Ink
|
|
10
|
+
import com.margelo.nitro.core.Promise
|
|
11
|
+
import kotlinx.coroutines.tasks.await
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Native Android implementation of DigitalInkRecognizer (MLKit Digital Ink).
|
|
15
|
+
* Ink strokes (points with optional timestamps) go in; candidate
|
|
16
|
+
* transcriptions come out. The per-language model downloads on first use.
|
|
17
|
+
*/
|
|
18
|
+
class HybridDigitalInkRecognizer : HybridDigitalInkRecognizerSpec() {
|
|
19
|
+
|
|
20
|
+
private val modelManager by lazy { RemoteModelManager.getInstance() }
|
|
21
|
+
|
|
22
|
+
private fun modelFor(languageTag: String): DigitalInkRecognitionModel {
|
|
23
|
+
val id = DigitalInkRecognitionModelIdentifier.fromLanguageTag(languageTag)
|
|
24
|
+
?: throw IllegalArgumentException("Unsupported language tag: $languageTag")
|
|
25
|
+
return DigitalInkRecognitionModel.builder(id).build()
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
private fun buildInk(strokes: Array<InkStroke>): Ink {
|
|
29
|
+
val ink = Ink.builder()
|
|
30
|
+
for (stroke in strokes) {
|
|
31
|
+
val sb = Ink.Stroke.builder()
|
|
32
|
+
for (p in stroke.points) {
|
|
33
|
+
val t = p.t
|
|
34
|
+
sb.addPoint(
|
|
35
|
+
if (t != null) Ink.Point.create(p.x.toFloat(), p.y.toFloat(), t.toLong())
|
|
36
|
+
else Ink.Point.create(p.x.toFloat(), p.y.toFloat()),
|
|
37
|
+
)
|
|
38
|
+
}
|
|
39
|
+
ink.addStroke(sb.build())
|
|
40
|
+
}
|
|
41
|
+
return ink.build()
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
override fun recognize(
|
|
45
|
+
strokes: Array<InkStroke>,
|
|
46
|
+
languageTag: String,
|
|
47
|
+
): Promise<Array<RecognitionCandidate>> {
|
|
48
|
+
return Promise.async {
|
|
49
|
+
val model = modelFor(languageTag)
|
|
50
|
+
modelManager.download(model, DownloadConditions.Builder().build()).await()
|
|
51
|
+
val recognizer = DigitalInkRecognition.getClient(
|
|
52
|
+
DigitalInkRecognizerOptions.builder(model).build(),
|
|
53
|
+
)
|
|
54
|
+
try {
|
|
55
|
+
val result = recognizer.recognize(buildInk(strokes)).await()
|
|
56
|
+
result.candidates
|
|
57
|
+
.map { RecognitionCandidate(it.text, it.score?.toDouble()) }
|
|
58
|
+
.toTypedArray()
|
|
59
|
+
} finally {
|
|
60
|
+
recognizer.close()
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
override fun downloadModel(languageTag: String): Promise<Unit> {
|
|
66
|
+
return Promise.async {
|
|
67
|
+
modelManager.download(modelFor(languageTag), DownloadConditions.Builder().build()).await()
|
|
68
|
+
Unit
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
override fun isModelDownloaded(languageTag: String): Promise<Boolean> {
|
|
73
|
+
return Promise.async { modelManager.isModelDownloaded(modelFor(languageTag)).await() }
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
override fun deleteModel(languageTag: String): Promise<Unit> {
|
|
77
|
+
return Promise.async {
|
|
78
|
+
modelManager.deleteDownloadedModel(modelFor(languageTag)).await()
|
|
79
|
+
Unit
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
override fun isAvailable(): Boolean = true
|
|
84
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
package com.nitromlkit.digitalink
|
|
2
|
+
|
|
3
|
+
import expo.modules.kotlin.modules.Module
|
|
4
|
+
import expo.modules.kotlin.modules.ModuleDefinition
|
|
5
|
+
import com.margelo.nitro.nitromlkit.digitalink.NitroMLKitDigitalInkOnLoad
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Loads the native library and registers the DigitalInkRecognizer HybridObject at
|
|
9
|
+
* startup via Expo autolinking. Exposes no JS API of its own.
|
|
10
|
+
*/
|
|
11
|
+
class NitroMLKitDigitalInkPackage : Module() {
|
|
12
|
+
init {
|
|
13
|
+
NitroMLKitDigitalInkOnLoad.initializeNative()
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
override fun definition() = ModuleDefinition {
|
|
17
|
+
Name("NitroMLKitDigitalInk")
|
|
18
|
+
}
|
|
19
|
+
}
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
import NitroModules
|
|
3
|
+
import MLKitCommon
|
|
4
|
+
import MLKitDigitalInkRecognition
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Native iOS implementation of DigitalInkRecognizer (MLKit Digital Ink).
|
|
8
|
+
* Conforms to the Nitrogen-generated HybridDigitalInkRecognizerSpec protocol.
|
|
9
|
+
*
|
|
10
|
+
* Ink strokes (points with optional timestamps) go in; candidate transcriptions
|
|
11
|
+
* come out. The per-language model downloads on first use. Model downloads are
|
|
12
|
+
* notification-based on iOS (unlike the Task-based Android API), so
|
|
13
|
+
* `ModelManager.download(_:conditions:)` is bridged to async via a
|
|
14
|
+
* NotificationCenter observer wrapped in a continuation.
|
|
15
|
+
*/
|
|
16
|
+
class HybridDigitalInkRecognizer: HybridDigitalInkRecognizerSpec {
|
|
17
|
+
|
|
18
|
+
// MARK: - HybridObject boilerplate
|
|
19
|
+
var memorySize: Int { MemoryLayout<HybridDigitalInkRecognizer>.size }
|
|
20
|
+
|
|
21
|
+
// MARK: - Lazy MLKit model manager
|
|
22
|
+
private lazy var modelManager: ModelManager = ModelManager.modelManager()
|
|
23
|
+
|
|
24
|
+
// MARK: - Helpers
|
|
25
|
+
|
|
26
|
+
/// Resolve a BCP-47 language tag to a `DigitalInkRecognitionModel`, or throw.
|
|
27
|
+
private func model(for languageTag: String) throws -> DigitalInkRecognitionModel {
|
|
28
|
+
// `modelIdentifierForLanguageTag:` is a failable (non-throwing) init on
|
|
29
|
+
// iOS — it returns nil for an unsupported tag.
|
|
30
|
+
guard let identifier = DigitalInkRecognitionModelIdentifier(forLanguageTag: languageTag) else {
|
|
31
|
+
throw RuntimeError.error(withMessage: "Unsupported language tag: \(languageTag)")
|
|
32
|
+
}
|
|
33
|
+
return DigitalInkRecognitionModel(modelIdentifier: identifier)
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/// Build a `Ink` from the Nitro InkStroke array.
|
|
37
|
+
private func buildInk(from strokes: [InkStroke]) -> Ink {
|
|
38
|
+
// On iOS these are top-level types (NS_SWIFT_NAME): StrokePoint / Stroke /
|
|
39
|
+
// Ink — NOT nested Ink.Point / Ink.Stroke (that's the Android spelling).
|
|
40
|
+
let mlStrokes: [Stroke] = strokes.map { stroke in
|
|
41
|
+
let points: [StrokePoint] = stroke.points.map { p in
|
|
42
|
+
if let t = p.t {
|
|
43
|
+
return StrokePoint(x: Float(p.x), y: Float(p.y), t: Int(t))
|
|
44
|
+
} else {
|
|
45
|
+
return StrokePoint(x: Float(p.x), y: Float(p.y))
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
return Stroke(points: points)
|
|
49
|
+
}
|
|
50
|
+
return Ink(strokes: mlStrokes)
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/// Bridge the notification-based ML Kit model download to async/await.
|
|
54
|
+
private func awaitModelDownload(_ model: DigitalInkRecognitionModel,
|
|
55
|
+
conditions: ModelDownloadConditions) async throws {
|
|
56
|
+
try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<Void, Error>) in
|
|
57
|
+
let center = NotificationCenter.default
|
|
58
|
+
var didResume = false
|
|
59
|
+
var successObserver: NSObjectProtocol?
|
|
60
|
+
var failObserver: NSObjectProtocol?
|
|
61
|
+
|
|
62
|
+
func cleanup() {
|
|
63
|
+
if let successObserver = successObserver { center.removeObserver(successObserver) }
|
|
64
|
+
if let failObserver = failObserver { center.removeObserver(failObserver) }
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
func isForThisModel(_ note: Notification) -> Bool {
|
|
68
|
+
guard let downloaded = note.userInfo?[ModelDownloadUserInfoKey.remoteModel.rawValue] as? NSObject else {
|
|
69
|
+
return false
|
|
70
|
+
}
|
|
71
|
+
return downloaded.isEqual(model)
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
successObserver = center.addObserver(forName: .mlkitModelDownloadDidSucceed,
|
|
75
|
+
object: nil, queue: .main) { note in
|
|
76
|
+
guard !didResume, isForThisModel(note) else { return }
|
|
77
|
+
didResume = true
|
|
78
|
+
cleanup()
|
|
79
|
+
continuation.resume()
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
failObserver = center.addObserver(forName: .mlkitModelDownloadDidFail,
|
|
83
|
+
object: nil, queue: .main) { note in
|
|
84
|
+
guard !didResume, isForThisModel(note) else { return }
|
|
85
|
+
didResume = true
|
|
86
|
+
cleanup()
|
|
87
|
+
let err = note.userInfo?[ModelDownloadUserInfoKey.error.rawValue] as? Error
|
|
88
|
+
continuation.resume(throwing: err ?? RuntimeError.error(
|
|
89
|
+
withMessage: "Failed to download digital-ink model"))
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// Kicks off the download; progress is reported via the observed notifications.
|
|
93
|
+
_ = self.modelManager.download(model, conditions: conditions)
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// MARK: - Protocol methods
|
|
98
|
+
|
|
99
|
+
func recognize(strokes: [InkStroke], languageTag: String) throws -> Promise<[RecognitionCandidate]> {
|
|
100
|
+
return Promise.async {
|
|
101
|
+
let model = try self.model(for: languageTag)
|
|
102
|
+
try await self.awaitModelDownload(model, conditions: ModelDownloadConditions())
|
|
103
|
+
|
|
104
|
+
let options = DigitalInkRecognizerOptions(model: model)
|
|
105
|
+
let recognizer = DigitalInkRecognizer.digitalInkRecognizer(options: options)
|
|
106
|
+
let ink = self.buildInk(from: strokes)
|
|
107
|
+
|
|
108
|
+
return try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<[RecognitionCandidate], Error>) in
|
|
109
|
+
recognizer.recognize(ink: ink) { result, error in
|
|
110
|
+
if let error = error {
|
|
111
|
+
continuation.resume(throwing: error)
|
|
112
|
+
} else {
|
|
113
|
+
let candidates = (result?.candidates ?? []).map {
|
|
114
|
+
RecognitionCandidate(text: $0.text, score: $0.score?.doubleValue)
|
|
115
|
+
}
|
|
116
|
+
continuation.resume(returning: candidates)
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
func downloadModel(languageTag: String) throws -> Promise<Void> {
|
|
124
|
+
return Promise.async {
|
|
125
|
+
let model = try self.model(for: languageTag)
|
|
126
|
+
try await self.awaitModelDownload(model, conditions: ModelDownloadConditions())
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
func isModelDownloaded(languageTag: String) throws -> Promise<Bool> {
|
|
131
|
+
return Promise.async {
|
|
132
|
+
let model = try self.model(for: languageTag)
|
|
133
|
+
// `isModelDownloaded(_:)` is synchronous on iOS.
|
|
134
|
+
return self.modelManager.isModelDownloaded(model)
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
func deleteModel(languageTag: String) throws -> Promise<Void> {
|
|
139
|
+
return Promise.async {
|
|
140
|
+
let model = try self.model(for: languageTag)
|
|
141
|
+
try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<Void, Error>) in
|
|
142
|
+
self.modelManager.deleteDownloadedModel(model) { error in
|
|
143
|
+
if let error = error {
|
|
144
|
+
continuation.resume(throwing: error)
|
|
145
|
+
} else {
|
|
146
|
+
continuation.resume()
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
func isAvailable() throws -> Bool {
|
|
154
|
+
return true
|
|
155
|
+
}
|
|
156
|
+
}
|
package/nitro.json
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"cxxNamespace": ["mlkit", "digitalink"],
|
|
3
|
+
"ios": {
|
|
4
|
+
"iosModuleName": "NitroMLKitDigitalInk"
|
|
5
|
+
},
|
|
6
|
+
"android": {
|
|
7
|
+
"androidNamespace": ["nitromlkit", "digitalink"],
|
|
8
|
+
"androidCxxLibName": "NitroMLKitDigitalInk"
|
|
9
|
+
},
|
|
10
|
+
"autolinking": {
|
|
11
|
+
"DigitalInkRecognizer": {
|
|
12
|
+
"ios": { "language": "swift", "implementationClassName": "HybridDigitalInkRecognizer" },
|
|
13
|
+
"android": { "language": "kotlin", "implementationClassName": "HybridDigitalInkRecognizer" }
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
** linguist-generated=true
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
#
|
|
2
|
+
# NitroMLKitDigitalInk+autolinking.cmake
|
|
3
|
+
# This file was generated by nitrogen. DO NOT MODIFY THIS FILE.
|
|
4
|
+
# https://github.com/mrousavy/nitro
|
|
5
|
+
# Copyright © Marc Rousavy @ Margelo
|
|
6
|
+
#
|
|
7
|
+
|
|
8
|
+
# This is a CMake file that adds all files generated by Nitrogen
|
|
9
|
+
# to the current CMake project.
|
|
10
|
+
#
|
|
11
|
+
# To use it, add this to your CMakeLists.txt:
|
|
12
|
+
# ```cmake
|
|
13
|
+
# include(${CMAKE_SOURCE_DIR}/../nitrogen/generated/android/NitroMLKitDigitalInk+autolinking.cmake)
|
|
14
|
+
# ```
|
|
15
|
+
|
|
16
|
+
# Define a flag to check if we are building properly
|
|
17
|
+
add_definitions(-DBUILDING_NITROMLKITDIGITALINK_WITH_GENERATED_CMAKE_PROJECT)
|
|
18
|
+
|
|
19
|
+
# Enable Raw Props parsing in react-native (for Nitro Views)
|
|
20
|
+
add_definitions(-DRN_SERIALIZABLE_STATE)
|
|
21
|
+
|
|
22
|
+
# Add all headers that were generated by Nitrogen
|
|
23
|
+
include_directories(
|
|
24
|
+
"../nitrogen/generated/shared/c++"
|
|
25
|
+
"../nitrogen/generated/android/c++"
|
|
26
|
+
"../nitrogen/generated/android/"
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
# Add all .cpp sources that were generated by Nitrogen
|
|
30
|
+
target_sources(
|
|
31
|
+
# CMake project name (Android C++ library name)
|
|
32
|
+
NitroMLKitDigitalInk PRIVATE
|
|
33
|
+
# Autolinking Setup
|
|
34
|
+
../nitrogen/generated/android/NitroMLKitDigitalInkOnLoad.cpp
|
|
35
|
+
# Shared Nitrogen C++ sources
|
|
36
|
+
../nitrogen/generated/shared/c++/HybridDigitalInkRecognizerSpec.cpp
|
|
37
|
+
# Android-specific Nitrogen C++ sources
|
|
38
|
+
../nitrogen/generated/android/c++/JHybridDigitalInkRecognizerSpec.cpp
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
# From node_modules/react-native/ReactAndroid/cmake-utils/folly-flags.cmake
|
|
42
|
+
# Used in node_modules/react-native/ReactAndroid/cmake-utils/ReactNative-application.cmake
|
|
43
|
+
target_compile_definitions(
|
|
44
|
+
NitroMLKitDigitalInk PRIVATE
|
|
45
|
+
-DFOLLY_NO_CONFIG=1
|
|
46
|
+
-DFOLLY_HAVE_CLOCK_GETTIME=1
|
|
47
|
+
-DFOLLY_USE_LIBCPP=1
|
|
48
|
+
-DFOLLY_CFG_NO_COROUTINES=1
|
|
49
|
+
-DFOLLY_MOBILE=1
|
|
50
|
+
-DFOLLY_HAVE_RECVMMSG=1
|
|
51
|
+
-DFOLLY_HAVE_PTHREAD=1
|
|
52
|
+
# Once we target android-23 above, we can comment
|
|
53
|
+
# the following line. NDK uses GNU style stderror_r() after API 23.
|
|
54
|
+
-DFOLLY_HAVE_XSI_STRERROR_R=1
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
# Add all libraries required by the generated specs
|
|
58
|
+
find_package(fbjni REQUIRED) # <-- Used for communication between Java <-> C++
|
|
59
|
+
find_package(ReactAndroid REQUIRED) # <-- Used to set up React Native bindings (e.g. CallInvoker/TurboModule)
|
|
60
|
+
find_package(react-native-nitro-modules REQUIRED) # <-- Used to create all HybridObjects and use the Nitro core library
|
|
61
|
+
|
|
62
|
+
# Link all libraries together
|
|
63
|
+
target_link_libraries(
|
|
64
|
+
NitroMLKitDigitalInk
|
|
65
|
+
fbjni::fbjni # <-- Facebook C++ JNI helpers
|
|
66
|
+
ReactAndroid::jsi # <-- RN: JSI
|
|
67
|
+
react-native-nitro-modules::NitroModules # <-- NitroModules Core :)
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
# Link react-native (different prefab between RN 0.75 and RN 0.76)
|
|
71
|
+
if(ReactAndroid_VERSION_MINOR GREATER_EQUAL 76)
|
|
72
|
+
target_link_libraries(
|
|
73
|
+
NitroMLKitDigitalInk
|
|
74
|
+
ReactAndroid::reactnative # <-- RN: Native Modules umbrella prefab
|
|
75
|
+
)
|
|
76
|
+
else()
|
|
77
|
+
target_link_libraries(
|
|
78
|
+
NitroMLKitDigitalInk
|
|
79
|
+
ReactAndroid::react_nativemodule_core # <-- RN: TurboModules Core
|
|
80
|
+
)
|
|
81
|
+
endif()
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
///
|
|
2
|
+
/// NitroMLKitDigitalInk+autolinking.gradle
|
|
3
|
+
/// This file was generated by nitrogen. DO NOT MODIFY THIS FILE.
|
|
4
|
+
/// https://github.com/mrousavy/nitro
|
|
5
|
+
/// Copyright © Marc Rousavy @ Margelo
|
|
6
|
+
///
|
|
7
|
+
|
|
8
|
+
/// This is a Gradle file that adds all files generated by Nitrogen
|
|
9
|
+
/// to the current Gradle project.
|
|
10
|
+
///
|
|
11
|
+
/// To use it, add this to your build.gradle:
|
|
12
|
+
/// ```gradle
|
|
13
|
+
/// apply from: '../nitrogen/generated/android/NitroMLKitDigitalInk+autolinking.gradle'
|
|
14
|
+
/// ```
|
|
15
|
+
|
|
16
|
+
logger.warn("[NitroModules] 🔥 NitroMLKitDigitalInk is boosted by nitro!")
|
|
17
|
+
|
|
18
|
+
android {
|
|
19
|
+
sourceSets {
|
|
20
|
+
main {
|
|
21
|
+
java.srcDirs += [
|
|
22
|
+
// Nitrogen files
|
|
23
|
+
"${project.projectDir}/../nitrogen/generated/android/kotlin"
|
|
24
|
+
]
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
}
|