@nitro-mlkit/pose-detection 0.1.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/NitroMLKitPose.podspec +34 -0
- package/README.md +57 -0
- package/android/CMakeLists.txt +12 -0
- package/android/build.gradle +77 -0
- package/android/src/main/AndroidManifest.xml +3 -0
- package/android/src/main/cpp/cpp-adapter.cpp +9 -0
- package/android/src/main/kotlin/com/margelo/nitro/nitromlkit/pose/HybridPoseDetector.kt +93 -0
- package/android/src/main/kotlin/com/nitromlkit/pose/NitroMLKitPosePackage.kt +19 -0
- package/expo-module.config.json +9 -0
- package/ios/HybridPoseDetector.swift +146 -0
- package/nitro.json +16 -0
- package/nitrogen/generated/.gitattributes +1 -0
- package/nitrogen/generated/android/NitroMLKitPose+autolinking.cmake +81 -0
- package/nitrogen/generated/android/NitroMLKitPose+autolinking.gradle +27 -0
- package/nitrogen/generated/android/NitroMLKitPoseOnLoad.cpp +54 -0
- package/nitrogen/generated/android/NitroMLKitPoseOnLoad.hpp +34 -0
- package/nitrogen/generated/android/c++/JBatchPoseResult.hpp +91 -0
- package/nitrogen/generated/android/c++/JHybridPoseDetectorSpec.cpp +123 -0
- package/nitrogen/generated/android/c++/JHybridPoseDetectorSpec.hpp +65 -0
- package/nitrogen/generated/android/c++/JPoseLandmark.hpp +73 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/nitromlkit/pose/BatchPoseResult.kt +66 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/nitromlkit/pose/HybridPoseDetectorSpec.kt +63 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/nitromlkit/pose/NitroMLKitPoseOnLoad.kt +35 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/nitromlkit/pose/PoseLandmark.kt +71 -0
- package/nitrogen/generated/ios/NitroMLKitPose+autolinking.rb +62 -0
- package/nitrogen/generated/ios/NitroMLKitPose-Swift-Cxx-Bridge.cpp +57 -0
- package/nitrogen/generated/ios/NitroMLKitPose-Swift-Cxx-Bridge.hpp +219 -0
- package/nitrogen/generated/ios/NitroMLKitPose-Swift-Cxx-Umbrella.hpp +53 -0
- package/nitrogen/generated/ios/NitroMLKitPoseAutolinking.mm +33 -0
- package/nitrogen/generated/ios/NitroMLKitPoseAutolinking.swift +26 -0
- package/nitrogen/generated/ios/c++/HybridPoseDetectorSpecSwift.cpp +11 -0
- package/nitrogen/generated/ios/c++/HybridPoseDetectorSpecSwift.hpp +106 -0
- package/nitrogen/generated/ios/swift/BatchPoseResult.swift +63 -0
- package/nitrogen/generated/ios/swift/Func_void_std__exception_ptr.swift +46 -0
- package/nitrogen/generated/ios/swift/Func_void_std__vector_BatchPoseResult_.swift +46 -0
- package/nitrogen/generated/ios/swift/Func_void_std__vector_PoseLandmark_.swift +46 -0
- package/nitrogen/generated/ios/swift/HybridPoseDetectorSpec.swift +57 -0
- package/nitrogen/generated/ios/swift/HybridPoseDetectorSpec_cxx.swift +188 -0
- package/nitrogen/generated/ios/swift/PoseLandmark.swift +49 -0
- package/nitrogen/generated/shared/c++/BatchPoseResult.hpp +99 -0
- package/nitrogen/generated/shared/c++/HybridPoseDetectorSpec.cpp +23 -0
- package/nitrogen/generated/shared/c++/HybridPoseDetectorSpec.hpp +71 -0
- package/nitrogen/generated/shared/c++/PoseLandmark.hpp +99 -0
- package/package.json +54 -0
- package/src/index.ts +18 -0
- package/src/specs/PoseDetector.nitro.ts +47 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Gonzalo Polo
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
|
|
3
|
+
package = JSON.parse(File.read(File.join(__dir__, "package.json")))
|
|
4
|
+
|
|
5
|
+
Pod::Spec.new do |s|
|
|
6
|
+
s.name = "NitroMLKitPose"
|
|
7
|
+
# Must match nitro.json's `ios.iosModuleName` — Nitrogen's generated bridge
|
|
8
|
+
# imports "NitroMLKitPose-Swift.h", so the compiled Swift module name has to be NitroMLKitPose.
|
|
9
|
+
s.module_name = "NitroMLKitPose"
|
|
10
|
+
s.version = package["version"]
|
|
11
|
+
s.summary = package["description"]
|
|
12
|
+
s.homepage = package["repository"]["url"]
|
|
13
|
+
s.license = package["license"]
|
|
14
|
+
s.authors = package["author"]
|
|
15
|
+
s.source = { :git => "https://github.com/pologonzalo/react-native-nitro-mlkit.git", :tag => s.version }
|
|
16
|
+
|
|
17
|
+
s.platforms = { :ios => "15.5" }
|
|
18
|
+
|
|
19
|
+
s.source_files = [
|
|
20
|
+
"ios/**/*.{swift,h,m,mm,cpp}"
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
s.dependency "GoogleMLKit/PoseDetection", "~> 7.0"
|
|
24
|
+
|
|
25
|
+
s.pod_target_xcconfig = {
|
|
26
|
+
"SWIFT_VERSION" => "5.9"
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
# Add all files generated by Nitrogen (generated sources + NitroModules dep +
|
|
30
|
+
# the C++/Swift-interop build flags — same setup as the working
|
|
31
|
+
# NitroMLKitFaceDetection podspec).
|
|
32
|
+
load 'nitrogen/generated/ios/NitroMLKitPose+autolinking.rb'
|
|
33
|
+
add_nitrogen_files(s)
|
|
34
|
+
end
|
package/README.md
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# React Native ML Kit — Pose Detection
|
|
2
|
+
|
|
3
|
+
**`@nitro-mlkit/pose-detection`** · on-device Google ML Kit via [Nitro Modules](https://github.com/mrousavy/nitro) — JSI, no bridge.
|
|
4
|
+
|
|
5
|
+
> ⚠️ **Beta (`0.1.0-beta.x`).** Android verified on-device; iOS builds & links but
|
|
6
|
+
> device runtime is pending — see [Platform status](#platform-status).
|
|
7
|
+
|
|
8
|
+
High-performance, on-device **pose detection** for React Native, built with
|
|
9
|
+
[Nitro Modules](https://github.com/mrousavy/nitro) (JSI, no bridge).
|
|
10
|
+
|
|
11
|
+
Powered by **Google ML Kit**. Returns the **33 skeletal landmarks** of the
|
|
12
|
+
primary body — each with a 3D position and an in-frame likelihood. **On-device.**
|
|
13
|
+
|
|
14
|
+
## Installation
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
npm install @nitro-mlkit/pose-detection@beta react-native-nitro-modules
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
No config plugin (autolinked Expo module). Just install and `npx expo prebuild`.
|
|
21
|
+
Not available in Expo Go.
|
|
22
|
+
|
|
23
|
+
## Usage
|
|
24
|
+
|
|
25
|
+
```ts
|
|
26
|
+
import { NitroPose } from "@nitro-mlkit/pose-detection";
|
|
27
|
+
|
|
28
|
+
const landmarks = await NitroPose.detect(imageUri);
|
|
29
|
+
// → [{ type, x, y, z, inFrameLikelihood }, ...] (33 landmarks, or [] if no body)
|
|
30
|
+
// type is the ML Kit landmark index (0 = nose, 11/12 = shoulders, 23/24 = hips, …)
|
|
31
|
+
|
|
32
|
+
// Native batch — one JSI call
|
|
33
|
+
const results = await NitroPose.detectBatch(uris, 4 /* concurrency */);
|
|
34
|
+
|
|
35
|
+
NitroPose.isAvailable(); // boolean
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Runs in SINGLE_IMAGE_MODE. All 33 landmarks are returned whenever a body is
|
|
39
|
+
found; joints outside the frame come back with a low `inFrameLikelihood`.
|
|
40
|
+
|
|
41
|
+
## Platform status
|
|
42
|
+
|
|
43
|
+
| Platform | Min | Status |
|
|
44
|
+
| -------- | --- | ------ |
|
|
45
|
+
| Android | API 21+ | ✅ Verified on-device (Pixel 9, API 36): 33 landmarks in ~430 ms (nose 96%, shoulders ~73%, out-of-frame hips 0%) |
|
|
46
|
+
| iOS | 15.5+ | ⚠️ Swift impl written; on-device build & run pending¹ |
|
|
47
|
+
| tvOS/macOS | — | 🔜 Planned |
|
|
48
|
+
|
|
49
|
+
¹ ML Kit's iOS pods ship no `arm64` Simulator slice; validate on a physical device.
|
|
50
|
+
|
|
51
|
+
## Part of `nitro-mlkit`
|
|
52
|
+
|
|
53
|
+
The full ML Kit suite on Nitro — see the other `@nitro-mlkit/*` packages.
|
|
54
|
+
|
|
55
|
+
## License
|
|
56
|
+
|
|
57
|
+
MIT © Gonzalo Polo
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
project(NitroMLKitPose)
|
|
2
|
+
cmake_minimum_required(VERSION 3.9.0)
|
|
3
|
+
|
|
4
|
+
set(CMAKE_CXX_STANDARD 20)
|
|
5
|
+
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
|
6
|
+
|
|
7
|
+
add_library(NitroMLKitPose SHARED
|
|
8
|
+
src/main/cpp/cpp-adapter.cpp
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
# Add all files generated by Nitrogen
|
|
12
|
+
include(${CMAKE_SOURCE_DIR}/../nitrogen/generated/android/NitroMLKitPose+autolinking.cmake)
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
buildscript {
|
|
2
|
+
ext.safeExtGet = { prop, fallback ->
|
|
3
|
+
rootProject.ext.has(prop) ? rootProject.ext.get(prop) : fallback
|
|
4
|
+
}
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
def reactNativeArchitectures() {
|
|
8
|
+
def value = rootProject.getProperties().get("reactNativeArchitectures")
|
|
9
|
+
return value ? value.split(",") : ["armeabi-v7a", "x86", "x86_64", "arm64-v8a"]
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
apply plugin: "com.android.library"
|
|
13
|
+
apply plugin: "kotlin-android"
|
|
14
|
+
|
|
15
|
+
android {
|
|
16
|
+
namespace "com.nitromlkit.pose"
|
|
17
|
+
compileSdkVersion safeExtGet("compileSdkVersion", 35)
|
|
18
|
+
ndkVersion safeExtGet("ndkVersion", "27.1.12297006")
|
|
19
|
+
|
|
20
|
+
defaultConfig {
|
|
21
|
+
minSdkVersion safeExtGet("minSdkVersion", 26)
|
|
22
|
+
targetSdkVersion safeExtGet("targetSdkVersion", 35)
|
|
23
|
+
|
|
24
|
+
externalNativeBuild {
|
|
25
|
+
cmake {
|
|
26
|
+
arguments "-DANDROID_STL=c++_shared"
|
|
27
|
+
abiFilters (*reactNativeArchitectures())
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
externalNativeBuild {
|
|
33
|
+
cmake {
|
|
34
|
+
path "CMakeLists.txt"
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
buildFeatures {
|
|
39
|
+
prefab true
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
compileOptions {
|
|
43
|
+
sourceCompatibility JavaVersion.VERSION_17
|
|
44
|
+
targetCompatibility JavaVersion.VERSION_17
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
kotlinOptions {
|
|
48
|
+
jvmTarget = "17"
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
sourceSets {
|
|
52
|
+
main {
|
|
53
|
+
java.srcDirs += [
|
|
54
|
+
"src/main/kotlin",
|
|
55
|
+
"../nitrogen/generated/android/kotlin"
|
|
56
|
+
]
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
dependencies {
|
|
62
|
+
// MLKit
|
|
63
|
+
implementation "com.google.mlkit:pose-detection:18.0.0-beta5"
|
|
64
|
+
|
|
65
|
+
// Nitro Modules
|
|
66
|
+
implementation project(":react-native-nitro-modules")
|
|
67
|
+
|
|
68
|
+
// Expo Modules (only used so autolinking calls our OnLoad at startup)
|
|
69
|
+
implementation project(":expo-modules-core")
|
|
70
|
+
|
|
71
|
+
// Kotlin coroutines
|
|
72
|
+
implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:1.9.0"
|
|
73
|
+
implementation "org.jetbrains.kotlinx:kotlinx-coroutines-play-services:1.9.0"
|
|
74
|
+
|
|
75
|
+
// React Native
|
|
76
|
+
implementation "com.facebook.react:react-android"
|
|
77
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
package com.margelo.nitro.nitromlkit.pose
|
|
2
|
+
|
|
3
|
+
import android.graphics.Bitmap
|
|
4
|
+
import android.graphics.BitmapFactory
|
|
5
|
+
import android.net.Uri
|
|
6
|
+
import com.google.mlkit.vision.common.InputImage
|
|
7
|
+
import com.google.mlkit.vision.pose.PoseDetection
|
|
8
|
+
import com.google.mlkit.vision.pose.defaults.PoseDetectorOptions
|
|
9
|
+
import com.margelo.nitro.NitroModules
|
|
10
|
+
import com.margelo.nitro.core.Promise
|
|
11
|
+
import kotlinx.coroutines.async
|
|
12
|
+
import kotlinx.coroutines.awaitAll
|
|
13
|
+
import kotlinx.coroutines.coroutineScope
|
|
14
|
+
import kotlinx.coroutines.tasks.await
|
|
15
|
+
import java.net.URL
|
|
16
|
+
import com.google.mlkit.vision.pose.PoseLandmark as MlkitLandmark
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Native Android implementation of PoseDetector (MLKit pose detection).
|
|
20
|
+
* Returns the 33 skeletal landmarks of the primary body in SINGLE_IMAGE_MODE.
|
|
21
|
+
*/
|
|
22
|
+
class HybridPoseDetector : HybridPoseDetectorSpec() {
|
|
23
|
+
|
|
24
|
+
private val detector by lazy {
|
|
25
|
+
PoseDetection.getClient(
|
|
26
|
+
PoseDetectorOptions.Builder()
|
|
27
|
+
.setDetectorMode(PoseDetectorOptions.SINGLE_IMAGE_MODE)
|
|
28
|
+
.build()
|
|
29
|
+
)
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
override fun detect(imageUri: String): Promise<Array<PoseLandmark>> {
|
|
33
|
+
return Promise.async { detectImage(imageUri) }
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
override fun detectBatch(imageUris: Array<String>, concurrency: Double): Promise<Array<BatchPoseResult>> {
|
|
37
|
+
return Promise.async {
|
|
38
|
+
val maxConcurrent = concurrency.toInt().coerceAtLeast(1)
|
|
39
|
+
val results = mutableListOf<BatchPoseResult>()
|
|
40
|
+
var start = 0
|
|
41
|
+
while (start < imageUris.size) {
|
|
42
|
+
val end = minOf(start + maxConcurrent, imageUris.size)
|
|
43
|
+
val chunk = coroutineScope {
|
|
44
|
+
(start until end).map { idx ->
|
|
45
|
+
async {
|
|
46
|
+
try {
|
|
47
|
+
BatchPoseResult(idx.toDouble(), detectImage(imageUris[idx]), true, null)
|
|
48
|
+
} catch (e: Exception) {
|
|
49
|
+
BatchPoseResult(idx.toDouble(), emptyArray(), false, e.message ?: "unknown error")
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}.awaitAll()
|
|
53
|
+
}
|
|
54
|
+
results.addAll(chunk)
|
|
55
|
+
start = end
|
|
56
|
+
}
|
|
57
|
+
results.toTypedArray()
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
override fun isAvailable(): Boolean = true
|
|
62
|
+
|
|
63
|
+
private suspend fun detectImage(uri: String): Array<PoseLandmark> {
|
|
64
|
+
val bitmap = loadBitmap(uri) ?: throw Error("Failed to load image: $uri")
|
|
65
|
+
val image = InputImage.fromBitmap(bitmap, 0)
|
|
66
|
+
val pose = detector.process(image).await()
|
|
67
|
+
return pose.allPoseLandmarks.map { lm ->
|
|
68
|
+
PoseLandmark(
|
|
69
|
+
type = lm.landmarkType.toDouble(),
|
|
70
|
+
x = lm.position3D.x.toDouble(),
|
|
71
|
+
y = lm.position3D.y.toDouble(),
|
|
72
|
+
z = lm.position3D.z.toDouble(),
|
|
73
|
+
inFrameLikelihood = lm.inFrameLikelihood.toDouble(),
|
|
74
|
+
)
|
|
75
|
+
}.toTypedArray()
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
private fun loadBitmap(uri: String): Bitmap? {
|
|
79
|
+
return try {
|
|
80
|
+
when {
|
|
81
|
+
uri.startsWith("file://") -> BitmapFactory.decodeFile(uri.removePrefix("file://"))
|
|
82
|
+
uri.startsWith("/") -> BitmapFactory.decodeFile(uri)
|
|
83
|
+
uri.startsWith("content://") -> {
|
|
84
|
+
val context = NitroModules.applicationContext ?: return null
|
|
85
|
+
context.contentResolver.openInputStream(Uri.parse(uri))?.use { BitmapFactory.decodeStream(it) }
|
|
86
|
+
}
|
|
87
|
+
else -> URL(uri).openStream().use { BitmapFactory.decodeStream(it) }
|
|
88
|
+
}
|
|
89
|
+
} catch (e: Exception) {
|
|
90
|
+
null
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
package com.nitromlkit.pose
|
|
2
|
+
|
|
3
|
+
import expo.modules.kotlin.modules.Module
|
|
4
|
+
import expo.modules.kotlin.modules.ModuleDefinition
|
|
5
|
+
import com.margelo.nitro.nitromlkit.pose.NitroMLKitPoseOnLoad
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Loads the native library and registers the PoseDetector HybridObject at startup via
|
|
9
|
+
* Expo autolinking. Exposes no JS API of its own.
|
|
10
|
+
*/
|
|
11
|
+
class NitroMLKitPosePackage : Module() {
|
|
12
|
+
init {
|
|
13
|
+
NitroMLKitPoseOnLoad.initializeNative()
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
override fun definition() = ModuleDefinition {
|
|
17
|
+
Name("NitroMLKitPose")
|
|
18
|
+
}
|
|
19
|
+
}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
import UIKit
|
|
3
|
+
import NitroModules
|
|
4
|
+
import MLKitPoseDetection
|
|
5
|
+
import MLKitPoseDetectionCommon
|
|
6
|
+
import MLKitVision
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Native iOS implementation of PoseDetector (MLKit pose detection).
|
|
10
|
+
* Returns the 33 skeletal landmarks of the primary detected body (SINGLE_IMAGE mode).
|
|
11
|
+
* Conforms to the Nitrogen-generated HybridPoseDetectorSpec protocol.
|
|
12
|
+
*/
|
|
13
|
+
class HybridPoseDetector: HybridPoseDetectorSpec {
|
|
14
|
+
|
|
15
|
+
// MARK: - HybridObject boilerplate
|
|
16
|
+
var memorySize: Int { MemoryLayout<HybridPoseDetector>.size }
|
|
17
|
+
|
|
18
|
+
// MARK: - Lazy MLKit detector
|
|
19
|
+
|
|
20
|
+
private lazy var detector: PoseDetector = {
|
|
21
|
+
let opts = PoseDetectorOptions()
|
|
22
|
+
opts.detectorMode = .singleImage
|
|
23
|
+
return PoseDetector.poseDetector(options: opts)
|
|
24
|
+
}()
|
|
25
|
+
|
|
26
|
+
/// Maps each iOS `PoseLandmarkType` to the integer index the Kotlin/JS side expects.
|
|
27
|
+
/// This mirrors the Android ML Kit `PoseLandmark.landmarkType` int constants (0..32,
|
|
28
|
+
/// the standard BlazePose ordering). iOS ships string-based landmark types, so we map
|
|
29
|
+
/// them back to the same ordinal.
|
|
30
|
+
private static let landmarkIndexMap: [PoseLandmarkType: Int] = [
|
|
31
|
+
.nose: 0,
|
|
32
|
+
.leftEyeInner: 1,
|
|
33
|
+
.leftEye: 2,
|
|
34
|
+
.leftEyeOuter: 3,
|
|
35
|
+
.rightEyeInner: 4,
|
|
36
|
+
.rightEye: 5,
|
|
37
|
+
.rightEyeOuter: 6,
|
|
38
|
+
.leftEar: 7,
|
|
39
|
+
.rightEar: 8,
|
|
40
|
+
.mouthLeft: 9,
|
|
41
|
+
.mouthRight: 10,
|
|
42
|
+
.leftShoulder: 11,
|
|
43
|
+
.rightShoulder: 12,
|
|
44
|
+
.leftElbow: 13,
|
|
45
|
+
.rightElbow: 14,
|
|
46
|
+
.leftWrist: 15,
|
|
47
|
+
.rightWrist: 16,
|
|
48
|
+
.leftPinkyFinger: 17,
|
|
49
|
+
.rightPinkyFinger: 18,
|
|
50
|
+
.leftIndexFinger: 19,
|
|
51
|
+
.rightIndexFinger: 20,
|
|
52
|
+
.leftThumb: 21,
|
|
53
|
+
.rightThumb: 22,
|
|
54
|
+
.leftHip: 23,
|
|
55
|
+
.rightHip: 24,
|
|
56
|
+
.leftKnee: 25,
|
|
57
|
+
.rightKnee: 26,
|
|
58
|
+
.leftAnkle: 27,
|
|
59
|
+
.rightAnkle: 28,
|
|
60
|
+
.leftHeel: 29,
|
|
61
|
+
.rightHeel: 30,
|
|
62
|
+
.leftToe: 31,
|
|
63
|
+
.rightToe: 32,
|
|
64
|
+
]
|
|
65
|
+
|
|
66
|
+
// MARK: - Protocol methods
|
|
67
|
+
|
|
68
|
+
func detect(imageUri: String) throws -> Promise<[PoseLandmark]> {
|
|
69
|
+
return Promise.async {
|
|
70
|
+
return try await self.detectImage(imageUri)
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
func detectBatch(imageUris: [String], concurrency: Double) throws -> Promise<[BatchPoseResult]> {
|
|
75
|
+
return Promise.async {
|
|
76
|
+
let maxConcurrent = max(1, Int(concurrency))
|
|
77
|
+
var results = [BatchPoseResult]()
|
|
78
|
+
results.reserveCapacity(imageUris.count)
|
|
79
|
+
|
|
80
|
+
for chunkStart in stride(from: 0, to: imageUris.count, by: maxConcurrent) {
|
|
81
|
+
let chunkEnd = min(chunkStart + maxConcurrent, imageUris.count)
|
|
82
|
+
let chunk = Array(imageUris[chunkStart..<chunkEnd])
|
|
83
|
+
|
|
84
|
+
let chunkResults = await withTaskGroup(of: (Int, BatchPoseResult).self) { group in
|
|
85
|
+
for (i, uri) in chunk.enumerated() {
|
|
86
|
+
let globalIdx = chunkStart + i
|
|
87
|
+
group.addTask {
|
|
88
|
+
do {
|
|
89
|
+
let landmarks = try await self.detectImage(uri)
|
|
90
|
+
return (globalIdx, BatchPoseResult(index: Double(globalIdx), landmarks: landmarks, success: true, error: nil))
|
|
91
|
+
} catch {
|
|
92
|
+
return (globalIdx, BatchPoseResult(index: Double(globalIdx), landmarks: [], success: false, error: error.localizedDescription))
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
var out = [(Int, BatchPoseResult)]()
|
|
98
|
+
for await result in group { out.append(result) }
|
|
99
|
+
return out.sorted(by: { $0.0 < $1.0 }).map(\.1)
|
|
100
|
+
}
|
|
101
|
+
results.append(contentsOf: chunkResults)
|
|
102
|
+
}
|
|
103
|
+
return results
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
func isAvailable() throws -> Bool {
|
|
108
|
+
return true
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// MARK: - Private helpers
|
|
112
|
+
|
|
113
|
+
private func detectImage(_ uri: String) async throws -> [PoseLandmark] {
|
|
114
|
+
guard let image = self.loadImage(from: uri) else {
|
|
115
|
+
throw RuntimeError.error(withMessage: "Failed to load image: \(uri)")
|
|
116
|
+
}
|
|
117
|
+
let visionImage = VisionImage(image: image)
|
|
118
|
+
visionImage.orientation = image.imageOrientation
|
|
119
|
+
|
|
120
|
+
let poses = try await detector.process(visionImage)
|
|
121
|
+
guard let pose = poses.first else { return [] }
|
|
122
|
+
|
|
123
|
+
return pose.landmarks.map { lm in
|
|
124
|
+
PoseLandmark(
|
|
125
|
+
type: Double(HybridPoseDetector.landmarkIndexMap[lm.type] ?? -1),
|
|
126
|
+
x: Double(lm.position.x),
|
|
127
|
+
y: Double(lm.position.y),
|
|
128
|
+
z: Double(lm.position.z),
|
|
129
|
+
inFrameLikelihood: Double(lm.inFrameLikelihood)
|
|
130
|
+
)
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
private func loadImage(from uri: String) -> UIImage? {
|
|
135
|
+
let path: String
|
|
136
|
+
if uri.hasPrefix("file://") {
|
|
137
|
+
path = String(uri.dropFirst(7))
|
|
138
|
+
} else if uri.hasPrefix("/") {
|
|
139
|
+
path = uri
|
|
140
|
+
} else {
|
|
141
|
+
guard let url = URL(string: uri), let data = try? Data(contentsOf: url) else { return nil }
|
|
142
|
+
return UIImage(data: data)
|
|
143
|
+
}
|
|
144
|
+
return UIImage(contentsOfFile: path)
|
|
145
|
+
}
|
|
146
|
+
}
|
package/nitro.json
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"cxxNamespace": ["mlkit", "pose"],
|
|
3
|
+
"ios": {
|
|
4
|
+
"iosModuleName": "NitroMLKitPose"
|
|
5
|
+
},
|
|
6
|
+
"android": {
|
|
7
|
+
"androidNamespace": ["nitromlkit", "pose"],
|
|
8
|
+
"androidCxxLibName": "NitroMLKitPose"
|
|
9
|
+
},
|
|
10
|
+
"autolinking": {
|
|
11
|
+
"PoseDetector": {
|
|
12
|
+
"ios": { "language": "swift", "implementationClassName": "HybridPoseDetector" },
|
|
13
|
+
"android": { "language": "kotlin", "implementationClassName": "HybridPoseDetector" }
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
** linguist-generated=true
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
#
|
|
2
|
+
# NitroMLKitPose+autolinking.cmake
|
|
3
|
+
# This file was generated by nitrogen. DO NOT MODIFY THIS FILE.
|
|
4
|
+
# https://github.com/mrousavy/nitro
|
|
5
|
+
# Copyright © Marc Rousavy @ Margelo
|
|
6
|
+
#
|
|
7
|
+
|
|
8
|
+
# This is a CMake file that adds all files generated by Nitrogen
|
|
9
|
+
# to the current CMake project.
|
|
10
|
+
#
|
|
11
|
+
# To use it, add this to your CMakeLists.txt:
|
|
12
|
+
# ```cmake
|
|
13
|
+
# include(${CMAKE_SOURCE_DIR}/../nitrogen/generated/android/NitroMLKitPose+autolinking.cmake)
|
|
14
|
+
# ```
|
|
15
|
+
|
|
16
|
+
# Define a flag to check if we are building properly
|
|
17
|
+
add_definitions(-DBUILDING_NITROMLKITPOSE_WITH_GENERATED_CMAKE_PROJECT)
|
|
18
|
+
|
|
19
|
+
# Enable Raw Props parsing in react-native (for Nitro Views)
|
|
20
|
+
add_definitions(-DRN_SERIALIZABLE_STATE)
|
|
21
|
+
|
|
22
|
+
# Add all headers that were generated by Nitrogen
|
|
23
|
+
include_directories(
|
|
24
|
+
"../nitrogen/generated/shared/c++"
|
|
25
|
+
"../nitrogen/generated/android/c++"
|
|
26
|
+
"../nitrogen/generated/android/"
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
# Add all .cpp sources that were generated by Nitrogen
|
|
30
|
+
target_sources(
|
|
31
|
+
# CMake project name (Android C++ library name)
|
|
32
|
+
NitroMLKitPose PRIVATE
|
|
33
|
+
# Autolinking Setup
|
|
34
|
+
../nitrogen/generated/android/NitroMLKitPoseOnLoad.cpp
|
|
35
|
+
# Shared Nitrogen C++ sources
|
|
36
|
+
../nitrogen/generated/shared/c++/HybridPoseDetectorSpec.cpp
|
|
37
|
+
# Android-specific Nitrogen C++ sources
|
|
38
|
+
../nitrogen/generated/android/c++/JHybridPoseDetectorSpec.cpp
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
# From node_modules/react-native/ReactAndroid/cmake-utils/folly-flags.cmake
|
|
42
|
+
# Used in node_modules/react-native/ReactAndroid/cmake-utils/ReactNative-application.cmake
|
|
43
|
+
target_compile_definitions(
|
|
44
|
+
NitroMLKitPose PRIVATE
|
|
45
|
+
-DFOLLY_NO_CONFIG=1
|
|
46
|
+
-DFOLLY_HAVE_CLOCK_GETTIME=1
|
|
47
|
+
-DFOLLY_USE_LIBCPP=1
|
|
48
|
+
-DFOLLY_CFG_NO_COROUTINES=1
|
|
49
|
+
-DFOLLY_MOBILE=1
|
|
50
|
+
-DFOLLY_HAVE_RECVMMSG=1
|
|
51
|
+
-DFOLLY_HAVE_PTHREAD=1
|
|
52
|
+
# Once we target android-23 above, we can comment
|
|
53
|
+
# the following line. NDK uses GNU style stderror_r() after API 23.
|
|
54
|
+
-DFOLLY_HAVE_XSI_STRERROR_R=1
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
# Add all libraries required by the generated specs
|
|
58
|
+
find_package(fbjni REQUIRED) # <-- Used for communication between Java <-> C++
|
|
59
|
+
find_package(ReactAndroid REQUIRED) # <-- Used to set up React Native bindings (e.g. CallInvoker/TurboModule)
|
|
60
|
+
find_package(react-native-nitro-modules REQUIRED) # <-- Used to create all HybridObjects and use the Nitro core library
|
|
61
|
+
|
|
62
|
+
# Link all libraries together
|
|
63
|
+
target_link_libraries(
|
|
64
|
+
NitroMLKitPose
|
|
65
|
+
fbjni::fbjni # <-- Facebook C++ JNI helpers
|
|
66
|
+
ReactAndroid::jsi # <-- RN: JSI
|
|
67
|
+
react-native-nitro-modules::NitroModules # <-- NitroModules Core :)
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
# Link react-native (different prefab between RN 0.75 and RN 0.76)
|
|
71
|
+
if(ReactAndroid_VERSION_MINOR GREATER_EQUAL 76)
|
|
72
|
+
target_link_libraries(
|
|
73
|
+
NitroMLKitPose
|
|
74
|
+
ReactAndroid::reactnative # <-- RN: Native Modules umbrella prefab
|
|
75
|
+
)
|
|
76
|
+
else()
|
|
77
|
+
target_link_libraries(
|
|
78
|
+
NitroMLKitPose
|
|
79
|
+
ReactAndroid::react_nativemodule_core # <-- RN: TurboModules Core
|
|
80
|
+
)
|
|
81
|
+
endif()
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
///
|
|
2
|
+
/// NitroMLKitPose+autolinking.gradle
|
|
3
|
+
/// This file was generated by nitrogen. DO NOT MODIFY THIS FILE.
|
|
4
|
+
/// https://github.com/mrousavy/nitro
|
|
5
|
+
/// Copyright © Marc Rousavy @ Margelo
|
|
6
|
+
///
|
|
7
|
+
|
|
8
|
+
/// This is a Gradle file that adds all files generated by Nitrogen
|
|
9
|
+
/// to the current Gradle project.
|
|
10
|
+
///
|
|
11
|
+
/// To use it, add this to your build.gradle:
|
|
12
|
+
/// ```gradle
|
|
13
|
+
/// apply from: '../nitrogen/generated/android/NitroMLKitPose+autolinking.gradle'
|
|
14
|
+
/// ```
|
|
15
|
+
|
|
16
|
+
logger.warn("[NitroModules] 🔥 NitroMLKitPose is boosted by nitro!")
|
|
17
|
+
|
|
18
|
+
android {
|
|
19
|
+
sourceSets {
|
|
20
|
+
main {
|
|
21
|
+
java.srcDirs += [
|
|
22
|
+
// Nitrogen files
|
|
23
|
+
"${project.projectDir}/../nitrogen/generated/android/kotlin"
|
|
24
|
+
]
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
///
|
|
2
|
+
/// NitroMLKitPoseOnLoad.cpp
|
|
3
|
+
/// This file was generated by nitrogen. DO NOT MODIFY THIS FILE.
|
|
4
|
+
/// https://github.com/mrousavy/nitro
|
|
5
|
+
/// Copyright © Marc Rousavy @ Margelo
|
|
6
|
+
///
|
|
7
|
+
|
|
8
|
+
#ifndef BUILDING_NITROMLKITPOSE_WITH_GENERATED_CMAKE_PROJECT
|
|
9
|
+
#error NitroMLKitPoseOnLoad.cpp is not being built with the autogenerated CMakeLists.txt project. Is a different CMakeLists.txt building this?
|
|
10
|
+
#endif
|
|
11
|
+
|
|
12
|
+
#include "NitroMLKitPoseOnLoad.hpp"
|
|
13
|
+
|
|
14
|
+
#include <jni.h>
|
|
15
|
+
#include <fbjni/fbjni.h>
|
|
16
|
+
#include <NitroModules/HybridObjectRegistry.hpp>
|
|
17
|
+
|
|
18
|
+
#include "JHybridPoseDetectorSpec.hpp"
|
|
19
|
+
#include <NitroModules/DefaultConstructableObject.hpp>
|
|
20
|
+
|
|
21
|
+
namespace margelo::nitro::mlkit::pose {
|
|
22
|
+
|
|
23
|
+
int initialize(JavaVM* vm) {
|
|
24
|
+
return facebook::jni::initialize(vm, []() {
|
|
25
|
+
::margelo::nitro::mlkit::pose::registerAllNatives();
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
struct JHybridPoseDetectorSpecImpl: public jni::JavaClass<JHybridPoseDetectorSpecImpl, JHybridPoseDetectorSpec::JavaPart> {
|
|
30
|
+
static constexpr auto kJavaDescriptor = "Lcom/margelo/nitro/nitromlkit/pose/HybridPoseDetector;";
|
|
31
|
+
static std::shared_ptr<JHybridPoseDetectorSpec> create() {
|
|
32
|
+
static const auto constructorFn = javaClassStatic()->getConstructor<JHybridPoseDetectorSpecImpl::javaobject()>();
|
|
33
|
+
jni::local_ref<JHybridPoseDetectorSpec::JavaPart> javaPart = javaClassStatic()->newObject(constructorFn);
|
|
34
|
+
return javaPart->getJHybridPoseDetectorSpec();
|
|
35
|
+
}
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
void registerAllNatives() {
|
|
39
|
+
using namespace margelo::nitro;
|
|
40
|
+
using namespace margelo::nitro::mlkit::pose;
|
|
41
|
+
|
|
42
|
+
// Register native JNI methods
|
|
43
|
+
margelo::nitro::mlkit::pose::JHybridPoseDetectorSpec::CxxPart::registerNatives();
|
|
44
|
+
|
|
45
|
+
// Register Nitro Hybrid Objects
|
|
46
|
+
HybridObjectRegistry::registerHybridObjectConstructor(
|
|
47
|
+
"PoseDetector",
|
|
48
|
+
[]() -> std::shared_ptr<HybridObject> {
|
|
49
|
+
return JHybridPoseDetectorSpecImpl::create();
|
|
50
|
+
}
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
} // namespace margelo::nitro::mlkit::pose
|