@dvai-bridge/android-mediapipe-core 4.0.0

package/android/src/main/java/co/deepvoiceai/bridge/mediapipe/core/PluginState.kt

@@ -0,0 +1,134 @@
+package co.deepvoiceai.bridge.mediapipe.core
+
+import android.content.Context
+import co.deepvoiceai.bridge.shared.core.CorsConfig
+import co.deepvoiceai.bridge.shared.core.HandlerContext
+import co.deepvoiceai.bridge.shared.core.HttpServer
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
+
+/**
+ * Owns the running state of the MediaPipe core: the [MediaPipeBridge]
+ * inference engine, the HTTP server, and model metadata. All access is
+ * serialized through the [Mutex] so concurrent start/stop calls from the
+ * bridge layer can never race against the underlying MediaPipe engine
+ * (which is not safe to construct twice in parallel).
+ *
+ * Capacitor-free: opts are plain [Map<String, Any?>] and return values are
+ * plain [Map<String, Any?>]. The Capacitor wrapper translates between
+ * JSObject and Map before/after calling into this class.
+ *
+ * Differs from [co.deepvoiceai.bridge.llama.core.PluginState] in three ways:
+ *  - No `mmprojPath` / `gpuLayers` / `contextSize` / `threads` opts —
+ *    LiteRT-LM doesn't expose those knobs.
+ *  - No `embeddingMode` opt — LiteRT-LM LLM has no embeddings path
+ *    (handler returns 400 with a redirect to the llama backend).
+ *  - Adds a `visionEnabled` opt that wires the bridge with
+ *    `EngineConfig.visionBackend` and toggles the [MediaPipeHandlers.visionCapable]
+ *    flag so `image_url` parts route through `Content.ImageBytes` instead of
+ *    returning a 400.
+ */
+class PluginState {
+    private val mutex = Mutex()
+    private var server: HttpServer? = null
+    private var bridge: MediaPipeBridge? = null
+    private var modelId: String = ""
+    private var isRunning: Boolean = false
+    private var baseUrl: String? = null
+    private var port: Int? = null
+
+    suspend fun start(opts: Map<String, Any?>, context: Context): Map<String, Any?> = mutex.withLock {
+        if (isRunning) stopInternal()
+
+        val modelPath = (opts["modelPath"] as? String)?.takeIf { it.isNotEmpty() }
+            ?: throw IllegalArgumentException("modelPath is required for mediapipe backend")
+
+        val maxTokens = (opts["maxTokens"] as? Number)?.toInt() ?: 2048
+        val visionEnabled = opts["visionEnabled"] as? Boolean ?: false
+        val httpBasePort = (opts["httpBasePort"] as? Number)?.toInt() ?: 38883
+        val httpMaxPortAttempts = (opts["httpMaxPortAttempts"] as? Number)?.toInt() ?: 16
+        val corsConfig = parseCors(opts["corsOrigin"])
+        val resolvedModelId = (opts["modelId"] as? String)?.takeIf { it.isNotEmpty() }
+            ?: deriveModelId(modelPath)
+
+        // Construct the MediaPipe bridge. The engine itself is loaded lazily
+        // on first use, so the handler set is wired before any heavy native
+        // initialization happens — failures then surface on first inference.
+        val newBridge = MediaPipeBridge(
+            context = context,
+            modelPath = modelPath,
+            maxTokens = maxTokens,
+            visionEnabled = visionEnabled,
+        )
+
+        val handlers = MediaPipeHandlers(
+            bridge = newBridge,
+            modelId = resolvedModelId,
+            visionCapable = visionEnabled,
+        )
+        val ctx = HandlerContext(modelId = resolvedModelId, backendName = "mediapipe")
+
+        val newServer = HttpServer()
+        val boundPort = try {
+            newServer.startWithRoutes(
+                basePort = httpBasePort,
+                maxAttempts = httpMaxPortAttempts,
+                host = "127.0.0.1",
+                handlers = handlers,
+                ctx = ctx,
+                corsConfig = corsConfig,
+            )
+        } catch (t: Throwable) {
+            // Bind failed — release the bridge engine if we already booted it
+            // so we don't leak native resources.
+            runCatching { newBridge.close() }
+            throw t
+        }
+
+        this.bridge = newBridge
+        this.server = newServer
+        this.modelId = resolvedModelId
+        this.port = boundPort
+        this.baseUrl = "http://127.0.0.1:$boundPort/v1"
+        this.isRunning = true
+
+        return@withLock mapOf(
+            "baseUrl" to "http://127.0.0.1:$boundPort/v1",
+            "port" to boundPort,
+            "backend" to "mediapipe",
+            "modelId" to resolvedModelId,
+        )
+    }
+
+    suspend fun stop() = mutex.withLock { stopInternal() }
+
+    private suspend fun stopInternal() {
+        server?.stop()
+        runCatching { bridge?.close() }
+        server = null
+        bridge = null
+        modelId = ""
+        baseUrl = null
+        port = null
+        isRunning = false
+    }
+
+    fun statusInfo(): Map<String, Any?> = buildMap {
+        put("running", isRunning)
+        baseUrl?.let { put("baseUrl", it) }
+        if (isRunning) put("backend", "mediapipe")
+    }
+
+    private fun parseCors(raw: Any?): CorsConfig = CorsConfig.fromOpt(raw)
+
+    /**
+     * Best-effort default model id from a model file path: strip the directory
+     * prefix and any `.litertlm` or legacy `.task` extension. Falls back to
+     * the literal path if no separators are present.
+     */
+    private fun deriveModelId(modelPath: String): String {
+        val name = modelPath.substringAfterLast('/').substringAfterLast('\\')
+        val stripped = name.removeSuffix(".litertlm").removeSuffix(".task")
+        return stripped.ifEmpty { "mediapipe-default" }
+    }
+}

package/android/src/main/res/xml/dvai_network_security_config.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="utf-8"?>
+<network-security-config>
+    <domain-config cleartextTrafficPermitted="true">
+        <domain includeSubdomains="false">localhost</domain>
+        <domain includeSubdomains="false">127.0.0.1</domain>
+    </domain-config>
+</network-security-config>

package/android/src/test/java/co/deepvoiceai/bridge/mediapipe/core/ImageDecoderTest.kt

@@ -0,0 +1,114 @@
+package co.deepvoiceai.bridge.mediapipe.core
+
+import okhttp3.OkHttpClient
+import okhttp3.mockwebserver.MockResponse
+import okhttp3.mockwebserver.MockWebServer
+import okio.Buffer
+import org.junit.After
+import org.junit.Assert.assertArrayEquals
+import org.junit.Assert.assertEquals
+import org.junit.Assert.fail
+import org.junit.Before
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.robolectric.RobolectricTestRunner
+import java.io.File
+import java.nio.file.Files
+
+@RunWith(RobolectricTestRunner::class)
+class ImageDecoderTest {
+    private lateinit var server: MockWebServer
+
+    @Before
+    fun setUp() {
+        server = MockWebServer().apply { start(0) }
+    }
+
+    @After
+    fun tearDown() {
+        server.shutdown()
+    }
+
+    /** `data:image/png;base64,...` decodes to bytes starting with the PNG magic header. */
+    @Test
+    fun `data URL base64 decodes to bytes`() {
+        val url = loadResourceText("tiny-test-base64.txt")
+        val bytes = ImageDecoder.resolve(url)
+        assertArrayEquals(
+            byteArrayOf(0x89.toByte(), 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A),
+            bytes.copyOfRange(0, 8),
+        )
+    }
+
+    /** `https://...` URLs fetch the body bytes verbatim. */
+    @Test
+    fun `https URL fetches bytes`() {
+        val payload = loadResource("tiny-test.png")
+        server.enqueue(MockResponse().setBody(Buffer().apply { write(payload) }))
+        val bytes = ImageDecoder.resolveWithClient(
+            server.url("/img.png").toString(),
+            OkHttpClient(),
+        )
+        assertArrayEquals(payload, bytes)
+    }
+
+    /** `file://` URLs read the file off disk. */
+    @Test
+    fun `file URL reads bytes`() {
+        val payload = loadResource("tiny-test.png")
+        val tmp = Files.createTempFile("dvai-image-", ".png").toFile().apply {
+            writeBytes(payload)
+            deleteOnExit()
+        }
+        try {
+            val bytes = ImageDecoder.resolve(tmp.toURI().toString())
+            assertArrayEquals(payload, bytes)
+        } finally {
+            tmp.delete()
+        }
+    }
+
+    /** Unsupported scheme → `InvalidScheme`. */
+    @Test(expected = ImageSourceError.InvalidScheme::class)
+    fun `unsupported scheme throws`() {
+        ImageDecoder.resolve("ftp://example.com/x.png")
+    }
+
+    /** `data:` URL with no comma → `MalformedDataURL`. */
+    @Test(expected = ImageSourceError.MalformedDataURL::class)
+    fun `malformed data URL throws`() {
+        ImageDecoder.resolve("data:image/png;base64")
+    }
+
+    /** `file:` URL with no path → `MalformedURL` (was previously misreported as `InvalidScheme`). */
+    @Test(expected = ImageSourceError.MalformedURL::class)
+    fun `file URL with no path throws MalformedURL`() {
+        ImageDecoder.resolve("file:")
+    }
+
+    /** HTTP non-2xx → `HttpError` carrying the status code. */
+    @Test
+    fun `http error throws HttpError`() {
+        server.enqueue(MockResponse().setResponseCode(404))
+        try {
+            ImageDecoder.resolveWithClient(
+                server.url("/missing.png").toString(),
+                OkHttpClient(),
+            )
+            fail("Expected ImageSourceError.HttpError")
+        } catch (e: ImageSourceError.HttpError) {
+            assertEquals(404, e.status)
+        }
+    }
+
+    private fun loadResource(name: String): ByteArray =
+        javaClass.getResourceAsStream("/images/$name")
+            ?.use { it.readBytes() }
+            ?: error("missing test resource: /images/$name")
+
+    private fun loadResourceText(name: String): String =
+        javaClass.getResourceAsStream("/images/$name")
+            ?.bufferedReader()
+            ?.use { it.readText().trim() }
+            ?: error("missing test resource: /images/$name")
+}