@capgo/capacitor-llm 8.0.11 → 8.0.12

package/android/src/main/java/ee/forgr/capgo_llm/LLM.java

@@ -4,20 +4,33 @@ import android.content.Context;
 import com.google.mediapipe.tasks.genai.llminference.LlmInference;
 import com.google.mediapipe.tasks.genai.llminference.LlmInference.LlmInferenceOptions;
 import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.Executor;
 import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.pytorch.executorch.extension.llm.LlmCallback;
+import org.pytorch.executorch.extension.llm.LlmModule;
 
 public class LLM {
 
+    private enum Engine {
+        MEDIAPIPE,
+        EXECUTORCH
+    }
+
     private static LLM instance;
-    private LlmInference llmInference;
+    private LlmInference mediaPipeInference;
+    private LlmModule executorchModule;
     private Map<String, ChatSession> chatSessions;
     private boolean isReady = false;
     private Context context;
     private Executor executor;
     private String modelPath = null;
+    private String tokenizerPath = null;
+    private Engine currentEngine = Engine.MEDIAPIPE;
 
     private LLM(Context context) {
         this.context = context;
@@ -34,28 +47,105 @@ public class LLM {
 
     // Model parameters
     private Integer maxTokens = 2048;
+    private Integer sequenceLength = 2048;
     private Integer topk = 40;
     private Float temperature = 0.1f;
-
-    public void setModel(String path, Integer maxTokens, Integer topk, Float temperature, ModelLoadCallback callback) {
+    private String modelType = null;
+
+    public void setModel(
+        String path,
+        String engine,
+        String modelType,
+        String tokenizerPath,
+        List<String> specialTokens,
+        Integer maxTokens,
+        Integer sequenceLength,
+        Integer topk,
+        Float temperature,
+        ModelLoadCallback callback
+    ) {
         this.modelPath = path;
+        this.tokenizerPath = tokenizerPath;
+        this.modelType = modelType;
         this.maxTokens = maxTokens;
+        this.sequenceLength = sequenceLength != null ? sequenceLength : maxTokens;
         this.topk = topk;
         this.temperature = temperature;
-        isReady = false;
-
-        // If LLM was already initialized, clean it up
-        if (llmInference != null) {
-            llmInference = null;
+        try {
+            this.currentEngine = resolveEngine(engine, path, tokenizerPath);
+        } catch (IllegalArgumentException e) {
+            isReady = false;
+            if (callback != null) {
+                callback.onError(e.getMessage());
+            }
+            return;
         }
+        isReady = false;
 
         android.util.Log.d(
             "LLM",
-            "setModel called with path: " + path + ", maxTokens: " + maxTokens + ", topk: " + topk + ", temperature: " + temperature
+            "setModel called with engine: " +
+                currentEngine +
+                ", path: " +
+                path +
+                ", tokenizerPath: " +
+                tokenizerPath +
+                ", maxTokens: " +
+                maxTokens +
+                ", sequenceLength: " +
+                this.sequenceLength +
+                ", topk: " +
+                topk +
+                ", temperature: " +
+                temperature +
+                ", specialTokens: " +
+                (specialTokens == null ? 0 : specialTokens.size())
         );
         initializeModel(callback);
     }
 
+    private Engine resolveEngine(String engine, String path, String tokenizerPath) {
+        String normalizedEngine = engine == null ? "auto" : engine.toLowerCase(Locale.US);
+        if ("executorch".equals(normalizedEngine)) {
+            return Engine.EXECUTORCH;
+        }
+        if ("mediapipe".equals(normalizedEngine)) {
+            return Engine.MEDIAPIPE;
+        }
+        if ("apple".equals(normalizedEngine)) {
+            throw new IllegalArgumentException("Apple Intelligence is only available on iOS");
+        }
+        if (tokenizerPath != null && !tokenizerPath.isEmpty()) {
+            return Engine.EXECUTORCH;
+        }
+        if (path != null && path.toLowerCase(Locale.US).endsWith(".pte")) {
+            return Engine.EXECUTORCH;
+        }
+        return Engine.MEDIAPIPE;
+    }
+
+    private void releaseCurrentModel() {
+        if (mediaPipeInference != null) {
+            try {
+                mediaPipeInference.close();
+            } catch (Exception e) {
+                android.util.Log.w("LLM", "Failed to close MediaPipe inference: " + e.getMessage());
+            } finally {
+                mediaPipeInference = null;
+            }
+        }
+
+        if (executorchModule != null) {
+            try {
+                executorchModule.stop();
+            } catch (Exception e) {
+                android.util.Log.w("LLM", "Failed to stop ExecuTorch module: " + e.getMessage());
+            } finally {
+                executorchModule = null;
+            }
+        }
+    }
+
     private void initializeModel(ModelLoadCallback callback) {
         if (modelPath == null) {
             if (callback != null) {
@@ -66,64 +156,12 @@ public class LLM {
 
         executor.execute(() -> {
             try {
-                String actualPath = modelPath;
-
-                // Debug logging
-                android.util.Log.d("LLM", "Original model path: " + modelPath);
-
-                // For /android_asset/ paths, we need to handle them specially
-                if (modelPath.startsWith("/android_asset/")) {
-                    // Extract the relative path from assets
-                    String assetPath = modelPath.substring("/android_asset/".length());
-                    android.util.Log.d("LLM", "Asset path: " + assetPath);
-
-                    // Check if the asset exists
-                    try {
-                        java.io.InputStream is = context.getAssets().open(assetPath);
-                        is.close();
-                        android.util.Log.d("LLM", "Asset exists: " + assetPath);
-
-                        // For MediaPipe, we need to copy the asset to a file
-                        java.io.File cacheDir = context.getCacheDir();
-                        java.io.File modelFile = new java.io.File(cacheDir, assetPath);
-
-                        // Create parent directories if needed
-                        modelFile.getParentFile().mkdirs();
-
-                        // Copy asset to cache
-                        copyAssetToFile(assetPath, modelFile);
-                        actualPath = modelFile.getAbsolutePath();
-                        android.util.Log.d("LLM", "Copied asset to: " + actualPath);
-
-                        // Also check for companion .litertlm file
-                        String litertlmPath = assetPath.replace(".task", ".litertlm");
-                        try {
-                            java.io.InputStream litertlmIs = context.getAssets().open(litertlmPath);
-                            litertlmIs.close();
-                            java.io.File litertlmFile = new java.io.File(cacheDir, litertlmPath);
-                            copyAssetToFile(litertlmPath, litertlmFile);
-                            android.util.Log.d("LLM", "Also copied companion file: " + litertlmFile.getAbsolutePath());
-                        } catch (Exception e) {
-                            android.util.Log.d("LLM", "No companion .litertlm file found");
-                        }
-                    } catch (Exception e) {
-                        android.util.Log.e("LLM", "Asset not found: " + assetPath, e);
-                        throw new RuntimeException("Asset not found: " + assetPath);
-                    }
+                releaseCurrentModel();
+                if (currentEngine == Engine.EXECUTORCH) {
+                    initializeExecutorchModel();
+                } else {
+                    initializeMediaPipeModel();
                 }
-
-                android.util.Log.d("LLM", "Final model path: " + actualPath);
-
-                // Create options for LLM inference
-                LlmInferenceOptions.Builder optionsBuilder = LlmInferenceOptions.builder()
-                    .setModelPath(actualPath)
-                    .setMaxTokens(maxTokens)
-                    .setMaxTopK(topk);
-
-                LlmInferenceOptions options = optionsBuilder.build();
-
-                // Initialize the LLM inference
-                llmInference = LlmInference.createFromOptions(context, options);
                 isReady = true;
 
                 if (callback != null) {
@@ -139,16 +177,109 @@ public class LLM {
         });
     }
 
+    private void initializeMediaPipeModel() throws Exception {
+        String actualPath = resolveModelPath(modelPath, true);
+        android.util.Log.d("LLM", "Final MediaPipe model path: " + actualPath);
+
+        LlmInferenceOptions.Builder optionsBuilder = LlmInferenceOptions.builder()
+            .setModelPath(actualPath)
+            .setMaxTokens(maxTokens)
+            .setMaxTopK(topk);
+
+        LlmInferenceOptions options = optionsBuilder.build();
+        mediaPipeInference = LlmInference.createFromOptions(context, options);
+    }
+
+    private void initializeExecutorchModel() throws Exception {
+        if (tokenizerPath == null || tokenizerPath.isEmpty()) {
+            throw new RuntimeException("tokenizerPath is required for ExecuTorch models");
+        }
+
+        String actualModelPath = resolveModelPath(modelPath, false);
+        String actualTokenizerPath = resolveModelPath(tokenizerPath, false);
+        int executorModelType = resolveExecutorchModelType(modelType);
+
+        android.util.Log.d("LLM", "Final ExecuTorch model path: " + actualModelPath);
+        android.util.Log.d("LLM", "Final ExecuTorch tokenizer path: " + actualTokenizerPath);
+
+        executorchModule = new LlmModule(executorModelType, actualModelPath, actualTokenizerPath, temperature);
+        executorchModule.load();
+    }
+
+    private int resolveExecutorchModelType(String modelType) {
+        if (modelType == null) {
+            return LlmModule.MODEL_TYPE_TEXT;
+        }
+        String normalized = modelType.toLowerCase(Locale.US);
+        if ("text-vision".equals(normalized) || "vision".equals(normalized)) {
+            return LlmModule.MODEL_TYPE_TEXT_VISION;
+        }
+        if ("multimodal".equals(normalized)) {
+            return LlmModule.MODEL_TYPE_MULTIMODAL;
+        }
+        return LlmModule.MODEL_TYPE_TEXT;
+    }
+
+    private String resolveModelPath(String path, boolean copyMediaPipeCompanion) throws Exception {
+        android.util.Log.d("LLM", "Original model path: " + path);
+
+        if (!path.startsWith("/android_asset/")) {
+            return path;
+        }
+
+        String assetPath = path.substring("/android_asset/".length());
+        android.util.Log.d("LLM", "Asset path: " + assetPath);
+
+        try (java.io.InputStream is = context.getAssets().open(assetPath)) {
+            android.util.Log.d("LLM", "Asset exists: " + assetPath);
+        } catch (Exception e) {
+            android.util.Log.e("LLM", "Asset not found: " + assetPath, e);
+            throw new RuntimeException("Asset not found: " + assetPath);
+        }
+
+        java.io.File cacheDir = context.getCacheDir();
+        java.io.File modelFile = new java.io.File(cacheDir, assetPath);
+        java.io.File parent = modelFile.getParentFile();
+        if (parent != null) {
+            parent.mkdirs();
+        }
+
+        copyAssetToFile(assetPath, modelFile);
+        android.util.Log.d("LLM", "Copied asset to: " + modelFile.getAbsolutePath());
+
+        if (copyMediaPipeCompanion && assetPath.endsWith(".task")) {
+            copyOptionalCompanion(assetPath, cacheDir);
+        }
+
+        return modelFile.getAbsolutePath();
+    }
+
+    private void copyOptionalCompanion(String assetPath, java.io.File cacheDir) {
+        String litertlmPath = assetPath.replace(".task", ".litertlm");
+        try (java.io.InputStream litertlmIs = context.getAssets().open(litertlmPath)) {
+            java.io.File litertlmFile = new java.io.File(cacheDir, litertlmPath);
+            java.io.File parent = litertlmFile.getParentFile();
+            if (parent != null) {
+                parent.mkdirs();
+            }
+            copyAssetToFile(litertlmPath, litertlmFile);
+            android.util.Log.d("LLM", "Also copied companion file: " + litertlmFile.getAbsolutePath());
+        } catch (Exception e) {
+            android.util.Log.d("LLM", "No companion .litertlm file found");
+        }
+    }
+
     private void copyAssetToFile(String assetPath, java.io.File destFile) throws Exception {
-        java.io.InputStream is = context.getAssets().open(assetPath);
-        java.io.FileOutputStream fos = new java.io.FileOutputStream(destFile);
-        byte[] buffer = new byte[1024];
-        int length;
-        while ((length = is.read(buffer)) > 0) {
-            fos.write(buffer, 0, length);
+        try (
+            java.io.InputStream is = context.getAssets().open(assetPath);
+            java.io.FileOutputStream fos = new java.io.FileOutputStream(destFile)
+        ) {
+            byte[] buffer = new byte[4096];
+            int length;
+            while ((length = is.read(buffer)) > 0) {
+                fos.write(buffer, 0, length);
+            }
         }
-        fos.close();
-        is.close();
     }
 
     public String createChat() {
@@ -165,11 +296,86 @@ public class LLM {
             return;
         }
 
-        if (!isReady || llmInference == null) {
+        if (!isReady) {
             callback.onError("Model not ready");
             return;
         }
 
+        if (currentEngine == Engine.EXECUTORCH) {
+            sendExecutorchMessage(chatId, message, session, callback);
+        } else {
+            sendMediaPipeMessage(chatId, message, session, callback);
+        }
+    }
+
+    private void sendExecutorchMessage(String chatId, String message, ChatSession session, MessageCallback callback) {
+        final LlmModule module = executorchModule;
+        final int generationSequenceLength = sequenceLength;
+        final float generationTemperature = temperature;
+
+        if (module == null) {
+            callback.onError("ExecuTorch model not ready");
+            return;
+        }
+
+        executor.execute(() -> {
+            try {
+                session.addMessage("user", message);
+                String fullPrompt = session.buildPrompt(message);
+                AtomicBoolean completed = new AtomicBoolean(false);
+                StringBuilder fullResponse = new StringBuilder();
+
+                module.resetContext();
+                int errorCode = module.generate(
+                    fullPrompt,
+                    generationSequenceLength,
+                    new LlmCallback() {
+                        @Override
+                        public void onResult(String result) {
+                            if (result == null || result.isEmpty()) {
+                                return;
+                            }
+                            callback.onTextReceived(chatId, result, true);
+                            fullResponse.append(result);
+                        }
+
+                        @Override
+                        public void onStats(String stats) {
+                            if (completed.compareAndSet(false, true)) {
+                                session.addMessage("assistant", fullResponse.toString());
+                                callback.onComplete(chatId);
+                            }
+                        }
+                    },
+                    false,
+                    generationTemperature,
+                    0,
+                    0
+                );
+
+                if (errorCode != 0) {
+                    completed.set(true);
+                    callback.onError("ExecuTorch error " + errorCode);
+                } else if (completed.compareAndSet(false, true)) {
+                    session.addMessage("assistant", fullResponse.toString());
+                    callback.onComplete(chatId);
+                }
+            } catch (Exception e) {
+                callback.onError(e.getMessage());
+            }
+        });
+    }
+
+    private void sendMediaPipeMessage(String chatId, String message, ChatSession session, MessageCallback callback) {
+        final LlmInference inference = mediaPipeInference;
+        final int sessionTopk = topk;
+        final float sessionTemperature = temperature;
+
+        if (inference == null) {
+            callback.onError("MediaPipe model not ready");
+            return;
+        }
+
         executor.execute(() -> {
             try {
                 // Add user message to history
@@ -182,12 +388,12 @@ public class LLM {
                 // Create a session with proper options
                 com.google.mediapipe.tasks.genai.llminference.LlmInferenceSession.LlmInferenceSessionOptions sessionOptions =
                     com.google.mediapipe.tasks.genai.llminference.LlmInferenceSession.LlmInferenceSessionOptions.builder()
-                        .setTopK(topk)
-                        .setTemperature(temperature)
+                        .setTopK(sessionTopk)
+                        .setTemperature(sessionTemperature)
                         .build();
 
                 com.google.mediapipe.tasks.genai.llminference.LlmInferenceSession inferenceSession =
-                    com.google.mediapipe.tasks.genai.llminference.LlmInferenceSession.createFromOptions(llmInference, sessionOptions);
+                    com.google.mediapipe.tasks.genai.llminference.LlmInferenceSession.createFromOptions(inference, sessionOptions);
 
                 // Add the query
                 inferenceSession.addQueryChunk(fullPrompt);
@@ -199,6 +405,7 @@ public class LLM {
                     new com.google.mediapipe.tasks.genai.llminference.ProgressListener<String>() {
                         private StringBuilder buffer = new StringBuilder();
                         private boolean hasStarted = false;
+                        private final AtomicBoolean completed = new AtomicBoolean(false);
 
                         @Override
                         public void run(String partialResult, boolean done) {
@@ -244,19 +451,17 @@ public class LLM {
                                 fullResponse.append(chunk);
                             }
 
-                            // Process any remaining content when done
-                            if (done && buffer.length() > 0) {
-                                String remaining = buffer.toString();
-                                if (!remaining.isEmpty()) {
+                            if (done && completed.compareAndSet(false, true)) {
+                                if (buffer.length() > 0) {
+                                    String remaining = buffer.toString();
                                     callback.onTextReceived(chatId, remaining, true);
                                     fullResponse.append(remaining);
+                                    buffer = new StringBuilder();
                                 }
 
-                                // Add complete response to history
                                 session.addMessage("assistant", fullResponse.toString());
                                 callback.onComplete(chatId);
 
-                                // Close the session
                                 try {
                                     inferenceSession.close();
                                 } catch (Exception e) {
@@ -310,8 +515,7 @@ public class LLM {
         }
 
         String buildPrompt(String newMessage) {
-            // For Gemma with MediaPipe, just return the message as-is
-            return newMessage;
+            return history.toString();
         }
     }
 }

package/android/src/main/java/ee/forgr/capgo_llm/LLMPlugin.java

@@ -1,15 +1,18 @@
 package ee.forgr.capgo_llm;
 
+import com.getcapacitor.JSArray;
 import com.getcapacitor.JSObject;
 import com.getcapacitor.Plugin;
 import com.getcapacitor.PluginCall;
 import com.getcapacitor.PluginMethod;
 import com.getcapacitor.annotation.CapacitorPlugin;
+import java.util.ArrayList;
+import java.util.List;
 
 @CapacitorPlugin(name = "CapgoLLM")
 public class LLMPlugin extends Plugin {
 
-    private final String pluginVersion = "8.0.11";
+    private final String pluginVersion = "8.0.12";
 
     private LLM llm;
 
@@ -94,14 +97,33 @@ public class LLMPlugin extends Plugin {
         }
 
         // Extract model parameters
+        String engine = call.getString("engine", "auto");
+        String modelType = call.getString("modelType");
+        String tokenizerPath = call.getString("tokenizerPath");
         Integer maxTokens = call.getInt("maxTokens", 2048);
+        Integer sequenceLength = call.getInt("sequenceLength", maxTokens);
         Integer topk = call.getInt("topk", 40);
         Float temperature = call.getFloat("temperature", 0.8f);
-        Integer randomSeed = call.getInt("randomSeed", 101);
+        List<String> specialTokens = getStringList(call.getArray("specialTokens"));
+
+        if (sequenceLength <= 0) {
+            call.reject("sequenceLength must be > 0");
+            return;
+        }
+
+        if (!specialTokens.isEmpty()) {
+            call.reject("specialTokens are only supported by ExecuTorch on iOS");
+            return;
+        }
 
         llm.setModel(
             path,
+            engine,
+            modelType,
+            tokenizerPath,
+            specialTokens,
             maxTokens,
+            sequenceLength,
             topk,
             temperature,
             new LLM.ModelLoadCallback() {
@@ -182,6 +204,21 @@ public class LLMPlugin extends Plugin {
             .start();
     }
 
+    private List<String> getStringList(JSArray values) {
+        List<String> result = new ArrayList<>();
+        if (values == null) {
+            return result;
+        }
+
+        for (int i = 0; i < values.length(); i++) {
+            String value = values.optString(i, null);
+            if (value != null) {
+                result.add(value);
+            }
+        }
+        return result;
+    }
+
     private void downloadFile(String urlString, java.io.File outputFile, ProgressCallback callback) throws Exception {
         java.net.URL url = new java.net.URL(urlString);
         java.net.HttpURLConnection connection = (java.net.HttpURLConnection) url.openConnection();

package/dist/docs.json

@@ -81,7 +81,7 @@
             "text": "Promise that resolves when model is loaded"
           }
         ],
-        "docs": "Sets the model configuration\n- iOS: Use \"Apple Intelligence\" as path for system model, or provide path to MediaPipe model\n- Android: Path to a MediaPipe model file (in assets or files directory)",
+        "docs": "Sets the model configuration\n- iOS: Use \"Apple Intelligence\" as path for system model, provide a MediaPipe model, or set engine to \"executorch\"\n- Android: Path to a MediaPipe or ExecuTorch model file (in assets or files directory)",
         "complexTypes": [
           "ModelOptions"
         ],
@@ -304,6 +304,13 @@
           "complexTypes": [],
           "type": "string"
         },
+        {
+          "name": "engine",
+          "tags": [],
+          "docs": "Runtime engine to use.\n- \"auto\": uses Apple Intelligence on iOS when path is \"Apple Intelligence\", ExecuTorch for `.pte` or tokenizerPath, otherwise MediaPipe\n- \"apple\": iOS Foundation Models / Apple Intelligence\n- \"mediapipe\": MediaPipe GenAI `.task` models\n- \"executorch\": ExecuTorch `.pte` models with a tokenizer file",
+          "complexTypes": [],
+          "type": "'auto' | 'apple' | 'mediapipe' | 'executorch' | undefined"
+        },
         {
           "name": "modelType",
           "tags": [],
@@ -311,6 +318,20 @@
           "complexTypes": [],
           "type": "string | undefined"
         },
+        {
+          "name": "tokenizerPath",
+          "tags": [],
+          "docs": "Tokenizer path for ExecuTorch models. Required when engine is \"executorch\".",
+          "complexTypes": [],
+          "type": "string | undefined"
+        },
+        {
+          "name": "specialTokens",
+          "tags": [],
+          "docs": "Optional special tokens passed to iOS ExecuTorch tokenizers.",
+          "complexTypes": [],
+          "type": "string[] | undefined"
+        },
         {
           "name": "maxTokens",
           "tags": [],
@@ -318,6 +339,13 @@
           "complexTypes": [],
           "type": "number | undefined"
         },
+        {
+          "name": "sequenceLength",
+          "tags": [],
+          "docs": "Sequence length for ExecuTorch generation. Defaults to maxTokens when omitted.",
+          "complexTypes": [],
+          "type": "number | undefined"
+        },
         {
           "name": "topk",
           "tags": [],

package/dist/esm/definitions.d.ts

@@ -28,8 +28,8 @@ export interface LLMPlugin {
     }>;
     /**
      * Sets the model configuration
-     * - iOS: Use "Apple Intelligence" as path for system model, or provide path to MediaPipe model
-     * - Android: Path to a MediaPipe model file (in assets or files directory)
+     * - iOS: Use "Apple Intelligence" as path for system model, provide a MediaPipe model, or set engine to "executorch"
+     * - Android: Path to a MediaPipe or ExecuTorch model file (in assets or files directory)
      * @param options - The model configuration
      * @returns Promise that resolves when model is loaded
      */
@@ -156,10 +156,24 @@ export interface ReadinessChangeEvent {
 export interface ModelOptions {
     /** Model path or "Apple Intelligence" for iOS system model */
     path: string;
+    /**
+     * Runtime engine to use.
+     * - "auto": uses Apple Intelligence on iOS when path is "Apple Intelligence", ExecuTorch for `.pte` or tokenizerPath, otherwise MediaPipe
+     * - "apple": iOS Foundation Models / Apple Intelligence
+     * - "mediapipe": MediaPipe GenAI `.task` models
+     * - "executorch": ExecuTorch `.pte` models with a tokenizer file
+     */
+    engine?: 'auto' | 'apple' | 'mediapipe' | 'executorch';
     /** Model file type/extension (e.g., "task", "bin", "litertlm"). If not provided, will be extracted from path. */
     modelType?: string;
+    /** Tokenizer path for ExecuTorch models. Required when engine is "executorch". */
+    tokenizerPath?: string;
+    /** Optional special tokens passed to iOS ExecuTorch tokenizers. */
+    specialTokens?: string[];
     /** Maximum number of tokens the model handles */
     maxTokens?: number;
+    /** Sequence length for ExecuTorch generation. Defaults to maxTokens when omitted. */
+    sequenceLength?: number;
     /** Number of tokens the model considers at each step */
     topk?: number;
     /** Amount of randomness in generation (0.0-1.0) */