cactus-react-native 0.2.1 → 0.2.2

package/README.md

@@ -1,13 +1,13 @@
 # Cactus React Native
 
-Running LLMs, VLMs, and TTS models directly on mobile devices.
+Run LLMs, VLMs, and TTS models directly on mobile devices.
 
 ## Installation
 
 ```json
 {
   "dependencies": {
-    "cactus-react-native": "^0.2.0",
+    "cactus-react-native": "^0.2.1",
     "react-native-fs": "^2.20.0"
   }
 }

package/android/src/main/jni.cpp

@@ -307,6 +307,7 @@ Java_com_cactus_LlamaContext_initContext(
     llama->is_load_interrupted = false;
     llama->loading_progress = 0;
 
+    callback_context *cb_ctx = nullptr;
     if (load_progress_callback != nullptr) {
         defaultParams.progress_callback = [](float progress, void * user_data) {
             callback_context *cb_ctx = (callback_context *)user_data;
@@ -323,7 +324,7 @@ Java_com_cactus_LlamaContext_initContext(
             return !llama->is_load_interrupted;
         };
 
-        callback_context *cb_ctx = new callback_context;
+        cb_ctx = new callback_context;
         cb_ctx->env = env;
         cb_ctx->llama = llama;
         cb_ctx->callback = env->NewGlobalRef(load_progress_callback);
@@ -339,17 +340,30 @@ Java_com_cactus_LlamaContext_initContext(
     env->ReleaseStringUTFChars(cache_type_v, cache_type_v_chars);
 
     LOGI("[CACTUS] is_model_loaded %s", (is_model_loaded ? "true" : "false"));
-    if (is_model_loaded) {
-        if (embedding && llama_model_has_encoder(llama->model) && llama_model_has_decoder(llama->model)) {
-            LOGI("[CACTUS] computing embeddings in encoder-decoder models is not supported");
-            llama_free(llama->ctx);
-            return -1;
+
+    if (!is_model_loaded) {
+        llama_free(llama->ctx);
+        if (cb_ctx) {
+            env->DeleteGlobalRef(cb_ctx->callback);
+            delete cb_ctx;
         }
-        context_map[(long) llama->ctx] = llama;
-    } else {
+        delete llama;
+        return -1;
+    }
+
+    if (embedding && llama_model_has_encoder(llama->model) && llama_model_has_decoder(llama->model)) {
+        LOGI("[CACTUS] computing embeddings in encoder-decoder models is not supported");
         llama_free(llama->ctx);
+        if (cb_ctx) {
+            env->DeleteGlobalRef(cb_ctx->callback);
+            delete cb_ctx;
+        }
+        delete llama;
+        return -1;
     }
 
+    context_map[(long) llama->ctx] = llama;
+
     std::vector<common_adapter_lora_info> lora;
     const char *lora_chars = env->GetStringUTFChars(lora_str, nullptr);
     if (lora_chars != nullptr && lora_chars[0] != '\0') {
@@ -378,9 +392,20 @@ Java_com_cactus_LlamaContext_initContext(
     env->ReleaseStringUTFChars(lora_str, lora_chars);
     int result = llama->applyLoraAdapters(lora);
     if (result != 0) {
-      LOGI("[Cactus] Failed to apply lora adapters");
-      llama_free(llama->ctx);
-      return -1;
+        LOGI("[Cactus] Failed to apply lora adapters");
+        context_map.erase((long) llama->ctx);
+        llama_free(llama->ctx);
+        if (cb_ctx) {
+            env->DeleteGlobalRef(cb_ctx->callback);
+            delete cb_ctx;
+        }
+        delete llama;
+        return -1;
+    }
+
+    if (cb_ctx) {
+        env->DeleteGlobalRef(cb_ctx->callback);
+        delete cb_ctx;
     }
 
     return reinterpret_cast<jlong>(llama->ctx);

package/lib/commonjs/lm.js

@@ -5,8 +5,11 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.CactusLM = void 0;
 var _index = require("./index");
+var _reactNative = require("react-native");
 var _telemetry = require("./telemetry");
 var _remote = require("./remote");
+// @ts-ignore
+
 class CactusLM {
   constructor(context) {
     this.context = context;
@@ -15,7 +18,13 @@ class CactusLM {
     if (cactusToken) {
       (0, _remote.setCactusToken)(cactusToken);
     }
-    const configs = [params, {
+
+    // Avoid two back-to-back loads on devices where GPU off-load is unsupported (Android).
+    const needGpuAttempt = _reactNative.Platform.OS !== 'android' && (params.n_gpu_layers ?? 0) > 0;
+    const configs = needGpuAttempt ? [params, {
+      ...params,
+      n_gpu_layers: 0
+    }] : [{
       ...params,
       n_gpu_layers: 0
     }];

package/lib/commonjs/lm.js.map

@@ -1 +1 @@
-{"version":3,"names":["_index","require","_telemetry","_remote","CactusLM","constructor","context","init","params","onProgress","cactusToken","setCactusToken","configs","n_gpu_layers","config","initLlama","lm","error","e","Telemetry","n_ctx","model","indexOf","length","Error","completion","messages","callback","embedding","text","mode","result","lastError","_handleRemoteEmbedding","_handleLocalEmbedding","remoteError","localError","embeddingValues","getVertexAIEmbedding","rewind","release","exports"],"sourceRoot":"../../src","sources":["lm.ts"],"mappings":";;;;;;AAAA,IAAAA,MAAA,GAAAC,OAAA;AASA,IAAAC,UAAA,GAAAD,OAAA;AACA,IAAAE,OAAA,GAAAF,OAAA;AAOO,MAAMG,QAAQ,CAAC;EAGZC,WAAWA,CAACC,OAAqB,EAAE;IACzC,IAAI,CAACA,OAAO,GAAGA,OAAO;EACxB;EAEA,aAAaC,IAAIA,CACfC,MAAqB,EACrBC,UAAuC,EACvCC,WAAoB,EACK;IACzB,IAAIA,WAAW,EAAE;MACf,IAAAC,sBAAc,EAACD,WAAW,CAAC;IAC7B;IAEA,MAAME,OAAO,GAAG,CACdJ,MAAM,EACN;MAAE,GAAGA,MAAM;MAAEK,YAAY,EAAE;IAAE,CAAC,CAC/B;IAED,KAAK,MAAMC,MAAM,IAAIF,OAAO,EAAE;MAC5B,IAAI;QACF,MAAMN,OAAO,GAAG,MAAM,IAAAS,gBAAS,EAACD,MAAM,EAAEL,UAAU,CAAC;QACnD,OAAO;UAAEO,EAAE,EAAE,IAAIZ,QAAQ,CAACE,OAAO,CAAC;UAAEW,KAAK,EAAE;QAAK,CAAC;MACnD,CAAC,CAAC,OAAOC,CAAC,EAAE;QACVC,oBAAS,CAACF,KAAK,CAACC,CAAC,EAAW;UAC1BL,YAAY,EAAEC,MAAM,CAACD,YAAY,IAAI,IAAI;UACzCO,KAAK,EAAEN,MAAM,CAACM,KAAK,IAAI,IAAI;UAC3BC,KAAK,EAAEP,MAAM,CAACO,KAAK,IAAI;QACzB,CAAC,CAAC;QACF,IAAIT,OAAO,CAACU,OAAO,CAACR,MAAM,CAAC,KAAKF,OAAO,CAACW,MAAM,GAAG,CAAC,EAAE;UAClD,OAAO;YAAEP,EAAE,EAAE,IAAI;YAAEC,KAAK,EAAEC;UAAW,CAAC;QACxC;MACF;IACF;IACA,OAAO;MAAEF,EAAE,EAAE,IAAI;MAAEC,KAAK,EAAE,IAAIO,KAAK,CAAC,+BAA+B;IAAE,CAAC;EACxE;EAEA,MAAMC,UAAUA,CACdC,QAAsC,EACtClB,MAAwB,GAAG,CAAC,CAAC,EAC7BmB,QAA8B,EACG;IACjC,OAAO,MAAM,IAAI,CAACrB,OAAO,CAACmB,UAAU,CAAC;MAAEC,QAAQ;MAAE,GAAGlB;IAAO,CAAC,EAAEmB,QAAQ,CAAC;EACzE;EAEA,MAAMC,SAASA,CACbC,IAAY,EACZrB,MAAwB,EACxBsB,IAAY,GAAG,OAAO,EACU;IAChC,IAAIC,MAA6B;IACjC,IAAIC,SAAuB,GAAG,IAAI;IAElC,IAAIF,IAAI,KAAK,QAAQ,EAAE;MACrBC,MAAM,GAAG,MAAM,IAAI,CAACE,sBAAsB,CAACJ,IAAI,CAAC;IAClD,CAAC,MAAM,IAAIC,IAAI,KAAK,OAAO,EAAE;MAC3BC,MAAM,GAAG,MAAM,IAAI,CAACG,qBAAqB,CAACL,IAAI,EAAErB,MAAM,CAAC;IACzD,CAAC,MAAM,IAAIsB,IAAI,KAAK,YAAY,EAAE;MAChC,IAAI;QACFC,MAAM,GAAG,MAAM,IAAI,CAACG,qBAAqB,CAACL,IAAI,EAAErB,MAAM,CAAC;MACzD,CAAC,CAAC,OAAOU,CAAC,EAAE;QACVc,SAAS,GAAGd,CAAU;QACtB,IAAI;UACFa,MAAM,GAAG,MAAM,IAAI,CAACE,sBAAsB,CAACJ,IAAI,CAAC;QAClD,CAAC,CAAC,OAAOM,WAAW,EAAE;UACpB,MAAMH,SAAS;QACjB;MACF;IACF,CAAC,MAAM,IAAIF,IAAI,KAAK,aAAa,EAAE;MACjC,IAAI;QACFC,MAAM,GAAG,MAAM,IAAI,CAACE,sBAAsB,CAACJ,IAAI,CAAC;MAClD,CAAC,CAAC,OAAOX,CAAC,EAAE;QACVc,SAAS,GAAGd,CAAU;QACtB,IAAI;UACFa,MAAM,GAAG,MAAM,IAAI,CAACG,qBAAqB,CAACL,IAAI,EAAErB,MAAM,CAAC;QACzD,CAAC,CAAC,OAAO4B,UAAU,EAAE;UACnB,MAAMJ,SAAS;QACjB;MACF;IACF,CAAC,MAAM;MACL,MAAM,IAAIR,KAAK,CAAC,gBAAgB,GAAGM,IAAI,GAAG,6DAA6D,CAAC;IAC1G;IACA,OAAOC,MAAM;EACf;EAEA,MAAcG,qBAAqBA,CAACL,IAAY,EAAErB,MAAwB,EAAkC;IAC1G,OAAO,IAAI,CAACF,OAAO,CAACsB,SAAS,CAACC,IAAI,EAAErB,MAAM,CAAC;EAC7C;EAEA,MAAcyB,sBAAsBA,CAACJ,IAAY,EAAkC;IACjF,MAAMQ,eAAe,GAAG,MAAM,IAAAC,4BAAoB,EAACT,IAAI,CAAC;IACxD,OAAO;MACLD,SAAS,EAAES;IACb,CAAC;EACH;EAEA,MAAME,MAAMA,CAAA,EAAkB;IAC5B;IACA,OAAO,IAAI,CAACjC,OAAO,EAAEiC,MAAM,CAAC,CAAC;EAC/B;EAEA,MAAMC,OAAOA,CAAA,EAAkB;IAC7B,OAAO,IAAI,CAAClC,OAAO,CAACkC,OAAO,CAAC,CAAC;EAC/B;AACF;AAACC,OAAA,CAAArC,QAAA,GAAAA,QAAA","ignoreList":[]}
+{"version":3,"names":["_index","require","_reactNative","_telemetry","_remote","CactusLM","constructor","context","init","params","onProgress","cactusToken","setCactusToken","needGpuAttempt","Platform","OS","n_gpu_layers","configs","config","initLlama","lm","error","e","Telemetry","n_ctx","model","indexOf","length","Error","completion","messages","callback","embedding","text","mode","result","lastError","_handleRemoteEmbedding","_handleLocalEmbedding","remoteError","localError","embeddingValues","getVertexAIEmbedding","rewind","release","exports"],"sourceRoot":"../../src","sources":["lm.ts"],"mappings":";;;;;;AAAA,IAAAA,MAAA,GAAAC,OAAA;AAEA,IAAAC,YAAA,GAAAD,OAAA;AASA,IAAAE,UAAA,GAAAF,OAAA;AACA,IAAAG,OAAA,GAAAH,OAAA;AAXA;;AAkBO,MAAMI,QAAQ,CAAC;EAGZC,WAAWA,CAACC,OAAqB,EAAE;IACzC,IAAI,CAACA,OAAO,GAAGA,OAAO;EACxB;EAEA,aAAaC,IAAIA,CACfC,MAAqB,EACrBC,UAAuC,EACvCC,WAAoB,EACK;IACzB,IAAIA,WAAW,EAAE;MACf,IAAAC,sBAAc,EAACD,WAAW,CAAC;IAC7B;;IAEA;IACA,MAAME,cAAc,GAAGC,qBAAQ,CAACC,EAAE,KAAK,SAAS,IAAI,CAACN,MAAM,CAACO,YAAY,IAAI,CAAC,IAAI,CAAC;IAClF,MAAMC,OAAO,GAAGJ,cAAc,GAC1B,CAACJ,MAAM,EAAE;MAAE,GAAGA,MAAM;MAAEO,YAAY,EAAE;IAAE,CAAC,CAAC,GACxC,CAAC;MAAE,GAAGP,MAAM;MAAEO,YAAY,EAAE;IAAE,CAAC,CAAC;IAEpC,KAAK,MAAME,MAAM,IAAID,OAAO,EAAE;MAC5B,IAAI;QACF,MAAMV,OAAO,GAAG,MAAM,IAAAY,gBAAS,EAACD,MAAM,EAAER,UAAU,CAAC;QACnD,OAAO;UAAEU,EAAE,EAAE,IAAIf,QAAQ,CAACE,OAAO,CAAC;UAAEc,KAAK,EAAE;QAAK,CAAC;MACnD,CAAC,CAAC,OAAOC,CAAC,EAAE;QACVC,oBAAS,CAACF,KAAK,CAACC,CAAC,EAAW;UAC1BN,YAAY,EAAEE,MAAM,CAACF,YAAY,IAAI,IAAI;UACzCQ,KAAK,EAAEN,MAAM,CAACM,KAAK,IAAI,IAAI;UAC3BC,KAAK,EAAEP,MAAM,CAACO,KAAK,IAAI;QACzB,CAAC,CAAC;QACF,IAAIR,OAAO,CAACS,OAAO,CAACR,MAAM,CAAC,KAAKD,OAAO,CAACU,MAAM,GAAG,CAAC,EAAE;UAClD,OAAO;YAAEP,EAAE,EAAE,IAAI;YAAEC,KAAK,EAAEC;UAAW,CAAC;QACxC;MACF;IACF;IACA,OAAO;MAAEF,EAAE,EAAE,IAAI;MAAEC,KAAK,EAAE,IAAIO,KAAK,CAAC,+BAA+B;IAAE,CAAC;EACxE;EAEA,MAAMC,UAAUA,CACdC,QAAsC,EACtCrB,MAAwB,GAAG,CAAC,CAAC,EAC7BsB,QAA8B,EACG;IACjC,OAAO,MAAM,IAAI,CAACxB,OAAO,CAACsB,UAAU,CAAC;MAAEC,QAAQ;MAAE,GAAGrB;IAAO,CAAC,EAAEsB,QAAQ,CAAC;EACzE;EAEA,MAAMC,SAASA,CACbC,IAAY,EACZxB,MAAwB,EACxByB,IAAY,GAAG,OAAO,EACU;IAChC,IAAIC,MAA6B;IACjC,IAAIC,SAAuB,GAAG,IAAI;IAElC,IAAIF,IAAI,KAAK,QAAQ,EAAE;MACrBC,MAAM,GAAG,MAAM,IAAI,CAACE,sBAAsB,CAACJ,IAAI,CAAC;IAClD,CAAC,MAAM,IAAIC,IAAI,KAAK,OAAO,EAAE;MAC3BC,MAAM,GAAG,MAAM,IAAI,CAACG,qBAAqB,CAACL,IAAI,EAAExB,MAAM,CAAC;IACzD,CAAC,MAAM,IAAIyB,IAAI,KAAK,YAAY,EAAE;MAChC,IAAI;QACFC,MAAM,GAAG,MAAM,IAAI,CAACG,qBAAqB,CAACL,IAAI,EAAExB,MAAM,CAAC;MACzD,CAAC,CAAC,OAAOa,CAAC,EAAE;QACVc,SAAS,GAAGd,CAAU;QACtB,IAAI;UACFa,MAAM,GAAG,MAAM,IAAI,CAACE,sBAAsB,CAACJ,IAAI,CAAC;QAClD,CAAC,CAAC,OAAOM,WAAW,EAAE;UACpB,MAAMH,SAAS;QACjB;MACF;IACF,CAAC,MAAM,IAAIF,IAAI,KAAK,aAAa,EAAE;MACjC,IAAI;QACFC,MAAM,GAAG,MAAM,IAAI,CAACE,sBAAsB,CAACJ,IAAI,CAAC;MAClD,CAAC,CAAC,OAAOX,CAAC,EAAE;QACVc,SAAS,GAAGd,CAAU;QACtB,IAAI;UACFa,MAAM,GAAG,MAAM,IAAI,CAACG,qBAAqB,CAACL,IAAI,EAAExB,MAAM,CAAC;QACzD,CAAC,CAAC,OAAO+B,UAAU,EAAE;UACnB,MAAMJ,SAAS;QACjB;MACF;IACF,CAAC,MAAM;MACL,MAAM,IAAIR,KAAK,CAAC,gBAAgB,GAAGM,IAAI,GAAG,6DAA6D,CAAC;IAC1G;IACA,OAAOC,MAAM;EACf;EAEA,MAAcG,qBAAqBA,CAACL,IAAY,EAAExB,MAAwB,EAAkC;IAC1G,OAAO,IAAI,CAACF,OAAO,CAACyB,SAAS,CAACC,IAAI,EAAExB,MAAM,CAAC;EAC7C;EAEA,MAAc4B,sBAAsBA,CAACJ,IAAY,EAAkC;IACjF,MAAMQ,eAAe,GAAG,MAAM,IAAAC,4BAAoB,EAACT,IAAI,CAAC;IACxD,OAAO;MACLD,SAAS,EAAES;IACb,CAAC;EACH;EAEA,MAAME,MAAMA,CAAA,EAAkB;IAC5B;IACA,OAAO,IAAI,CAACpC,OAAO,EAAEoC,MAAM,CAAC,CAAC;EAC/B;EAEA,MAAMC,OAAOA,CAAA,EAAkB;IAC7B,OAAO,IAAI,CAACrC,OAAO,CAACqC,OAAO,CAAC,CAAC;EAC/B;AACF;AAACC,OAAA,CAAAxC,QAAA,GAAAA,QAAA","ignoreList":[]}

package/lib/module/lm.js

@@ -1,6 +1,8 @@
 "use strict";
 
 import { initLlama } from './index';
+// @ts-ignore
+import { Platform } from 'react-native';
 import { Telemetry } from './telemetry';
 import { setCactusToken, getVertexAIEmbedding } from './remote';
 export class CactusLM {
@@ -11,7 +13,13 @@ export class CactusLM {
     if (cactusToken) {
       setCactusToken(cactusToken);
     }
-    const configs = [params, {
+
+    // Avoid two back-to-back loads on devices where GPU off-load is unsupported (Android).
+    const needGpuAttempt = Platform.OS !== 'android' && (params.n_gpu_layers ?? 0) > 0;
+    const configs = needGpuAttempt ? [params, {
+      ...params,
+      n_gpu_layers: 0
+    }] : [{
       ...params,
       n_gpu_layers: 0
     }];

package/lib/module/lm.js.map

@@ -1 +1 @@
-{"version":3,"names":["initLlama","Telemetry","setCactusToken","getVertexAIEmbedding","CactusLM","constructor","context","init","params","onProgress","cactusToken","configs","n_gpu_layers","config","lm","error","e","n_ctx","model","indexOf","length","Error","completion","messages","callback","embedding","text","mode","result","lastError","_handleRemoteEmbedding","_handleLocalEmbedding","remoteError","localError","embeddingValues","rewind","release"],"sourceRoot":"../../src","sources":["lm.ts"],"mappings":";;AAAA,SAASA,SAAS,QAAsB,SAAS;AASjD,SAASC,SAAS,QAAQ,aAAa;AACvC,SAASC,cAAc,EAAEC,oBAAoB,QAAQ,UAAU;AAO/D,OAAO,MAAMC,QAAQ,CAAC;EAGZC,WAAWA,CAACC,OAAqB,EAAE;IACzC,IAAI,CAACA,OAAO,GAAGA,OAAO;EACxB;EAEA,aAAaC,IAAIA,CACfC,MAAqB,EACrBC,UAAuC,EACvCC,WAAoB,EACK;IACzB,IAAIA,WAAW,EAAE;MACfR,cAAc,CAACQ,WAAW,CAAC;IAC7B;IAEA,MAAMC,OAAO,GAAG,CACdH,MAAM,EACN;MAAE,GAAGA,MAAM;MAAEI,YAAY,EAAE;IAAE,CAAC,CAC/B;IAED,KAAK,MAAMC,MAAM,IAAIF,OAAO,EAAE;MAC5B,IAAI;QACF,MAAML,OAAO,GAAG,MAAMN,SAAS,CAACa,MAAM,EAAEJ,UAAU,CAAC;QACnD,OAAO;UAAEK,EAAE,EAAE,IAAIV,QAAQ,CAACE,OAAO,CAAC;UAAES,KAAK,EAAE;QAAK,CAAC;MACnD,CAAC,CAAC,OAAOC,CAAC,EAAE;QACVf,SAAS,CAACc,KAAK,CAACC,CAAC,EAAW;UAC1BJ,YAAY,EAAEC,MAAM,CAACD,YAAY,IAAI,IAAI;UACzCK,KAAK,EAAEJ,MAAM,CAACI,KAAK,IAAI,IAAI;UAC3BC,KAAK,EAAEL,MAAM,CAACK,KAAK,IAAI;QACzB,CAAC,CAAC;QACF,IAAIP,OAAO,CAACQ,OAAO,CAACN,MAAM,CAAC,KAAKF,OAAO,CAACS,MAAM,GAAG,CAAC,EAAE;UAClD,OAAO;YAAEN,EAAE,EAAE,IAAI;YAAEC,KAAK,EAAEC;UAAW,CAAC;QACxC;MACF;IACF;IACA,OAAO;MAAEF,EAAE,EAAE,IAAI;MAAEC,KAAK,EAAE,IAAIM,KAAK,CAAC,+BAA+B;IAAE,CAAC;EACxE;EAEA,MAAMC,UAAUA,CACdC,QAAsC,EACtCf,MAAwB,GAAG,CAAC,CAAC,EAC7BgB,QAA8B,EACG;IACjC,OAAO,MAAM,IAAI,CAAClB,OAAO,CAACgB,UAAU,CAAC;MAAEC,QAAQ;MAAE,GAAGf;IAAO,CAAC,EAAEgB,QAAQ,CAAC;EACzE;EAEA,MAAMC,SAASA,CACbC,IAAY,EACZlB,MAAwB,EACxBmB,IAAY,GAAG,OAAO,EACU;IAChC,IAAIC,MAA6B;IACjC,IAAIC,SAAuB,GAAG,IAAI;IAElC,IAAIF,IAAI,KAAK,QAAQ,EAAE;MACrBC,MAAM,GAAG,MAAM,IAAI,CAACE,sBAAsB,CAACJ,IAAI,CAAC;IAClD,CAAC,MAAM,IAAIC,IAAI,KAAK,OAAO,EAAE;MAC3BC,MAAM,GAAG,MAAM,IAAI,CAACG,qBAAqB,CAACL,IAAI,EAAElB,MAAM,CAAC;IACzD,CAAC,MAAM,IAAImB,IAAI,KAAK,YAAY,EAAE;MAChC,IAAI;QACFC,MAAM,GAAG,MAAM,IAAI,CAACG,qBAAqB,CAACL,IAAI,EAAElB,MAAM,CAAC;MACzD,CAAC,CAAC,OAAOQ,CAAC,EAAE;QACVa,SAAS,GAAGb,CAAU;QACtB,IAAI;UACFY,MAAM,GAAG,MAAM,IAAI,CAACE,sBAAsB,CAACJ,IAAI,CAAC;QAClD,CAAC,CAAC,OAAOM,WAAW,EAAE;UACpB,MAAMH,SAAS;QACjB;MACF;IACF,CAAC,MAAM,IAAIF,IAAI,KAAK,aAAa,EAAE;MACjC,IAAI;QACFC,MAAM,GAAG,MAAM,IAAI,CAACE,sBAAsB,CAACJ,IAAI,CAAC;MAClD,CAAC,CAAC,OAAOV,CAAC,EAAE;QACVa,SAAS,GAAGb,CAAU;QACtB,IAAI;UACFY,MAAM,GAAG,MAAM,IAAI,CAACG,qBAAqB,CAACL,IAAI,EAAElB,MAAM,CAAC;QACzD,CAAC,CAAC,OAAOyB,UAAU,EAAE;UACnB,MAAMJ,SAAS;QACjB;MACF;IACF,CAAC,MAAM;MACL,MAAM,IAAIR,KAAK,CAAC,gBAAgB,GAAGM,IAAI,GAAG,6DAA6D,CAAC;IAC1G;IACA,OAAOC,MAAM;EACf;EAEA,MAAcG,qBAAqBA,CAACL,IAAY,EAAElB,MAAwB,EAAkC;IAC1G,OAAO,IAAI,CAACF,OAAO,CAACmB,SAAS,CAACC,IAAI,EAAElB,MAAM,CAAC;EAC7C;EAEA,MAAcsB,sBAAsBA,CAACJ,IAAY,EAAkC;IACjF,MAAMQ,eAAe,GAAG,MAAM/B,oBAAoB,CAACuB,IAAI,CAAC;IACxD,OAAO;MACLD,SAAS,EAAES;IACb,CAAC;EACH;EAEA,MAAMC,MAAMA,CAAA,EAAkB;IAC5B;IACA,OAAO,IAAI,CAAC7B,OAAO,EAAE6B,MAAM,CAAC,CAAC;EAC/B;EAEA,MAAMC,OAAOA,CAAA,EAAkB;IAC7B,OAAO,IAAI,CAAC9B,OAAO,CAAC8B,OAAO,CAAC,CAAC;EAC/B;AACF","ignoreList":[]}
+{"version":3,"names":["initLlama","Platform","Telemetry","setCactusToken","getVertexAIEmbedding","CactusLM","constructor","context","init","params","onProgress","cactusToken","needGpuAttempt","OS","n_gpu_layers","configs","config","lm","error","e","n_ctx","model","indexOf","length","Error","completion","messages","callback","embedding","text","mode","result","lastError","_handleRemoteEmbedding","_handleLocalEmbedding","remoteError","localError","embeddingValues","rewind","release"],"sourceRoot":"../../src","sources":["lm.ts"],"mappings":";;AAAA,SAASA,SAAS,QAAsB,SAAS;AACjD;AACA,SAASC,QAAQ,QAAQ,cAAc;AASvC,SAASC,SAAS,QAAQ,aAAa;AACvC,SAASC,cAAc,EAAEC,oBAAoB,QAAQ,UAAU;AAO/D,OAAO,MAAMC,QAAQ,CAAC;EAGZC,WAAWA,CAACC,OAAqB,EAAE;IACzC,IAAI,CAACA,OAAO,GAAGA,OAAO;EACxB;EAEA,aAAaC,IAAIA,CACfC,MAAqB,EACrBC,UAAuC,EACvCC,WAAoB,EACK;IACzB,IAAIA,WAAW,EAAE;MACfR,cAAc,CAACQ,WAAW,CAAC;IAC7B;;IAEA;IACA,MAAMC,cAAc,GAAGX,QAAQ,CAACY,EAAE,KAAK,SAAS,IAAI,CAACJ,MAAM,CAACK,YAAY,IAAI,CAAC,IAAI,CAAC;IAClF,MAAMC,OAAO,GAAGH,cAAc,GAC1B,CAACH,MAAM,EAAE;MAAE,GAAGA,MAAM;MAAEK,YAAY,EAAE;IAAE,CAAC,CAAC,GACxC,CAAC;MAAE,GAAGL,MAAM;MAAEK,YAAY,EAAE;IAAE,CAAC,CAAC;IAEpC,KAAK,MAAME,MAAM,IAAID,OAAO,EAAE;MAC5B,IAAI;QACF,MAAMR,OAAO,GAAG,MAAMP,SAAS,CAACgB,MAAM,EAAEN,UAAU,CAAC;QACnD,OAAO;UAAEO,EAAE,EAAE,IAAIZ,QAAQ,CAACE,OAAO,CAAC;UAAEW,KAAK,EAAE;QAAK,CAAC;MACnD,CAAC,CAAC,OAAOC,CAAC,EAAE;QACVjB,SAAS,CAACgB,KAAK,CAACC,CAAC,EAAW;UAC1BL,YAAY,EAAEE,MAAM,CAACF,YAAY,IAAI,IAAI;UACzCM,KAAK,EAAEJ,MAAM,CAACI,KAAK,IAAI,IAAI;UAC3BC,KAAK,EAAEL,MAAM,CAACK,KAAK,IAAI;QACzB,CAAC,CAAC;QACF,IAAIN,OAAO,CAACO,OAAO,CAACN,MAAM,CAAC,KAAKD,OAAO,CAACQ,MAAM,GAAG,CAAC,EAAE;UAClD,OAAO;YAAEN,EAAE,EAAE,IAAI;YAAEC,KAAK,EAAEC;UAAW,CAAC;QACxC;MACF;IACF;IACA,OAAO;MAAEF,EAAE,EAAE,IAAI;MAAEC,KAAK,EAAE,IAAIM,KAAK,CAAC,+BAA+B;IAAE,CAAC;EACxE;EAEA,MAAMC,UAAUA,CACdC,QAAsC,EACtCjB,MAAwB,GAAG,CAAC,CAAC,EAC7BkB,QAA8B,EACG;IACjC,OAAO,MAAM,IAAI,CAACpB,OAAO,CAACkB,UAAU,CAAC;MAAEC,QAAQ;MAAE,GAAGjB;IAAO,CAAC,EAAEkB,QAAQ,CAAC;EACzE;EAEA,MAAMC,SAASA,CACbC,IAAY,EACZpB,MAAwB,EACxBqB,IAAY,GAAG,OAAO,EACU;IAChC,IAAIC,MAA6B;IACjC,IAAIC,SAAuB,GAAG,IAAI;IAElC,IAAIF,IAAI,KAAK,QAAQ,EAAE;MACrBC,MAAM,GAAG,MAAM,IAAI,CAACE,sBAAsB,CAACJ,IAAI,CAAC;IAClD,CAAC,MAAM,IAAIC,IAAI,KAAK,OAAO,EAAE;MAC3BC,MAAM,GAAG,MAAM,IAAI,CAACG,qBAAqB,CAACL,IAAI,EAAEpB,MAAM,CAAC;IACzD,CAAC,MAAM,IAAIqB,IAAI,KAAK,YAAY,EAAE;MAChC,IAAI;QACFC,MAAM,GAAG,MAAM,IAAI,CAACG,qBAAqB,CAACL,IAAI,EAAEpB,MAAM,CAAC;MACzD,CAAC,CAAC,OAAOU,CAAC,EAAE;QACVa,SAAS,GAAGb,CAAU;QACtB,IAAI;UACFY,MAAM,GAAG,MAAM,IAAI,CAACE,sBAAsB,CAACJ,IAAI,CAAC;QAClD,CAAC,CAAC,OAAOM,WAAW,EAAE;UACpB,MAAMH,SAAS;QACjB;MACF;IACF,CAAC,MAAM,IAAIF,IAAI,KAAK,aAAa,EAAE;MACjC,IAAI;QACFC,MAAM,GAAG,MAAM,IAAI,CAACE,sBAAsB,CAACJ,IAAI,CAAC;MAClD,CAAC,CAAC,OAAOV,CAAC,EAAE;QACVa,SAAS,GAAGb,CAAU;QACtB,IAAI;UACFY,MAAM,GAAG,MAAM,IAAI,CAACG,qBAAqB,CAACL,IAAI,EAAEpB,MAAM,CAAC;QACzD,CAAC,CAAC,OAAO2B,UAAU,EAAE;UACnB,MAAMJ,SAAS;QACjB;MACF;IACF,CAAC,MAAM;MACL,MAAM,IAAIR,KAAK,CAAC,gBAAgB,GAAGM,IAAI,GAAG,6DAA6D,CAAC;IAC1G;IACA,OAAOC,MAAM;EACf;EAEA,MAAcG,qBAAqBA,CAACL,IAAY,EAAEpB,MAAwB,EAAkC;IAC1G,OAAO,IAAI,CAACF,OAAO,CAACqB,SAAS,CAACC,IAAI,EAAEpB,MAAM,CAAC;EAC7C;EAEA,MAAcwB,sBAAsBA,CAACJ,IAAY,EAAkC;IACjF,MAAMQ,eAAe,GAAG,MAAMjC,oBAAoB,CAACyB,IAAI,CAAC;IACxD,OAAO;MACLD,SAAS,EAAES;IACb,CAAC;EACH;EAEA,MAAMC,MAAMA,CAAA,EAAkB;IAC5B;IACA,OAAO,IAAI,CAAC/B,OAAO,EAAE+B,MAAM,CAAC,CAAC;EAC/B;EAEA,MAAMC,OAAOA,CAAA,EAAkB;IAC7B,OAAO,IAAI,CAAChC,OAAO,CAACgC,OAAO,CAAC,CAAC;EAC/B;AACF","ignoreList":[]}

package/lib/typescript/lm.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"lm.d.ts","sourceRoot":"","sources":["../../src/lm.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,aAAa,EACb,gBAAgB,EAChB,0BAA0B,EAC1B,sBAAsB,EACtB,eAAe,EACf,qBAAqB,EACtB,MAAM,SAAS,CAAA;AAIhB,UAAU,cAAc;IACtB,EAAE,EAAE,QAAQ,GAAG,IAAI,CAAA;IACnB,KAAK,EAAE,KAAK,GAAG,IAAI,CAAA;CACpB;AAED,qBAAa,QAAQ;IACnB,OAAO,CAAC,OAAO,CAAc;IAE7B,OAAO;WAIM,IAAI,CACf,MAAM,EAAE,aAAa,EACrB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,EACvC,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CAAC,cAAc,CAAC;IA4BpB,UAAU,CACd,QAAQ,EAAE,0BAA0B,EAAE,EACtC,MAAM,GAAE,gBAAqB,EAC7B,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,KAAK,IAAI,GAC7B,OAAO,CAAC,sBAAsB,CAAC;IAI5B,SAAS,CACb,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,eAAe,EACxB,IAAI,GAAE,MAAgB,GACrB,OAAO,CAAC,qBAAqB,CAAC;YAoCnB,qBAAqB;YAIrB,sBAAsB;IAO9B,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAKvB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B"}
+{"version":3,"file":"lm.d.ts","sourceRoot":"","sources":["../../src/lm.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,aAAa,EACb,gBAAgB,EAChB,0BAA0B,EAC1B,sBAAsB,EACtB,eAAe,EACf,qBAAqB,EACtB,MAAM,SAAS,CAAA;AAIhB,UAAU,cAAc;IACtB,EAAE,EAAE,QAAQ,GAAG,IAAI,CAAA;IACnB,KAAK,EAAE,KAAK,GAAG,IAAI,CAAA;CACpB;AAED,qBAAa,QAAQ;IACnB,OAAO,CAAC,OAAO,CAAc;IAE7B,OAAO;WAIM,IAAI,CACf,MAAM,EAAE,aAAa,EACrB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,EACvC,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CAAC,cAAc,CAAC;IA6BpB,UAAU,CACd,QAAQ,EAAE,0BAA0B,EAAE,EACtC,MAAM,GAAE,gBAAqB,EAC7B,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,KAAK,IAAI,GAC7B,OAAO,CAAC,sBAAsB,CAAC;IAI5B,SAAS,CACb,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,eAAe,EACxB,IAAI,GAAE,MAAgB,GACrB,OAAO,CAAC,qBAAqB,CAAC;YAoCnB,qBAAqB;YAIrB,sBAAsB;IAO9B,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAKvB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cactus-react-native",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "description": "Run AI models locally on mobile devices",
   "main": "lib/commonjs/index",
   "module": "lib/module/index",

package/src/lm.ts

@@ -1,4 +1,6 @@
 import { initLlama, LlamaContext } from './index'
+// @ts-ignore
+import { Platform } from 'react-native'
 import type {
   ContextParams,
   CompletionParams,
@@ -31,10 +33,11 @@ export class CactusLM {
       setCactusToken(cactusToken);
     }
 
-    const configs = [
-      params,
-      { ...params, n_gpu_layers: 0 } 
-    ];
+    // Avoid two back-to-back loads on devices where GPU off-load is unsupported (Android).
+    const needGpuAttempt = Platform.OS !== 'android' && (params.n_gpu_layers ?? 0) > 0
+    const configs = needGpuAttempt
+      ? [params, { ...params, n_gpu_layers: 0 }]
+      : [{ ...params, n_gpu_layers: 0 }]
 
     for (const config of configs) {
       try {