cui-llama.rn 1.2.6 → 1.3.0

package/lib/module/index.js

@@ -3,8 +3,8 @@ import RNLlama from './NativeRNLlama';
 import { SchemaGrammarConverter, convertJsonSchemaToGrammar } from './grammar';
 import { formatChat } from './chat';
 export { SchemaGrammarConverter, convertJsonSchemaToGrammar };
+const EVENT_ON_INIT_CONTEXT_PROGRESS = '@RNLlama_onInitContextProgress';
 const EVENT_ON_TOKEN = '@RNLlama_onToken';
-const EVENT_ON_MODEL_PROGRESS = '@RNLlama_onModelProgress';
 let EventEmitter;
 if (Platform.OS === 'ios') {
   // @ts-ignore
@@ -45,16 +45,18 @@ export class LlamaContext {
   async saveSession(filepath, options) {
     return RNLlama.saveSession(this.id, filepath, (options === null || options === void 0 ? void 0 : options.tokenSize) || -1);
   }
-  async getFormattedChat(messages) {
+  async getFormattedChat(messages, template) {
     var _this$model;
     const chat = formatChat(messages);
-    return RNLlama.getFormattedChat(this.id, chat, (_this$model = this.model) !== null && _this$model !== void 0 && _this$model.isChatTemplateSupported ? undefined : 'chatml');
+    let tmpl = (_this$model = this.model) !== null && _this$model !== void 0 && _this$model.isChatTemplateSupported ? undefined : 'chatml';
+    if (template) tmpl = template; // Force replace if provided
+    return RNLlama.getFormattedChat(this.id, chat, tmpl);
   }
   async completion(params, callback) {
     let finalPrompt = params.prompt;
     if (params.messages) {
       // messages always win
-      finalPrompt = await this.getFormattedChat(params.messages);
+      finalPrompt = await this.getFormattedChat(params.messages, params.chatTemplate);
     }
     let tokenListener = callback && EventEmitter.addListener(EVENT_ON_TOKEN, evt => {
       const {
@@ -94,8 +96,8 @@ export class LlamaContext {
   detokenize(tokens) {
     return RNLlama.detokenize(this.id, tokens);
   }
-  embedding(text) {
-    return RNLlama.embedding(this.id, text);
+  embedding(text, params) {
+    return RNLlama.embedding(this.id, text, params || {});
   }
   async bench(pp, tg, pl, nr) {
     const result = await RNLlama.bench(this.id, pp, tg, pl, nr);
@@ -120,30 +122,64 @@ export async function getCpuFeatures() {
 export async function setContextLimit(limit) {
   return RNLlama.setContextLimit(limit);
 }
-export async function initLlama(_ref2, progressCallback) {
+let contextIdCounter = 0;
+const contextIdRandom = () => process.env.NODE_ENV === 'test' ? 0 : Math.floor(Math.random() * 100000);
+const modelInfoSkip = [
+// Large fields
+'tokenizer.ggml.tokens', 'tokenizer.ggml.token_type', 'tokenizer.ggml.merges'];
+export async function loadLlamaModelInfo(model) {
+  let path = model;
+  if (path.startsWith('file://')) path = path.slice(7);
+  return RNLlama.modelInfo(path, modelInfoSkip);
+}
+const poolTypeMap = {
+  // -1 is unspecified as undefined
+  none: 0,
+  mean: 1,
+  cls: 2,
+  last: 3,
+  rank: 4
+};
+export async function initLlama(_ref2, onProgress) {
+  var _loraPath, _removeProgressListen2;
   let {
     model,
     is_model_asset: isModelAsset,
+    pooling_type: poolingType,
+    lora,
     ...rest
   } = _ref2;
   let path = model;
   if (path.startsWith('file://')) path = path.slice(7);
-  const modelProgressListener = EventEmitter.addListener(EVENT_ON_MODEL_PROGRESS, event => {
-    if (event.progress && progressCallback) progressCallback(event.progress);
-    if (event.progress === 100) {
-      modelProgressListener.remove();
-    }
-  });
+  let loraPath = lora;
+  if ((_loraPath = loraPath) !== null && _loraPath !== void 0 && _loraPath.startsWith('file://')) loraPath = loraPath.slice(7);
+  const contextId = contextIdCounter + contextIdRandom();
+  contextIdCounter += 1;
+  let removeProgressListener = null;
+  if (onProgress) {
+    removeProgressListener = EventEmitter.addListener(EVENT_ON_INIT_CONTEXT_PROGRESS, evt => {
+      if (evt.contextId !== contextId) return;
+      onProgress(evt.progress);
+    });
+  }
+  const poolType = poolTypeMap[poolingType];
   const {
-    contextId,
     gpu,
     reasonNoGPU,
     model: modelDetails
-  } = await RNLlama.initContext({
+  } = await RNLlama.initContext(contextId, {
     model: path,
     is_model_asset: !!isModelAsset,
+    use_progress_callback: !!onProgress,
+    pooling_type: poolType,
+    lora: loraPath,
     ...rest
+  }).catch(err => {
+    var _removeProgressListen;
+    (_removeProgressListen = removeProgressListener) === null || _removeProgressListen === void 0 ? void 0 : _removeProgressListen.remove();
+    throw err;
   });
+  (_removeProgressListen2 = removeProgressListener) === null || _removeProgressListen2 === void 0 ? void 0 : _removeProgressListen2.remove();
   return new LlamaContext({
     contextId,
     gpu,

package/lib/module/index.js.map

@@ -1 +1 @@
-{"version":3,"names":["NativeEventEmitter","DeviceEventEmitter","Platform","RNLlama","SchemaGrammarConverter","convertJsonSchemaToGrammar","formatChat","EVENT_ON_TOKEN","EVENT_ON_MODEL_PROGRESS","EventEmitter","OS","LlamaContext","gpu","reasonNoGPU","model","constructor","_ref","contextId","id","loadSession","filepath","path","startsWith","slice","saveSession","options","tokenSize","getFormattedChat","messages","_this$model","chat","isChatTemplateSupported","undefined","completion","params","callback","finalPrompt","prompt","tokenListener","addListener","evt","tokenResult","Error","promise","emit_partial_completion","then","completionResult","_tokenListener","remove","catch","err","_tokenListener2","stopCompletion","tokenizeAsync","text","tokenizeSync","detokenize","tokens","embedding","bench","pp","tg","pl","nr","result","modelDesc","modelSize","modelNParams","ppAvg","ppStd","tgAvg","tgStd","JSON","parse","release","releaseContext","getCpuFeatures","setContextLimit","limit","initLlama","_ref2","progressCallback","is_model_asset","isModelAsset","rest","modelProgressListener","event","progress","modelDetails","initContext","releaseAllLlama","releaseAllContexts"],"sourceRoot":"..\\..\\src","sources":["index.ts"],"mappings":"AAAA,SAASA,kBAAkB,EAAEC,kBAAkB,EAAEC,QAAQ,QAAQ,cAAc;AAE/E,OAAOC,OAAO,MAAM,iBAAiB;AAYrC,SAASC,sBAAsB,EAAEC,0BAA0B,QAAQ,WAAW;AAE9E,SAASC,UAAU,QAAQ,QAAQ;AAEnC,SAASF,sBAAsB,EAAEC,0BAA0B;AAE3D,MAAME,cAAc,GAAG,kBAAkB;AAEzC,MAAMC,uBAAuB,GAAG,0BAA0B;AAE1D,IAAIC,YAA2D;AAC/D,IAAIP,QAAQ,CAACQ,EAAE,KAAK,KAAK,EAAE;EACzB;EACAD,YAAY,GAAG,IAAIT,kBAAkB,CAACG,OAAO,CAAC;AAChD;AACA,IAAID,QAAQ,CAACQ,EAAE,KAAK,SAAS,EAAE;EAC7BD,YAAY,GAAGR,kBAAkB;AACnC;AAgCA,OAAO,MAAMU,YAAY,CAAC;EAGxBC,GAAG,GAAY,KAAK;EAEpBC,WAAW,GAAW,EAAE;EAExBC,KAAK,GAED,CAAC,CAAC;EAENC,WAAWA,CAAAC,IAAA,EAA6D;IAAA,IAA5D;MAAEC,SAAS;MAAEL,GAAG;MAAEC,WAAW;MAAEC;IAA0B,CAAC,GAAAE,IAAA;IACpE,IAAI,CAACE,EAAE,GAAGD,SAAS;IACnB,IAAI,CAACL,GAAG,GAAGA,GAAG;IACd,IAAI,CAACC,WAAW,GAAGA,WAAW;IAC9B,IAAI,CAACC,KAAK,GAAGA,KAAK;EACpB;;EAEA;AACF;AACA;EACE,MAAMK,WAAWA,CAACC,QAAgB,EAAoC;IACpE,IAAIC,IAAI,GAAGD,QAAQ;IACnB,IAAIC,IAAI,CAACC,UAAU,CAAC,SAAS,CAAC,EAAED,IAAI,GAAGA,IAAI,CAACE,KAAK,CAAC,CAAC,CAAC;IACpD,OAAOpB,OAAO,CAACgB,WAAW,CAAC,IAAI,CAACD,EAAE,EAAEG,IAAI,CAAC;EAC3C;;EAEA;AACF;AACA;EACE,MAAMG,WAAWA,CACfJ,QAAgB,EAChBK,OAA+B,EACd;IACjB,OAAOtB,OAAO,CAACqB,WAAW,CAAC,IAAI,CAACN,EAAE,EAAEE,QAAQ,EAAE,CAAAK,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEC,SAAS,KAAI,CAAC,CAAC,CAAC;EACzE;EAEA,MAAMC,gBAAgBA,CACpBC,QAAuC,EACtB;IAAA,IAAAC,WAAA;IACjB,MAAMC,IAAI,GAAGxB,UAAU,CAACsB,QAAQ,CAAC;IACjC,OAAOzB,OAAO,CAACwB,gBAAgB,CAC7B,IAAI,CAACT,EAAE,EACPY,IAAI,EACJ,CAAAD,WAAA,OAAI,CAACf,KAAK,cAAAe,WAAA,eAAVA,WAAA,CAAYE,uBAAuB,GAAGC,SAAS,GAAG,QACpD,CAAC;EACH;EAEA,MAAMC,UAAUA,CACdC,MAAwB,EACxBC,QAAoC,EACH;IAEjC,IAAIC,WAAW,GAAGF,MAAM,CAACG,MAAM;IAC/B,IAAIH,MAAM,CAACN,QAAQ,EAAE;MAAE;MACrBQ,WAAW,GAAG,MAAM,IAAI,CAACT,gBAAgB,CAACO,MAAM,CAACN,QAAQ,CAAC;IAC5D;IAEA,IAAIU,aAAkB,GACpBH,QAAQ,IACR1B,YAAY,CAAC8B,WAAW,CAAChC,cAAc,EAAGiC,GAAqB,IAAK;MAClE,MAAM;QAAEvB,SAAS;QAAEwB;MAAY,CAAC,GAAGD,GAAG;MACtC,IAAIvB,SAAS,KAAK,IAAI,CAACC,EAAE,EAAE;MAC3BiB,QAAQ,CAACM,WAAW,CAAC;IACvB,CAAC,CAAC;IAEJ,IAAI,CAACL,WAAW,EAAE,MAAM,IAAIM,KAAK,CAAC,oBAAoB,CAAC;IACvD,MAAMC,OAAO,GAAGxC,OAAO,CAAC8B,UAAU,CAAC,IAAI,CAACf,EAAE,EAAE;MAC1C,GAAGgB,MAAM;MACTG,MAAM,EAAED,WAAW;MACnBQ,uBAAuB,EAAE,CAAC,CAACT;IAC7B,CAAC,CAAC;IACF,OAAOQ,OAAO,CACXE,IAAI,CAAEC,gBAAgB,IAAK;MAAA,IAAAC,cAAA;MAC1B,CAAAA,cAAA,GAAAT,aAAa,cAAAS,cAAA,uBAAbA,cAAA,CAAeC,MAAM,CAAC,CAAC;MACvBV,aAAa,GAAG,IAAI;MACpB,OAAOQ,gBAAgB;IACzB,CAAC,CAAC,CACDG,KAAK,CAAEC,GAAQ,IAAK;MAAA,IAAAC,eAAA;MACnB,CAAAA,eAAA,GAAAb,aAAa,cAAAa,eAAA,uBAAbA,eAAA,CAAeH,MAAM,CAAC,CAAC;MACvBV,aAAa,GAAG,IAAI;MACpB,MAAMY,GAAG;IACX,CAAC,CAAC;EACN;EAEAE,cAAcA,CAAA,EAAkB;IAC9B,OAAOjD,OAAO,CAACiD,cAAc,CAAC,IAAI,CAAClC,EAAE,CAAC;EACxC;EAEAmC,aAAaA,CAACC,IAAY,EAAiC;IACzD,OAAOnD,OAAO,CAACkD,aAAa,CAAC,IAAI,CAACnC,EAAE,EAAEoC,IAAI,CAAC;EAC7C;EAEAC,YAAYA,CAACD,IAAY,EAAwB;IAC/C,OAAOnD,OAAO,CAACoD,YAAY,CAAC,IAAI,CAACrC,EAAE,EAAEoC,IAAI,CAAC;EAC5C;EAEAE,UAAUA,CAACC,MAAgB,EAAmB;IAC5C,OAAOtD,OAAO,CAACqD,UAAU,CAAC,IAAI,CAACtC,EAAE,EAAEuC,MAAM,CAAC;EAC5C;EAEAC,SAASA,CAACJ,IAAY,EAAkC;IACtD,OAAOnD,OAAO,CAACuD,SAAS,CAAC,IAAI,CAACxC,EAAE,EAAEoC,IAAI,CAAC;EACzC;EAEA,MAAMK,KAAKA,CACTC,EAAU,EACVC,EAAU,EACVC,EAAU,EACVC,EAAU,EACY;IACtB,MAAMC,MAAM,GAAG,MAAM7D,OAAO,CAACwD,KAAK,CAAC,IAAI,CAACzC,EAAE,EAAE0C,EAAE,EAAEC,EAAE,EAAEC,EAAE,EAAEC,EAAE,CAAC;IAC3D,MAAM,CAACE,SAAS,EAAEC,SAAS,EAAEC,YAAY,EAAEC,KAAK,EAAEC,KAAK,EAAEC,KAAK,EAAEC,KAAK,CAAC,GACpEC,IAAI,CAACC,KAAK,CAACT,MAAM,CAAC;IACpB,OAAO;MACLC,SAAS;MACTC,SAAS;MACTC,YAAY;MACZC,KAAK;MACLC,KAAK;MACLC,KAAK;MACLC;IACF,CAAC;EACH;EAEA,MAAMG,OAAOA,CAAA,EAAkB;IAC7B,OAAOvE,OAAO,CAACwE,cAAc,CAAC,IAAI,CAACzD,EAAE,CAAC;EACxC;AACF;AAEA,OAAO,eAAe0D,cAAcA,CAAA,EAAgC;EAClE,OAAOzE,OAAO,CAACyE,cAAc,CAAC,CAAC;AACjC;AAEA,OAAO,eAAeC,eAAeA,CAACC,KAAa,EAAiB;EAClE,OAAO3E,OAAO,CAAC0E,eAAe,CAACC,KAAK,CAAC;AACvC;AAEA,OAAO,eAAeC,SAASA,CAAAC,KAAA,EAK7BC,gBAA6C,EACtB;EAAA,IANO;IAC5BnE,KAAK;IACLoE,cAAc,EAAEC,YAAY;IAC5B,GAAGC;EACU,CAAC,GAAAJ,KAAA;EAGhB,IAAI3D,IAAI,GAAGP,KAAK;EAChB,IAAIO,IAAI,CAACC,UAAU,CAAC,SAAS,CAAC,EAAED,IAAI,GAAGA,IAAI,CAACE,KAAK,CAAC,CAAC,CAAC;EAEpD,MAAM8D,qBAAqB,GAAG5E,YAAY,CAAC8B,WAAW,CAAC/B,uBAAuB,EAAG8E,KAAK,IAAK;IACzF,IAAGA,KAAK,CAACC,QAAQ,IAAIN,gBAAgB,EACnCA,gBAAgB,CAACK,KAAK,CAACC,QAAQ,CAAC;IAClC,IAAGD,KAAK,CAACC,QAAQ,KAAK,GAAG,EAAE;MACzBF,qBAAqB,CAACrC,MAAM,CAAC,CAAC;IAChC;EACF,CAAC,CAAC;EAEF,MAAM;IACJ/B,SAAS;IACTL,GAAG;IACHC,WAAW;IACXC,KAAK,EAAE0E;EACT,CAAC,GAAG,MAAMrF,OAAO,CAACsF,WAAW,CAAC;IAC5B3E,KAAK,EAAEO,IAAI;IACX6D,cAAc,EAAE,CAAC,CAACC,YAAY;IAC9B,GAAGC;EACL,CAAC,CAAC;EAEF,OAAO,IAAIzE,YAAY,CAAC;IAAEM,SAAS;IAAEL,GAAG;IAAEC,WAAW;IAAEC,KAAK,EAAE0E;EAAa,CAAC,CAAC;AAC/E;AAEA,OAAO,eAAeE,eAAeA,CAAA,EAAkB;EACrD,OAAOvF,OAAO,CAACwF,kBAAkB,CAAC,CAAC;AACrC"}
+{"version":3,"names":["NativeEventEmitter","DeviceEventEmitter","Platform","RNLlama","SchemaGrammarConverter","convertJsonSchemaToGrammar","formatChat","EVENT_ON_INIT_CONTEXT_PROGRESS","EVENT_ON_TOKEN","EventEmitter","OS","LlamaContext","gpu","reasonNoGPU","model","constructor","_ref","contextId","id","loadSession","filepath","path","startsWith","slice","saveSession","options","tokenSize","getFormattedChat","messages","template","_this$model","chat","tmpl","isChatTemplateSupported","undefined","completion","params","callback","finalPrompt","prompt","chatTemplate","tokenListener","addListener","evt","tokenResult","Error","promise","emit_partial_completion","then","completionResult","_tokenListener","remove","catch","err","_tokenListener2","stopCompletion","tokenizeAsync","text","tokenizeSync","detokenize","tokens","embedding","bench","pp","tg","pl","nr","result","modelDesc","modelSize","modelNParams","ppAvg","ppStd","tgAvg","tgStd","JSON","parse","release","releaseContext","getCpuFeatures","setContextLimit","limit","contextIdCounter","contextIdRandom","process","env","NODE_ENV","Math","floor","random","modelInfoSkip","loadLlamaModelInfo","modelInfo","poolTypeMap","none","mean","cls","last","rank","initLlama","_ref2","onProgress","_loraPath","_removeProgressListen2","is_model_asset","isModelAsset","pooling_type","poolingType","lora","rest","loraPath","removeProgressListener","progress","poolType","modelDetails","initContext","use_progress_callback","_removeProgressListen","releaseAllLlama","releaseAllContexts"],"sourceRoot":"..\\..\\src","sources":["index.ts"],"mappings":"AAAA,SAASA,kBAAkB,EAAEC,kBAAkB,EAAEC,QAAQ,QAAQ,cAAc;AAE/E,OAAOC,OAAO,MAAM,iBAAiB;AAarC,SAASC,sBAAsB,EAAEC,0BAA0B,QAAQ,WAAW;AAE9E,SAASC,UAAU,QAAQ,QAAQ;AAEnC,SAASF,sBAAsB,EAAEC,0BAA0B;AAE3D,MAAME,8BAA8B,GAAG,gCAAgC;AACvE,MAAMC,cAAc,GAAG,kBAAkB;AAEzC,IAAIC,YAA2D;AAC/D,IAAIP,QAAQ,CAACQ,EAAE,KAAK,KAAK,EAAE;EACzB;EACAD,YAAY,GAAG,IAAIT,kBAAkB,CAACG,OAAO,CAAC;AAChD;AACA,IAAID,QAAQ,CAACQ,EAAE,KAAK,SAAS,EAAE;EAC7BD,YAAY,GAAGR,kBAAkB;AACnC;AA0CA,OAAO,MAAMU,YAAY,CAAC;EAGxBC,GAAG,GAAY,KAAK;EAEpBC,WAAW,GAAW,EAAE;EAExBC,KAAK,GAED,CAAC,CAAC;EAENC,WAAWA,CAAAC,IAAA,EAA6D;IAAA,IAA5D;MAAEC,SAAS;MAAEL,GAAG;MAAEC,WAAW;MAAEC;IAA0B,CAAC,GAAAE,IAAA;IACpE,IAAI,CAACE,EAAE,GAAGD,SAAS;IACnB,IAAI,CAACL,GAAG,GAAGA,GAAG;IACd,IAAI,CAACC,WAAW,GAAGA,WAAW;IAC9B,IAAI,CAACC,KAAK,GAAGA,KAAK;EACpB;;EAEA;AACF;AACA;EACE,MAAMK,WAAWA,CAACC,QAAgB,EAAoC;IACpE,IAAIC,IAAI,GAAGD,QAAQ;IACnB,IAAIC,IAAI,CAACC,UAAU,CAAC,SAAS,CAAC,EAAED,IAAI,GAAGA,IAAI,CAACE,KAAK,CAAC,CAAC,CAAC;IACpD,OAAOpB,OAAO,CAACgB,WAAW,CAAC,IAAI,CAACD,EAAE,EAAEG,IAAI,CAAC;EAC3C;;EAEA;AACF;AACA;EACE,MAAMG,WAAWA,CACfJ,QAAgB,EAChBK,OAA+B,EACd;IACjB,OAAOtB,OAAO,CAACqB,WAAW,CAAC,IAAI,CAACN,EAAE,EAAEE,QAAQ,EAAE,CAAAK,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEC,SAAS,KAAI,CAAC,CAAC,CAAC;EACzE;EAEA,MAAMC,gBAAgBA,CACpBC,QAAuC,EACvCC,QAAiB,EACA;IAAA,IAAAC,WAAA;IACjB,MAAMC,IAAI,GAAGzB,UAAU,CAACsB,QAAQ,CAAC;IACjC,IAAII,IAAI,GAAG,CAAAF,WAAA,OAAI,CAAChB,KAAK,cAAAgB,WAAA,eAAVA,WAAA,CAAYG,uBAAuB,GAAGC,SAAS,GAAG,QAAQ;IACrE,IAAIL,QAAQ,EAAEG,IAAI,GAAGH,QAAQ,EAAC;IAC9B,OAAO1B,OAAO,CAACwB,gBAAgB,CAAC,IAAI,CAACT,EAAE,EAAEa,IAAI,EAAEC,IAAI,CAAC;EACtD;EAEA,MAAMG,UAAUA,CACdC,MAAwB,EACxBC,QAAoC,EACH;IACjC,IAAIC,WAAW,GAAGF,MAAM,CAACG,MAAM;IAC/B,IAAIH,MAAM,CAACR,QAAQ,EAAE;MACnB;MACAU,WAAW,GAAG,MAAM,IAAI,CAACX,gBAAgB,CAACS,MAAM,CAACR,QAAQ,EAAEQ,MAAM,CAACI,YAAY,CAAC;IACjF;IAEA,IAAIC,aAAkB,GACpBJ,QAAQ,IACR5B,YAAY,CAACiC,WAAW,CAAClC,cAAc,EAAGmC,GAAqB,IAAK;MAClE,MAAM;QAAE1B,SAAS;QAAE2B;MAAY,CAAC,GAAGD,GAAG;MACtC,IAAI1B,SAAS,KAAK,IAAI,CAACC,EAAE,EAAE;MAC3BmB,QAAQ,CAACO,WAAW,CAAC;IACvB,CAAC,CAAC;IAEJ,IAAI,CAACN,WAAW,EAAE,MAAM,IAAIO,KAAK,CAAC,oBAAoB,CAAC;IACvD,MAAMC,OAAO,GAAG3C,OAAO,CAACgC,UAAU,CAAC,IAAI,CAACjB,EAAE,EAAE;MAC1C,GAAGkB,MAAM;MACTG,MAAM,EAAED,WAAW;MACnBS,uBAAuB,EAAE,CAAC,CAACV;IAC7B,CAAC,CAAC;IACF,OAAOS,OAAO,CACXE,IAAI,CAAEC,gBAAgB,IAAK;MAAA,IAAAC,cAAA;MAC1B,CAAAA,cAAA,GAAAT,aAAa,cAAAS,cAAA,uBAAbA,cAAA,CAAeC,MAAM,CAAC,CAAC;MACvBV,aAAa,GAAG,IAAI;MACpB,OAAOQ,gBAAgB;IACzB,CAAC,CAAC,CACDG,KAAK,CAAEC,GAAQ,IAAK;MAAA,IAAAC,eAAA;MACnB,CAAAA,eAAA,GAAAb,aAAa,cAAAa,eAAA,uBAAbA,eAAA,CAAeH,MAAM,CAAC,CAAC;MACvBV,aAAa,GAAG,IAAI;MACpB,MAAMY,GAAG;IACX,CAAC,CAAC;EACN;EAEAE,cAAcA,CAAA,EAAkB;IAC9B,OAAOpD,OAAO,CAACoD,cAAc,CAAC,IAAI,CAACrC,EAAE,CAAC;EACxC;EAEAsC,aAAaA,CAACC,IAAY,EAAiC;IACzD,OAAOtD,OAAO,CAACqD,aAAa,CAAC,IAAI,CAACtC,EAAE,EAAEuC,IAAI,CAAC;EAC7C;EAEAC,YAAYA,CAACD,IAAY,EAAwB;IAC/C,OAAOtD,OAAO,CAACuD,YAAY,CAAC,IAAI,CAACxC,EAAE,EAAEuC,IAAI,CAAC;EAC5C;EAEAE,UAAUA,CAACC,MAAgB,EAAmB;IAC5C,OAAOzD,OAAO,CAACwD,UAAU,CAAC,IAAI,CAACzC,EAAE,EAAE0C,MAAM,CAAC;EAC5C;EAEAC,SAASA,CACPJ,IAAY,EACZrB,MAAwB,EACQ;IAChC,OAAOjC,OAAO,CAAC0D,SAAS,CAAC,IAAI,CAAC3C,EAAE,EAAEuC,IAAI,EAAErB,MAAM,IAAI,CAAC,CAAC,CAAC;EACvD;EAEA,MAAM0B,KAAKA,CACTC,EAAU,EACVC,EAAU,EACVC,EAAU,EACVC,EAAU,EACY;IACtB,MAAMC,MAAM,GAAG,MAAMhE,OAAO,CAAC2D,KAAK,CAAC,IAAI,CAAC5C,EAAE,EAAE6C,EAAE,EAAEC,EAAE,EAAEC,EAAE,EAAEC,EAAE,CAAC;IAC3D,MAAM,CAACE,SAAS,EAAEC,SAAS,EAAEC,YAAY,EAAEC,KAAK,EAAEC,KAAK,EAAEC,KAAK,EAAEC,KAAK,CAAC,GACpEC,IAAI,CAACC,KAAK,CAACT,MAAM,CAAC;IACpB,OAAO;MACLC,SAAS;MACTC,SAAS;MACTC,YAAY;MACZC,KAAK;MACLC,KAAK;MACLC,KAAK;MACLC;IACF,CAAC;EACH;EAEA,MAAMG,OAAOA,CAAA,EAAkB;IAC7B,OAAO1E,OAAO,CAAC2E,cAAc,CAAC,IAAI,CAAC5D,EAAE,CAAC;EACxC;AACF;AAEA,OAAO,eAAe6D,cAAcA,CAAA,EAAgC;EAClE,OAAO5E,OAAO,CAAC4E,cAAc,CAAC,CAAC;AACjC;AAEA,OAAO,eAAeC,eAAeA,CAACC,KAAa,EAAiB;EAClE,OAAO9E,OAAO,CAAC6E,eAAe,CAACC,KAAK,CAAC;AACvC;AAEA,IAAIC,gBAAgB,GAAG,CAAC;AACxB,MAAMC,eAAe,GAAGA,CAAA,KACtBC,OAAO,CAACC,GAAG,CAACC,QAAQ,KAAK,MAAM,GAAG,CAAC,GAAGC,IAAI,CAACC,KAAK,CAACD,IAAI,CAACE,MAAM,CAAC,CAAC,GAAG,MAAM,CAAC;AAE1E,MAAMC,aAAa,GAAG;AACpB;AACA,uBAAuB,EACvB,2BAA2B,EAC3B,uBAAuB,CACxB;AACD,OAAO,eAAeC,kBAAkBA,CAAC7E,KAAa,EAAmB;EACvE,IAAIO,IAAI,GAAGP,KAAK;EAChB,IAAIO,IAAI,CAACC,UAAU,CAAC,SAAS,CAAC,EAAED,IAAI,GAAGA,IAAI,CAACE,KAAK,CAAC,CAAC,CAAC;EACpD,OAAOpB,OAAO,CAACyF,SAAS,CAACvE,IAAI,EAAEqE,aAAa,CAAC;AAC/C;AAEA,MAAMG,WAAW,GAAG;EAClB;EACAC,IAAI,EAAE,CAAC;EACPC,IAAI,EAAE,CAAC;EACPC,GAAG,EAAE,CAAC;EACNC,IAAI,EAAE,CAAC;EACPC,IAAI,EAAE;AACR,CAAC;AAED,OAAO,eAAeC,SAASA,CAAAC,KAAA,EAQ7BC,UAAuC,EAChB;EAAA,IAAAC,SAAA,EAAAC,sBAAA;EAAA,IARvB;IACEzF,KAAK;IACL0F,cAAc,EAAEC,YAAY;IAC5BC,YAAY,EAAEC,WAAW;IACzBC,IAAI;IACJ,GAAGC;EACU,CAAC,GAAAT,KAAA;EAGhB,IAAI/E,IAAI,GAAGP,KAAK;EAChB,IAAIO,IAAI,CAACC,UAAU,CAAC,SAAS,CAAC,EAAED,IAAI,GAAGA,IAAI,CAACE,KAAK,CAAC,CAAC,CAAC;EAEpD,IAAIuF,QAAQ,GAAGF,IAAI;EACnB,KAAAN,SAAA,GAAIQ,QAAQ,cAAAR,SAAA,eAARA,SAAA,CAAUhF,UAAU,CAAC,SAAS,CAAC,EAAEwF,QAAQ,GAAGA,QAAQ,CAACvF,KAAK,CAAC,CAAC,CAAC;EAEjE,MAAMN,SAAS,GAAGiE,gBAAgB,GAAGC,eAAe,CAAC,CAAC;EACtDD,gBAAgB,IAAI,CAAC;EAErB,IAAI6B,sBAA2B,GAAG,IAAI;EACtC,IAAIV,UAAU,EAAE;IACdU,sBAAsB,GAAGtG,YAAY,CAACiC,WAAW,CAC/CnC,8BAA8B,EAC7BoC,GAA4C,IAAK;MAChD,IAAIA,GAAG,CAAC1B,SAAS,KAAKA,SAAS,EAAE;MACjCoF,UAAU,CAAC1D,GAAG,CAACqE,QAAQ,CAAC;IAC1B,CACF,CAAC;EACH;EAEA,MAAMC,QAAQ,GAAGpB,WAAW,CAACc,WAAW,CAA6B;EACrE,MAAM;IACJ/F,GAAG;IACHC,WAAW;IACXC,KAAK,EAAEoG;EACT,CAAC,GAAG,MAAM/G,OAAO,CAACgH,WAAW,CAAClG,SAAS,EAAE;IACvCH,KAAK,EAAEO,IAAI;IACXmF,cAAc,EAAE,CAAC,CAACC,YAAY;IAC9BW,qBAAqB,EAAE,CAAC,CAACf,UAAU;IACnCK,YAAY,EAAEO,QAAQ;IACtBL,IAAI,EAAEE,QAAQ;IACd,GAAGD;EACL,CAAC,CAAC,CAACzD,KAAK,CAAEC,GAAQ,IAAK;IAAA,IAAAgE,qBAAA;IACrB,CAAAA,qBAAA,GAAAN,sBAAsB,cAAAM,qBAAA,uBAAtBA,qBAAA,CAAwBlE,MAAM,CAAC,CAAC;IAChC,MAAME,GAAG;EACX,CAAC,CAAC;EACF,CAAAkD,sBAAA,GAAAQ,sBAAsB,cAAAR,sBAAA,uBAAtBA,sBAAA,CAAwBpD,MAAM,CAAC,CAAC;EAChC,OAAO,IAAIxC,YAAY,CAAC;IAAEM,SAAS;IAAEL,GAAG;IAAEC,WAAW;IAAEC,KAAK,EAAEoG;EAAa,CAAC,CAAC;AAC/E;AAEA,OAAO,eAAeI,eAAeA,CAAA,EAAkB;EACrD,OAAOnH,OAAO,CAACoH,kBAAkB,CAAC,CAAC;AACrC"}

package/lib/typescript/NativeRNLlama.d.ts

@@ -1,12 +1,27 @@
 import type { TurboModule } from 'react-native';
+export type NativeEmbeddingParams = {
+    embd_normalize?: number;
+};
 export type NativeContextParams = {
     model: string;
     is_model_asset?: boolean;
-    embedding?: boolean;
+    use_progress_callback?: boolean;
     n_ctx?: number;
     n_batch?: number;
     n_threads?: number;
     n_gpu_layers?: number;
+    /**
+     * Enable flash attention, only recommended in GPU device (Experimental in llama.cpp)
+     */
+    flash_attn?: boolean;
+    /**
+     * KV cache data type for the K (Experimental in llama.cpp)
+     */
+    cache_type_k?: string;
+    /**
+     * KV cache data type for the V (Experimental in llama.cpp)
+     */
+    cache_type_v?: string;
     use_mlock?: boolean;
     use_mmap?: boolean;
     vocab_only?: boolean;
@@ -14,6 +29,9 @@ export type NativeContextParams = {
     lora_scaled?: number;
     rope_freq_base?: number;
     rope_freq_scale?: number;
+    pooling_type?: number;
+    embedding?: boolean;
+    embd_normalize?: number;
 };
 export type NativeCompletionParams = {
     prompt: string;
@@ -25,8 +43,8 @@ export type NativeCompletionParams = {
     top_k?: number;
     top_p?: number;
     min_p?: number;
-    xtc_t?: number;
-    xtc_p?: number;
+    xtc_threshold?: number;
+    xtc_probability?: number;
     typical_p?: number;
     temperature?: number;
     penalty_last_n?: number;
@@ -38,6 +56,11 @@ export type NativeCompletionParams = {
     mirostat_eta?: number;
     penalize_nl?: boolean;
     seed?: number;
+    dry_multiplier?: number;
+    dry_base?: number;
+    dry_allowed_length?: number;
+    dry_penalty_last_n?: number;
+    dry_sequence_breakers?: Array<string>;
     ignore_eos?: boolean;
     logit_bias?: Array<Array<number>>;
     emit_partial_completion: boolean;
@@ -100,7 +123,8 @@ export type NativeCPUFeatures = {
 };
 export interface Spec extends TurboModule {
     setContextLimit(limit: number): Promise<void>;
-    initContext(params: NativeContextParams): Promise<NativeLlamaContext>;
+    modelInfo(path: string, skip?: string[]): Promise<Object>;
+    initContext(contextId: number, params: NativeContextParams): Promise<NativeLlamaContext>;
     loadSession(contextId: number, filepath: string): Promise<NativeSessionLoadResult>;
     saveSession(contextId: number, filepath: string, size: number): Promise<number>;
     completion(contextId: number, params: NativeCompletionParams): Promise<NativeCompletionResult>;
@@ -110,7 +134,7 @@ export interface Spec extends TurboModule {
     getCpuFeatures(): Promise<NativeCPUFeatures>;
     getFormattedChat(contextId: number, messages: NativeLlamaChatMessage[], chatTemplate?: string): Promise<string>;
     detokenize(contextId: number, tokens: number[]): Promise<string>;
-    embedding(contextId: number, text: string): Promise<NativeEmbeddingResult>;
+    embedding(contextId: number, text: string, params: NativeEmbeddingParams): Promise<NativeEmbeddingResult>;
     bench(contextId: number, pp: number, tg: number, pl: number, nr: number): Promise<string>;
     releaseContext(contextId: number): Promise<void>;
     releaseAllContexts(): Promise<void>;

package/lib/typescript/NativeRNLlama.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"NativeRNLlama.d.ts","sourceRoot":"","sources":["../../src/NativeRNLlama.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAA;AAG/C,MAAM,MAAM,mBAAmB,GAAG;IAChC,KAAK,EAAE,MAAM,CAAA;IACb,cAAc,CAAC,EAAE,OAAO,CAAA;IAExB,SAAS,CAAC,EAAE,OAAO,CAAA;IAEnB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,OAAO,CAAC,EAAE,MAAM,CAAA;IAEhB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,YAAY,CAAC,EAAE,MAAM,CAAA;IAErB,SAAS,CAAC,EAAE,OAAO,CAAA;IACnB,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,UAAU,CAAC,EAAE,OAAO,CAAA;IAEpB,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,WAAW,CAAC,EAAE,MAAM,CAAA;IAEpB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,eAAe,CAAC,EAAE,MAAM,CAAA;CACzB,CAAA;AAED,MAAM,MAAM,sBAAsB,GAAG;IACnC,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,IAAI,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;IAEpB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,WAAW,CAAC,EAAE,OAAO,CAAA;IACrB,IAAI,CAAC,EAAE,MAAM,CAAA;IAEb,UAAU,CAAC,EAAE,OAAO,CAAA;IACpB,UAAU,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;IAEjC,uBAAuB,EAAE,OAAO,CAAA;CACjC,CAAA;AAED,MAAM,MAAM,6BAA6B,GAAG;IAC1C,OAAO,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,MAAM,CAAA;CACb,CAAA;AAED,MAAM,MAAM,yBAAyB,GAAG;IACtC,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,KAAK,CAAC,6BAA6B,CAAC,CAAA;CAC5C,CAAA;AAED,MAAM,MAAM,6BAA6B,GAAG;IAC1C,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,mBAAmB,EAAE,MAAM,CAAA;IAC3B,iBAAiB,EAAE,MAAM,CAAA;IACzB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,sBAAsB,EAAE,MAAM,CAAA;IAC9B,oBAAoB,EAAE,MAAM,CAAA;CAC7B,CAAA;AAED,MAAM,MAAM,sBAAsB,GAAG;IACnC,IAAI,EAAE,MAAM,CAAA;IAEZ,gBAAgB,EAAE,MAAM,CAAA;IACxB,gBAAgB,EAAE,MAAM,CAAA;IACxB,SAAS,EAAE,OAAO,CAAA;IAClB,WAAW,EAAE,OAAO,CAAA;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,aAAa,EAAE,MAAM,CAAA;IACrB,aAAa,EAAE,MAAM,CAAA;IACrB,aAAa,EAAE,MAAM,CAAA;IACrB,OAAO,EAAE,6BAA6B,CAAA;IAEtC,wBAAwB,CAAC,EAAE,KAAK,CAAC,yBAAyB,CAAC,CAAA;CAC5D,CAAA;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;CACtB,CAAA;AAED,MAAM,MAAM,qBAAqB,GAAG;IAClC,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;CACzB,CAAA;AAED,MAAM,MAAM,kBAAkB,GAAG;IAC/B,SAAS,EAAE,MAAM,CAAA;IACjB,GAAG,EAAE,OAAO,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;IACnB,KAAK,EAAE,MAAM,CAAA;CACd,CAAA;AAED,MAAM,MAAM,uBAAuB,GAAG;IACpC,aAAa,EAAE,MAAM,CAAA;IACrB,MAAM,EAAE,MAAM,CAAA;CACf,CAAA;AAED,MAAM,MAAM,sBAAsB,GAAG;IACnC,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;CAChB,CAAA;AAED,MAAM,MAAM,iBAAiB,GAAG;IAC9B,KAAK,EAAE,OAAO,CAAA;IACd,IAAI,EAAE,OAAO,CAAA;IACb,OAAO,EAAE,OAAO,CAAA;CACjB,CAAA;AAED,MAAM,WAAW,IAAK,SAAQ,WAAW;IACvC,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC7C,WAAW,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAA;IAErE,WAAW,CACT,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,uBAAuB,CAAC,CAAA;IACnC,WAAW,CACT,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,MAAM,CAAC,CAAA;IAClB,UAAU,CACR,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,sBAAsB,GAC7B,OAAO,CAAC,sBAAsB,CAAC,CAAA;IAClC,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAChD,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAA;IAC7E,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,oBAAoB,CAAA;IACnE,cAAc,IAAK,OAAO,CAAC,iBAAiB,CAAC,CAAA;IAC7C,gBAAgB,CACd,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,sBAAsB,EAAE,EAClC,YAAY,CAAC,EAAE,MAAM,GACpB,OAAO,CAAC,MAAM,CAAC,CAAA;IAClB,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;IAChE,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAA;IAC1E,KAAK,CACH,SAAS,EAAE,MAAM,EACjB,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,GACT,OAAO,CAAC,MAAM,CAAC,CAAA;IAElB,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAEhD,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACpC;;AAED,wBAA+D"}
+{"version":3,"file":"NativeRNLlama.d.ts","sourceRoot":"","sources":["../../src/NativeRNLlama.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAA;AAG/C,MAAM,MAAM,qBAAqB,GAAG;IAClC,cAAc,CAAC,EAAE,MAAM,CAAA;CACxB,CAAA;AAED,MAAM,MAAM,mBAAmB,GAAG;IAChC,KAAK,EAAE,MAAM,CAAA;IACb,cAAc,CAAC,EAAE,OAAO,CAAA;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAA;IAE/B,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,OAAO,CAAC,EAAE,MAAM,CAAA;IAEhB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,YAAY,CAAC,EAAE,MAAM,CAAA;IAErB;;OAEG;IACH,UAAU,CAAC,EAAE,OAAO,CAAA;IAEpB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;IAErB,SAAS,CAAC,EAAE,OAAO,CAAA;IACnB,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,UAAU,CAAC,EAAE,OAAO,CAAA;IAEpB,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,WAAW,CAAC,EAAE,MAAM,CAAA;IAEpB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,eAAe,CAAC,EAAE,MAAM,CAAA;IAExB,YAAY,CAAC,EAAE,MAAM,CAAA;IAGrB,SAAS,CAAC,EAAE,OAAO,CAAA;IACnB,cAAc,CAAC,EAAE,MAAM,CAAA;CACxB,CAAA;AAED,MAAM,MAAM,sBAAsB,GAAG;IACnC,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,IAAI,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;IAEpB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,WAAW,CAAC,EAAE,OAAO,CAAA;IACrB,IAAI,CAAC,EAAE,MAAM,CAAA;IAEb,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,qBAAqB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;IAErC,UAAU,CAAC,EAAE,OAAO,CAAA;IACpB,UAAU,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;IAEjC,uBAAuB,EAAE,OAAO,CAAA;CACjC,CAAA;AAED,MAAM,MAAM,6BAA6B,GAAG;IAC1C,OAAO,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,MAAM,CAAA;CACb,CAAA;AAED,MAAM,MAAM,yBAAyB,GAAG;IACtC,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,KAAK,CAAC,6BAA6B,CAAC,CAAA;CAC5C,CAAA;AAED,MAAM,MAAM,6BAA6B,GAAG;IAC1C,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,mBAAmB,EAAE,MAAM,CAAA;IAC3B,iBAAiB,EAAE,MAAM,CAAA;IACzB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,sBAAsB,EAAE,MAAM,CAAA;IAC9B,oBAAoB,EAAE,MAAM,CAAA;CAC7B,CAAA;AAED,MAAM,MAAM,sBAAsB,GAAG;IACnC,IAAI,EAAE,MAAM,CAAA;IAEZ,gBAAgB,EAAE,MAAM,CAAA;IACxB,gBAAgB,EAAE,MAAM,CAAA;IACxB,SAAS,EAAE,OAAO,CAAA;IAClB,WAAW,EAAE,OAAO,CAAA;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,aAAa,EAAE,MAAM,CAAA;IACrB,aAAa,EAAE,MAAM,CAAA;IACrB,aAAa,EAAE,MAAM,CAAA;IACrB,OAAO,EAAE,6BAA6B,CAAA;IAEtC,wBAAwB,CAAC,EAAE,KAAK,CAAC,yBAAyB,CAAC,CAAA;CAC5D,CAAA;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;CACtB,CAAA;AAED,MAAM,MAAM,qBAAqB,GAAG;IAClC,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;CACzB,CAAA;AAED,MAAM,MAAM,kBAAkB,GAAG;IAC/B,SAAS,EAAE,MAAM,CAAA;IACjB,GAAG,EAAE,OAAO,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;IACnB,KAAK,EAAE,MAAM,CAAA;CACd,CAAA;AAED,MAAM,MAAM,uBAAuB,GAAG;IACpC,aAAa,EAAE,MAAM,CAAA;IACrB,MAAM,EAAE,MAAM,CAAA;CACf,CAAA;AAED,MAAM,MAAM,sBAAsB,GAAG;IACnC,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;CAChB,CAAA;AAED,MAAM,MAAM,iBAAiB,GAAG;IAC9B,KAAK,EAAE,OAAO,CAAA;IACd,IAAI,EAAE,OAAO,CAAA;IACb,OAAO,EAAE,OAAO,CAAA;CACjB,CAAA;AAED,MAAM,WAAW,IAAK,SAAQ,WAAW;IACvC,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAE7C,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;IACzD,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAA;IAExF,WAAW,CACT,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,uBAAuB,CAAC,CAAA;IACnC,WAAW,CACT,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,MAAM,CAAC,CAAA;IAClB,UAAU,CACR,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,sBAAsB,GAC7B,OAAO,CAAC,sBAAsB,CAAC,CAAA;IAClC,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAChD,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAA;IAC7E,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,oBAAoB,CAAA;IACnE,cAAc,IAAK,OAAO,CAAC,iBAAiB,CAAC,CAAA;IAC7C,gBAAgB,CACd,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,sBAAsB,EAAE,EAClC,YAAY,CAAC,EAAE,MAAM,GACpB,OAAO,CAAC,MAAM,CAAC,CAAA;IAClB,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;IAChE,SAAS,CACP,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,qBAAqB,GAC5B,OAAO,CAAC,qBAAqB,CAAC,CAAA;IACjC,KAAK,CACH,SAAS,EAAE,MAAM,EACjB,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,GACT,OAAO,CAAC,MAAM,CAAC,CAAA;IAElB,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAEhD,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACpC;;AAED,wBAA+D"}

package/lib/typescript/index.d.ts

@@ -1,4 +1,4 @@
-import type { NativeContextParams, NativeLlamaContext, NativeCompletionParams, NativeCompletionTokenProb, NativeCompletionResult, NativeTokenizeResult, NativeEmbeddingResult, NativeSessionLoadResult, NativeCPUFeatures } from './NativeRNLlama';
+import type { NativeContextParams, NativeLlamaContext, NativeCompletionParams, NativeCompletionTokenProb, NativeCompletionResult, NativeTokenizeResult, NativeEmbeddingResult, NativeSessionLoadResult, NativeCPUFeatures, NativeEmbeddingParams } from './NativeRNLlama';
 import { SchemaGrammarConverter, convertJsonSchemaToGrammar } from './grammar';
 import type { RNLlamaOAICompatibleMessage } from './chat';
 export { SchemaGrammarConverter, convertJsonSchemaToGrammar };
@@ -6,10 +6,16 @@ export type TokenData = {
     token: string;
     completion_probabilities?: Array<NativeCompletionTokenProb>;
 };
-export type ContextParams = NativeContextParams;
+export type ContextParams = Omit<NativeContextParams, 'cache_type_k' | 'cache_type_v' | 'pooling_type'> & {
+    cache_type_k?: 'f16' | 'f32' | 'q8_0' | 'q4_0' | 'q4_1' | 'iq4_nl' | 'q5_0' | 'q5_1';
+    cache_type_v?: 'f16' | 'f32' | 'q8_0' | 'q4_0' | 'q4_1' | 'iq4_nl' | 'q5_0' | 'q5_1';
+    pooling_type?: 'none' | 'mean' | 'cls' | 'last' | 'rank';
+};
+export type EmbeddingParams = NativeEmbeddingParams;
 export type CompletionParams = Omit<NativeCompletionParams, 'emit_partial_completion' | 'prompt'> & {
     prompt?: string;
     messages?: RNLlamaOAICompatibleMessage[];
+    chatTemplate?: string;
 };
 export type BenchResult = {
     modelDesc: string;
@@ -38,18 +44,19 @@ export declare class LlamaContext {
     saveSession(filepath: string, options?: {
         tokenSize: number;
     }): Promise<number>;
-    getFormattedChat(messages: RNLlamaOAICompatibleMessage[]): Promise<string>;
+    getFormattedChat(messages: RNLlamaOAICompatibleMessage[], template?: string): Promise<string>;
     completion(params: CompletionParams, callback?: (data: TokenData) => void): Promise<NativeCompletionResult>;
     stopCompletion(): Promise<void>;
     tokenizeAsync(text: string): Promise<NativeTokenizeResult>;
     tokenizeSync(text: string): NativeTokenizeResult;
     detokenize(tokens: number[]): Promise<string>;
-    embedding(text: string): Promise<NativeEmbeddingResult>;
+    embedding(text: string, params?: EmbeddingParams): Promise<NativeEmbeddingResult>;
     bench(pp: number, tg: number, pl: number, nr: number): Promise<BenchResult>;
     release(): Promise<void>;
 }
 export declare function getCpuFeatures(): Promise<NativeCPUFeatures>;
 export declare function setContextLimit(limit: number): Promise<void>;
-export declare function initLlama({ model, is_model_asset: isModelAsset, ...rest }: ContextParams, progressCallback?: (progress: number) => void): Promise<LlamaContext>;
+export declare function loadLlamaModelInfo(model: string): Promise<Object>;
+export declare function initLlama({ model, is_model_asset: isModelAsset, pooling_type: poolingType, lora, ...rest }: ContextParams, onProgress?: (progress: number) => void): Promise<LlamaContext>;
 export declare function releaseAllLlama(): Promise<void>;
 //# sourceMappingURL=index.d.ts.map

package/lib/typescript/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,mBAAmB,EACnB,kBAAkB,EAClB,sBAAsB,EACtB,yBAAyB,EACzB,sBAAsB,EACtB,oBAAoB,EACpB,qBAAqB,EACrB,uBAAuB,EACvB,iBAAiB,EAClB,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAAE,sBAAsB,EAAE,0BAA0B,EAAE,MAAM,WAAW,CAAA;AAC9E,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,QAAQ,CAAA;AAGzD,OAAO,EAAE,sBAAsB,EAAE,0BAA0B,EAAE,CAAA;AAe7D,MAAM,MAAM,SAAS,GAAG;IACtB,KAAK,EAAE,MAAM,CAAA;IACb,wBAAwB,CAAC,EAAE,KAAK,CAAC,yBAAyB,CAAC,CAAA;CAC5D,CAAA;AAOD,MAAM,MAAM,aAAa,GAAG,mBAAmB,CAAA;AAE/C,MAAM,MAAM,gBAAgB,GAAG,IAAI,CACjC,sBAAsB,EACtB,yBAAyB,GAAG,QAAQ,CACrC,GAAG;IACF,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,2BAA2B,EAAE,CAAA;CACzC,CAAA;AAED,MAAM,MAAM,WAAW,GAAG;IACxB,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,YAAY,EAAE,MAAM,CAAA;IACpB,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,MAAM,CAAA;CACd,CAAA;AAED,qBAAa,YAAY;IACvB,EAAE,EAAE,MAAM,CAAA;IAEV,GAAG,EAAE,OAAO,CAAQ;IAEpB,WAAW,EAAE,MAAM,CAAK;IAExB,KAAK,EAAE;QACL,uBAAuB,CAAC,EAAE,OAAO,CAAA;KAClC,CAAK;gBAEM,EAAE,SAAS,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,EAAE,kBAAkB;IAOtE;;OAEG;IACG,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,uBAAuB,CAAC;IAMrE;;OAEG;IACG,WAAW,CACf,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE;QAAE,SAAS,EAAE,MAAM,CAAA;KAAE,GAC9B,OAAO,CAAC,MAAM,CAAC;IAIZ,gBAAgB,CACpB,QAAQ,EAAE,2BAA2B,EAAE,GACtC,OAAO,CAAC,MAAM,CAAC;IASZ,UAAU,CACd,MAAM,EAAE,gBAAgB,EACxB,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,SAAS,KAAK,IAAI,GACnC,OAAO,CAAC,sBAAsB,CAAC;IAkClC,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAI/B,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAI1D,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,oBAAoB;IAIhD,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAI7C,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAIjD,KAAK,CACT,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,GACT,OAAO,CAAC,WAAW,CAAC;IAejB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B;AAED,wBAAsB,cAAc,IAAK,OAAO,CAAC,iBAAiB,CAAC,CAElE;AAED,wBAAsB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAElE;AAED,wBAAsB,SAAS,CAAC,EAC5B,KAAK,EACL,cAAc,EAAE,YAAY,EAC5B,GAAG,IAAI,EACR,EAAE,aAAa,EAChB,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,GAC5C,OAAO,CAAC,YAAY,CAAC,CAwBvB;AAED,wBAAsB,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC,CAErD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,mBAAmB,EACnB,kBAAkB,EAClB,sBAAsB,EACtB,yBAAyB,EACzB,sBAAsB,EACtB,oBAAoB,EACpB,qBAAqB,EACrB,uBAAuB,EACvB,iBAAiB,EACjB,qBAAqB,EACtB,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAAE,sBAAsB,EAAE,0BAA0B,EAAE,MAAM,WAAW,CAAA;AAC9E,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,QAAQ,CAAA;AAGzD,OAAO,EAAE,sBAAsB,EAAE,0BAA0B,EAAE,CAAA;AAc7D,MAAM,MAAM,SAAS,GAAG;IACtB,KAAK,EAAE,MAAM,CAAA;IACb,wBAAwB,CAAC,EAAE,KAAK,CAAC,yBAAyB,CAAC,CAAA;CAC5D,CAAA;AAOD,MAAM,MAAM,aAAa,GAAG,IAAI,CAC9B,mBAAmB,EACnB,cAAc,GAAG,cAAc,GAAI,cAAc,CAClD,GAAG;IACF,YAAY,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,CAAA;IACpF,YAAY,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,CAAA;IACpF,YAAY,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,KAAK,GAAG,MAAM,GAAG,MAAM,CAAA;CACzD,CAAA;AAED,MAAM,MAAM,eAAe,GAAG,qBAAqB,CAAA;AAEnD,MAAM,MAAM,gBAAgB,GAAG,IAAI,CACjC,sBAAsB,EACtB,yBAAyB,GAAG,QAAQ,CACrC,GAAG;IACF,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,2BAA2B,EAAE,CAAA;IACxC,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB,CAAA;AAED,MAAM,MAAM,WAAW,GAAG;IACxB,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,YAAY,EAAE,MAAM,CAAA;IACpB,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,MAAM,CAAA;CACd,CAAA;AAED,qBAAa,YAAY;IACvB,EAAE,EAAE,MAAM,CAAA;IAEV,GAAG,EAAE,OAAO,CAAQ;IAEpB,WAAW,EAAE,MAAM,CAAK;IAExB,KAAK,EAAE;QACL,uBAAuB,CAAC,EAAE,OAAO,CAAA;KAClC,CAAK;gBAEM,EAAE,SAAS,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,EAAE,kBAAkB;IAOtE;;OAEG;IACG,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,uBAAuB,CAAC;IAMrE;;OAEG;IACG,WAAW,CACf,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE;QAAE,SAAS,EAAE,MAAM,CAAA;KAAE,GAC9B,OAAO,CAAC,MAAM,CAAC;IAIZ,gBAAgB,CACpB,QAAQ,EAAE,2BAA2B,EAAE,EACvC,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,MAAM,CAAC;IAOZ,UAAU,CACd,MAAM,EAAE,gBAAgB,EACxB,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,SAAS,KAAK,IAAI,GACnC,OAAO,CAAC,sBAAsB,CAAC;IAkClC,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAI/B,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAI1D,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,oBAAoB;IAIhD,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAI7C,SAAS,CACP,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,eAAe,GACvB,OAAO,CAAC,qBAAqB,CAAC;IAI3B,KAAK,CACT,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,GACT,OAAO,CAAC,WAAW,CAAC;IAejB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B;AAED,wBAAsB,cAAc,IAAK,OAAO,CAAC,iBAAiB,CAAC,CAElE;AAED,wBAAsB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAElE;AAYD,wBAAsB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAIvE;AAWD,wBAAsB,SAAS,CAC7B,EACE,KAAK,EACL,cAAc,EAAE,YAAY,EAC5B,YAAY,EAAE,WAAW,EACzB,IAAI,EACJ,GAAG,IAAI,EACR,EAAE,aAAa,EAChB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,GACtC,OAAO,CAAC,YAAY,CAAC,CAuCvB;AAED,wBAAsB,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC,CAErD"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cui-llama.rn",
-  "version": "1.2.6",
+  "version": "1.3.0",
   "description": "Fork of llama.rn for ChatterUI",
   "main": "lib/commonjs/index",
   "module": "lib/module/index",

package/src/NativeRNLlama.ts

@@ -1,11 +1,14 @@
 import type { TurboModule } from 'react-native'
 import { TurboModuleRegistry } from 'react-native'
 
+export type NativeEmbeddingParams = {
+  embd_normalize?: number
+}
+
 export type NativeContextParams = {
   model: string
   is_model_asset?: boolean
-
-  embedding?: boolean
+  use_progress_callback?: boolean
 
   n_ctx?: number
   n_batch?: number
@@ -13,6 +16,20 @@ export type NativeContextParams = {
   n_threads?: number
   n_gpu_layers?: number
 
+  /**
+   * Enable flash attention, only recommended in GPU device (Experimental in llama.cpp)
+   */
+  flash_attn?: boolean
+
+  /**
+   * KV cache data type for the K (Experimental in llama.cpp)
+   */
+  cache_type_k?: string
+  /**
+   * KV cache data type for the V (Experimental in llama.cpp)
+   */
+  cache_type_v?: string
+
   use_mlock?: boolean
   use_mmap?: boolean
   vocab_only?: boolean
@@ -22,6 +39,12 @@ export type NativeContextParams = {
 
   rope_freq_base?: number
   rope_freq_scale?: number
+
+  pooling_type?: number
+
+  // Embedding params
+  embedding?: boolean
+  embd_normalize?: number
 }
 
 export type NativeCompletionParams = {
@@ -35,8 +58,8 @@ export type NativeCompletionParams = {
   top_k?: number
   top_p?: number
   min_p?: number
-  xtc_t?: number
-  xtc_p?: number
+  xtc_threshold?: number
+  xtc_probability?: number
   typical_p?: number
   temperature?: number // -> temp
   penalty_last_n?: number
@@ -49,6 +72,12 @@ export type NativeCompletionParams = {
   penalize_nl?: boolean
   seed?: number
 
+  dry_multiplier?: number
+  dry_base?: number
+  dry_allowed_length?: number    
+  dry_penalty_last_n?: number
+  dry_sequence_breakers?: Array<string>
+
   ignore_eos?: boolean
   logit_bias?: Array<Array<number>>
 
@@ -125,7 +154,9 @@ export type NativeCPUFeatures = {
 
 export interface Spec extends TurboModule {
   setContextLimit(limit: number): Promise<void>
-  initContext(params: NativeContextParams): Promise<NativeLlamaContext>
+
+  modelInfo(path: string, skip?: string[]): Promise<Object>
+  initContext(contextId: number, params: NativeContextParams): Promise<NativeLlamaContext>
 
   loadSession(
     contextId: number,
@@ -150,7 +181,11 @@ export interface Spec extends TurboModule {
     chatTemplate?: string,
   ): Promise<string>
   detokenize(contextId: number, tokens: number[]): Promise<string>
-  embedding(contextId: number, text: string): Promise<NativeEmbeddingResult>
+  embedding(
+    contextId: number,
+    text: string,
+    params: NativeEmbeddingParams,
+  ): Promise<NativeEmbeddingResult>
   bench(
     contextId: number,
     pp: number,

package/src/index.ts

@@ -11,6 +11,7 @@ import type {
   NativeEmbeddingResult,
   NativeSessionLoadResult,
   NativeCPUFeatures,
+  NativeEmbeddingParams,
 } from './NativeRNLlama'
 import { SchemaGrammarConverter, convertJsonSchemaToGrammar } from './grammar'
 import type { RNLlamaOAICompatibleMessage } from './chat'
@@ -18,10 +19,9 @@ import { formatChat } from './chat'
 
 export { SchemaGrammarConverter, convertJsonSchemaToGrammar }
 
+const EVENT_ON_INIT_CONTEXT_PROGRESS = '@RNLlama_onInitContextProgress'
 const EVENT_ON_TOKEN = '@RNLlama_onToken'
 
-const EVENT_ON_MODEL_PROGRESS = '@RNLlama_onModelProgress'
-
 let EventEmitter: NativeEventEmitter | DeviceEventEmitterStatic
 if (Platform.OS === 'ios') {
   // @ts-ignore
@@ -41,7 +41,16 @@ type TokenNativeEvent = {
   tokenResult: TokenData
 }
 
-export type ContextParams = NativeContextParams
+export type ContextParams = Omit<
+  NativeContextParams,
+  'cache_type_k' | 'cache_type_v' |  'pooling_type'
+> & {
+  cache_type_k?: 'f16' | 'f32' | 'q8_0' | 'q4_0' | 'q4_1' | 'iq4_nl' | 'q5_0' | 'q5_1'
+  cache_type_v?: 'f16' | 'f32' | 'q8_0' | 'q4_0' | 'q4_1' | 'iq4_nl' | 'q5_0' | 'q5_1'
+  pooling_type?: 'none' | 'mean' | 'cls' | 'last' | 'rank'
+}
+
+export type EmbeddingParams = NativeEmbeddingParams
 
 export type CompletionParams = Omit<
   NativeCompletionParams,
@@ -49,6 +58,7 @@ export type CompletionParams = Omit<
 > & {
   prompt?: string
   messages?: RNLlamaOAICompatibleMessage[]
+  chatTemplate?: string
 }
 
 export type BenchResult = {
@@ -100,23 +110,22 @@ export class LlamaContext {
 
   async getFormattedChat(
     messages: RNLlamaOAICompatibleMessage[],
+    template?: string,
   ): Promise<string> {
     const chat = formatChat(messages)
-    return RNLlama.getFormattedChat(
-      this.id,
-      chat,
-      this.model?.isChatTemplateSupported ? undefined : 'chatml',
-    )
+    let tmpl = this.model?.isChatTemplateSupported ? undefined : 'chatml'
+    if (template) tmpl = template // Force replace if provided
+    return RNLlama.getFormattedChat(this.id, chat, tmpl)
   }
 
   async completion(
     params: CompletionParams,
     callback?: (data: TokenData) => void,
   ): Promise<NativeCompletionResult> {
-
     let finalPrompt = params.prompt
-    if (params.messages) { // messages always win
-      finalPrompt = await this.getFormattedChat(params.messages)
+    if (params.messages) {
+      // messages always win
+      finalPrompt = await this.getFormattedChat(params.messages, params.chatTemplate)
     }
 
     let tokenListener: any =
@@ -162,8 +171,11 @@ export class LlamaContext {
     return RNLlama.detokenize(this.id, tokens)
   }
 
-  embedding(text: string): Promise<NativeEmbeddingResult> {
-    return RNLlama.embedding(this.id, text)
+  embedding(
+    text: string,
+    params?: EmbeddingParams,
+  ): Promise<NativeEmbeddingResult> {
+    return RNLlama.embedding(this.id, text, params || {})
   }
 
   async bench(
@@ -199,35 +211,78 @@ export async function setContextLimit(limit: number): Promise<void> {
   return RNLlama.setContextLimit(limit)
 }
 
-export async function initLlama({
+let contextIdCounter = 0
+const contextIdRandom = () =>
+  process.env.NODE_ENV === 'test' ? 0 : Math.floor(Math.random() * 100000)
+
+const modelInfoSkip = [
+  // Large fields
+  'tokenizer.ggml.tokens',
+  'tokenizer.ggml.token_type',
+  'tokenizer.ggml.merges',
+]
+export async function loadLlamaModelInfo(model: string): Promise<Object> {
+  let path = model
+  if (path.startsWith('file://')) path = path.slice(7)
+  return RNLlama.modelInfo(path, modelInfoSkip)
+}
+
+const poolTypeMap = {
+  // -1 is unspecified as undefined
+  none: 0,
+  mean: 1,
+  cls: 2,
+  last: 3,
+  rank: 4,
+}
+
+export async function initLlama(
+  {
     model,
     is_model_asset: isModelAsset,
+    pooling_type: poolingType,
+    lora,
     ...rest
-  }: ContextParams, 
-  progressCallback?: (progress: number) => void
+  }: ContextParams,
+  onProgress?: (progress: number) => void,
 ): Promise<LlamaContext> {
   let path = model
   if (path.startsWith('file://')) path = path.slice(7)
-  
-  const modelProgressListener = EventEmitter.addListener(EVENT_ON_MODEL_PROGRESS, (event) => {
-    if(event.progress && progressCallback)
-      progressCallback(event.progress)
-    if(event.progress === 100) {
-      modelProgressListener.remove()
-    }
-  })
 
+  let loraPath = lora
+  if (loraPath?.startsWith('file://')) loraPath = loraPath.slice(7)
+
+  const contextId = contextIdCounter + contextIdRandom()
+  contextIdCounter += 1
+
+  let removeProgressListener: any = null
+  if (onProgress) {
+    removeProgressListener = EventEmitter.addListener(
+      EVENT_ON_INIT_CONTEXT_PROGRESS,
+      (evt: { contextId: number; progress: number }) => {
+        if (evt.contextId !== contextId) return
+        onProgress(evt.progress)
+      },
+    )
+  }
+
+  const poolType = poolTypeMap[poolingType as keyof typeof poolTypeMap]
   const {
-    contextId,
     gpu,
     reasonNoGPU,
     model: modelDetails,
-  } = await RNLlama.initContext({
+  } = await RNLlama.initContext(contextId, {
     model: path,
     is_model_asset: !!isModelAsset,
+    use_progress_callback: !!onProgress,
+    pooling_type: poolType,
+    lora: loraPath,
     ...rest,
+  }).catch((err: any) => {
+    removeProgressListener?.remove()
+    throw err
   })
-
+  removeProgressListener?.remove()
   return new LlamaContext({ contextId, gpu, reasonNoGPU, model: modelDetails })
 }