cactus-react-native 0.2.4 → 0.2.5

package/android/src/main/jni.cpp

@@ -706,7 +706,7 @@ Java_com_cactus_LlamaContext_doCompletion(
     UNUSED(thiz);
     auto llama = context_map[(long) context_ptr];
 
-    llama->rewind();
+    // llama->rewind();
 
     //llama_reset_timings(llama->ctx);
 

package/ios/CactusContext.mm

@@ -417,7 +417,7 @@
 - (NSDictionary *)completion:(NSDictionary *)params
     onToken:(void (^)(NSMutableDictionary * tokenResult))onToken
 {
-    llama->rewind();
+    // llama->rewind();
 
     //llama_reset_timings(llama->ctx);
 
@@ -877,7 +877,7 @@
         @throw [NSException exceptionWithName:@"LlamaException" reason:@"Multimodal is not enabled" userInfo:nil];
     }
     
-    llama->rewind();
+    // llama->rewind();
     
     llama->params.prompt = [prompt UTF8String];
     llama->params.sampling.seed = params[@"seed"] ? [params[@"seed"] intValue] : -1;

package/lib/commonjs/chat.js

@@ -65,6 +65,9 @@ class ConversationHistoryManager {
   reset() {
     this.history = [];
   }
+  getMessages() {
+    return this.history;
+  }
 }
 exports.ConversationHistoryManager = ConversationHistoryManager;
 //# sourceMappingURL=chat.js.map

package/lib/commonjs/chat.js.map

@@ -1 +1 @@
-{"version":3,"names":["formatChat","messages","chat","forEach","currMsg","role","content","Array","isArray","part","text","TypeError","Error","push","ConversationHistoryManager","history","processNewMessages","fullMessageHistory","divergent","length","i","JSON","stringify","newMessages","requiresReset","slice","update","assistantResponse","reset","exports"],"sourceRoot":"../../src","sources":["chat.ts"],"mappings":";;;;;;;AAWO,SAASA,UAAUA,CACxBC,QAAsC,EACZ;EAC1B,MAAMC,IAA8B,GAAG,EAAE;EAEzCD,QAAQ,CAACE,OAAO,CAAEC,OAAO,IAAK;IAC5B,MAAMC,IAAY,GAAGD,OAAO,CAACC,IAAI,IAAI,EAAE;IAEvC,IAAIC,OAAe,GAAG,EAAE;IACxB,IAAI,SAAS,IAAIF,OAAO,EAAE;MACxB,IAAI,OAAOA,OAAO,CAACE,OAAO,KAAK,QAAQ,EAAE;QACvC;QAAC,CAAC;UAAEA;QAAQ,CAAC,GAAGF,OAAO;MACzB,CAAC,MAAM,IAAIG,KAAK,CAACC,OAAO,CAACJ,OAAO,CAACE,OAAO,CAAC,EAAE;QACzCF,OAAO,CAACE,OAAO,CAACH,OAAO,CAAEM,IAAI,IAAK;UAChC,IAAI,MAAM,IAAIA,IAAI,EAAE;YAClBH,OAAO,IAAI,GAAGA,OAAO,GAAG,IAAI,GAAG,EAAE,GAAGG,IAAI,CAACC,IAAI,EAAE;UACjD;QACF,CAAC,CAAC;MACJ,CAAC,MAAM;QACL,MAAM,IAAIC,SAAS,CACjB,kFACF,CAAC;MACH;IACF,CAAC,MAAM;MACL,MAAM,IAAIC,KAAK,CACb,6EACF,CAAC;IACH;IAEAV,IAAI,CAACW,IAAI,CAAC;MAAER,IAAI;MAAEC;IAAQ,CAAC,CAAC;EAC9B,CAAC,CAAC;EACF,OAAOJ,IAAI;AACb;AAOO,MAAMY,0BAA0B,CAAC;EAC9BC,OAAO,GAAiC,EAAE;EAE3CC,kBAAkBA,CACvBC,kBAAgD,EAC7B;IACnB,IAAIC,SAAS,GAAGD,kBAAkB,CAACE,MAAM,GAAG,IAAI,CAACJ,OAAO,CAACI,MAAM;IAC/D,IAAI,CAACD,SAAS,EAAE;MACd,KAAK,IAAIE,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG,IAAI,CAACL,OAAO,CAACI,MAAM,EAAEC,CAAC,EAAE,EAAE;QAC5C,IAAIC,IAAI,CAACC,SAAS,CAAC,IAAI,CAACP,OAAO,CAACK,CAAC,CAAC,CAAC,KAAKC,IAAI,CAACC,SAAS,CAACL,kBAAkB,CAACG,CAAC,CAAC,CAAC,EAAE;UAC7EF,SAAS,GAAG,IAAI;UAChB;QACF;MACF;IACF;IAEA,IAAIA,SAAS,EAAE;MACb,OAAO;QAAEK,WAAW,EAAEN,kBAAkB;QAAEO,aAAa,EAAE;MAAK,CAAC;IACjE;IAEA,MAAMD,WAAW,GAAGN,kBAAkB,CAACQ,KAAK,CAAC,IAAI,CAACV,OAAO,CAACI,MAAM,CAAC;IACjE,OAAO;MAAEI,WAAW;MAAEC,aAAa,EAAE;IAAM,CAAC;EAC9C;EAEOE,MAAMA,CACXH,WAAyC,EACzCI,iBAA6C,EAC7C;IACA,IAAI,CAACZ,OAAO,CAACF,IAAI,CAAC,GAAGU,WAAW,EAAEI,iBAAiB,CAAC;EACtD;EAEOC,KAAKA,CAAA,EAAG;IACb,IAAI,CAACb,OAAO,GAAG,EAAE;EACnB;AACF;AAACc,OAAA,CAAAf,0BAAA,GAAAA,0BAAA","ignoreList":[]}
+{"version":3,"names":["formatChat","messages","chat","forEach","currMsg","role","content","Array","isArray","part","text","TypeError","Error","push","ConversationHistoryManager","history","processNewMessages","fullMessageHistory","divergent","length","i","JSON","stringify","newMessages","requiresReset","slice","update","assistantResponse","reset","getMessages","exports"],"sourceRoot":"../../src","sources":["chat.ts"],"mappings":";;;;;;;AAWO,SAASA,UAAUA,CACxBC,QAAsC,EACZ;EAC1B,MAAMC,IAA8B,GAAG,EAAE;EAEzCD,QAAQ,CAACE,OAAO,CAAEC,OAAO,IAAK;IAC5B,MAAMC,IAAY,GAAGD,OAAO,CAACC,IAAI,IAAI,EAAE;IAEvC,IAAIC,OAAe,GAAG,EAAE;IACxB,IAAI,SAAS,IAAIF,OAAO,EAAE;MACxB,IAAI,OAAOA,OAAO,CAACE,OAAO,KAAK,QAAQ,EAAE;QACvC;QAAC,CAAC;UAAEA;QAAQ,CAAC,GAAGF,OAAO;MACzB,CAAC,MAAM,IAAIG,KAAK,CAACC,OAAO,CAACJ,OAAO,CAACE,OAAO,CAAC,EAAE;QACzCF,OAAO,CAACE,OAAO,CAACH,OAAO,CAAEM,IAAI,IAAK;UAChC,IAAI,MAAM,IAAIA,IAAI,EAAE;YAClBH,OAAO,IAAI,GAAGA,OAAO,GAAG,IAAI,GAAG,EAAE,GAAGG,IAAI,CAACC,IAAI,EAAE;UACjD;QACF,CAAC,CAAC;MACJ,CAAC,MAAM;QACL,MAAM,IAAIC,SAAS,CACjB,kFACF,CAAC;MACH;IACF,CAAC,MAAM;MACL,MAAM,IAAIC,KAAK,CACb,6EACF,CAAC;IACH;IAEAV,IAAI,CAACW,IAAI,CAAC;MAAER,IAAI;MAAEC;IAAQ,CAAC,CAAC;EAC9B,CAAC,CAAC;EACF,OAAOJ,IAAI;AACb;AAOO,MAAMY,0BAA0B,CAAC;EAC9BC,OAAO,GAAiC,EAAE;EAE3CC,kBAAkBA,CACvBC,kBAAgD,EAC7B;IACnB,IAAIC,SAAS,GAAGD,kBAAkB,CAACE,MAAM,GAAG,IAAI,CAACJ,OAAO,CAACI,MAAM;IAC/D,IAAI,CAACD,SAAS,EAAE;MACd,KAAK,IAAIE,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG,IAAI,CAACL,OAAO,CAACI,MAAM,EAAEC,CAAC,EAAE,EAAE;QAC5C,IAAIC,IAAI,CAACC,SAAS,CAAC,IAAI,CAACP,OAAO,CAACK,CAAC,CAAC,CAAC,KAAKC,IAAI,CAACC,SAAS,CAACL,kBAAkB,CAACG,CAAC,CAAC,CAAC,EAAE;UAC7EF,SAAS,GAAG,IAAI;UAChB;QACF;MACF;IACF;IAEA,IAAIA,SAAS,EAAE;MACb,OAAO;QAAEK,WAAW,EAAEN,kBAAkB;QAAEO,aAAa,EAAE;MAAK,CAAC;IACjE;IAEA,MAAMD,WAAW,GAAGN,kBAAkB,CAACQ,KAAK,CAAC,IAAI,CAACV,OAAO,CAACI,MAAM,CAAC;IACjE,OAAO;MAAEI,WAAW;MAAEC,aAAa,EAAE;IAAM,CAAC;EAC9C;EAEOE,MAAMA,CACXH,WAAyC,EACzCI,iBAA6C,EAC7C;IACA,IAAI,CAACZ,OAAO,CAACF,IAAI,CAAC,GAAGU,WAAW,EAAEI,iBAAiB,CAAC;EACtD;EAEOC,KAAKA,CAAA,EAAG;IACb,IAAI,CAACb,OAAO,GAAG,EAAE;EACnB;EAEOc,WAAWA,CAAA,EAAiC;IACjD,OAAO,IAAI,CAACd,OAAO;EACrB;AACF;AAACe,OAAA,CAAAhB,0BAAA,GAAAA,0BAAA","ignoreList":[]}

package/lib/commonjs/lm.js

@@ -11,6 +11,16 @@ var _chat = require("./chat");
 // @ts-ignore
 
 class CactusLM {
+  // the initPromise enables a "async singleton" initialization pattern which
+  // protects against a race condition in the event of multiple init attempts
+  static _initCache = new Map();
+  static getCacheKey(params, cactusToken, retryOptions) {
+    return JSON.stringify({
+      params,
+      cactusToken,
+      retryOptions
+    });
+  }
   constructor(context) {
     this.context = context;
     this.conversationHistoryManager = new _chat.ConversationHistoryManager();
@@ -19,62 +29,75 @@ class CactusLM {
     if (cactusToken) {
       (0, _remote.setCactusToken)(cactusToken);
     }
-    const maxRetries = retryOptions?.maxRetries ?? 3;
-    const delayMs = retryOptions?.delayMs ?? 1000;
-    const configs = [params, {
-      ...params,
-      n_gpu_layers: 0
-    }];
-    const sleep = ms => {
-      return new Promise(resolve => {
-        const start = Date.now();
-        const wait = () => {
-          if (Date.now() - start >= ms) {
-            resolve();
-          } else {
-            Promise.resolve().then(wait);
+    const key = CactusLM.getCacheKey(params, cactusToken, retryOptions);
+    if (CactusLM._initCache.has(key)) {
+      // concurrent initialization calls with the same params all get the same cached Promise
+      return CactusLM._initCache.get(key);
+    }
+    const initPromise = (async () => {
+      const maxRetries = retryOptions?.maxRetries ?? 3;
+      const delayMs = retryOptions?.delayMs ?? 1000;
+      const configs = [params, {
+        ...params,
+        n_gpu_layers: 0
+      }];
+      const sleep = ms => {
+        return new Promise(resolve => {
+          const start = Date.now();
+          const wait = () => {
+            if (Date.now() - start >= ms) {
+              resolve();
+            } else {
+              Promise.resolve().then(wait);
+            }
+          };
+          wait();
+        });
+      };
+      for (const config of configs) {
+        let lastError = null;
+        for (let attempt = 1; attempt <= maxRetries; attempt++) {
+          try {
+            const context = await (0, _index.initLlama)(config, onProgress);
+            return {
+              lm: new CactusLM(context),
+              error: null
+            };
+          } catch (e) {
+            lastError = e;
+            const isLastConfig = configs.indexOf(config) === configs.length - 1;
+            const isLastAttempt = attempt === maxRetries;
+            _telemetry.Telemetry.error(e, {
+              n_gpu_layers: config.n_gpu_layers ?? null,
+              n_ctx: config.n_ctx ?? null,
+              model: config.model ?? null
+            });
+            if (!isLastAttempt) {
+              const delay = delayMs * Math.pow(2, attempt - 1);
+              await sleep(delay);
+            } else if (!isLastConfig) {
+              break;
+            }
           }
-        };
-        wait();
-      });
-    };
-    for (const config of configs) {
-      let lastError = null;
-      for (let attempt = 1; attempt <= maxRetries; attempt++) {
-        try {
-          const context = await (0, _index.initLlama)(config, onProgress);
+        }
+        if (configs.indexOf(config) === configs.length - 1 && lastError) {
           return {
-            lm: new CactusLM(context),
-            error: null
+            lm: null,
+            error: lastError
           };
-        } catch (e) {
-          lastError = e;
-          const isLastConfig = configs.indexOf(config) === configs.length - 1;
-          const isLastAttempt = attempt === maxRetries;
-          _telemetry.Telemetry.error(e, {
-            n_gpu_layers: config.n_gpu_layers ?? null,
-            n_ctx: config.n_ctx ?? null,
-            model: config.model ?? null
-          });
-          if (!isLastAttempt) {
-            const delay = delayMs * Math.pow(2, attempt - 1);
-            await sleep(delay);
-          } else if (!isLastConfig) {
-            break;
-          }
         }
       }
-      if (configs.indexOf(config) === configs.length - 1 && lastError) {
-        return {
-          lm: null,
-          error: lastError
-        };
-      }
+      return {
+        lm: null,
+        error: new Error('Failed to initialize CactusLM after all retries')
+      };
+    })();
+    CactusLM._initCache.set(key, initPromise);
+    const result = await initPromise;
+    if (result.error) {
+      CactusLM._initCache.delete(key); // Reset on failure to allow retries
     }
-    return {
-      lm: null,
-      error: new Error('Failed to initialize CactusLM after all retries')
-    };
+    return result;
   }
   completion = async (messages, params = {}, callback) => {
     const {
@@ -147,6 +170,9 @@ class CactusLM {
   async release() {
     return this.context.release();
   }
+  async stopCompletion() {
+    return await this.context.stopCompletion();
+  }
 }
 exports.CactusLM = CactusLM;
 //# sourceMappingURL=lm.js.map

package/lib/commonjs/lm.js.map

@@ -1 +1 @@
-{"version":3,"names":["_index","require","_telemetry","_remote","_chat","CactusLM","constructor","context","conversationHistoryManager","ConversationHistoryManager","init","params","onProgress","cactusToken","retryOptions","setCactusToken","maxRetries","delayMs","configs","n_gpu_layers","sleep","ms","Promise","resolve","start","Date","now","wait","then","config","lastError","attempt","initLlama","lm","error","e","isLastConfig","indexOf","length","isLastAttempt","Telemetry","n_ctx","model","delay","Math","pow","Error","completion","messages","callback","newMessages","requiresReset","processNewMessages","rewind","reset","console","warn","result","update","role","content","embedding","text","mode","_handleRemoteEmbedding","_handleLocalEmbedding","remoteError","localError","embeddingValues","getVertexAIEmbedding","release","exports"],"sourceRoot":"../../src","sources":["lm.ts"],"mappings":";;;;;;AAAA,IAAAA,MAAA,GAAAC,OAAA;AAYA,IAAAC,UAAA,GAAAD,OAAA;AACA,IAAAE,OAAA,GAAAF,OAAA;AACA,IAAAG,KAAA,GAAAH,OAAA;AAbA;;AAoBO,MAAMI,QAAQ,CAAC;EAIVC,WAAWA,CAACC,OAAqB,EAAE;IAC3C,IAAI,CAACA,OAAO,GAAGA,OAAO;IACtB,IAAI,CAACC,0BAA0B,GAAG,IAAIC,gCAA0B,CAAC,CAAC;EACpE;EAEA,aAAaC,IAAIA,CACfC,MAAqB,EACrBC,UAAuC,EACvCC,WAAoB,EACpBC,YAAwD,EAC/B;IACzB,IAAID,WAAW,EAAE;MACf,IAAAE,sBAAc,EAACF,WAAW,CAAC;IAC7B;IAEA,MAAMG,UAAU,GAAGF,YAAY,EAAEE,UAAU,IAAI,CAAC;IAChD,MAAMC,OAAO,GAAGH,YAAY,EAAEG,OAAO,IAAI,IAAI;IAE7C,MAAMC,OAAO,GAAG,CACdP,MAAM,EACN;MAAE,GAAGA,MAAM;MAAEQ,YAAY,EAAE;IAAE,CAAC,CAC/B;IAED,MAAMC,KAAK,GAAIC,EAAU,IAAoB;MAC3C,OAAO,IAAIC,OAAO,CAACC,OAAO,IAAI;QAC5B,MAAMC,KAAK,GAAGC,IAAI,CAACC,GAAG,CAAC,CAAC;QACxB,MAAMC,IAAI,GAAGA,CAAA,KAAM;UACjB,IAAIF,IAAI,CAACC,GAAG,CAAC,CAAC,GAAGF,KAAK,IAAIH,EAAE,EAAE;YAC5BE,OAAO,CAAC,CAAC;UACX,CAAC,MAAM;YACLD,OAAO,CAACC,OAAO,CAAC,CAAC,CAACK,IAAI,CAACD,IAAI,CAAC;UAC9B;QACF,CAAC;QACDA,IAAI,CAAC,CAAC;MACR,CAAC,CAAC;IACJ,CAAC;IAED,KAAK,MAAME,MAAM,IAAIX,OAAO,EAAE;MAC5B,IAAIY,SAAuB,GAAG,IAAI;MAElC,KAAK,IAAIC,OAAO,GAAG,CAAC,EAAEA,OAAO,IAAIf,UAAU,EAAEe,OAAO,EAAE,EAAE;QACtD,IAAI;UACF,MAAMxB,OAAO,GAAG,MAAM,IAAAyB,gBAAS,EAACH,MAAM,EAAEjB,UAAU,CAAC;UACnD,OAAO;YAAEqB,EAAE,EAAE,IAAI5B,QAAQ,CAACE,OAAO,CAAC;YAAE2B,KAAK,EAAE;UAAK,CAAC;QACnD,CAAC,CAAC,OAAOC,CAAC,EAAE;UACVL,SAAS,GAAGK,CAAU;UACtB,MAAMC,YAAY,GAAGlB,OAAO,CAACmB,OAAO,CAACR,MAAM,CAAC,KAAKX,OAAO,CAACoB,MAAM,GAAG,CAAC;UACnE,MAAMC,aAAa,GAAGR,OAAO,KAAKf,UAAU;UAE5CwB,oBAAS,CAACN,KAAK,CAACC,CAAC,EAAW;YAC1BhB,YAAY,EAAEU,MAAM,CAACV,YAAY,IAAI,IAAI;YACzCsB,KAAK,EAAEZ,MAAM,CAACY,KAAK,IAAI,IAAI;YAC3BC,KAAK,EAAEb,MAAM,CAACa,KAAK,IAAI;UACzB,CAAC,CAAC;UAEF,IAAI,CAACH,aAAa,EAAE;YAClB,MAAMI,KAAK,GAAG1B,OAAO,GAAG2B,IAAI,CAACC,GAAG,CAAC,CAAC,EAAEd,OAAO,GAAG,CAAC,CAAC;YAChD,MAAMX,KAAK,CAACuB,KAAK,CAAC;UACpB,CAAC,MAAM,IAAI,CAACP,YAAY,EAAE;YACxB;UACF;QACF;MACF;MAEA,IAAIlB,OAAO,CAACmB,OAAO,CAACR,MAAM,CAAC,KAAKX,OAAO,CAACoB,MAAM,GAAG,CAAC,IAAIR,SAAS,EAAE;QAC/D,OAAO;UAAEG,EAAE,EAAE,IAAI;UAAEC,KAAK,EAAEJ;QAAU,CAAC;MACvC;IACF;IACA,OAAO;MAAEG,EAAE,EAAE,IAAI;MAAEC,KAAK,EAAE,IAAIY,KAAK,CAAC,iDAAiD;IAAE,CAAC;EAC1F;EAEAC,UAAU,GAAG,MAAAA,CACXC,QAAsC,EACtCrC,MAAwB,GAAG,CAAC,CAAC,EAC7BsC,QAA8B,KACM;IACpC,MAAM;MAAEC,WAAW;MAAEC;IAAc,CAAC,GAClC,IAAI,CAAC3C,0BAA0B,CAAC4C,kBAAkB,CAACJ,QAAQ,CAAC;IAE9D,IAAIG,aAAa,EAAE;MACjB,IAAI,CAAC5C,OAAO,EAAE8C,MAAM,CAAC,CAAC;MACtB,IAAI,CAAC7C,0BAA0B,CAAC8C,KAAK,CAAC,CAAC;IACzC;IAEA,IAAIJ,WAAW,CAACZ,MAAM,KAAK,CAAC,EAAE;MAC5BiB,OAAO,CAACC,IAAI,CAAC,0BAA0B,CAAC;IAC1C;IAEA,MAAMC,MAAM,GAAG,MAAM,IAAI,CAAClD,OAAO,CAACwC,UAAU,CAAC;MAAEC,QAAQ,EAAEE,WAAW;MAAE,GAAGvC;IAAO,CAAC,EAAEsC,QAAQ,CAAC;IAE5F,IAAI,CAACzC,0BAA0B,CAACkD,MAAM,CAACR,WAAW,EAAE;MAClDS,IAAI,EAAE,WAAW;MACjBC,OAAO,EAAEH,MAAM,CAACG;IAClB,CAAC,CAAC;IAEF,OAAOH,MAAM;EACf,CAAC;EAED,MAAMI,SAASA,CACbC,IAAY,EACZnD,MAAwB,EACxBoD,IAAY,GAAG,OAAO,EACU;IAChC,IAAIN,MAA6B;IACjC,IAAI3B,SAAuB,GAAG,IAAI;IAElC,IAAIiC,IAAI,KAAK,QAAQ,EAAE;MACrBN,MAAM,GAAG,MAAM,IAAI,CAACO,sBAAsB,CAACF,IAAI,CAAC;IAClD,CAAC,MAAM,IAAIC,IAAI,KAAK,OAAO,EAAE;MAC3BN,MAAM,GAAG,MAAM,IAAI,CAACQ,qBAAqB,CAACH,IAAI,EAAEnD,MAAM,CAAC;IACzD,CAAC,MAAM,IAAIoD,IAAI,KAAK,YAAY,EAAE;MAChC,IAAI;QACFN,MAAM,GAAG,MAAM,IAAI,CAACQ,qBAAqB,CAACH,IAAI,EAAEnD,MAAM,CAAC;MACzD,CAAC,CAAC,OAAOwB,CAAC,EAAE;QACVL,SAAS,GAAGK,CAAU;QACtB,IAAI;UACFsB,MAAM,GAAG,MAAM,IAAI,CAACO,sBAAsB,CAACF,IAAI,CAAC;QAClD,CAAC,CAAC,OAAOI,WAAW,EAAE;UACpB,MAAMpC,SAAS;QACjB;MACF;IACF,CAAC,MAAM,IAAIiC,IAAI,KAAK,aAAa,EAAE;MACjC,IAAI;QACFN,MAAM,GAAG,MAAM,IAAI,CAACO,sBAAsB,CAACF,IAAI,CAAC;MAClD,CAAC,CAAC,OAAO3B,CAAC,EAAE;QACVL,SAAS,GAAGK,CAAU;QACtB,IAAI;UACFsB,MAAM,GAAG,MAAM,IAAI,CAACQ,qBAAqB,CAACH,IAAI,EAAEnD,MAAM,CAAC;QACzD,CAAC,CAAC,OAAOwD,UAAU,EAAE;UACnB,MAAMrC,SAAS;QACjB;MACF;IACF,CAAC,MAAM;MACL,MAAM,IAAIgB,KAAK,CAAC,gBAAgB,GAAGiB,IAAI,GAAG,6DAA6D,CAAC;IAC1G;IACA,OAAON,MAAM;EACf;EAEA,MAAgBQ,qBAAqBA,CAACH,IAAY,EAAEnD,MAAwB,EAAkC;IAC5G,OAAO,IAAI,CAACJ,OAAO,CAACsD,SAAS,CAACC,IAAI,EAAEnD,MAAM,CAAC;EAC7C;EAEA,MAAgBqD,sBAAsBA,CAACF,IAAY,EAAkC;IACnF,MAAMM,eAAe,GAAG,MAAM,IAAAC,4BAAoB,EAACP,IAAI,CAAC;IACxD,OAAO;MACLD,SAAS,EAAEO;IACb,CAAC;EACH;EAEAf,MAAM,GAAG,MAAAA,CAAA,KAA2B;IAClC,OAAO,IAAI,CAAC9C,OAAO,EAAE8C,MAAM,CAAC,CAAC;EAC/B,CAAC;EAED,MAAMiB,OAAOA,CAAA,EAAkB;IAC7B,OAAO,IAAI,CAAC/D,OAAO,CAAC+D,OAAO,CAAC,CAAC;EAC/B;AACF;AAACC,OAAA,CAAAlE,QAAA,GAAAA,QAAA","ignoreList":[]}
+{"version":3,"names":["_index","require","_telemetry","_remote","_chat","CactusLM","_initCache","Map","getCacheKey","params","cactusToken","retryOptions","JSON","stringify","constructor","context","conversationHistoryManager","ConversationHistoryManager","init","onProgress","setCactusToken","key","has","get","initPromise","maxRetries","delayMs","configs","n_gpu_layers","sleep","ms","Promise","resolve","start","Date","now","wait","then","config","lastError","attempt","initLlama","lm","error","e","isLastConfig","indexOf","length","isLastAttempt","Telemetry","n_ctx","model","delay","Math","pow","Error","set","result","delete","completion","messages","callback","newMessages","requiresReset","processNewMessages","rewind","reset","console","warn","update","role","content","embedding","text","mode","_handleRemoteEmbedding","_handleLocalEmbedding","remoteError","localError","embeddingValues","getVertexAIEmbedding","release","stopCompletion","exports"],"sourceRoot":"../../src","sources":["lm.ts"],"mappings":";;;;;;AAAA,IAAAA,MAAA,GAAAC,OAAA;AAYA,IAAAC,UAAA,GAAAD,OAAA;AACA,IAAAE,OAAA,GAAAF,OAAA;AACA,IAAAG,KAAA,GAAAH,OAAA;AAbA;;AAoBO,MAAMI,QAAQ,CAAC;EAIpB;EACA;EACA,OAAeC,UAAU,GAAyC,IAAIC,GAAG,CAAC,CAAC;EAE3E,OAAeC,WAAWA,CAACC,MAAqB,EAAEC,WAAoB,EAAEC,YAAwD,EAAU;IACxI,OAAOC,IAAI,CAACC,SAAS,CAAC;MAAEJ,MAAM;MAAEC,WAAW;MAAEC;IAAa,CAAC,CAAC;EAC9D;EAEUG,WAAWA,CAACC,OAAqB,EAAE;IAC3C,IAAI,CAACA,OAAO,GAAGA,OAAO;IACtB,IAAI,CAACC,0BAA0B,GAAG,IAAIC,gCAA0B,CAAC,CAAC;EACpE;EAEA,aAAaC,IAAIA,CACfT,MAAqB,EACrBU,UAAuC,EACvCT,WAAoB,EACpBC,YAAwD,EAC/B;IAEzB,IAAID,WAAW,EAAE;MACf,IAAAU,sBAAc,EAACV,WAAW,CAAC;IAC7B;IAEA,MAAMW,GAAG,GAAGhB,QAAQ,CAACG,WAAW,CAACC,MAAM,EAAEC,WAAW,EAAEC,YAAY,CAAC;IACnE,IAAIN,QAAQ,CAACC,UAAU,CAACgB,GAAG,CAACD,GAAG,CAAC,EAAE;MAChC;MACA,OAAOhB,QAAQ,CAACC,UAAU,CAACiB,GAAG,CAACF,GAAG,CAAC;IACrC;IAEA,MAAMG,WAAW,GAAG,CAAC,YAAY;MAC/B,MAAMC,UAAU,GAAGd,YAAY,EAAEc,UAAU,IAAI,CAAC;MAChD,MAAMC,OAAO,GAAGf,YAAY,EAAEe,OAAO,IAAI,IAAI;MAE7C,MAAMC,OAAO,GAAG,CACdlB,MAAM,EACN;QAAE,GAAGA,MAAM;QAAEmB,YAAY,EAAE;MAAE,CAAC,CAC/B;MAED,MAAMC,KAAK,GAAIC,EAAU,IAAoB;QAC3C,OAAO,IAAIC,OAAO,CAACC,OAAO,IAAI;UAC5B,MAAMC,KAAK,GAAGC,IAAI,CAACC,GAAG,CAAC,CAAC;UACxB,MAAMC,IAAI,GAAGA,CAAA,KAAM;YACjB,IAAIF,IAAI,CAACC,GAAG,CAAC,CAAC,GAAGF,KAAK,IAAIH,EAAE,EAAE;cAC5BE,OAAO,CAAC,CAAC;YACX,CAAC,MAAM;cACLD,OAAO,CAACC,OAAO,CAAC,CAAC,CAACK,IAAI,CAACD,IAAI,CAAC;YAC9B;UACF,CAAC;UACDA,IAAI,CAAC,CAAC;QACR,CAAC,CAAC;MACJ,CAAC;MAED,KAAK,MAAME,MAAM,IAAIX,OAAO,EAAE;QAC5B,IAAIY,SAAuB,GAAG,IAAI;QAElC,KAAK,IAAIC,OAAO,GAAG,CAAC,EAAEA,OAAO,IAAIf,UAAU,EAAEe,OAAO,EAAE,EAAE;UACtD,IAAI;YACF,MAAMzB,OAAO,GAAG,MAAM,IAAA0B,gBAAS,EAACH,MAAM,EAAEnB,UAAU,CAAC;YACnD,OAAO;cAAEuB,EAAE,EAAE,IAAIrC,QAAQ,CAACU,OAAO,CAAC;cAAE4B,KAAK,EAAE;YAAK,CAAC;UACnD,CAAC,CAAC,OAAOC,CAAC,EAAE;YACVL,SAAS,GAAGK,CAAU;YACtB,MAAMC,YAAY,GAAGlB,OAAO,CAACmB,OAAO,CAACR,MAAM,CAAC,KAAKX,OAAO,CAACoB,MAAM,GAAG,CAAC;YACnE,MAAMC,aAAa,GAAGR,OAAO,KAAKf,UAAU;YAE5CwB,oBAAS,CAACN,KAAK,CAACC,CAAC,EAAW;cAC1BhB,YAAY,EAAEU,MAAM,CAACV,YAAY,IAAI,IAAI;cACzCsB,KAAK,EAAEZ,MAAM,CAACY,KAAK,IAAI,IAAI;cAC3BC,KAAK,EAAEb,MAAM,CAACa,KAAK,IAAI;YACzB,CAAC,CAAC;YAEF,IAAI,CAACH,aAAa,EAAE;cAClB,MAAMI,KAAK,GAAG1B,OAAO,GAAG2B,IAAI,CAACC,GAAG,CAAC,CAAC,EAAEd,OAAO,GAAG,CAAC,CAAC;cAChD,MAAMX,KAAK,CAACuB,KAAK,CAAC;YACpB,CAAC,MAAM,IAAI,CAACP,YAAY,EAAE;cACxB;YACF;UACF;QACF;QAEA,IAAIlB,OAAO,CAACmB,OAAO,CAACR,MAAM,CAAC,KAAKX,OAAO,CAACoB,MAAM,GAAG,CAAC,IAAIR,SAAS,EAAE;UAC/D,OAAO;YAAEG,EAAE,EAAE,IAAI;YAAEC,KAAK,EAAEJ;UAAU,CAAC;QACvC;MACF;MACA,OAAO;QAAEG,EAAE,EAAE,IAAI;QAAEC,KAAK,EAAE,IAAIY,KAAK,CAAC,iDAAiD;MAAE,CAAC;IAC1F,CAAC,EAAE,CAAC;IAEJlD,QAAQ,CAACC,UAAU,CAACkD,GAAG,CAACnC,GAAG,EAAEG,WAAW,CAAC;IAEzC,MAAMiC,MAAM,GAAG,MAAMjC,WAAW;IAChC,IAAIiC,MAAM,CAACd,KAAK,EAAE;MAChBtC,QAAQ,CAACC,UAAU,CAACoD,MAAM,CAACrC,GAAG,CAAC,CAAC,CAAC;IACnC;IACA,OAAOoC,MAAM;EACf;EAEAE,UAAU,GAAG,MAAAA,CACXC,QAAsC,EACtCnD,MAAwB,GAAG,CAAC,CAAC,EAC7BoD,QAA8B,KACM;IACpC,MAAM;MAAEC,WAAW;MAAEC;IAAc,CAAC,GAClC,IAAI,CAAC/C,0BAA0B,CAACgD,kBAAkB,CAACJ,QAAQ,CAAC;IAE9D,IAAIG,aAAa,EAAE;MACjB,IAAI,CAAChD,OAAO,EAAEkD,MAAM,CAAC,CAAC;MACtB,IAAI,CAACjD,0BAA0B,CAACkD,KAAK,CAAC,CAAC;IACzC;IAEA,IAAIJ,WAAW,CAACf,MAAM,KAAK,CAAC,EAAE;MAC5BoB,OAAO,CAACC,IAAI,CAAC,0BAA0B,CAAC;IAC1C;IAEA,MAAMX,MAAM,GAAG,MAAM,IAAI,CAAC1C,OAAO,CAAC4C,UAAU,CAAC;MAAEC,QAAQ,EAAEE,WAAW;MAAE,GAAGrD;IAAO,CAAC,EAAEoD,QAAQ,CAAC;IAE5F,IAAI,CAAC7C,0BAA0B,CAACqD,MAAM,CAACP,WAAW,EAAE;MAClDQ,IAAI,EAAE,WAAW;MACjBC,OAAO,EAAEd,MAAM,CAACc;IAClB,CAAC,CAAC;IAEF,OAAOd,MAAM;EACf,CAAC;EAED,MAAMe,SAASA,CACbC,IAAY,EACZhE,MAAwB,EACxBiE,IAAY,GAAG,OAAO,EACU;IAChC,IAAIjB,MAA6B;IACjC,IAAIlB,SAAuB,GAAG,IAAI;IAElC,IAAImC,IAAI,KAAK,QAAQ,EAAE;MACrBjB,MAAM,GAAG,MAAM,IAAI,CAACkB,sBAAsB,CAACF,IAAI,CAAC;IAClD,CAAC,MAAM,IAAIC,IAAI,KAAK,OAAO,EAAE;MAC3BjB,MAAM,GAAG,MAAM,IAAI,CAACmB,qBAAqB,CAACH,IAAI,EAAEhE,MAAM,CAAC;IACzD,CAAC,MAAM,IAAIiE,IAAI,KAAK,YAAY,EAAE;MAChC,IAAI;QACFjB,MAAM,GAAG,MAAM,IAAI,CAACmB,qBAAqB,CAACH,IAAI,EAAEhE,MAAM,CAAC;MACzD,CAAC,CAAC,OAAOmC,CAAC,EAAE;QACVL,SAAS,GAAGK,CAAU;QACtB,IAAI;UACFa,MAAM,GAAG,MAAM,IAAI,CAACkB,sBAAsB,CAACF,IAAI,CAAC;QAClD,CAAC,CAAC,OAAOI,WAAW,EAAE;UACpB,MAAMtC,SAAS;QACjB;MACF;IACF,CAAC,MAAM,IAAImC,IAAI,KAAK,aAAa,EAAE;MACjC,IAAI;QACFjB,MAAM,GAAG,MAAM,IAAI,CAACkB,sBAAsB,CAACF,IAAI,CAAC;MAClD,CAAC,CAAC,OAAO7B,CAAC,EAAE;QACVL,SAAS,GAAGK,CAAU;QACtB,IAAI;UACFa,MAAM,GAAG,MAAM,IAAI,CAACmB,qBAAqB,CAACH,IAAI,EAAEhE,MAAM,CAAC;QACzD,CAAC,CAAC,OAAOqE,UAAU,EAAE;UACnB,MAAMvC,SAAS;QACjB;MACF;IACF,CAAC,MAAM;MACL,MAAM,IAAIgB,KAAK,CAAC,gBAAgB,GAAGmB,IAAI,GAAG,6DAA6D,CAAC;IAC1G;IACA,OAAOjB,MAAM;EACf;EAEA,MAAgBmB,qBAAqBA,CAACH,IAAY,EAAEhE,MAAwB,EAAkC;IAC5G,OAAO,IAAI,CAACM,OAAO,CAACyD,SAAS,CAACC,IAAI,EAAEhE,MAAM,CAAC;EAC7C;EAEA,MAAgBkE,sBAAsBA,CAACF,IAAY,EAAkC;IACnF,MAAMM,eAAe,GAAG,MAAM,IAAAC,4BAAoB,EAACP,IAAI,CAAC;IACxD,OAAO;MACLD,SAAS,EAAEO;IACb,CAAC;EACH;EAEAd,MAAM,GAAG,MAAAA,CAAA,KAA2B;IAClC,OAAO,IAAI,CAAClD,OAAO,EAAEkD,MAAM,CAAC,CAAC;EAC/B,CAAC;EAED,MAAMgB,OAAOA,CAAA,EAAkB;IAC7B,OAAO,IAAI,CAAClE,OAAO,CAACkE,OAAO,CAAC,CAAC;EAC/B;EAEA,MAAMC,cAAcA,CAAA,EAAkB;IACpC,OAAO,MAAM,IAAI,CAACnE,OAAO,CAACmE,cAAc,CAAC,CAAC;EAC5C;AAEF;AAACC,OAAA,CAAA9E,QAAA,GAAAA,QAAA","ignoreList":[]}

package/lib/commonjs/vlm.js

@@ -9,44 +9,92 @@ var _telemetry = require("./telemetry");
 var _remote = require("./remote");
 var _chat = require("./chat");
 class CactusVLM {
+  // see CactusLM for detailed docs
+  static _initCache = new Map();
+  static getCacheKey(params, cactusToken, retryOptions) {
+    return JSON.stringify({
+      params,
+      cactusToken,
+      retryOptions
+    });
+  }
   constructor(context) {
     this.context = context;
     this.conversationHistoryManager = new _chat.ConversationHistoryManager();
   }
-  static async init(params, onProgress, cactusToken) {
+  static async init(params, onProgress, cactusToken, retryOptions) {
     if (cactusToken) {
       (0, _remote.setCactusToken)(cactusToken);
     }
-    const configs = [params, {
-      ...params,
-      n_gpu_layers: 0
-    }];
-    for (const config of configs) {
-      try {
-        const context = await (0, _index.initLlama)(config, onProgress);
-        await (0, _index.initMultimodal)(context.id, params.mmproj, false);
-        return {
-          vlm: new CactusVLM(context),
-          error: null
-        };
-      } catch (e) {
-        _telemetry.Telemetry.error(e, {
-          n_gpu_layers: config.n_gpu_layers ?? null,
-          n_ctx: config.n_ctx ?? null,
-          model: config.model ?? null
+    const key = CactusVLM.getCacheKey(params, cactusToken, retryOptions);
+    if (CactusVLM._initCache.has(key)) {
+      return CactusVLM._initCache.get(key);
+    }
+    const initPromise = (async () => {
+      const maxRetries = retryOptions?.maxRetries ?? 3;
+      const delayMs = retryOptions?.delayMs ?? 1000;
+      const configs = [params, {
+        ...params,
+        n_gpu_layers: 0
+      }];
+      const sleep = ms => {
+        return new Promise(resolve => {
+          const start = Date.now();
+          const wait = () => {
+            if (Date.now() - start >= ms) {
+              resolve();
+            } else {
+              Promise.resolve().then(wait);
+            }
+          };
+          wait();
         });
-        if (configs.indexOf(config) === configs.length - 1) {
+      };
+      for (const config of configs) {
+        let lastError = null;
+        for (let attempt = 1; attempt <= maxRetries; attempt++) {
+          try {
+            const context = await (0, _index.initLlama)(config, onProgress);
+            await (0, _index.initMultimodal)(context.id, params.mmproj, false);
+            return {
+              vlm: new CactusVLM(context),
+              error: null
+            };
+          } catch (e) {
+            lastError = e;
+            const isLastConfig = configs.indexOf(config) === configs.length - 1;
+            const isLastAttempt = attempt === maxRetries;
+            _telemetry.Telemetry.error(e, {
+              n_gpu_layers: config.n_gpu_layers ?? null,
+              n_ctx: config.n_ctx ?? null,
+              model: config.model ?? null
+            });
+            if (!isLastAttempt) {
+              const delay = delayMs * Math.pow(2, attempt - 1);
+              await sleep(delay);
+            } else if (!isLastConfig) {
+              break;
+            }
+          }
+        }
+        if (configs.indexOf(config) === configs.length - 1 && lastError) {
           return {
             vlm: null,
-            error: e
+            error: lastError
           };
         }
       }
+      return {
+        vlm: null,
+        error: new Error('Failed to initialize CactusVLM')
+      };
+    })();
+    CactusVLM._initCache.set(key, initPromise);
+    const result = await initPromise;
+    if (result.error) {
+      CactusVLM._initCache.delete(key);
     }
-    return {
-      vlm: null,
-      error: new Error('Failed to initialize CactusVLM')
-    };
+    return result;
   }
   async completion(messages, params = {}, callback) {
     const mode = params.mode || 'local';
@@ -163,6 +211,9 @@ class CactusVLM {
   async release() {
     return this.context.release();
   }
+  async stopCompletion() {
+    return await this.context.stopCompletion();
+  }
 }
 exports.CactusVLM = CactusVLM;
 //# sourceMappingURL=vlm.js.map

package/lib/commonjs/vlm.js.map

@@ -1 +1 @@
-{"version":3,"names":["_index","require","_telemetry","_remote","_chat","CactusVLM","constructor","context","conversationHistoryManager","ConversationHistoryManager","init","params","onProgress","cactusToken","setCactusToken","configs","n_gpu_layers","config","initLlama","initMultimodal","id","mmproj","vlm","error","e","Telemetry","n_ctx","model","indexOf","length","Error","completion","messages","callback","mode","result","lastError","_handleRemoteCompletion","_handleLocalCompletion","remoteError","localError","newMessages","requiresReset","processNewMessages","rewind","reset","console","warn","images","formattedPrompt","getFormattedChat","prompt","multimodalCompletion","emit_partial_completion","update","role","content","text","map","m","join","imagePath","responseText","getVisionCompletion","getTextCompletion","i","token","reasoning_content","tool_calls","tokens_predicted","split","tokens_evaluated","truncated","stopped_eos","stopped_word","stopped_limit","stopping_word","tokens_cached","timings","prompt_n","prompt_ms","prompt_per_token_ms","prompt_per_second","predicted_n","predicted_ms","predicted_per_token_ms","predicted_per_second","release","exports"],"sourceRoot":"../../src","sources":["vlm.ts"],"mappings":";;;;;;AAAA,IAAAA,MAAA,GAAAC,OAAA;AAWA,IAAAC,UAAA,GAAAD,OAAA;AACA,IAAAE,OAAA,GAAAF,OAAA;AACA,IAAAG,KAAA,GAAAH,OAAA;AAgBO,MAAMI,SAAS,CAAC;EAIbC,WAAWA,CAACC,OAAqB,EAAE;IACzC,IAAI,CAACA,OAAO,GAAGA,OAAO;IACtB,IAAI,CAACC,0BAA0B,GAAG,IAAIC,gCAA0B,CAAC,CAAC;EACpE;EAEA,aAAaC,IAAIA,CACfC,MAAwB,EACxBC,UAAuC,EACvCC,WAAoB,EACM;IAC1B,IAAIA,WAAW,EAAE;MACf,IAAAC,sBAAc,EAACD,WAAW,CAAC;IAC7B;IAEA,MAAME,OAAO,GAAG,CACdJ,MAAM,EACN;MAAE,GAAGA,MAAM;MAAEK,YAAY,EAAE;IAAE,CAAC,CAC/B;IAED,KAAK,MAAMC,MAAM,IAAIF,OAAO,EAAE;MAC5B,IAAI;QACF,MAAMR,OAAO,GAAG,MAAM,IAAAW,gBAAS,EAACD,MAAM,EAAEL,UAAU,CAAC;QACnD,MAAM,IAAAO,qBAAc,EAACZ,OAAO,CAACa,EAAE,EAAET,MAAM,CAACU,MAAM,EAAE,KAAK,CAAC;QACtD,OAAO;UAACC,GAAG,EAAE,IAAIjB,SAAS,CAACE,OAAO,CAAC;UAAEgB,KAAK,EAAE;QAAI,CAAC;MACnD,CAAC,CAAC,OAAOC,CAAC,EAAE;QACVC,oBAAS,CAACF,KAAK,CAACC,CAAC,EAAW;UAC1BR,YAAY,EAAEC,MAAM,CAACD,YAAY,IAAI,IAAI;UACzCU,KAAK,EAAET,MAAM,CAACS,KAAK,IAAI,IAAI;UAC3BC,KAAK,EAAEV,MAAM,CAACU,KAAK,IAAI;QACzB,CAAC,CAAC;QACF,IAAIZ,OAAO,CAACa,OAAO,CAACX,MAAM,CAAC,KAAKF,OAAO,CAACc,MAAM,GAAG,CAAC,EAAE;UAClD,OAAO;YAACP,GAAG,EAAE,IAAI;YAAEC,KAAK,EAAEC;UAAU,CAAC;QACvC;MACF;IACF;IAEA,OAAO;MAACF,GAAG,EAAE,IAAI;MAAEC,KAAK,EAAE,IAAIO,KAAK,CAAC,gCAAgC;IAAC,CAAC;EACxE;EAEA,MAAMC,UAAUA,CACdC,QAAsC,EACtCrB,MAA2B,GAAG,CAAC,CAAC,EAChCsB,QAA8B,EACG;IACjC,MAAMC,IAAI,GAAGvB,MAAM,CAACuB,IAAI,IAAI,OAAO;IAEnC,IAAIC,MAA8B;IAClC,IAAIC,SAAuB,GAAG,IAAI;IAElC,IAAIF,IAAI,KAAK,QAAQ,EAAE;MACrBC,MAAM,GAAG,MAAM,IAAI,CAACE,uBAAuB,CAACL,QAAQ,EAAErB,MAAM,EAAEsB,QAAQ,CAAC;IACzE,CAAC,MAAM,IAAIC,IAAI,KAAK,OAAO,EAAE;MAC3BC,MAAM,GAAG,MAAM,IAAI,CAACG,sBAAsB,CAACN,QAAQ,EAAErB,MAAM,EAAEsB,QAAQ,CAAC;IACxE,CAAC,MAAM,IAAIC,IAAI,KAAK,YAAY,EAAE;MAChC,IAAI;QACFC,MAAM,GAAG,MAAM,IAAI,CAACG,sBAAsB,CAACN,QAAQ,EAAErB,MAAM,EAAEsB,QAAQ,CAAC;MACxE,CAAC,CAAC,OAAOT,CAAC,EAAE;QACVY,SAAS,GAAGZ,CAAU;QACtB,IAAI;UACFW,MAAM,GAAG,MAAM,IAAI,CAACE,uBAAuB,CAACL,QAAQ,EAAErB,MAAM,EAAEsB,QAAQ,CAAC;QACzE,CAAC,CAAC,OAAOM,WAAW,EAAE;UACpB,MAAMH,SAAS;QACjB;MACF;IACF,CAAC,MAAM,IAAIF,IAAI,KAAK,aAAa,EAAE;MACjC,IAAI;QACFC,MAAM,GAAG,MAAM,IAAI,CAACE,uBAAuB,CAACL,QAAQ,EAAErB,MAAM,EAAEsB,QAAQ,CAAC;MACzE,CAAC,CAAC,OAAOT,CAAC,EAAE;QACVY,SAAS,GAAGZ,CAAU;QACtB,IAAI;UACFW,MAAM,GAAG,MAAM,IAAI,CAACG,sBAAsB,CAACN,QAAQ,EAAErB,MAAM,EAAEsB,QAAQ,CAAC;QACxE,CAAC,CAAC,OAAOO,UAAU,EAAE;UACnB,MAAMJ,SAAS;QACjB;MACF;IACF,CAAC,MAAM;MACL,MAAM,IAAIN,KAAK,CAAC,gBAAgB,GAAGI,IAAI,GAAG,6DAA6D,CAAC;IAC1G;IAEA,OAAOC,MAAM;EACf;EAEQG,sBAAsB,GAAG,MAAAA,CAC/BN,QAAsC,EACtCrB,MAA2B,EAC3BsB,QAA8B,KACM;IACpC,MAAM;MAAEQ,WAAW;MAAEC;IAAc,CAAC,GAClC,IAAI,CAAClC,0BAA0B,CAACmC,kBAAkB,CAACX,QAAQ,CAAC;IAE9D,IAAIU,aAAa,EAAE;MACjB,IAAI,CAACnC,OAAO,EAAEqC,MAAM,CAAC,CAAC;MACtB,IAAI,CAACpC,0BAA0B,CAACqC,KAAK,CAAC,CAAC;IACzC;IAEA,IAAIJ,WAAW,CAACZ,MAAM,KAAK,CAAC,EAAE;MAC5BiB,OAAO,CAACC,IAAI,CAAC,0BAA0B,CAAC;IAC1C;IAEA,IAAIZ,MAA8B;IAElC,IAAIxB,MAAM,CAACqC,MAAM,IAAIrC,MAAM,CAACqC,MAAM,CAACnB,MAAM,GAAG,CAAC,EAAE;MAC7C,MAAMoB,eAAe,GAAG,MAAM,IAAI,CAAC1C,OAAO,CAAC2C,gBAAgB,CAACT,WAAW,CAAC;MACxE,MAAMU,MAAM,GACV,OAAOF,eAAe,KAAK,QAAQ,GAC/BA,eAAe,GACfA,eAAe,CAACE,MAAM;MAC5BhB,MAAM,GAAG,MAAM,IAAAiB,2BAAoB,EACjC,IAAI,CAAC7C,OAAO,CAACa,EAAE,EACf+B,MAAM,EACNxC,MAAM,CAACqC,MAAM,EACb;QAAE,GAAGrC,MAAM;QAAEwC,MAAM;QAAEE,uBAAuB,EAAE,CAAC,CAACpB;MAAS,CAC3D,CAAC;IACH,CAAC,MAAM;MACLE,MAAM,GAAG,MAAM,IAAI,CAAC5B,OAAO,CAACwB,UAAU,CAAC;QAAEC,QAAQ,EAAES,WAAW;QAAE,GAAG9B;MAAO,CAAC,EAAEsB,QAAQ,CAAC;IACxF;IAEA,IAAI,CAACzB,0BAA0B,CAAC8C,MAAM,CAACb,WAAW,EAAE;MAClDc,IAAI,EAAE,WAAW;MACjBC,OAAO,EAAErB,MAAM,CAACqB,OAAO,IAAIrB,MAAM,CAACsB;IACpC,CAAC,CAAC;IAEF,OAAOtB,MAAM;EACf,CAAC;EAED,MAAcE,uBAAuBA,CACnCL,QAAsC,EACtCrB,MAA2B,EAC3BsB,QAA8B,EACG;IACjC,MAAMkB,MAAM,GAAGnB,QAAQ,CAAC0B,GAAG,CAAEC,CAAC,IAAK,GAAGA,CAAC,CAACJ,IAAI,KAAKI,CAAC,CAACH,OAAO,EAAE,CAAC,CAACI,IAAI,CAAC,IAAI,CAAC;IACxE,MAAMC,SAAS,GAAGlD,MAAM,CAACqC,MAAM,IAAIrC,MAAM,CAACqC,MAAM,CAACnB,MAAM,GAAG,CAAC,GAAGlB,MAAM,CAACqC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE;IAEnF,IAAIc,YAAoB;IACxB,IAAID,SAAS,EAAE;MACbC,YAAY,GAAG,MAAM,IAAAC,2BAAmB,EAACZ,MAAM,EAAEU,SAAS,CAAC;IAC7D,CAAC,MAAM;MACLC,YAAY,GAAG,MAAM,IAAAE,yBAAiB,EAACb,MAAM,CAAC;IAChD;IAEA,IAAIlB,QAAQ,EAAE;MACZ,KAAK,IAAIgC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGH,YAAY,CAACjC,MAAM,EAAEoC,CAAC,EAAE,EAAE;QAC5ChC,QAAQ,CAAC;UAAEiC,KAAK,EAAEJ,YAAY,CAACG,CAAC;QAAE,CAAC,CAAC;MACtC;IACF;IAEA,OAAO;MACLR,IAAI,EAAEK,YAAY;MAClBK,iBAAiB,EAAE,EAAE;MACrBC,UAAU,EAAE,EAAE;MACdZ,OAAO,EAAEM,YAAY;MACrBO,gBAAgB,EAAEP,YAAY,CAACQ,KAAK,CAAC,GAAG,CAAC,CAACzC,MAAM;MAChD0C,gBAAgB,EAAEpB,MAAM,CAACmB,KAAK,CAAC,GAAG,CAAC,CAACzC,MAAM;MAC1C2C,SAAS,EAAE,KAAK;MAChBC,WAAW,EAAE,IAAI;MACjBC,YAAY,EAAE,EAAE;MAChBC,aAAa,EAAE,CAAC;MAChBC,aAAa,EAAE,EAAE;MACjBC,aAAa,EAAE,CAAC;MAChBC,OAAO,EAAE;QACPC,QAAQ,EAAE5B,MAAM,CAACmB,KAAK,CAAC,GAAG,CAAC,CAACzC,MAAM;QAClCmD,SAAS,EAAE,CAAC;QACZC,mBAAmB,EAAE,CAAC;QACtBC,iBAAiB,EAAE,CAAC;QACpBC,WAAW,EAAErB,YAAY,CAACQ,KAAK,CAAC,GAAG,CAAC,CAACzC,MAAM;QAC3CuD,YAAY,EAAE,CAAC;QACfC,sBAAsB,EAAE,CAAC;QACzBC,oBAAoB,EAAE;MACxB;IACF,CAAC;EACH;EAEA,MAAM1C,MAAMA,CAAA,EAAkB;IAC5B,OAAO,IAAI,CAACrC,OAAO,EAAEqC,MAAM,CAAC,CAAC;EAC/B;EAEA,MAAM2C,OAAOA,CAAA,EAAkB;IAC7B,OAAO,IAAI,CAAChF,OAAO,CAACgF,OAAO,CAAC,CAAC;EAC/B;AACF;AAACC,OAAA,CAAAnF,SAAA,GAAAA,SAAA","ignoreList":[]}
+{"version":3,"names":["_index","require","_telemetry","_remote","_chat","CactusVLM","_initCache","Map","getCacheKey","params","cactusToken","retryOptions","JSON","stringify","constructor","context","conversationHistoryManager","ConversationHistoryManager","init","onProgress","setCactusToken","key","has","get","initPromise","maxRetries","delayMs","configs","n_gpu_layers","sleep","ms","Promise","resolve","start","Date","now","wait","then","config","lastError","attempt","initLlama","initMultimodal","id","mmproj","vlm","error","e","isLastConfig","indexOf","length","isLastAttempt","Telemetry","n_ctx","model","delay","Math","pow","Error","set","result","delete","completion","messages","callback","mode","_handleRemoteCompletion","_handleLocalCompletion","remoteError","localError","newMessages","requiresReset","processNewMessages","rewind","reset","console","warn","images","formattedPrompt","getFormattedChat","prompt","multimodalCompletion","emit_partial_completion","update","role","content","text","map","m","join","imagePath","responseText","getVisionCompletion","getTextCompletion","i","token","reasoning_content","tool_calls","tokens_predicted","split","tokens_evaluated","truncated","stopped_eos","stopped_word","stopped_limit","stopping_word","tokens_cached","timings","prompt_n","prompt_ms","prompt_per_token_ms","prompt_per_second","predicted_n","predicted_ms","predicted_per_token_ms","predicted_per_second","release","stopCompletion","exports"],"sourceRoot":"../../src","sources":["vlm.ts"],"mappings":";;;;;;AAAA,IAAAA,MAAA,GAAAC,OAAA;AAWA,IAAAC,UAAA,GAAAD,OAAA;AACA,IAAAE,OAAA,GAAAF,OAAA;AACA,IAAAG,KAAA,GAAAH,OAAA;AAgBO,MAAMI,SAAS,CAAC;EAIrB;EACA,OAAeC,UAAU,GAA0C,IAAIC,GAAG,CAAC,CAAC;EAE5E,OAAeC,WAAWA,CAACC,MAAwB,EAAEC,WAAoB,EAAEC,YAAwD,EAAU;IAC3I,OAAOC,IAAI,CAACC,SAAS,CAAC;MAAEJ,MAAM;MAAEC,WAAW;MAAEC;IAAa,CAAC,CAAC;EAC9D;EAEQG,WAAWA,CAACC,OAAqB,EAAE;IACzC,IAAI,CAACA,OAAO,GAAGA,OAAO;IACtB,IAAI,CAACC,0BAA0B,GAAG,IAAIC,gCAA0B,CAAC,CAAC;EACpE;EAEA,aAAaC,IAAIA,CACfT,MAAwB,EACxBU,UAAuC,EACvCT,WAAoB,EACpBC,YAAwD,EAC9B;IAC1B,IAAID,WAAW,EAAE;MACf,IAAAU,sBAAc,EAACV,WAAW,CAAC;IAC7B;IAEA,MAAMW,GAAG,GAAGhB,SAAS,CAACG,WAAW,CAACC,MAAM,EAAEC,WAAW,EAAEC,YAAY,CAAC;IACpE,IAAIN,SAAS,CAACC,UAAU,CAACgB,GAAG,CAACD,GAAG,CAAC,EAAE;MACjC,OAAOhB,SAAS,CAACC,UAAU,CAACiB,GAAG,CAACF,GAAG,CAAC;IACtC;IAEA,MAAMG,WAAW,GAAG,CAAC,YAAY;MAC/B,MAAMC,UAAU,GAAGd,YAAY,EAAEc,UAAU,IAAI,CAAC;MAChD,MAAMC,OAAO,GAAGf,YAAY,EAAEe,OAAO,IAAI,IAAI;MAE7C,MAAMC,OAAO,GAAG,CACdlB,MAAM,EACN;QAAE,GAAGA,MAAM;QAAEmB,YAAY,EAAE;MAAE,CAAC,CAC/B;MAED,MAAMC,KAAK,GAAIC,EAAU,IAAoB;QAC3C,OAAO,IAAIC,OAAO,CAACC,OAAO,IAAI;UAC5B,MAAMC,KAAK,GAAGC,IAAI,CAACC,GAAG,CAAC,CAAC;UACxB,MAAMC,IAAI,GAAGA,CAAA,KAAM;YACjB,IAAIF,IAAI,CAACC,GAAG,CAAC,CAAC,GAAGF,KAAK,IAAIH,EAAE,EAAE;cAC5BE,OAAO,CAAC,CAAC;YACX,CAAC,MAAM;cACLD,OAAO,CAACC,OAAO,CAAC,CAAC,CAACK,IAAI,CAACD,IAAI,CAAC;YAC9B;UACF,CAAC;UACDA,IAAI,CAAC,CAAC;QACR,CAAC,CAAC;MACJ,CAAC;MAED,KAAK,MAAME,MAAM,IAAIX,OAAO,EAAE;QAC5B,IAAIY,SAAuB,GAAG,IAAI;QAElC,KAAK,IAAIC,OAAO,GAAG,CAAC,EAAEA,OAAO,IAAIf,UAAU,EAAEe,OAAO,EAAE,EAAE;UACtD,IAAI;YACF,MAAMzB,OAAO,GAAG,MAAM,IAAA0B,gBAAS,EAACH,MAAM,EAAEnB,UAAU,CAAC;YACnD,MAAM,IAAAuB,qBAAc,EAAC3B,OAAO,CAAC4B,EAAE,EAAElC,MAAM,CAACmC,MAAM,EAAE,KAAK,CAAC;YACtD,OAAO;cAACC,GAAG,EAAE,IAAIxC,SAAS,CAACU,OAAO,CAAC;cAAE+B,KAAK,EAAE;YAAI,CAAC;UACnD,CAAC,CAAC,OAAOC,CAAC,EAAE;YACVR,SAAS,GAAGQ,CAAU;YACtB,MAAMC,YAAY,GAAGrB,OAAO,CAACsB,OAAO,CAACX,MAAM,CAAC,KAAKX,OAAO,CAACuB,MAAM,GAAG,CAAC;YACnE,MAAMC,aAAa,GAAGX,OAAO,KAAKf,UAAU;YAE5C2B,oBAAS,CAACN,KAAK,CAACC,CAAC,EAAW;cAC1BnB,YAAY,EAAEU,MAAM,CAACV,YAAY,IAAI,IAAI;cACzCyB,KAAK,EAAEf,MAAM,CAACe,KAAK,IAAI,IAAI;cAC3BC,KAAK,EAAEhB,MAAM,CAACgB,KAAK,IAAI;YACzB,CAAC,CAAC;YAEF,IAAI,CAACH,aAAa,EAAE;cAClB,MAAMI,KAAK,GAAG7B,OAAO,GAAG8B,IAAI,CAACC,GAAG,CAAC,CAAC,EAAEjB,OAAO,GAAG,CAAC,CAAC;cAChD,MAAMX,KAAK,CAAC0B,KAAK,CAAC;YACpB,CAAC,MAAM,IAAI,CAACP,YAAY,EAAE;cACxB;YACF;UACF;QACF;QAEA,IAAIrB,OAAO,CAACsB,OAAO,CAACX,MAAM,CAAC,KAAKX,OAAO,CAACuB,MAAM,GAAG,CAAC,IAAIX,SAAS,EAAE;UAC/D,OAAO;YAACM,GAAG,EAAE,IAAI;YAAEC,KAAK,EAAEP;UAAS,CAAC;QACtC;MACF;MAEA,OAAO;QAACM,GAAG,EAAE,IAAI;QAAEC,KAAK,EAAE,IAAIY,KAAK,CAAC,gCAAgC;MAAC,CAAC;IACxE,CAAC,EAAE,CAAC;IAEJrD,SAAS,CAACC,UAAU,CAACqD,GAAG,CAACtC,GAAG,EAAEG,WAAW,CAAC;IAE1C,MAAMoC,MAAM,GAAG,MAAMpC,WAAW;IAChC,IAAIoC,MAAM,CAACd,KAAK,EAAE;MAChBzC,SAAS,CAACC,UAAU,CAACuD,MAAM,CAACxC,GAAG,CAAC;IAClC;IACA,OAAOuC,MAAM;EACf;EAEA,MAAME,UAAUA,CACdC,QAAsC,EACtCtD,MAA2B,GAAG,CAAC,CAAC,EAChCuD,QAA8B,EACG;IACjC,MAAMC,IAAI,GAAGxD,MAAM,CAACwD,IAAI,IAAI,OAAO;IAEnC,IAAIL,MAA8B;IAClC,IAAIrB,SAAuB,GAAG,IAAI;IAElC,IAAI0B,IAAI,KAAK,QAAQ,EAAE;MACrBL,MAAM,GAAG,MAAM,IAAI,CAACM,uBAAuB,CAACH,QAAQ,EAAEtD,MAAM,EAAEuD,QAAQ,CAAC;IACzE,CAAC,MAAM,IAAIC,IAAI,KAAK,OAAO,EAAE;MAC3BL,MAAM,GAAG,MAAM,IAAI,CAACO,sBAAsB,CAACJ,QAAQ,EAAEtD,MAAM,EAAEuD,QAAQ,CAAC;IACxE,CAAC,MAAM,IAAIC,IAAI,KAAK,YAAY,EAAE;MAChC,IAAI;QACFL,MAAM,GAAG,MAAM,IAAI,CAACO,sBAAsB,CAACJ,QAAQ,EAAEtD,MAAM,EAAEuD,QAAQ,CAAC;MACxE,CAAC,CAAC,OAAOjB,CAAC,EAAE;QACVR,SAAS,GAAGQ,CAAU;QACtB,IAAI;UACFa,MAAM,GAAG,MAAM,IAAI,CAACM,uBAAuB,CAACH,QAAQ,EAAEtD,MAAM,EAAEuD,QAAQ,CAAC;QACzE,CAAC,CAAC,OAAOI,WAAW,EAAE;UACpB,MAAM7B,SAAS;QACjB;MACF;IACF,CAAC,MAAM,IAAI0B,IAAI,KAAK,aAAa,EAAE;MACjC,IAAI;QACFL,MAAM,GAAG,MAAM,IAAI,CAACM,uBAAuB,CAACH,QAAQ,EAAEtD,MAAM,EAAEuD,QAAQ,CAAC;MACzE,CAAC,CAAC,OAAOjB,CAAC,EAAE;QACVR,SAAS,GAAGQ,CAAU;QACtB,IAAI;UACFa,MAAM,GAAG,MAAM,IAAI,CAACO,sBAAsB,CAACJ,QAAQ,EAAEtD,MAAM,EAAEuD,QAAQ,CAAC;QACxE,CAAC,CAAC,OAAOK,UAAU,EAAE;UACnB,MAAM9B,SAAS;QACjB;MACF;IACF,CAAC,MAAM;MACL,MAAM,IAAImB,KAAK,CAAC,gBAAgB,GAAGO,IAAI,GAAG,6DAA6D,CAAC;IAC1G;IAEA,OAAOL,MAAM;EACf;EAEQO,sBAAsB,GAAG,MAAAA,CAC/BJ,QAAsC,EACtCtD,MAA2B,EAC3BuD,QAA8B,KACM;IACpC,MAAM;MAAEM,WAAW;MAAEC;IAAc,CAAC,GAClC,IAAI,CAACvD,0BAA0B,CAACwD,kBAAkB,CAACT,QAAQ,CAAC;IAE9D,IAAIQ,aAAa,EAAE;MACjB,IAAI,CAACxD,OAAO,EAAE0D,MAAM,CAAC,CAAC;MACtB,IAAI,CAACzD,0BAA0B,CAAC0D,KAAK,CAAC,CAAC;IACzC;IAEA,IAAIJ,WAAW,CAACpB,MAAM,KAAK,CAAC,EAAE;MAC5ByB,OAAO,CAACC,IAAI,CAAC,0BAA0B,CAAC;IAC1C;IAEA,IAAIhB,MAA8B;IAElC,IAAInD,MAAM,CAACoE,MAAM,IAAIpE,MAAM,CAACoE,MAAM,CAAC3B,MAAM,GAAG,CAAC,EAAE;MAC7C,MAAM4B,eAAe,GAAG,MAAM,IAAI,CAAC/D,OAAO,CAACgE,gBAAgB,CAACT,WAAW,CAAC;MACxE,MAAMU,MAAM,GACV,OAAOF,eAAe,KAAK,QAAQ,GAC/BA,eAAe,GACfA,eAAe,CAACE,MAAM;MAC5BpB,MAAM,GAAG,MAAM,IAAAqB,2BAAoB,EACjC,IAAI,CAAClE,OAAO,CAAC4B,EAAE,EACfqC,MAAM,EACNvE,MAAM,CAACoE,MAAM,EACb;QAAE,GAAGpE,MAAM;QAAEuE,MAAM;QAAEE,uBAAuB,EAAE,CAAC,CAAClB;MAAS,CAC3D,CAAC;IACH,CAAC,MAAM;MACLJ,MAAM,GAAG,MAAM,IAAI,CAAC7C,OAAO,CAAC+C,UAAU,CAAC;QAAEC,QAAQ,EAAEO,WAAW;QAAE,GAAG7D;MAAO,CAAC,EAAEuD,QAAQ,CAAC;IACxF;IAEA,IAAI,CAAChD,0BAA0B,CAACmE,MAAM,CAACb,WAAW,EAAE;MAClDc,IAAI,EAAE,WAAW;MACjBC,OAAO,EAAEzB,MAAM,CAACyB,OAAO,IAAIzB,MAAM,CAAC0B;IACpC,CAAC,CAAC;IAEF,OAAO1B,MAAM;EACf,CAAC;EAED,MAAcM,uBAAuBA,CACnCH,QAAsC,EACtCtD,MAA2B,EAC3BuD,QAA8B,EACG;IACjC,MAAMgB,MAAM,GAAGjB,QAAQ,CAACwB,GAAG,CAAEC,CAAC,IAAK,GAAGA,CAAC,CAACJ,IAAI,KAAKI,CAAC,CAACH,OAAO,EAAE,CAAC,CAACI,IAAI,CAAC,IAAI,CAAC;IACxE,MAAMC,SAAS,GAAGjF,MAAM,CAACoE,MAAM,IAAIpE,MAAM,CAACoE,MAAM,CAAC3B,MAAM,GAAG,CAAC,GAAGzC,MAAM,CAACoE,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE;IAEnF,IAAIc,YAAoB;IACxB,IAAID,SAAS,EAAE;MACbC,YAAY,GAAG,MAAM,IAAAC,2BAAmB,EAACZ,MAAM,EAAEU,SAAS,CAAC;IAC7D,CAAC,MAAM;MACLC,YAAY,GAAG,MAAM,IAAAE,yBAAiB,EAACb,MAAM,CAAC;IAChD;IAEA,IAAIhB,QAAQ,EAAE;MACZ,KAAK,IAAI8B,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGH,YAAY,CAACzC,MAAM,EAAE4C,CAAC,EAAE,EAAE;QAC5C9B,QAAQ,CAAC;UAAE+B,KAAK,EAAEJ,YAAY,CAACG,CAAC;QAAE,CAAC,CAAC;MACtC;IACF;IAEA,OAAO;MACLR,IAAI,EAAEK,YAAY;MAClBK,iBAAiB,EAAE,EAAE;MACrBC,UAAU,EAAE,EAAE;MACdZ,OAAO,EAAEM,YAAY;MACrBO,gBAAgB,EAAEP,YAAY,CAACQ,KAAK,CAAC,GAAG,CAAC,CAACjD,MAAM;MAChDkD,gBAAgB,EAAEpB,MAAM,CAACmB,KAAK,CAAC,GAAG,CAAC,CAACjD,MAAM;MAC1CmD,SAAS,EAAE,KAAK;MAChBC,WAAW,EAAE,IAAI;MACjBC,YAAY,EAAE,EAAE;MAChBC,aAAa,EAAE,CAAC;MAChBC,aAAa,EAAE,EAAE;MACjBC,aAAa,EAAE,CAAC;MAChBC,OAAO,EAAE;QACPC,QAAQ,EAAE5B,MAAM,CAACmB,KAAK,CAAC,GAAG,CAAC,CAACjD,MAAM;QAClC2D,SAAS,EAAE,CAAC;QACZC,mBAAmB,EAAE,CAAC;QACtBC,iBAAiB,EAAE,CAAC;QACpBC,WAAW,EAAErB,YAAY,CAACQ,KAAK,CAAC,GAAG,CAAC,CAACjD,MAAM;QAC3C+D,YAAY,EAAE,CAAC;QACfC,sBAAsB,EAAE,CAAC;QACzBC,oBAAoB,EAAE;MACxB;IACF,CAAC;EACH;EAEA,MAAM1C,MAAMA,CAAA,EAAkB;IAC5B,OAAO,IAAI,CAAC1D,OAAO,EAAE0D,MAAM,CAAC,CAAC;EAC/B;EAEA,MAAM2C,OAAOA,CAAA,EAAkB;IAC7B,OAAO,IAAI,CAACrG,OAAO,CAACqG,OAAO,CAAC,CAAC;EAC/B;EAEA,MAAMC,cAAcA,CAAA,EAAkB;IACpC,OAAO,MAAM,IAAI,CAACtG,OAAO,CAACsG,cAAc,CAAC,CAAC;EAC5C;AACF;AAACC,OAAA,CAAAjH,SAAA,GAAAA,SAAA","ignoreList":[]}

package/lib/module/chat.js

@@ -60,5 +60,8 @@ export class ConversationHistoryManager {
   reset() {
     this.history = [];
   }
+  getMessages() {
+    return this.history;
+  }
 }
 //# sourceMappingURL=chat.js.map

package/lib/module/chat.js.map

@@ -1 +1 @@
-{"version":3,"names":["formatChat","messages","chat","forEach","currMsg","role","content","Array","isArray","part","text","TypeError","Error","push","ConversationHistoryManager","history","processNewMessages","fullMessageHistory","divergent","length","i","JSON","stringify","newMessages","requiresReset","slice","update","assistantResponse","reset"],"sourceRoot":"../../src","sources":["chat.ts"],"mappings":";;AAWA,OAAO,SAASA,UAAUA,CACxBC,QAAsC,EACZ;EAC1B,MAAMC,IAA8B,GAAG,EAAE;EAEzCD,QAAQ,CAACE,OAAO,CAAEC,OAAO,IAAK;IAC5B,MAAMC,IAAY,GAAGD,OAAO,CAACC,IAAI,IAAI,EAAE;IAEvC,IAAIC,OAAe,GAAG,EAAE;IACxB,IAAI,SAAS,IAAIF,OAAO,EAAE;MACxB,IAAI,OAAOA,OAAO,CAACE,OAAO,KAAK,QAAQ,EAAE;QACvC;QAAC,CAAC;UAAEA;QAAQ,CAAC,GAAGF,OAAO;MACzB,CAAC,MAAM,IAAIG,KAAK,CAACC,OAAO,CAACJ,OAAO,CAACE,OAAO,CAAC,EAAE;QACzCF,OAAO,CAACE,OAAO,CAACH,OAAO,CAAEM,IAAI,IAAK;UAChC,IAAI,MAAM,IAAIA,IAAI,EAAE;YAClBH,OAAO,IAAI,GAAGA,OAAO,GAAG,IAAI,GAAG,EAAE,GAAGG,IAAI,CAACC,IAAI,EAAE;UACjD;QACF,CAAC,CAAC;MACJ,CAAC,MAAM;QACL,MAAM,IAAIC,SAAS,CACjB,kFACF,CAAC;MACH;IACF,CAAC,MAAM;MACL,MAAM,IAAIC,KAAK,CACb,6EACF,CAAC;IACH;IAEAV,IAAI,CAACW,IAAI,CAAC;MAAER,IAAI;MAAEC;IAAQ,CAAC,CAAC;EAC9B,CAAC,CAAC;EACF,OAAOJ,IAAI;AACb;AAOA,OAAO,MAAMY,0BAA0B,CAAC;EAC9BC,OAAO,GAAiC,EAAE;EAE3CC,kBAAkBA,CACvBC,kBAAgD,EAC7B;IACnB,IAAIC,SAAS,GAAGD,kBAAkB,CAACE,MAAM,GAAG,IAAI,CAACJ,OAAO,CAACI,MAAM;IAC/D,IAAI,CAACD,SAAS,EAAE;MACd,KAAK,IAAIE,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG,IAAI,CAACL,OAAO,CAACI,MAAM,EAAEC,CAAC,EAAE,EAAE;QAC5C,IAAIC,IAAI,CAACC,SAAS,CAAC,IAAI,CAACP,OAAO,CAACK,CAAC,CAAC,CAAC,KAAKC,IAAI,CAACC,SAAS,CAACL,kBAAkB,CAACG,CAAC,CAAC,CAAC,EAAE;UAC7EF,SAAS,GAAG,IAAI;UAChB;QACF;MACF;IACF;IAEA,IAAIA,SAAS,EAAE;MACb,OAAO;QAAEK,WAAW,EAAEN,kBAAkB;QAAEO,aAAa,EAAE;MAAK,CAAC;IACjE;IAEA,MAAMD,WAAW,GAAGN,kBAAkB,CAACQ,KAAK,CAAC,IAAI,CAACV,OAAO,CAACI,MAAM,CAAC;IACjE,OAAO;MAAEI,WAAW;MAAEC,aAAa,EAAE;IAAM,CAAC;EAC9C;EAEOE,MAAMA,CACXH,WAAyC,EACzCI,iBAA6C,EAC7C;IACA,IAAI,CAACZ,OAAO,CAACF,IAAI,CAAC,GAAGU,WAAW,EAAEI,iBAAiB,CAAC;EACtD;EAEOC,KAAKA,CAAA,EAAG;IACb,IAAI,CAACb,OAAO,GAAG,EAAE;EACnB;AACF","ignoreList":[]}
+{"version":3,"names":["formatChat","messages","chat","forEach","currMsg","role","content","Array","isArray","part","text","TypeError","Error","push","ConversationHistoryManager","history","processNewMessages","fullMessageHistory","divergent","length","i","JSON","stringify","newMessages","requiresReset","slice","update","assistantResponse","reset","getMessages"],"sourceRoot":"../../src","sources":["chat.ts"],"mappings":";;AAWA,OAAO,SAASA,UAAUA,CACxBC,QAAsC,EACZ;EAC1B,MAAMC,IAA8B,GAAG,EAAE;EAEzCD,QAAQ,CAACE,OAAO,CAAEC,OAAO,IAAK;IAC5B,MAAMC,IAAY,GAAGD,OAAO,CAACC,IAAI,IAAI,EAAE;IAEvC,IAAIC,OAAe,GAAG,EAAE;IACxB,IAAI,SAAS,IAAIF,OAAO,EAAE;MACxB,IAAI,OAAOA,OAAO,CAACE,OAAO,KAAK,QAAQ,EAAE;QACvC;QAAC,CAAC;UAAEA;QAAQ,CAAC,GAAGF,OAAO;MACzB,CAAC,MAAM,IAAIG,KAAK,CAACC,OAAO,CAACJ,OAAO,CAACE,OAAO,CAAC,EAAE;QACzCF,OAAO,CAACE,OAAO,CAACH,OAAO,CAAEM,IAAI,IAAK;UAChC,IAAI,MAAM,IAAIA,IAAI,EAAE;YAClBH,OAAO,IAAI,GAAGA,OAAO,GAAG,IAAI,GAAG,EAAE,GAAGG,IAAI,CAACC,IAAI,EAAE;UACjD;QACF,CAAC,CAAC;MACJ,CAAC,MAAM;QACL,MAAM,IAAIC,SAAS,CACjB,kFACF,CAAC;MACH;IACF,CAAC,MAAM;MACL,MAAM,IAAIC,KAAK,CACb,6EACF,CAAC;IACH;IAEAV,IAAI,CAACW,IAAI,CAAC;MAAER,IAAI;MAAEC;IAAQ,CAAC,CAAC;EAC9B,CAAC,CAAC;EACF,OAAOJ,IAAI;AACb;AAOA,OAAO,MAAMY,0BAA0B,CAAC;EAC9BC,OAAO,GAAiC,EAAE;EAE3CC,kBAAkBA,CACvBC,kBAAgD,EAC7B;IACnB,IAAIC,SAAS,GAAGD,kBAAkB,CAACE,MAAM,GAAG,IAAI,CAACJ,OAAO,CAACI,MAAM;IAC/D,IAAI,CAACD,SAAS,EAAE;MACd,KAAK,IAAIE,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG,IAAI,CAACL,OAAO,CAACI,MAAM,EAAEC,CAAC,EAAE,EAAE;QAC5C,IAAIC,IAAI,CAACC,SAAS,CAAC,IAAI,CAACP,OAAO,CAACK,CAAC,CAAC,CAAC,KAAKC,IAAI,CAACC,SAAS,CAACL,kBAAkB,CAACG,CAAC,CAAC,CAAC,EAAE;UAC7EF,SAAS,GAAG,IAAI;UAChB;QACF;MACF;IACF;IAEA,IAAIA,SAAS,EAAE;MACb,OAAO;QAAEK,WAAW,EAAEN,kBAAkB;QAAEO,aAAa,EAAE;MAAK,CAAC;IACjE;IAEA,MAAMD,WAAW,GAAGN,kBAAkB,CAACQ,KAAK,CAAC,IAAI,CAACV,OAAO,CAACI,MAAM,CAAC;IACjE,OAAO;MAAEI,WAAW;MAAEC,aAAa,EAAE;IAAM,CAAC;EAC9C;EAEOE,MAAMA,CACXH,WAAyC,EACzCI,iBAA6C,EAC7C;IACA,IAAI,CAACZ,OAAO,CAACF,IAAI,CAAC,GAAGU,WAAW,EAAEI,iBAAiB,CAAC;EACtD;EAEOC,KAAKA,CAAA,EAAG;IACb,IAAI,CAACb,OAAO,GAAG,EAAE;EACnB;EAEOc,WAAWA,CAAA,EAAiC;IACjD,OAAO,IAAI,CAACd,OAAO;EACrB;AACF","ignoreList":[]}

package/lib/module/lm.js

@@ -7,6 +7,16 @@ import { Telemetry } from './telemetry';
 import { setCactusToken, getVertexAIEmbedding } from './remote';
 import { ConversationHistoryManager } from './chat';
 export class CactusLM {
+  // the initPromise enables a "async singleton" initialization pattern which
+  // protects against a race condition in the event of multiple init attempts
+  static _initCache = new Map();
+  static getCacheKey(params, cactusToken, retryOptions) {
+    return JSON.stringify({
+      params,
+      cactusToken,
+      retryOptions
+    });
+  }
   constructor(context) {
     this.context = context;
     this.conversationHistoryManager = new ConversationHistoryManager();
@@ -15,62 +25,75 @@ export class CactusLM {
     if (cactusToken) {
       setCactusToken(cactusToken);
     }
-    const maxRetries = retryOptions?.maxRetries ?? 3;
-    const delayMs = retryOptions?.delayMs ?? 1000;
-    const configs = [params, {
-      ...params,
-      n_gpu_layers: 0
-    }];
-    const sleep = ms => {
-      return new Promise(resolve => {
-        const start = Date.now();
-        const wait = () => {
-          if (Date.now() - start >= ms) {
-            resolve();
-          } else {
-            Promise.resolve().then(wait);
+    const key = CactusLM.getCacheKey(params, cactusToken, retryOptions);
+    if (CactusLM._initCache.has(key)) {
+      // concurrent initialization calls with the same params all get the same cached Promise
+      return CactusLM._initCache.get(key);
+    }
+    const initPromise = (async () => {
+      const maxRetries = retryOptions?.maxRetries ?? 3;
+      const delayMs = retryOptions?.delayMs ?? 1000;
+      const configs = [params, {
+        ...params,
+        n_gpu_layers: 0
+      }];
+      const sleep = ms => {
+        return new Promise(resolve => {
+          const start = Date.now();
+          const wait = () => {
+            if (Date.now() - start >= ms) {
+              resolve();
+            } else {
+              Promise.resolve().then(wait);
+            }
+          };
+          wait();
+        });
+      };
+      for (const config of configs) {
+        let lastError = null;
+        for (let attempt = 1; attempt <= maxRetries; attempt++) {
+          try {
+            const context = await initLlama(config, onProgress);
+            return {
+              lm: new CactusLM(context),
+              error: null
+            };
+          } catch (e) {
+            lastError = e;
+            const isLastConfig = configs.indexOf(config) === configs.length - 1;
+            const isLastAttempt = attempt === maxRetries;
+            Telemetry.error(e, {
+              n_gpu_layers: config.n_gpu_layers ?? null,
+              n_ctx: config.n_ctx ?? null,
+              model: config.model ?? null
+            });
+            if (!isLastAttempt) {
+              const delay = delayMs * Math.pow(2, attempt - 1);
+              await sleep(delay);
+            } else if (!isLastConfig) {
+              break;
+            }
           }
-        };
-        wait();
-      });
-    };
-    for (const config of configs) {
-      let lastError = null;
-      for (let attempt = 1; attempt <= maxRetries; attempt++) {
-        try {
-          const context = await initLlama(config, onProgress);
+        }
+        if (configs.indexOf(config) === configs.length - 1 && lastError) {
           return {
-            lm: new CactusLM(context),
-            error: null
+            lm: null,
+            error: lastError
           };
-        } catch (e) {
-          lastError = e;
-          const isLastConfig = configs.indexOf(config) === configs.length - 1;
-          const isLastAttempt = attempt === maxRetries;
-          Telemetry.error(e, {
-            n_gpu_layers: config.n_gpu_layers ?? null,
-            n_ctx: config.n_ctx ?? null,
-            model: config.model ?? null
-          });
-          if (!isLastAttempt) {
-            const delay = delayMs * Math.pow(2, attempt - 1);
-            await sleep(delay);
-          } else if (!isLastConfig) {
-            break;
-          }
         }
       }
-      if (configs.indexOf(config) === configs.length - 1 && lastError) {
-        return {
-          lm: null,
-          error: lastError
-        };
-      }
+      return {
+        lm: null,
+        error: new Error('Failed to initialize CactusLM after all retries')
+      };
+    })();
+    CactusLM._initCache.set(key, initPromise);
+    const result = await initPromise;
+    if (result.error) {
+      CactusLM._initCache.delete(key); // Reset on failure to allow retries
     }
-    return {
-      lm: null,
-      error: new Error('Failed to initialize CactusLM after all retries')
-    };
+    return result;
   }
   completion = async (messages, params = {}, callback) => {
     const {
@@ -143,5 +166,8 @@ export class CactusLM {
   async release() {
     return this.context.release();
   }
+  async stopCompletion() {
+    return await this.context.stopCompletion();
+  }
 }
 //# sourceMappingURL=lm.js.map

package/lib/module/lm.js.map

@@ -1 +1 @@
-{"version":3,"names":["initLlama","Telemetry","setCactusToken","getVertexAIEmbedding","ConversationHistoryManager","CactusLM","constructor","context","conversationHistoryManager","init","params","onProgress","cactusToken","retryOptions","maxRetries","delayMs","configs","n_gpu_layers","sleep","ms","Promise","resolve","start","Date","now","wait","then","config","lastError","attempt","lm","error","e","isLastConfig","indexOf","length","isLastAttempt","n_ctx","model","delay","Math","pow","Error","completion","messages","callback","newMessages","requiresReset","processNewMessages","rewind","reset","console","warn","result","update","role","content","embedding","text","mode","_handleRemoteEmbedding","_handleLocalEmbedding","remoteError","localError","embeddingValues","release"],"sourceRoot":"../../src","sources":["lm.ts"],"mappings":";;AAAA,SAASA,SAAS,QAAsB,SAAS;AACjD;;AAWA,SAASC,SAAS,QAAQ,aAAa;AACvC,SAASC,cAAc,EAAEC,oBAAoB,QAAQ,UAAU;AAC/D,SAASC,0BAA0B,QAAQ,QAAQ;AAOnD,OAAO,MAAMC,QAAQ,CAAC;EAIVC,WAAWA,CAACC,OAAqB,EAAE;IAC3C,IAAI,CAACA,OAAO,GAAGA,OAAO;IACtB,IAAI,CAACC,0BAA0B,GAAG,IAAIJ,0BAA0B,CAAC,CAAC;EACpE;EAEA,aAAaK,IAAIA,CACfC,MAAqB,EACrBC,UAAuC,EACvCC,WAAoB,EACpBC,YAAwD,EAC/B;IACzB,IAAID,WAAW,EAAE;MACfV,cAAc,CAACU,WAAW,CAAC;IAC7B;IAEA,MAAME,UAAU,GAAGD,YAAY,EAAEC,UAAU,IAAI,CAAC;IAChD,MAAMC,OAAO,GAAGF,YAAY,EAAEE,OAAO,IAAI,IAAI;IAE7C,MAAMC,OAAO,GAAG,CACdN,MAAM,EACN;MAAE,GAAGA,MAAM;MAAEO,YAAY,EAAE;IAAE,CAAC,CAC/B;IAED,MAAMC,KAAK,GAAIC,EAAU,IAAoB;MAC3C,OAAO,IAAIC,OAAO,CAACC,OAAO,IAAI;QAC5B,MAAMC,KAAK,GAAGC,IAAI,CAACC,GAAG,CAAC,CAAC;QACxB,MAAMC,IAAI,GAAGA,CAAA,KAAM;UACjB,IAAIF,IAAI,CAACC,GAAG,CAAC,CAAC,GAAGF,KAAK,IAAIH,EAAE,EAAE;YAC5BE,OAAO,CAAC,CAAC;UACX,CAAC,MAAM;YACLD,OAAO,CAACC,OAAO,CAAC,CAAC,CAACK,IAAI,CAACD,IAAI,CAAC;UAC9B;QACF,CAAC;QACDA,IAAI,CAAC,CAAC;MACR,CAAC,CAAC;IACJ,CAAC;IAED,KAAK,MAAME,MAAM,IAAIX,OAAO,EAAE;MAC5B,IAAIY,SAAuB,GAAG,IAAI;MAElC,KAAK,IAAIC,OAAO,GAAG,CAAC,EAAEA,OAAO,IAAIf,UAAU,EAAEe,OAAO,EAAE,EAAE;QACtD,IAAI;UACF,MAAMtB,OAAO,GAAG,MAAMP,SAAS,CAAC2B,MAAM,EAAEhB,UAAU,CAAC;UACnD,OAAO;YAAEmB,EAAE,EAAE,IAAIzB,QAAQ,CAACE,OAAO,CAAC;YAAEwB,KAAK,EAAE;UAAK,CAAC;QACnD,CAAC,CAAC,OAAOC,CAAC,EAAE;UACVJ,SAAS,GAAGI,CAAU;UACtB,MAAMC,YAAY,GAAGjB,OAAO,CAACkB,OAAO,CAACP,MAAM,CAAC,KAAKX,OAAO,CAACmB,MAAM,GAAG,CAAC;UACnE,MAAMC,aAAa,GAAGP,OAAO,KAAKf,UAAU;UAE5Cb,SAAS,CAAC8B,KAAK,CAACC,CAAC,EAAW;YAC1Bf,YAAY,EAAEU,MAAM,CAACV,YAAY,IAAI,IAAI;YACzCoB,KAAK,EAAEV,MAAM,CAACU,KAAK,IAAI,IAAI;YAC3BC,KAAK,EAAEX,MAAM,CAACW,KAAK,IAAI;UACzB,CAAC,CAAC;UAEF,IAAI,CAACF,aAAa,EAAE;YAClB,MAAMG,KAAK,GAAGxB,OAAO,GAAGyB,IAAI,CAACC,GAAG,CAAC,CAAC,EAAEZ,OAAO,GAAG,CAAC,CAAC;YAChD,MAAMX,KAAK,CAACqB,KAAK,CAAC;UACpB,CAAC,MAAM,IAAI,CAACN,YAAY,EAAE;YACxB;UACF;QACF;MACF;MAEA,IAAIjB,OAAO,CAACkB,OAAO,CAACP,MAAM,CAAC,KAAKX,OAAO,CAACmB,MAAM,GAAG,CAAC,IAAIP,SAAS,EAAE;QAC/D,OAAO;UAAEE,EAAE,EAAE,IAAI;UAAEC,KAAK,EAAEH;QAAU,CAAC;MACvC;IACF;IACA,OAAO;MAAEE,EAAE,EAAE,IAAI;MAAEC,KAAK,EAAE,IAAIW,KAAK,CAAC,iDAAiD;IAAE,CAAC;EAC1F;EAEAC,UAAU,GAAG,MAAAA,CACXC,QAAsC,EACtClC,MAAwB,GAAG,CAAC,CAAC,EAC7BmC,QAA8B,KACM;IACpC,MAAM;MAAEC,WAAW;MAAEC;IAAc,CAAC,GAClC,IAAI,CAACvC,0BAA0B,CAACwC,kBAAkB,CAACJ,QAAQ,CAAC;IAE9D,IAAIG,aAAa,EAAE;MACjB,IAAI,CAACxC,OAAO,EAAE0C,MAAM,CAAC,CAAC;MACtB,IAAI,CAACzC,0BAA0B,CAAC0C,KAAK,CAAC,CAAC;IACzC;IAEA,IAAIJ,WAAW,CAACX,MAAM,KAAK,CAAC,EAAE;MAC5BgB,OAAO,CAACC,IAAI,CAAC,0BAA0B,CAAC;IAC1C;IAEA,MAAMC,MAAM,GAAG,MAAM,IAAI,CAAC9C,OAAO,CAACoC,UAAU,CAAC;MAAEC,QAAQ,EAAEE,WAAW;MAAE,GAAGpC;IAAO,CAAC,EAAEmC,QAAQ,CAAC;IAE5F,IAAI,CAACrC,0BAA0B,CAAC8C,MAAM,CAACR,WAAW,EAAE;MAClDS,IAAI,EAAE,WAAW;MACjBC,OAAO,EAAEH,MAAM,CAACG;IAClB,CAAC,CAAC;IAEF,OAAOH,MAAM;EACf,CAAC;EAED,MAAMI,SAASA,CACbC,IAAY,EACZhD,MAAwB,EACxBiD,IAAY,GAAG,OAAO,EACU;IAChC,IAAIN,MAA6B;IACjC,IAAIzB,SAAuB,GAAG,IAAI;IAElC,IAAI+B,IAAI,KAAK,QAAQ,EAAE;MACrBN,MAAM,GAAG,MAAM,IAAI,CAACO,sBAAsB,CAACF,IAAI,CAAC;IAClD,CAAC,MAAM,IAAIC,IAAI,KAAK,OAAO,EAAE;MAC3BN,MAAM,GAAG,MAAM,IAAI,CAACQ,qBAAqB,CAACH,IAAI,EAAEhD,MAAM,CAAC;IACzD,CAAC,MAAM,IAAIiD,IAAI,KAAK,YAAY,EAAE;MAChC,IAAI;QACFN,MAAM,GAAG,MAAM,IAAI,CAACQ,qBAAqB,CAACH,IAAI,EAAEhD,MAAM,CAAC;MACzD,CAAC,CAAC,OAAOsB,CAAC,EAAE;QACVJ,SAAS,GAAGI,CAAU;QACtB,IAAI;UACFqB,MAAM,GAAG,MAAM,IAAI,CAACO,sBAAsB,CAACF,IAAI,CAAC;QAClD,CAAC,CAAC,OAAOI,WAAW,EAAE;UACpB,MAAMlC,SAAS;QACjB;MACF;IACF,CAAC,MAAM,IAAI+B,IAAI,KAAK,aAAa,EAAE;MACjC,IAAI;QACFN,MAAM,GAAG,MAAM,IAAI,CAACO,sBAAsB,CAACF,IAAI,CAAC;MAClD,CAAC,CAAC,OAAO1B,CAAC,EAAE;QACVJ,SAAS,GAAGI,CAAU;QACtB,IAAI;UACFqB,MAAM,GAAG,MAAM,IAAI,CAACQ,qBAAqB,CAACH,IAAI,EAAEhD,MAAM,CAAC;QACzD,CAAC,CAAC,OAAOqD,UAAU,EAAE;UACnB,MAAMnC,SAAS;QACjB;MACF;IACF,CAAC,MAAM;MACL,MAAM,IAAIc,KAAK,CAAC,gBAAgB,GAAGiB,IAAI,GAAG,6DAA6D,CAAC;IAC1G;IACA,OAAON,MAAM;EACf;EAEA,MAAgBQ,qBAAqBA,CAACH,IAAY,EAAEhD,MAAwB,EAAkC;IAC5G,OAAO,IAAI,CAACH,OAAO,CAACkD,SAAS,CAACC,IAAI,EAAEhD,MAAM,CAAC;EAC7C;EAEA,MAAgBkD,sBAAsBA,CAACF,IAAY,EAAkC;IACnF,MAAMM,eAAe,GAAG,MAAM7D,oBAAoB,CAACuD,IAAI,CAAC;IACxD,OAAO;MACLD,SAAS,EAAEO;IACb,CAAC;EACH;EAEAf,MAAM,GAAG,MAAAA,CAAA,KAA2B;IAClC,OAAO,IAAI,CAAC1C,OAAO,EAAE0C,MAAM,CAAC,CAAC;EAC/B,CAAC;EAED,MAAMgB,OAAOA,CAAA,EAAkB;IAC7B,OAAO,IAAI,CAAC1D,OAAO,CAAC0D,OAAO,CAAC,CAAC;EAC/B;AACF","ignoreList":[]}
+{"version":3,"names":["initLlama","Telemetry","setCactusToken","getVertexAIEmbedding","ConversationHistoryManager","CactusLM","_initCache","Map","getCacheKey","params","cactusToken","retryOptions","JSON","stringify","constructor","context","conversationHistoryManager","init","onProgress","key","has","get","initPromise","maxRetries","delayMs","configs","n_gpu_layers","sleep","ms","Promise","resolve","start","Date","now","wait","then","config","lastError","attempt","lm","error","e","isLastConfig","indexOf","length","isLastAttempt","n_ctx","model","delay","Math","pow","Error","set","result","delete","completion","messages","callback","newMessages","requiresReset","processNewMessages","rewind","reset","console","warn","update","role","content","embedding","text","mode","_handleRemoteEmbedding","_handleLocalEmbedding","remoteError","localError","embeddingValues","release","stopCompletion"],"sourceRoot":"../../src","sources":["lm.ts"],"mappings":";;AAAA,SAASA,SAAS,QAAsB,SAAS;AACjD;;AAWA,SAASC,SAAS,QAAQ,aAAa;AACvC,SAASC,cAAc,EAAEC,oBAAoB,QAAQ,UAAU;AAC/D,SAASC,0BAA0B,QAAQ,QAAQ;AAOnD,OAAO,MAAMC,QAAQ,CAAC;EAIpB;EACA;EACA,OAAeC,UAAU,GAAyC,IAAIC,GAAG,CAAC,CAAC;EAE3E,OAAeC,WAAWA,CAACC,MAAqB,EAAEC,WAAoB,EAAEC,YAAwD,EAAU;IACxI,OAAOC,IAAI,CAACC,SAAS,CAAC;MAAEJ,MAAM;MAAEC,WAAW;MAAEC;IAAa,CAAC,CAAC;EAC9D;EAEUG,WAAWA,CAACC,OAAqB,EAAE;IAC3C,IAAI,CAACA,OAAO,GAAGA,OAAO;IACtB,IAAI,CAACC,0BAA0B,GAAG,IAAIZ,0BAA0B,CAAC,CAAC;EACpE;EAEA,aAAaa,IAAIA,CACfR,MAAqB,EACrBS,UAAuC,EACvCR,WAAoB,EACpBC,YAAwD,EAC/B;IAEzB,IAAID,WAAW,EAAE;MACfR,cAAc,CAACQ,WAAW,CAAC;IAC7B;IAEA,MAAMS,GAAG,GAAGd,QAAQ,CAACG,WAAW,CAACC,MAAM,EAAEC,WAAW,EAAEC,YAAY,CAAC;IACnE,IAAIN,QAAQ,CAACC,UAAU,CAACc,GAAG,CAACD,GAAG,CAAC,EAAE;MAChC;MACA,OAAOd,QAAQ,CAACC,UAAU,CAACe,GAAG,CAACF,GAAG,CAAC;IACrC;IAEA,MAAMG,WAAW,GAAG,CAAC,YAAY;MAC/B,MAAMC,UAAU,GAAGZ,YAAY,EAAEY,UAAU,IAAI,CAAC;MAChD,MAAMC,OAAO,GAAGb,YAAY,EAAEa,OAAO,IAAI,IAAI;MAE7C,MAAMC,OAAO,GAAG,CACdhB,MAAM,EACN;QAAE,GAAGA,MAAM;QAAEiB,YAAY,EAAE;MAAE,CAAC,CAC/B;MAED,MAAMC,KAAK,GAAIC,EAAU,IAAoB;QAC3C,OAAO,IAAIC,OAAO,CAACC,OAAO,IAAI;UAC5B,MAAMC,KAAK,GAAGC,IAAI,CAACC,GAAG,CAAC,CAAC;UACxB,MAAMC,IAAI,GAAGA,CAAA,KAAM;YACjB,IAAIF,IAAI,CAACC,GAAG,CAAC,CAAC,GAAGF,KAAK,IAAIH,EAAE,EAAE;cAC5BE,OAAO,CAAC,CAAC;YACX,CAAC,MAAM;cACLD,OAAO,CAACC,OAAO,CAAC,CAAC,CAACK,IAAI,CAACD,IAAI,CAAC;YAC9B;UACF,CAAC;UACDA,IAAI,CAAC,CAAC;QACR,CAAC,CAAC;MACJ,CAAC;MAED,KAAK,MAAME,MAAM,IAAIX,OAAO,EAAE;QAC5B,IAAIY,SAAuB,GAAG,IAAI;QAElC,KAAK,IAAIC,OAAO,GAAG,CAAC,EAAEA,OAAO,IAAIf,UAAU,EAAEe,OAAO,EAAE,EAAE;UACtD,IAAI;YACF,MAAMvB,OAAO,GAAG,MAAMf,SAAS,CAACoC,MAAM,EAAElB,UAAU,CAAC;YACnD,OAAO;cAAEqB,EAAE,EAAE,IAAIlC,QAAQ,CAACU,OAAO,CAAC;cAAEyB,KAAK,EAAE;YAAK,CAAC;UACnD,CAAC,CAAC,OAAOC,CAAC,EAAE;YACVJ,SAAS,GAAGI,CAAU;YACtB,MAAMC,YAAY,GAAGjB,OAAO,CAACkB,OAAO,CAACP,MAAM,CAAC,KAAKX,OAAO,CAACmB,MAAM,GAAG,CAAC;YACnE,MAAMC,aAAa,GAAGP,OAAO,KAAKf,UAAU;YAE5CtB,SAAS,CAACuC,KAAK,CAACC,CAAC,EAAW;cAC1Bf,YAAY,EAAEU,MAAM,CAACV,YAAY,IAAI,IAAI;cACzCoB,KAAK,EAAEV,MAAM,CAACU,KAAK,IAAI,IAAI;cAC3BC,KAAK,EAAEX,MAAM,CAACW,KAAK,IAAI;YACzB,CAAC,CAAC;YAEF,IAAI,CAACF,aAAa,EAAE;cAClB,MAAMG,KAAK,GAAGxB,OAAO,GAAGyB,IAAI,CAACC,GAAG,CAAC,CAAC,EAAEZ,OAAO,GAAG,CAAC,CAAC;cAChD,MAAMX,KAAK,CAACqB,KAAK,CAAC;YACpB,CAAC,MAAM,IAAI,CAACN,YAAY,EAAE;cACxB;YACF;UACF;QACF;QAEA,IAAIjB,OAAO,CAACkB,OAAO,CAACP,MAAM,CAAC,KAAKX,OAAO,CAACmB,MAAM,GAAG,CAAC,IAAIP,SAAS,EAAE;UAC/D,OAAO;YAAEE,EAAE,EAAE,IAAI;YAAEC,KAAK,EAAEH;UAAU,CAAC;QACvC;MACF;MACA,OAAO;QAAEE,EAAE,EAAE,IAAI;QAAEC,KAAK,EAAE,IAAIW,KAAK,CAAC,iDAAiD;MAAE,CAAC;IAC1F,CAAC,EAAE,CAAC;IAEJ9C,QAAQ,CAACC,UAAU,CAAC8C,GAAG,CAACjC,GAAG,EAAEG,WAAW,CAAC;IAEzC,MAAM+B,MAAM,GAAG,MAAM/B,WAAW;IAChC,IAAI+B,MAAM,CAACb,KAAK,EAAE;MAChBnC,QAAQ,CAACC,UAAU,CAACgD,MAAM,CAACnC,GAAG,CAAC,CAAC,CAAC;IACnC;IACA,OAAOkC,MAAM;EACf;EAEAE,UAAU,GAAG,MAAAA,CACXC,QAAsC,EACtC/C,MAAwB,GAAG,CAAC,CAAC,EAC7BgD,QAA8B,KACM;IACpC,MAAM;MAAEC,WAAW;MAAEC;IAAc,CAAC,GAClC,IAAI,CAAC3C,0BAA0B,CAAC4C,kBAAkB,CAACJ,QAAQ,CAAC;IAE9D,IAAIG,aAAa,EAAE;MACjB,IAAI,CAAC5C,OAAO,EAAE8C,MAAM,CAAC,CAAC;MACtB,IAAI,CAAC7C,0BAA0B,CAAC8C,KAAK,CAAC,CAAC;IACzC;IAEA,IAAIJ,WAAW,CAACd,MAAM,KAAK,CAAC,EAAE;MAC5BmB,OAAO,CAACC,IAAI,CAAC,0BAA0B,CAAC;IAC1C;IAEA,MAAMX,MAAM,GAAG,MAAM,IAAI,CAACtC,OAAO,CAACwC,UAAU,CAAC;MAAEC,QAAQ,EAAEE,WAAW;MAAE,GAAGjD;IAAO,CAAC,EAAEgD,QAAQ,CAAC;IAE5F,IAAI,CAACzC,0BAA0B,CAACiD,MAAM,CAACP,WAAW,EAAE;MAClDQ,IAAI,EAAE,WAAW;MACjBC,OAAO,EAAEd,MAAM,CAACc;IAClB,CAAC,CAAC;IAEF,OAAOd,MAAM;EACf,CAAC;EAED,MAAMe,SAASA,CACbC,IAAY,EACZ5D,MAAwB,EACxB6D,IAAY,GAAG,OAAO,EACU;IAChC,IAAIjB,MAA6B;IACjC,IAAIhB,SAAuB,GAAG,IAAI;IAElC,IAAIiC,IAAI,KAAK,QAAQ,EAAE;MACrBjB,MAAM,GAAG,MAAM,IAAI,CAACkB,sBAAsB,CAACF,IAAI,CAAC;IAClD,CAAC,MAAM,IAAIC,IAAI,KAAK,OAAO,EAAE;MAC3BjB,MAAM,GAAG,MAAM,IAAI,CAACmB,qBAAqB,CAACH,IAAI,EAAE5D,MAAM,CAAC;IACzD,CAAC,MAAM,IAAI6D,IAAI,KAAK,YAAY,EAAE;MAChC,IAAI;QACFjB,MAAM,GAAG,MAAM,IAAI,CAACmB,qBAAqB,CAACH,IAAI,EAAE5D,MAAM,CAAC;MACzD,CAAC,CAAC,OAAOgC,CAAC,EAAE;QACVJ,SAAS,GAAGI,CAAU;QACtB,IAAI;UACFY,MAAM,GAAG,MAAM,IAAI,CAACkB,sBAAsB,CAACF,IAAI,CAAC;QAClD,CAAC,CAAC,OAAOI,WAAW,EAAE;UACpB,MAAMpC,SAAS;QACjB;MACF;IACF,CAAC,MAAM,IAAIiC,IAAI,KAAK,aAAa,EAAE;MACjC,IAAI;QACFjB,MAAM,GAAG,MAAM,IAAI,CAACkB,sBAAsB,CAACF,IAAI,CAAC;MAClD,CAAC,CAAC,OAAO5B,CAAC,EAAE;QACVJ,SAAS,GAAGI,CAAU;QACtB,IAAI;UACFY,MAAM,GAAG,MAAM,IAAI,CAACmB,qBAAqB,CAACH,IAAI,EAAE5D,MAAM,CAAC;QACzD,CAAC,CAAC,OAAOiE,UAAU,EAAE;UACnB,MAAMrC,SAAS;QACjB;MACF;IACF,CAAC,MAAM;MACL,MAAM,IAAIc,KAAK,CAAC,gBAAgB,GAAGmB,IAAI,GAAG,6DAA6D,CAAC;IAC1G;IACA,OAAOjB,MAAM;EACf;EAEA,MAAgBmB,qBAAqBA,CAACH,IAAY,EAAE5D,MAAwB,EAAkC;IAC5G,OAAO,IAAI,CAACM,OAAO,CAACqD,SAAS,CAACC,IAAI,EAAE5D,MAAM,CAAC;EAC7C;EAEA,MAAgB8D,sBAAsBA,CAACF,IAAY,EAAkC;IACnF,MAAMM,eAAe,GAAG,MAAMxE,oBAAoB,CAACkE,IAAI,CAAC;IACxD,OAAO;MACLD,SAAS,EAAEO;IACb,CAAC;EACH;EAEAd,MAAM,GAAG,MAAAA,CAAA,KAA2B;IAClC,OAAO,IAAI,CAAC9C,OAAO,EAAE8C,MAAM,CAAC,CAAC;EAC/B,CAAC;EAED,MAAMe,OAAOA,CAAA,EAAkB;IAC7B,OAAO,IAAI,CAAC7D,OAAO,CAAC6D,OAAO,CAAC,CAAC;EAC/B;EAEA,MAAMC,cAAcA,CAAA,EAAkB;IACpC,OAAO,MAAM,IAAI,CAAC9D,OAAO,CAAC8D,cAAc,CAAC,CAAC;EAC5C;AAEF","ignoreList":[]}

package/lib/module/vlm.js

@@ -5,44 +5,92 @@ import { Telemetry } from './telemetry';
 import { setCactusToken, getTextCompletion, getVisionCompletion } from './remote';
 import { ConversationHistoryManager } from './chat';
 export class CactusVLM {
+  // see CactusLM for detailed docs
+  static _initCache = new Map();
+  static getCacheKey(params, cactusToken, retryOptions) {
+    return JSON.stringify({
+      params,
+      cactusToken,
+      retryOptions
+    });
+  }
   constructor(context) {
     this.context = context;
     this.conversationHistoryManager = new ConversationHistoryManager();
   }
-  static async init(params, onProgress, cactusToken) {
+  static async init(params, onProgress, cactusToken, retryOptions) {
     if (cactusToken) {
       setCactusToken(cactusToken);
     }
-    const configs = [params, {
-      ...params,
-      n_gpu_layers: 0
-    }];
-    for (const config of configs) {
-      try {
-        const context = await initLlama(config, onProgress);
-        await initMultimodal(context.id, params.mmproj, false);
-        return {
-          vlm: new CactusVLM(context),
-          error: null
-        };
-      } catch (e) {
-        Telemetry.error(e, {
-          n_gpu_layers: config.n_gpu_layers ?? null,
-          n_ctx: config.n_ctx ?? null,
-          model: config.model ?? null
+    const key = CactusVLM.getCacheKey(params, cactusToken, retryOptions);
+    if (CactusVLM._initCache.has(key)) {
+      return CactusVLM._initCache.get(key);
+    }
+    const initPromise = (async () => {
+      const maxRetries = retryOptions?.maxRetries ?? 3;
+      const delayMs = retryOptions?.delayMs ?? 1000;
+      const configs = [params, {
+        ...params,
+        n_gpu_layers: 0
+      }];
+      const sleep = ms => {
+        return new Promise(resolve => {
+          const start = Date.now();
+          const wait = () => {
+            if (Date.now() - start >= ms) {
+              resolve();
+            } else {
+              Promise.resolve().then(wait);
+            }
+          };
+          wait();
         });
-        if (configs.indexOf(config) === configs.length - 1) {
+      };
+      for (const config of configs) {
+        let lastError = null;
+        for (let attempt = 1; attempt <= maxRetries; attempt++) {
+          try {
+            const context = await initLlama(config, onProgress);
+            await initMultimodal(context.id, params.mmproj, false);
+            return {
+              vlm: new CactusVLM(context),
+              error: null
+            };
+          } catch (e) {
+            lastError = e;
+            const isLastConfig = configs.indexOf(config) === configs.length - 1;
+            const isLastAttempt = attempt === maxRetries;
+            Telemetry.error(e, {
+              n_gpu_layers: config.n_gpu_layers ?? null,
+              n_ctx: config.n_ctx ?? null,
+              model: config.model ?? null
+            });
+            if (!isLastAttempt) {
+              const delay = delayMs * Math.pow(2, attempt - 1);
+              await sleep(delay);
+            } else if (!isLastConfig) {
+              break;
+            }
+          }
+        }
+        if (configs.indexOf(config) === configs.length - 1 && lastError) {
           return {
             vlm: null,
-            error: e
+            error: lastError
           };
         }
       }
+      return {
+        vlm: null,
+        error: new Error('Failed to initialize CactusVLM')
+      };
+    })();
+    CactusVLM._initCache.set(key, initPromise);
+    const result = await initPromise;
+    if (result.error) {
+      CactusVLM._initCache.delete(key);
     }
-    return {
-      vlm: null,
-      error: new Error('Failed to initialize CactusVLM')
-    };
+    return result;
   }
   async completion(messages, params = {}, callback) {
     const mode = params.mode || 'local';
@@ -159,5 +207,8 @@ export class CactusVLM {
   async release() {
     return this.context.release();
   }
+  async stopCompletion() {
+    return await this.context.stopCompletion();
+  }
 }
 //# sourceMappingURL=vlm.js.map

package/lib/module/vlm.js.map

@@ -1 +1 @@
-{"version":3,"names":["initLlama","initMultimodal","multimodalCompletion","Telemetry","setCactusToken","getTextCompletion","getVisionCompletion","ConversationHistoryManager","CactusVLM","constructor","context","conversationHistoryManager","init","params","onProgress","cactusToken","configs","n_gpu_layers","config","id","mmproj","vlm","error","e","n_ctx","model","indexOf","length","Error","completion","messages","callback","mode","result","lastError","_handleRemoteCompletion","_handleLocalCompletion","remoteError","localError","newMessages","requiresReset","processNewMessages","rewind","reset","console","warn","images","formattedPrompt","getFormattedChat","prompt","emit_partial_completion","update","role","content","text","map","m","join","imagePath","responseText","i","token","reasoning_content","tool_calls","tokens_predicted","split","tokens_evaluated","truncated","stopped_eos","stopped_word","stopped_limit","stopping_word","tokens_cached","timings","prompt_n","prompt_ms","prompt_per_token_ms","prompt_per_second","predicted_n","predicted_ms","predicted_per_token_ms","predicted_per_second","release"],"sourceRoot":"../../src","sources":["vlm.ts"],"mappings":";;AAAA,SACEA,SAAS,EACTC,cAAc,EACdC,oBAAoB,QAMf,SAAS;AAEhB,SAASC,SAAS,QAAQ,aAAa;AACvC,SAASC,cAAc,EAAEC,iBAAiB,EAAEC,mBAAmB,QAAQ,UAAU;AACjF,SAASC,0BAA0B,QAAQ,QAAQ;AAgBnD,OAAO,MAAMC,SAAS,CAAC;EAIbC,WAAWA,CAACC,OAAqB,EAAE;IACzC,IAAI,CAACA,OAAO,GAAGA,OAAO;IACtB,IAAI,CAACC,0BAA0B,GAAG,IAAIJ,0BAA0B,CAAC,CAAC;EACpE;EAEA,aAAaK,IAAIA,CACfC,MAAwB,EACxBC,UAAuC,EACvCC,WAAoB,EACM;IAC1B,IAAIA,WAAW,EAAE;MACfX,cAAc,CAACW,WAAW,CAAC;IAC7B;IAEA,MAAMC,OAAO,GAAG,CACdH,MAAM,EACN;MAAE,GAAGA,MAAM;MAAEI,YAAY,EAAE;IAAE,CAAC,CAC/B;IAED,KAAK,MAAMC,MAAM,IAAIF,OAAO,EAAE;MAC5B,IAAI;QACF,MAAMN,OAAO,GAAG,MAAMV,SAAS,CAACkB,MAAM,EAAEJ,UAAU,CAAC;QACnD,MAAMb,cAAc,CAACS,OAAO,CAACS,EAAE,EAAEN,MAAM,CAACO,MAAM,EAAE,KAAK,CAAC;QACtD,OAAO;UAACC,GAAG,EAAE,IAAIb,SAAS,CAACE,OAAO,CAAC;UAAEY,KAAK,EAAE;QAAI,CAAC;MACnD,CAAC,CAAC,OAAOC,CAAC,EAAE;QACVpB,SAAS,CAACmB,KAAK,CAACC,CAAC,EAAW;UAC1BN,YAAY,EAAEC,MAAM,CAACD,YAAY,IAAI,IAAI;UACzCO,KAAK,EAAEN,MAAM,CAACM,KAAK,IAAI,IAAI;UAC3BC,KAAK,EAAEP,MAAM,CAACO,KAAK,IAAI;QACzB,CAAC,CAAC;QACF,IAAIT,OAAO,CAACU,OAAO,CAACR,MAAM,CAAC,KAAKF,OAAO,CAACW,MAAM,GAAG,CAAC,EAAE;UAClD,OAAO;YAACN,GAAG,EAAE,IAAI;YAAEC,KAAK,EAAEC;UAAU,CAAC;QACvC;MACF;IACF;IAEA,OAAO;MAACF,GAAG,EAAE,IAAI;MAAEC,KAAK,EAAE,IAAIM,KAAK,CAAC,gCAAgC;IAAC,CAAC;EACxE;EAEA,MAAMC,UAAUA,CACdC,QAAsC,EACtCjB,MAA2B,GAAG,CAAC,CAAC,EAChCkB,QAA8B,EACG;IACjC,MAAMC,IAAI,GAAGnB,MAAM,CAACmB,IAAI,IAAI,OAAO;IAEnC,IAAIC,MAA8B;IAClC,IAAIC,SAAuB,GAAG,IAAI;IAElC,IAAIF,IAAI,KAAK,QAAQ,EAAE;MACrBC,MAAM,GAAG,MAAM,IAAI,CAACE,uBAAuB,CAACL,QAAQ,EAAEjB,MAAM,EAAEkB,QAAQ,CAAC;IACzE,CAAC,MAAM,IAAIC,IAAI,KAAK,OAAO,EAAE;MAC3BC,MAAM,GAAG,MAAM,IAAI,CAACG,sBAAsB,CAACN,QAAQ,EAAEjB,MAAM,EAAEkB,QAAQ,CAAC;IACxE,CAAC,MAAM,IAAIC,IAAI,KAAK,YAAY,EAAE;MAChC,IAAI;QACFC,MAAM,GAAG,MAAM,IAAI,CAACG,sBAAsB,CAACN,QAAQ,EAAEjB,MAAM,EAAEkB,QAAQ,CAAC;MACxE,CAAC,CAAC,OAAOR,CAAC,EAAE;QACVW,SAAS,GAAGX,CAAU;QACtB,IAAI;UACFU,MAAM,GAAG,MAAM,IAAI,CAACE,uBAAuB,CAACL,QAAQ,EAAEjB,MAAM,EAAEkB,QAAQ,CAAC;QACzE,CAAC,CAAC,OAAOM,WAAW,EAAE;UACpB,MAAMH,SAAS;QACjB;MACF;IACF,CAAC,MAAM,IAAIF,IAAI,KAAK,aAAa,EAAE;MACjC,IAAI;QACFC,MAAM,GAAG,MAAM,IAAI,CAACE,uBAAuB,CAACL,QAAQ,EAAEjB,MAAM,EAAEkB,QAAQ,CAAC;MACzE,CAAC,CAAC,OAAOR,CAAC,EAAE;QACVW,SAAS,GAAGX,CAAU;QACtB,IAAI;UACFU,MAAM,GAAG,MAAM,IAAI,CAACG,sBAAsB,CAACN,QAAQ,EAAEjB,MAAM,EAAEkB,QAAQ,CAAC;QACxE,CAAC,CAAC,OAAOO,UAAU,EAAE;UACnB,MAAMJ,SAAS;QACjB;MACF;IACF,CAAC,MAAM;MACL,MAAM,IAAIN,KAAK,CAAC,gBAAgB,GAAGI,IAAI,GAAG,6DAA6D,CAAC;IAC1G;IAEA,OAAOC,MAAM;EACf;EAEQG,sBAAsB,GAAG,MAAAA,CAC/BN,QAAsC,EACtCjB,MAA2B,EAC3BkB,QAA8B,KACM;IACpC,MAAM;MAAEQ,WAAW;MAAEC;IAAc,CAAC,GAClC,IAAI,CAAC7B,0BAA0B,CAAC8B,kBAAkB,CAACX,QAAQ,CAAC;IAE9D,IAAIU,aAAa,EAAE;MACjB,IAAI,CAAC9B,OAAO,EAAEgC,MAAM,CAAC,CAAC;MACtB,IAAI,CAAC/B,0BAA0B,CAACgC,KAAK,CAAC,CAAC;IACzC;IAEA,IAAIJ,WAAW,CAACZ,MAAM,KAAK,CAAC,EAAE;MAC5BiB,OAAO,CAACC,IAAI,CAAC,0BAA0B,CAAC;IAC1C;IAEA,IAAIZ,MAA8B;IAElC,IAAIpB,MAAM,CAACiC,MAAM,IAAIjC,MAAM,CAACiC,MAAM,CAACnB,MAAM,GAAG,CAAC,EAAE;MAC7C,MAAMoB,eAAe,GAAG,MAAM,IAAI,CAACrC,OAAO,CAACsC,gBAAgB,CAACT,WAAW,CAAC;MACxE,MAAMU,MAAM,GACV,OAAOF,eAAe,KAAK,QAAQ,GAC/BA,eAAe,GACfA,eAAe,CAACE,MAAM;MAC5BhB,MAAM,GAAG,MAAM/B,oBAAoB,CACjC,IAAI,CAACQ,OAAO,CAACS,EAAE,EACf8B,MAAM,EACNpC,MAAM,CAACiC,MAAM,EACb;QAAE,GAAGjC,MAAM;QAAEoC,MAAM;QAAEC,uBAAuB,EAAE,CAAC,CAACnB;MAAS,CAC3D,CAAC;IACH,CAAC,MAAM;MACLE,MAAM,GAAG,MAAM,IAAI,CAACvB,OAAO,CAACmB,UAAU,CAAC;QAAEC,QAAQ,EAAES,WAAW;QAAE,GAAG1B;MAAO,CAAC,EAAEkB,QAAQ,CAAC;IACxF;IAEA,IAAI,CAACpB,0BAA0B,CAACwC,MAAM,CAACZ,WAAW,EAAE;MAClDa,IAAI,EAAE,WAAW;MACjBC,OAAO,EAAEpB,MAAM,CAACoB,OAAO,IAAIpB,MAAM,CAACqB;IACpC,CAAC,CAAC;IAEF,OAAOrB,MAAM;EACf,CAAC;EAED,MAAcE,uBAAuBA,CACnCL,QAAsC,EACtCjB,MAA2B,EAC3BkB,QAA8B,EACG;IACjC,MAAMkB,MAAM,GAAGnB,QAAQ,CAACyB,GAAG,CAAEC,CAAC,IAAK,GAAGA,CAAC,CAACJ,IAAI,KAAKI,CAAC,CAACH,OAAO,EAAE,CAAC,CAACI,IAAI,CAAC,IAAI,CAAC;IACxE,MAAMC,SAAS,GAAG7C,MAAM,CAACiC,MAAM,IAAIjC,MAAM,CAACiC,MAAM,CAACnB,MAAM,GAAG,CAAC,GAAGd,MAAM,CAACiC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE;IAEnF,IAAIa,YAAoB;IACxB,IAAID,SAAS,EAAE;MACbC,YAAY,GAAG,MAAMrD,mBAAmB,CAAC2C,MAAM,EAAES,SAAS,CAAC;IAC7D,CAAC,MAAM;MACLC,YAAY,GAAG,MAAMtD,iBAAiB,CAAC4C,MAAM,CAAC;IAChD;IAEA,IAAIlB,QAAQ,EAAE;MACZ,KAAK,IAAI6B,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGD,YAAY,CAAChC,MAAM,EAAEiC,CAAC,EAAE,EAAE;QAC5C7B,QAAQ,CAAC;UAAE8B,KAAK,EAAEF,YAAY,CAACC,CAAC;QAAE,CAAC,CAAC;MACtC;IACF;IAEA,OAAO;MACLN,IAAI,EAAEK,YAAY;MAClBG,iBAAiB,EAAE,EAAE;MACrBC,UAAU,EAAE,EAAE;MACdV,OAAO,EAAEM,YAAY;MACrBK,gBAAgB,EAAEL,YAAY,CAACM,KAAK,CAAC,GAAG,CAAC,CAACtC,MAAM;MAChDuC,gBAAgB,EAAEjB,MAAM,CAACgB,KAAK,CAAC,GAAG,CAAC,CAACtC,MAAM;MAC1CwC,SAAS,EAAE,KAAK;MAChBC,WAAW,EAAE,IAAI;MACjBC,YAAY,EAAE,EAAE;MAChBC,aAAa,EAAE,CAAC;MAChBC,aAAa,EAAE,EAAE;MACjBC,aAAa,EAAE,CAAC;MAChBC,OAAO,EAAE;QACPC,QAAQ,EAAEzB,MAAM,CAACgB,KAAK,CAAC,GAAG,CAAC,CAACtC,MAAM;QAClCgD,SAAS,EAAE,CAAC;QACZC,mBAAmB,EAAE,CAAC;QACtBC,iBAAiB,EAAE,CAAC;QACpBC,WAAW,EAAEnB,YAAY,CAACM,KAAK,CAAC,GAAG,CAAC,CAACtC,MAAM;QAC3CoD,YAAY,EAAE,CAAC;QACfC,sBAAsB,EAAE,CAAC;QACzBC,oBAAoB,EAAE;MACxB;IACF,CAAC;EACH;EAEA,MAAMvC,MAAMA,CAAA,EAAkB;IAC5B,OAAO,IAAI,CAAChC,OAAO,EAAEgC,MAAM,CAAC,CAAC;EAC/B;EAEA,MAAMwC,OAAOA,CAAA,EAAkB;IAC7B,OAAO,IAAI,CAACxE,OAAO,CAACwE,OAAO,CAAC,CAAC;EAC/B;AACF","ignoreList":[]}
+{"version":3,"names":["initLlama","initMultimodal","multimodalCompletion","Telemetry","setCactusToken","getTextCompletion","getVisionCompletion","ConversationHistoryManager","CactusVLM","_initCache","Map","getCacheKey","params","cactusToken","retryOptions","JSON","stringify","constructor","context","conversationHistoryManager","init","onProgress","key","has","get","initPromise","maxRetries","delayMs","configs","n_gpu_layers","sleep","ms","Promise","resolve","start","Date","now","wait","then","config","lastError","attempt","id","mmproj","vlm","error","e","isLastConfig","indexOf","length","isLastAttempt","n_ctx","model","delay","Math","pow","Error","set","result","delete","completion","messages","callback","mode","_handleRemoteCompletion","_handleLocalCompletion","remoteError","localError","newMessages","requiresReset","processNewMessages","rewind","reset","console","warn","images","formattedPrompt","getFormattedChat","prompt","emit_partial_completion","update","role","content","text","map","m","join","imagePath","responseText","i","token","reasoning_content","tool_calls","tokens_predicted","split","tokens_evaluated","truncated","stopped_eos","stopped_word","stopped_limit","stopping_word","tokens_cached","timings","prompt_n","prompt_ms","prompt_per_token_ms","prompt_per_second","predicted_n","predicted_ms","predicted_per_token_ms","predicted_per_second","release","stopCompletion"],"sourceRoot":"../../src","sources":["vlm.ts"],"mappings":";;AAAA,SACEA,SAAS,EACTC,cAAc,EACdC,oBAAoB,QAMf,SAAS;AAEhB,SAASC,SAAS,QAAQ,aAAa;AACvC,SAASC,cAAc,EAAEC,iBAAiB,EAAEC,mBAAmB,QAAQ,UAAU;AACjF,SAASC,0BAA0B,QAAQ,QAAQ;AAgBnD,OAAO,MAAMC,SAAS,CAAC;EAIrB;EACA,OAAeC,UAAU,GAA0C,IAAIC,GAAG,CAAC,CAAC;EAE5E,OAAeC,WAAWA,CAACC,MAAwB,EAAEC,WAAoB,EAAEC,YAAwD,EAAU;IAC3I,OAAOC,IAAI,CAACC,SAAS,CAAC;MAAEJ,MAAM;MAAEC,WAAW;MAAEC;IAAa,CAAC,CAAC;EAC9D;EAEQG,WAAWA,CAACC,OAAqB,EAAE;IACzC,IAAI,CAACA,OAAO,GAAGA,OAAO;IACtB,IAAI,CAACC,0BAA0B,GAAG,IAAIZ,0BAA0B,CAAC,CAAC;EACpE;EAEA,aAAaa,IAAIA,CACfR,MAAwB,EACxBS,UAAuC,EACvCR,WAAoB,EACpBC,YAAwD,EAC9B;IAC1B,IAAID,WAAW,EAAE;MACfT,cAAc,CAACS,WAAW,CAAC;IAC7B;IAEA,MAAMS,GAAG,GAAGd,SAAS,CAACG,WAAW,CAACC,MAAM,EAAEC,WAAW,EAAEC,YAAY,CAAC;IACpE,IAAIN,SAAS,CAACC,UAAU,CAACc,GAAG,CAACD,GAAG,CAAC,EAAE;MACjC,OAAOd,SAAS,CAACC,UAAU,CAACe,GAAG,CAACF,GAAG,CAAC;IACtC;IAEA,MAAMG,WAAW,GAAG,CAAC,YAAY;MAC/B,MAAMC,UAAU,GAAGZ,YAAY,EAAEY,UAAU,IAAI,CAAC;MAChD,MAAMC,OAAO,GAAGb,YAAY,EAAEa,OAAO,IAAI,IAAI;MAE7C,MAAMC,OAAO,GAAG,CACdhB,MAAM,EACN;QAAE,GAAGA,MAAM;QAAEiB,YAAY,EAAE;MAAE,CAAC,CAC/B;MAED,MAAMC,KAAK,GAAIC,EAAU,IAAoB;QAC3C,OAAO,IAAIC,OAAO,CAACC,OAAO,IAAI;UAC5B,MAAMC,KAAK,GAAGC,IAAI,CAACC,GAAG,CAAC,CAAC;UACxB,MAAMC,IAAI,GAAGA,CAAA,KAAM;YACjB,IAAIF,IAAI,CAACC,GAAG,CAAC,CAAC,GAAGF,KAAK,IAAIH,EAAE,EAAE;cAC5BE,OAAO,CAAC,CAAC;YACX,CAAC,MAAM;cACLD,OAAO,CAACC,OAAO,CAAC,CAAC,CAACK,IAAI,CAACD,IAAI,CAAC;YAC9B;UACF,CAAC;UACDA,IAAI,CAAC,CAAC;QACR,CAAC,CAAC;MACJ,CAAC;MAED,KAAK,MAAME,MAAM,IAAIX,OAAO,EAAE;QAC5B,IAAIY,SAAuB,GAAG,IAAI;QAElC,KAAK,IAAIC,OAAO,GAAG,CAAC,EAAEA,OAAO,IAAIf,UAAU,EAAEe,OAAO,EAAE,EAAE;UACtD,IAAI;YACF,MAAMvB,OAAO,GAAG,MAAMlB,SAAS,CAACuC,MAAM,EAAElB,UAAU,CAAC;YACnD,MAAMpB,cAAc,CAACiB,OAAO,CAACwB,EAAE,EAAE9B,MAAM,CAAC+B,MAAM,EAAE,KAAK,CAAC;YACtD,OAAO;cAACC,GAAG,EAAE,IAAIpC,SAAS,CAACU,OAAO,CAAC;cAAE2B,KAAK,EAAE;YAAI,CAAC;UACnD,CAAC,CAAC,OAAOC,CAAC,EAAE;YACVN,SAAS,GAAGM,CAAU;YACtB,MAAMC,YAAY,GAAGnB,OAAO,CAACoB,OAAO,CAACT,MAAM,CAAC,KAAKX,OAAO,CAACqB,MAAM,GAAG,CAAC;YACnE,MAAMC,aAAa,GAAGT,OAAO,KAAKf,UAAU;YAE5CvB,SAAS,CAAC0C,KAAK,CAACC,CAAC,EAAW;cAC1BjB,YAAY,EAAEU,MAAM,CAACV,YAAY,IAAI,IAAI;cACzCsB,KAAK,EAAEZ,MAAM,CAACY,KAAK,IAAI,IAAI;cAC3BC,KAAK,EAAEb,MAAM,CAACa,KAAK,IAAI;YACzB,CAAC,CAAC;YAEF,IAAI,CAACF,aAAa,EAAE;cAClB,MAAMG,KAAK,GAAG1B,OAAO,GAAG2B,IAAI,CAACC,GAAG,CAAC,CAAC,EAAEd,OAAO,GAAG,CAAC,CAAC;cAChD,MAAMX,KAAK,CAACuB,KAAK,CAAC;YACpB,CAAC,MAAM,IAAI,CAACN,YAAY,EAAE;cACxB;YACF;UACF;QACF;QAEA,IAAInB,OAAO,CAACoB,OAAO,CAACT,MAAM,CAAC,KAAKX,OAAO,CAACqB,MAAM,GAAG,CAAC,IAAIT,SAAS,EAAE;UAC/D,OAAO;YAACI,GAAG,EAAE,IAAI;YAAEC,KAAK,EAAEL;UAAS,CAAC;QACtC;MACF;MAEA,OAAO;QAACI,GAAG,EAAE,IAAI;QAAEC,KAAK,EAAE,IAAIW,KAAK,CAAC,gCAAgC;MAAC,CAAC;IACxE,CAAC,EAAE,CAAC;IAEJhD,SAAS,CAACC,UAAU,CAACgD,GAAG,CAACnC,GAAG,EAAEG,WAAW,CAAC;IAE1C,MAAMiC,MAAM,GAAG,MAAMjC,WAAW;IAChC,IAAIiC,MAAM,CAACb,KAAK,EAAE;MAChBrC,SAAS,CAACC,UAAU,CAACkD,MAAM,CAACrC,GAAG,CAAC;IAClC;IACA,OAAOoC,MAAM;EACf;EAEA,MAAME,UAAUA,CACdC,QAAsC,EACtCjD,MAA2B,GAAG,CAAC,CAAC,EAChCkD,QAA8B,EACG;IACjC,MAAMC,IAAI,GAAGnD,MAAM,CAACmD,IAAI,IAAI,OAAO;IAEnC,IAAIL,MAA8B;IAClC,IAAIlB,SAAuB,GAAG,IAAI;IAElC,IAAIuB,IAAI,KAAK,QAAQ,EAAE;MACrBL,MAAM,GAAG,MAAM,IAAI,CAACM,uBAAuB,CAACH,QAAQ,EAAEjD,MAAM,EAAEkD,QAAQ,CAAC;IACzE,CAAC,MAAM,IAAIC,IAAI,KAAK,OAAO,EAAE;MAC3BL,MAAM,GAAG,MAAM,IAAI,CAACO,sBAAsB,CAACJ,QAAQ,EAAEjD,MAAM,EAAEkD,QAAQ,CAAC;IACxE,CAAC,MAAM,IAAIC,IAAI,KAAK,YAAY,EAAE;MAChC,IAAI;QACFL,MAAM,GAAG,MAAM,IAAI,CAACO,sBAAsB,CAACJ,QAAQ,EAAEjD,MAAM,EAAEkD,QAAQ,CAAC;MACxE,CAAC,CAAC,OAAOhB,CAAC,EAAE;QACVN,SAAS,GAAGM,CAAU;QACtB,IAAI;UACFY,MAAM,GAAG,MAAM,IAAI,CAACM,uBAAuB,CAACH,QAAQ,EAAEjD,MAAM,EAAEkD,QAAQ,CAAC;QACzE,CAAC,CAAC,OAAOI,WAAW,EAAE;UACpB,MAAM1B,SAAS;QACjB;MACF;IACF,CAAC,MAAM,IAAIuB,IAAI,KAAK,aAAa,EAAE;MACjC,IAAI;QACFL,MAAM,GAAG,MAAM,IAAI,CAACM,uBAAuB,CAACH,QAAQ,EAAEjD,MAAM,EAAEkD,QAAQ,CAAC;MACzE,CAAC,CAAC,OAAOhB,CAAC,EAAE;QACVN,SAAS,GAAGM,CAAU;QACtB,IAAI;UACFY,MAAM,GAAG,MAAM,IAAI,CAACO,sBAAsB,CAACJ,QAAQ,EAAEjD,MAAM,EAAEkD,QAAQ,CAAC;QACxE,CAAC,CAAC,OAAOK,UAAU,EAAE;UACnB,MAAM3B,SAAS;QACjB;MACF;IACF,CAAC,MAAM;MACL,MAAM,IAAIgB,KAAK,CAAC,gBAAgB,GAAGO,IAAI,GAAG,6DAA6D,CAAC;IAC1G;IAEA,OAAOL,MAAM;EACf;EAEQO,sBAAsB,GAAG,MAAAA,CAC/BJ,QAAsC,EACtCjD,MAA2B,EAC3BkD,QAA8B,KACM;IACpC,MAAM;MAAEM,WAAW;MAAEC;IAAc,CAAC,GAClC,IAAI,CAAClD,0BAA0B,CAACmD,kBAAkB,CAACT,QAAQ,CAAC;IAE9D,IAAIQ,aAAa,EAAE;MACjB,IAAI,CAACnD,OAAO,EAAEqD,MAAM,CAAC,CAAC;MACtB,IAAI,CAACpD,0BAA0B,CAACqD,KAAK,CAAC,CAAC;IACzC;IAEA,IAAIJ,WAAW,CAACnB,MAAM,KAAK,CAAC,EAAE;MAC5BwB,OAAO,CAACC,IAAI,CAAC,0BAA0B,CAAC;IAC1C;IAEA,IAAIhB,MAA8B;IAElC,IAAI9C,MAAM,CAAC+D,MAAM,IAAI/D,MAAM,CAAC+D,MAAM,CAAC1B,MAAM,GAAG,CAAC,EAAE;MAC7C,MAAM2B,eAAe,GAAG,MAAM,IAAI,CAAC1D,OAAO,CAAC2D,gBAAgB,CAACT,WAAW,CAAC;MACxE,MAAMU,MAAM,GACV,OAAOF,eAAe,KAAK,QAAQ,GAC/BA,eAAe,GACfA,eAAe,CAACE,MAAM;MAC5BpB,MAAM,GAAG,MAAMxD,oBAAoB,CACjC,IAAI,CAACgB,OAAO,CAACwB,EAAE,EACfoC,MAAM,EACNlE,MAAM,CAAC+D,MAAM,EACb;QAAE,GAAG/D,MAAM;QAAEkE,MAAM;QAAEC,uBAAuB,EAAE,CAAC,CAACjB;MAAS,CAC3D,CAAC;IACH,CAAC,MAAM;MACLJ,MAAM,GAAG,MAAM,IAAI,CAACxC,OAAO,CAAC0C,UAAU,CAAC;QAAEC,QAAQ,EAAEO,WAAW;QAAE,GAAGxD;MAAO,CAAC,EAAEkD,QAAQ,CAAC;IACxF;IAEA,IAAI,CAAC3C,0BAA0B,CAAC6D,MAAM,CAACZ,WAAW,EAAE;MAClDa,IAAI,EAAE,WAAW;MACjBC,OAAO,EAAExB,MAAM,CAACwB,OAAO,IAAIxB,MAAM,CAACyB;IACpC,CAAC,CAAC;IAEF,OAAOzB,MAAM;EACf,CAAC;EAED,MAAcM,uBAAuBA,CACnCH,QAAsC,EACtCjD,MAA2B,EAC3BkD,QAA8B,EACG;IACjC,MAAMgB,MAAM,GAAGjB,QAAQ,CAACuB,GAAG,CAAEC,CAAC,IAAK,GAAGA,CAAC,CAACJ,IAAI,KAAKI,CAAC,CAACH,OAAO,EAAE,CAAC,CAACI,IAAI,CAAC,IAAI,CAAC;IACxE,MAAMC,SAAS,GAAG3E,MAAM,CAAC+D,MAAM,IAAI/D,MAAM,CAAC+D,MAAM,CAAC1B,MAAM,GAAG,CAAC,GAAGrC,MAAM,CAAC+D,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE;IAEnF,IAAIa,YAAoB;IACxB,IAAID,SAAS,EAAE;MACbC,YAAY,GAAG,MAAMlF,mBAAmB,CAACwE,MAAM,EAAES,SAAS,CAAC;IAC7D,CAAC,MAAM;MACLC,YAAY,GAAG,MAAMnF,iBAAiB,CAACyE,MAAM,CAAC;IAChD;IAEA,IAAIhB,QAAQ,EAAE;MACZ,KAAK,IAAI2B,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGD,YAAY,CAACvC,MAAM,EAAEwC,CAAC,EAAE,EAAE;QAC5C3B,QAAQ,CAAC;UAAE4B,KAAK,EAAEF,YAAY,CAACC,CAAC;QAAE,CAAC,CAAC;MACtC;IACF;IAEA,OAAO;MACLN,IAAI,EAAEK,YAAY;MAClBG,iBAAiB,EAAE,EAAE;MACrBC,UAAU,EAAE,EAAE;MACdV,OAAO,EAAEM,YAAY;MACrBK,gBAAgB,EAAEL,YAAY,CAACM,KAAK,CAAC,GAAG,CAAC,CAAC7C,MAAM;MAChD8C,gBAAgB,EAAEjB,MAAM,CAACgB,KAAK,CAAC,GAAG,CAAC,CAAC7C,MAAM;MAC1C+C,SAAS,EAAE,KAAK;MAChBC,WAAW,EAAE,IAAI;MACjBC,YAAY,EAAE,EAAE;MAChBC,aAAa,EAAE,CAAC;MAChBC,aAAa,EAAE,EAAE;MACjBC,aAAa,EAAE,CAAC;MAChBC,OAAO,EAAE;QACPC,QAAQ,EAAEzB,MAAM,CAACgB,KAAK,CAAC,GAAG,CAAC,CAAC7C,MAAM;QAClCuD,SAAS,EAAE,CAAC;QACZC,mBAAmB,EAAE,CAAC;QACtBC,iBAAiB,EAAE,CAAC;QACpBC,WAAW,EAAEnB,YAAY,CAACM,KAAK,CAAC,GAAG,CAAC,CAAC7C,MAAM;QAC3C2D,YAAY,EAAE,CAAC;QACfC,sBAAsB,EAAE,CAAC;QACzBC,oBAAoB,EAAE;MACxB;IACF,CAAC;EACH;EAEA,MAAMvC,MAAMA,CAAA,EAAkB;IAC5B,OAAO,IAAI,CAACrD,OAAO,EAAEqD,MAAM,CAAC,CAAC;EAC/B;EAEA,MAAMwC,OAAOA,CAAA,EAAkB;IAC7B,OAAO,IAAI,CAAC7F,OAAO,CAAC6F,OAAO,CAAC,CAAC;EAC/B;EAEA,MAAMC,cAAcA,CAAA,EAAkB;IACpC,OAAO,MAAM,IAAI,CAAC9F,OAAO,CAAC8F,cAAc,CAAC,CAAC;EAC5C;AACF","ignoreList":[]}

package/lib/typescript/chat.d.ts

@@ -16,5 +16,6 @@ export declare class ConversationHistoryManager {
     processNewMessages(fullMessageHistory: CactusOAICompatibleMessage[]): ProcessedMessages;
     update(newMessages: CactusOAICompatibleMessage[], assistantResponse: CactusOAICompatibleMessage): void;
     reset(): void;
+    getMessages(): CactusOAICompatibleMessage[];
 }
 //# sourceMappingURL=chat.d.ts.map

package/lib/typescript/chat.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAA;AAE5D,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAA;CACd,CAAA;AAED,MAAM,MAAM,0BAA0B,GAAG;IACvC,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,CAAC,EAAE,MAAM,GAAG,iBAAiB,EAAE,GAAG,GAAG,CAAA;CAC7C,CAAA;AAED,wBAAgB,UAAU,CACxB,QAAQ,EAAE,0BAA0B,EAAE,GACrC,sBAAsB,EAAE,CA8B1B;AAED,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,0BAA0B,EAAE,CAAC;IAC1C,aAAa,EAAE,OAAO,CAAC;CACxB;AAED,qBAAa,0BAA0B;IACrC,OAAO,CAAC,OAAO,CAAoC;IAE5C,kBAAkB,CACvB,kBAAkB,EAAE,0BAA0B,EAAE,GAC/C,iBAAiB;IAmBb,MAAM,CACX,WAAW,EAAE,0BAA0B,EAAE,EACzC,iBAAiB,EAAE,0BAA0B;IAKxC,KAAK;CAGb"}
+{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAA;AAE5D,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAA;CACd,CAAA;AAED,MAAM,MAAM,0BAA0B,GAAG;IACvC,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,CAAC,EAAE,MAAM,GAAG,iBAAiB,EAAE,GAAG,GAAG,CAAA;CAC7C,CAAA;AAED,wBAAgB,UAAU,CACxB,QAAQ,EAAE,0BAA0B,EAAE,GACrC,sBAAsB,EAAE,CA8B1B;AAED,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,0BAA0B,EAAE,CAAC;IAC1C,aAAa,EAAE,OAAO,CAAC;CACxB;AAED,qBAAa,0BAA0B;IACrC,OAAO,CAAC,OAAO,CAAoC;IAE5C,kBAAkB,CACvB,kBAAkB,EAAE,0BAA0B,EAAE,GAC/C,iBAAiB;IAmBb,MAAM,CACX,WAAW,EAAE,0BAA0B,EAAE,EACzC,iBAAiB,EAAE,0BAA0B;IAKxC,KAAK;IAIL,WAAW,IAAI,0BAA0B,EAAE;CAGnD"}

package/lib/typescript/lm.d.ts

@@ -8,6 +8,8 @@ interface CactusLMReturn {
 export declare class CactusLM {
     protected context: LlamaContext;
     protected conversationHistoryManager: ConversationHistoryManager;
+    private static _initCache;
+    private static getCacheKey;
     protected constructor(context: LlamaContext);
     static init(params: ContextParams, onProgress?: (progress: number) => void, cactusToken?: string, retryOptions?: {
         maxRetries?: number;
@@ -19,6 +21,7 @@ export declare class CactusLM {
     protected _handleRemoteEmbedding(text: string): Promise<NativeEmbeddingResult>;
     rewind: () => Promise<void>;
     release(): Promise<void>;
+    stopCompletion(): Promise<void>;
 }
 export {};
 //# sourceMappingURL=lm.d.ts.map

package/lib/typescript/lm.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"lm.d.ts","sourceRoot":"","sources":["../../src/lm.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,YAAY,EAAE,MAAM,SAAS,CAAA;AAGjD,OAAO,KAAK,EACV,aAAa,EACb,gBAAgB,EAChB,0BAA0B,EAC1B,sBAAsB,EACtB,eAAe,EACf,qBAAqB,EACtB,MAAM,SAAS,CAAA;AAIhB,OAAO,EAAE,0BAA0B,EAAE,MAAM,QAAQ,CAAA;AAEnD,UAAU,cAAc;IACtB,EAAE,EAAE,QAAQ,GAAG,IAAI,CAAA;IACnB,KAAK,EAAE,KAAK,GAAG,IAAI,CAAA;CACpB;AAED,qBAAa,QAAQ;IACnB,SAAS,CAAC,OAAO,EAAE,YAAY,CAAA;IAC/B,SAAS,CAAC,0BAA0B,EAAE,0BAA0B,CAAA;IAEhE,SAAS,aAAa,OAAO,EAAE,YAAY;WAK9B,IAAI,CACf,MAAM,EAAE,aAAa,EACrB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,EACvC,WAAW,CAAC,EAAE,MAAM,EACpB,YAAY,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,GACvD,OAAO,CAAC,cAAc,CAAC;IA6D1B,UAAU,GACR,UAAU,0BAA0B,EAAE,EACtC,SAAQ,gBAAqB,EAC7B,WAAW,CAAC,IAAI,EAAE,GAAG,KAAK,IAAI,KAC7B,OAAO,CAAC,sBAAsB,CAAC,CAqBjC;IAEK,SAAS,CACb,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,eAAe,EACxB,IAAI,GAAE,MAAgB,GACrB,OAAO,CAAC,qBAAqB,CAAC;cAoCjB,qBAAqB,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,qBAAqB,CAAC;cAI7F,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAOpF,MAAM,QAAa,OAAO,CAAC,IAAI,CAAC,CAE/B;IAEK,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B"}
+{"version":3,"file":"lm.d.ts","sourceRoot":"","sources":["../../src/lm.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,YAAY,EAAE,MAAM,SAAS,CAAA;AAGjD,OAAO,KAAK,EACV,aAAa,EACb,gBAAgB,EAChB,0BAA0B,EAC1B,sBAAsB,EACtB,eAAe,EACf,qBAAqB,EACtB,MAAM,SAAS,CAAA;AAIhB,OAAO,EAAE,0BAA0B,EAAE,MAAM,QAAQ,CAAA;AAEnD,UAAU,cAAc;IACtB,EAAE,EAAE,QAAQ,GAAG,IAAI,CAAA;IACnB,KAAK,EAAE,KAAK,GAAG,IAAI,CAAA;CACpB;AAED,qBAAa,QAAQ;IACnB,SAAS,CAAC,OAAO,EAAE,YAAY,CAAA;IAC/B,SAAS,CAAC,0BAA0B,EAAE,0BAA0B,CAAA;IAIhE,OAAO,CAAC,MAAM,CAAC,UAAU,CAAmD;IAE5E,OAAO,CAAC,MAAM,CAAC,WAAW;IAI1B,SAAS,aAAa,OAAO,EAAE,YAAY;WAK9B,IAAI,CACf,MAAM,EAAE,aAAa,EACrB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,EACvC,WAAW,CAAC,EAAE,MAAM,EACpB,YAAY,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,GACvD,OAAO,CAAC,cAAc,CAAC;IA8E1B,UAAU,GACR,UAAU,0BAA0B,EAAE,EACtC,SAAQ,gBAAqB,EAC7B,WAAW,CAAC,IAAI,EAAE,GAAG,KAAK,IAAI,KAC7B,OAAO,CAAC,sBAAsB,CAAC,CAqBjC;IAEK,SAAS,CACb,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,eAAe,EACxB,IAAI,GAAE,MAAgB,GACrB,OAAO,CAAC,qBAAqB,CAAC;cAoCjB,qBAAqB,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,qBAAqB,CAAC;cAI7F,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAOpF,MAAM,QAAa,OAAO,CAAC,IAAI,CAAC,CAE/B;IAEK,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAIxB,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;CAItC"}

package/lib/typescript/vlm.d.ts

@@ -14,13 +14,19 @@ export type VLMCompletionParams = Omit<CompletionParams, 'prompt'> & {
 export declare class CactusVLM {
     private context;
     protected conversationHistoryManager: ConversationHistoryManager;
+    private static _initCache;
+    private static getCacheKey;
     private constructor();
-    static init(params: VLMContextParams, onProgress?: (progress: number) => void, cactusToken?: string): Promise<CactusVLMReturn>;
+    static init(params: VLMContextParams, onProgress?: (progress: number) => void, cactusToken?: string, retryOptions?: {
+        maxRetries?: number;
+        delayMs?: number;
+    }): Promise<CactusVLMReturn>;
     completion(messages: CactusOAICompatibleMessage[], params?: VLMCompletionParams, callback?: (data: any) => void): Promise<NativeCompletionResult>;
     private _handleLocalCompletion;
     private _handleRemoteCompletion;
     rewind(): Promise<void>;
     release(): Promise<void>;
+    stopCompletion(): Promise<void>;
 }
 export {};
 //# sourceMappingURL=vlm.d.ts.map

package/lib/typescript/vlm.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"vlm.d.ts","sourceRoot":"","sources":["../../src/vlm.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,KAAK,aAAa,EAClB,KAAK,gBAAgB,EACrB,KAAK,0BAA0B,EAC/B,KAAK,sBAAsB,EAC5B,MAAM,SAAS,CAAA;AAIhB,OAAO,EAAE,0BAA0B,EAAE,MAAM,QAAQ,CAAA;AAEnD,UAAU,eAAe;IACvB,GAAG,EAAE,SAAS,GAAG,IAAI,CAAA;IACrB,KAAK,EAAE,KAAK,GAAG,IAAI,CAAA;CACpB;AAED,MAAM,MAAM,gBAAgB,GAAG,aAAa,GAAG;IAC7C,MAAM,EAAE,MAAM,CAAA;CACf,CAAA;AAED,MAAM,MAAM,mBAAmB,GAAG,IAAI,CAAC,gBAAgB,EAAE,QAAQ,CAAC,GAAG;IACnE,MAAM,CAAC,EAAE,MAAM,EAAE,CAAA;IACjB,IAAI,CAAC,EAAE,MAAM,CAAA;CACd,CAAA;AAED,qBAAa,SAAS;IACpB,OAAO,CAAC,OAAO,CAAc;IAC7B,SAAS,CAAC,0BAA0B,EAAE,0BAA0B,CAAA;IAEhE,OAAO;WAKM,IAAI,CACf,MAAM,EAAE,gBAAgB,EACxB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,EACvC,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CAAC,eAAe,CAAC;IA8BrB,UAAU,CACd,QAAQ,EAAE,0BAA0B,EAAE,EACtC,MAAM,GAAE,mBAAwB,EAChC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,KAAK,IAAI,GAC7B,OAAO,CAAC,sBAAsB,CAAC;IAuClC,OAAO,CAAC,sBAAsB,CAyC7B;YAEa,uBAAuB;IA+C/B,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAIvB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B"}
+{"version":3,"file":"vlm.d.ts","sourceRoot":"","sources":["../../src/vlm.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,KAAK,aAAa,EAClB,KAAK,gBAAgB,EACrB,KAAK,0BAA0B,EAC/B,KAAK,sBAAsB,EAC5B,MAAM,SAAS,CAAA;AAIhB,OAAO,EAAE,0BAA0B,EAAE,MAAM,QAAQ,CAAA;AAEnD,UAAU,eAAe;IACvB,GAAG,EAAE,SAAS,GAAG,IAAI,CAAA;IACrB,KAAK,EAAE,KAAK,GAAG,IAAI,CAAA;CACpB;AAED,MAAM,MAAM,gBAAgB,GAAG,aAAa,GAAG;IAC7C,MAAM,EAAE,MAAM,CAAA;CACf,CAAA;AAED,MAAM,MAAM,mBAAmB,GAAG,IAAI,CAAC,gBAAgB,EAAE,QAAQ,CAAC,GAAG;IACnE,MAAM,CAAC,EAAE,MAAM,EAAE,CAAA;IACjB,IAAI,CAAC,EAAE,MAAM,CAAA;CACd,CAAA;AAED,qBAAa,SAAS;IACpB,OAAO,CAAC,OAAO,CAAc;IAC7B,SAAS,CAAC,0BAA0B,EAAE,0BAA0B,CAAA;IAGhE,OAAO,CAAC,MAAM,CAAC,UAAU,CAAoD;IAE7E,OAAO,CAAC,MAAM,CAAC,WAAW;IAI1B,OAAO;WAKM,IAAI,CACf,MAAM,EAAE,gBAAgB,EACxB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,EACvC,WAAW,CAAC,EAAE,MAAM,EACpB,YAAY,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,GACvD,OAAO,CAAC,eAAe,CAAC;IA8ErB,UAAU,CACd,QAAQ,EAAE,0BAA0B,EAAE,EACtC,MAAM,GAAE,mBAAwB,EAChC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,KAAK,IAAI,GAC7B,OAAO,CAAC,sBAAsB,CAAC;IAuClC,OAAO,CAAC,sBAAsB,CAyC7B;YAEa,uBAAuB;IA+C/B,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAIvB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAIxB,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;CAGtC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cactus-react-native",
-  "version": "0.2.4",
+  "version": "0.2.5",
   "description": "Run AI models locally on mobile devices",
   "main": "lib/commonjs/index",
   "module": "lib/module/index",

package/src/chat.ts

@@ -82,4 +82,8 @@ export class ConversationHistoryManager {
   public reset() {
     this.history = [];
   }
+
+  public getMessages(): CactusOAICompatibleMessage[] {
+    return this.history;
+  }
 }

package/src/lm.ts

@@ -23,6 +23,14 @@ export class CactusLM {
   protected context: LlamaContext
   protected conversationHistoryManager: ConversationHistoryManager
 
+  // the initPromise enables a "async singleton" initialization pattern which
+  // protects against a race condition in the event of multiple init attempts
+  private static _initCache: Map<string, Promise<CactusLMReturn>> = new Map();
+
+  private static getCacheKey(params: ContextParams, cactusToken?: string, retryOptions?: { maxRetries?: number; delayMs?: number }): string {
+    return JSON.stringify({ params, cactusToken, retryOptions });
+  }
+
   protected constructor(context: LlamaContext) {
     this.context = context
     this.conversationHistoryManager = new ConversationHistoryManager()
@@ -34,64 +42,81 @@ export class CactusLM {
     cactusToken?: string,
     retryOptions?: { maxRetries?: number; delayMs?: number },
   ): Promise<CactusLMReturn> {
+
     if (cactusToken) {
       setCactusToken(cactusToken);
     }
 
-    const maxRetries = retryOptions?.maxRetries ?? 3;
-    const delayMs = retryOptions?.delayMs ?? 1000;
-
-    const configs = [
-      params,
-      { ...params, n_gpu_layers: 0 } 
-    ];
-
-    const sleep = (ms: number): Promise<void> => {
-      return new Promise(resolve => {
-        const start = Date.now();
-        const wait = () => {
-          if (Date.now() - start >= ms) {
-            resolve();
-          } else {
-            Promise.resolve().then(wait);
-          }
-        };
-        wait();
-      });
-    };
+    const key = CactusLM.getCacheKey(params, cactusToken, retryOptions);
+    if (CactusLM._initCache.has(key)) {
+      // concurrent initialization calls with the same params all get the same cached Promise
+      return CactusLM._initCache.get(key)!;
+    }
 
-    for (const config of configs) {
-      let lastError: Error | null = null;
-      
-      for (let attempt = 1; attempt <= maxRetries; attempt++) {
-        try {
-          const context = await initLlama(config, onProgress);
-          return { lm: new CactusLM(context), error: null };
-        } catch (e) {
-          lastError = e as Error;
-          const isLastConfig = configs.indexOf(config) === configs.length - 1;
-          const isLastAttempt = attempt === maxRetries;
-          
-          Telemetry.error(e as Error, {
-            n_gpu_layers: config.n_gpu_layers ?? null,
-            n_ctx: config.n_ctx ?? null,
-            model: config.model ?? null,
-          });
-          
-          if (!isLastAttempt) {
-            const delay = delayMs * Math.pow(2, attempt - 1);
-            await sleep(delay);
-          } else if (!isLastConfig) {
-            break;
+    const initPromise = (async () => {
+      const maxRetries = retryOptions?.maxRetries ?? 3;
+      const delayMs = retryOptions?.delayMs ?? 1000;
+
+      const configs = [
+        params,
+        { ...params, n_gpu_layers: 0 }
+      ];
+
+      const sleep = (ms: number): Promise<void> => {
+        return new Promise(resolve => {
+          const start = Date.now();
+          const wait = () => {
+            if (Date.now() - start >= ms) {
+              resolve();
+            } else {
+              Promise.resolve().then(wait);
+            }
+          };
+          wait();
+        });
+      };
+
+      for (const config of configs) {
+        let lastError: Error | null = null;
+
+        for (let attempt = 1; attempt <= maxRetries; attempt++) {
+          try {
+            const context = await initLlama(config, onProgress);
+            return { lm: new CactusLM(context), error: null };
+          } catch (e) {
+            lastError = e as Error;
+            const isLastConfig = configs.indexOf(config) === configs.length - 1;
+            const isLastAttempt = attempt === maxRetries;
+
+            Telemetry.error(e as Error, {
+              n_gpu_layers: config.n_gpu_layers ?? null,
+              n_ctx: config.n_ctx ?? null,
+              model: config.model ?? null,
+            });
+
+            if (!isLastAttempt) {
+              const delay = delayMs * Math.pow(2, attempt - 1);
+              await sleep(delay);
+            } else if (!isLastConfig) {
+              break;
+            }
           }
         }
+
+        if (configs.indexOf(config) === configs.length - 1 && lastError) {
+          return { lm: null, error: lastError };
+        }
       }
-      
-      if (configs.indexOf(config) === configs.length - 1 && lastError) {
-        return { lm: null, error: lastError };
-      }
+      return { lm: null, error: new Error('Failed to initialize CactusLM after all retries') };
+    })();
+
+    CactusLM._initCache.set(key, initPromise);
+
+    const result = await initPromise;
+    if (result.error) {
+      CactusLM._initCache.delete(key); // Reset on failure to allow retries
     }
-    return { lm: null, error: new Error('Failed to initialize CactusLM after all retries') };
+    return result;
   }
 
   completion = async (
@@ -179,4 +204,9 @@ export class CactusLM {
   async release(): Promise<void> {
     return this.context.release()
   }
+
+  async stopCompletion(): Promise<void> {
+    return await this.context.stopCompletion()
+  }
+
 } 

package/src/vlm.ts

@@ -31,6 +31,13 @@ export class CactusVLM {
   private context: LlamaContext
   protected conversationHistoryManager: ConversationHistoryManager
 
+  // see CactusLM for detailed docs
+  private static _initCache: Map<string, Promise<CactusVLMReturn>> = new Map();
+
+  private static getCacheKey(params: VLMContextParams, cactusToken?: string, retryOptions?: { maxRetries?: number; delayMs?: number }): string {
+    return JSON.stringify({ params, cactusToken, retryOptions });
+  }
+
   private constructor(context: LlamaContext) {
     this.context = context
     this.conversationHistoryManager = new ConversationHistoryManager()
@@ -40,34 +47,83 @@ export class CactusVLM {
     params: VLMContextParams,
     onProgress?: (progress: number) => void,
     cactusToken?: string,
+    retryOptions?: { maxRetries?: number; delayMs?: number },
   ): Promise<CactusVLMReturn> {
     if (cactusToken) {
       setCactusToken(cactusToken);
     }
 
-    const configs = [
-      params,
-      { ...params, n_gpu_layers: 0 } 
-    ];
+    const key = CactusVLM.getCacheKey(params, cactusToken, retryOptions);
+    if (CactusVLM._initCache.has(key)) {
+      return CactusVLM._initCache.get(key)!;
+    }
 
-    for (const config of configs) {
-      try {
-        const context = await initLlama(config, onProgress)
-        await initMultimodal(context.id, params.mmproj, false)
-        return {vlm: new CactusVLM(context), error: null}
-      } catch (e) {
-        Telemetry.error(e as Error, {
-          n_gpu_layers: config.n_gpu_layers ?? null,
-          n_ctx: config.n_ctx ?? null,
-          model: config.model ?? null,
+    const initPromise = (async () => {
+      const maxRetries = retryOptions?.maxRetries ?? 3;
+      const delayMs = retryOptions?.delayMs ?? 1000;
+
+      const configs = [
+        params,
+        { ...params, n_gpu_layers: 0 } 
+      ];
+
+      const sleep = (ms: number): Promise<void> => {
+        return new Promise(resolve => {
+          const start = Date.now();
+          const wait = () => {
+            if (Date.now() - start >= ms) {
+              resolve();
+            } else {
+              Promise.resolve().then(wait);
+            }
+          };
+          wait();
         });
-        if (configs.indexOf(config) === configs.length - 1) {
-          return {vlm: null, error: e as Error}
+      };
+
+      for (const config of configs) {
+        let lastError: Error | null = null;
+
+        for (let attempt = 1; attempt <= maxRetries; attempt++) {
+          try {
+            const context = await initLlama(config, onProgress)
+            await initMultimodal(context.id, params.mmproj, false)
+            return {vlm: new CactusVLM(context), error: null}
+          } catch (e) {
+            lastError = e as Error;
+            const isLastConfig = configs.indexOf(config) === configs.length - 1;
+            const isLastAttempt = attempt === maxRetries;
+
+            Telemetry.error(e as Error, {
+              n_gpu_layers: config.n_gpu_layers ?? null,
+              n_ctx: config.n_ctx ?? null,
+              model: config.model ?? null,
+            });
+
+            if (!isLastAttempt) {
+              const delay = delayMs * Math.pow(2, attempt - 1);
+              await sleep(delay);
+            } else if (!isLastConfig) {
+              break;
+            }
+          }
+        }
+
+        if (configs.indexOf(config) === configs.length - 1 && lastError) {
+          return {vlm: null, error: lastError}
         }
       }
-    }
 
-    return {vlm: null, error: new Error('Failed to initialize CactusVLM')}
+      return {vlm: null, error: new Error('Failed to initialize CactusVLM')}
+    })();
+
+    CactusVLM._initCache.set(key, initPromise);
+
+    const result = await initPromise;
+    if (result.error) {
+      CactusVLM._initCache.delete(key); 
+    }
+    return result;
   }
 
   async completion(
@@ -210,4 +266,8 @@ export class CactusVLM {
   async release(): Promise<void> {
     return this.context.release()
   }
+
+  async stopCompletion(): Promise<void> {
+    return await this.context.stopCompletion()
+  }
 }