@fugood/llama.node 1.1.10 → 1.2.0-rc.0

package/src/LlamaContext.h

@@ -1,7 +1,11 @@
 #include "common.hpp"
 #include "tools/mtmd/clip.h"
 #include "tools/mtmd/mtmd.h"
-#include "tts_utils.h"
+#include "rn-llama/rn-llama.h"
+#include "rn-llama/rn-completion.h"
+#include "rn-llama/rn-tts.h"
+
+using namespace rnllama;
 
 class LlamaCompletionWorker;
 
@@ -43,7 +47,7 @@ private:
   void ReleaseMultimodal(const Napi::CallbackInfo &info);
 
   // TTS methods
-  tts_type getTTSType(Napi::Env env, nlohmann::json speaker = nullptr);
+  rnllama::tts_type getTTSType(Napi::Env env, nlohmann::json speaker = nullptr);
   Napi::Value InitVocoder(const Napi::CallbackInfo &info);
   void ReleaseVocoder(const Napi::CallbackInfo &info);
   Napi::Value IsVocoderEnabled(const Napi::CallbackInfo &info);
@@ -53,17 +57,8 @@ private:
 
   std::string _info;
   Napi::Object _meta;
-  LlamaSessionPtr _sess = nullptr;
-  common_chat_templates_ptr _templates;
-  std::vector<common_adapter_lora_info> _lora;
   LlamaCompletionWorker *_wip = nullptr;
 
-  // Multimodal support
-  mtmd_context *_mtmd_ctx = nullptr;
-  bool _has_multimodal = false;
-
-  // Vocoder support
-  tts_type _tts_type = UNKNOWN;
-  vocoder_context _vocoder;
-  bool _has_vocoder = false;
+  // Use rn-llama context instead of direct llama.cpp types
+  llama_rn_context *_rn_ctx = nullptr;
 };

package/src/LoadSessionWorker.cpp

@@ -2,31 +2,34 @@
 #include "LlamaContext.h"
 
 LoadSessionWorker::LoadSessionWorker(const Napi::CallbackInfo &info,
-                                     LlamaSessionPtr &sess)
+                                     rnllama::llama_rn_context* rn_ctx)
     : AsyncWorker(info.Env()), Deferred(info.Env()), _path(info[0].ToString()),
-      _sess(sess) {}
+      _rn_ctx(rn_ctx) {}
 
 void LoadSessionWorker::Execute() {
-  _sess->get_mutex().lock();
-  // reserve the maximum number of tokens for capacity
-  std::vector<llama_token> tokens;
-  tokens.reserve(_sess->params().n_ctx);
+  try {
+    if (!_rn_ctx || !_rn_ctx->ctx) {
+      SetError("Context not available");
+      return;
+    }
 
-  // Find LLAMA_TOKEN_NULL in the tokens and resize the array to the index of
-  // the null token
-  auto null_token_iter =
-      std::find(tokens.begin(), tokens.end(), LLAMA_TOKEN_NULL);
-  if (null_token_iter != tokens.end()) {
-    tokens.resize(std::distance(tokens.begin(), null_token_iter));
-  }
+    // reserve the maximum number of tokens for capacity
+    std::vector<llama_token> tokens;
+    tokens.reserve(_rn_ctx->n_ctx);
 
-  if (!llama_state_load_file(_sess->context(), _path.c_str(), tokens.data(),
-                             tokens.capacity(), &count)) {
-    SetError("Failed to load session");
+    if (!llama_state_load_file(_rn_ctx->ctx, _path.c_str(), tokens.data(),
+                               tokens.capacity(), &count)) {
+      SetError("Failed to load session");
+      return;
+    }
+    
+    tokens.resize(count);
+    
+    _rn_ctx->completion->embd = std::move(tokens);
+    _rn_ctx->completion->n_past = count;
+  } catch (const std::exception &e) {
+    SetError(e.what());
   }
-  tokens.resize(count);
-  _sess->set_tokens(std::move(tokens));
-  _sess->get_mutex().unlock();
 }
 
 void LoadSessionWorker::OnOK() { Resolve(AsyncWorker::Env().Undefined()); }

package/src/LoadSessionWorker.h

@@ -1,9 +1,10 @@
 #include "common.hpp"
+#include "rn-llama/rn-llama.h"
 
 class LoadSessionWorker : public Napi::AsyncWorker,
                           public Napi::Promise::Deferred {
 public:
-  LoadSessionWorker(const Napi::CallbackInfo &info, LlamaSessionPtr &sess);
+  LoadSessionWorker(const Napi::CallbackInfo &info, rnllama::llama_rn_context* rn_ctx);
 
 protected:
   void Execute();
@@ -12,6 +13,6 @@ protected:
 
 private:
   std::string _path;
-  LlamaSessionPtr _sess;
+  rnllama::llama_rn_context* _rn_ctx;
   size_t count = 0;
 };

package/src/RerankWorker.h

@@ -1,4 +1,5 @@
 #include "common.hpp"
+#include "rn-llama/rn-llama.h"
 #include <vector>
 
 struct RerankResult {
@@ -8,7 +9,7 @@ struct RerankResult {
 class RerankWorker : public Napi::AsyncWorker,
                      public Napi::Promise::Deferred {
 public:
-  RerankWorker(const Napi::CallbackInfo &info, LlamaSessionPtr &sess,
+  RerankWorker(const Napi::CallbackInfo &info, rnllama::llama_rn_context* rn_ctx,
                std::string query, std::vector<std::string> documents, 
                common_params &params);
 
@@ -18,7 +19,7 @@ protected:
   void OnError(const Napi::Error &err);
 
 private:
-  LlamaSessionPtr _sess;
+  rnllama::llama_rn_context* _rn_ctx;
   std::string _query;
   std::vector<std::string> _documents;
   common_params _params;

package/src/SaveSessionWorker.cpp

@@ -2,30 +2,33 @@
 #include "LlamaContext.h"
 
 SaveSessionWorker::SaveSessionWorker(const Napi::CallbackInfo &info,
-                                     LlamaSessionPtr &sess)
+                                     rnllama::llama_rn_context* rn_ctx)
     : AsyncWorker(info.Env()), Deferred(info.Env()), _path(info[0].ToString()),
-      _sess(sess) {}
+      _rn_ctx(rn_ctx) {}
 
 void SaveSessionWorker::Execute() {
-  _sess->get_mutex().lock();
-  auto tokens = _sess->tokens_ptr();
-  auto tokens_to_save =
-      std::vector<llama_token>(tokens->begin(), tokens->end());
+  try {
+    if (!_rn_ctx || !_rn_ctx->ctx) {
+      SetError("Context not available");
+      return;
+    }
 
-  // Find LLAMA_TOKEN_NULL in the tokens and resize the array to the index of
-  // the null token
-  auto null_token_iter =
-      std::find(tokens_to_save.begin(), tokens_to_save.end(), LLAMA_TOKEN_NULL);
-  if (null_token_iter != tokens_to_save.end()) {
-    tokens_to_save.resize(
-        std::distance(tokens_to_save.begin(), null_token_iter));
+    // For rn-llama, we save the context state directly
+    if (_rn_ctx->completion && !_rn_ctx->completion->embd.empty()) {
+      auto &tokens = _rn_ctx->completion->embd;
+      if (!llama_state_save_file(_rn_ctx->ctx, _path.c_str(),
+                                tokens.data(), tokens.size())) {
+        SetError("Failed to save session");
+      }
+    } else {
+      // Save empty session if no tokens available
+      if (!llama_state_save_file(_rn_ctx->ctx, _path.c_str(), nullptr, 0)) {
+        SetError("Failed to save session");
+      }
+    }
+  } catch (const std::exception &e) {
+    SetError(e.what());
   }
-
-  if (!llama_state_save_file(_sess->context(), _path.c_str(),
-                             tokens_to_save.data(), tokens_to_save.size())) {
-    SetError("Failed to save session");
-  }
-  _sess->get_mutex().unlock();
 }
 
 void SaveSessionWorker::OnOK() { Resolve(AsyncWorker::Env().Undefined()); }

package/src/SaveSessionWorker.h

@@ -1,9 +1,10 @@
 #include "common.hpp"
+#include "rn-llama/rn-llama.h"
 
 class SaveSessionWorker : public Napi::AsyncWorker,
                           public Napi::Promise::Deferred {
 public:
-  SaveSessionWorker(const Napi::CallbackInfo &info, LlamaSessionPtr &sess);
+  SaveSessionWorker(const Napi::CallbackInfo &info, rnllama::llama_rn_context* rn_ctx);
 
 protected:
   void Execute();
@@ -12,5 +13,5 @@ protected:
 
 private:
   std::string _path;
-  LlamaSessionPtr _sess;
+  rnllama::llama_rn_context* _rn_ctx;
 };

package/src/TokenizeWorker.cpp

@@ -2,59 +2,62 @@
 #include "LlamaContext.h"
 
 TokenizeWorker::TokenizeWorker(const Napi::CallbackInfo &info,
-                               LlamaSessionPtr &sess, std::string text,
+                               rnllama::llama_rn_context* rn_ctx, std::string text,
                                std::vector<std::string> media_paths)
-    : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess), _text(text),
+    : AsyncWorker(info.Env()), Deferred(info.Env()), _rn_ctx(rn_ctx), _text(text),
       _media_paths(media_paths) {}
 
 void TokenizeWorker::Execute() {
-  auto mtmd_ctx = _sess->get_mtmd_ctx();
-  if (!_media_paths.empty()) {
-    try {
-      _result = tokenizeWithMedia(mtmd_ctx, _text, _media_paths);
-      mtmd_input_chunks_free(_result.chunks);
-    } catch (const std::exception &e) {
-      SetError(e.what());
+  try {
+    // Use rn-llama tokenize API directly
+    auto result = _rn_ctx->tokenize(_text, _media_paths);
+    
+    // Convert llama_token to int32_t
+    _result.tokens.resize(result.tokens.size());
+    for (size_t i = 0; i < result.tokens.size(); i++) {
+      _result.tokens[i] = static_cast<int32_t>(result.tokens[i]);
     }
-  } else {
-    const auto tokens = common_tokenize(_sess->context(), _text, false);
-    _result.tokens = tokens;
-    _result.has_media = false;
+    
+    _result.has_media = result.has_media;
+    _result.bitmap_hashes = result.bitmap_hashes;
+    _result.chunk_pos = result.chunk_pos;
+    _result.chunk_pos_media = result.chunk_pos_media;
+  } catch (const std::exception &e) {
+    SetError(e.what());
   }
 }
 
 void TokenizeWorker::OnOK() {
-  Napi::HandleScope scope(Napi::AsyncWorker::Env());
-  auto result = Napi::Object::New(Napi::AsyncWorker::Env());
-  auto tokens =
-      Napi::Int32Array::New(Napi::AsyncWorker::Env(), _result.tokens.size());
-  memcpy(tokens.Data(), _result.tokens.data(),
-         _result.tokens.size() * sizeof(llama_token));
-  result.Set("tokens", tokens);
-  result.Set("has_media", _result.has_media);
+  Napi::Env env = Napi::AsyncWorker::Env();
+  Napi::Object ret = Napi::Object::New(env);
+  auto tokens = Napi::Int32Array::New(env, _result.tokens.size());
+  memcpy(tokens.Data(), _result.tokens.data(), _result.tokens.size() * sizeof(int32_t));
+  ret.Set("tokens", tokens);
+  ret.Set("has_media", Napi::Boolean::New(env, _result.has_media));
+
   if (_result.has_media) {
-    auto bitmap_hashes = Napi::Array::New(Napi::AsyncWorker::Env(),
-                                          _result.bitmap_hashes.size());
+    auto bitmap_hashes = Napi::Array::New(env, _result.bitmap_hashes.size());
     for (size_t i = 0; i < _result.bitmap_hashes.size(); i++) {
-      bitmap_hashes.Set(i, _result.bitmap_hashes[i]);
+      bitmap_hashes.Set(i, Napi::String::New(env, _result.bitmap_hashes[i]));
     }
-    result.Set("bitmap_hashes", bitmap_hashes);
-    auto chunk_pos =
-        Napi::Array::New(Napi::AsyncWorker::Env(), _result.chunk_pos.size());
+    ret.Set("bitmap_hashes", bitmap_hashes);
+    
+    auto chunk_pos = Napi::Array::New(env, _result.chunk_pos.size());
     for (size_t i = 0; i < _result.chunk_pos.size(); i++) {
-      chunk_pos.Set(i, _result.chunk_pos[i]);
+      chunk_pos.Set(i, Napi::Number::New(env, static_cast<double>(_result.chunk_pos[i])));
     }
-    result.Set("chunk_pos", chunk_pos);
-    auto chunk_pos_media = Napi::Array::New(Napi::AsyncWorker::Env(),
-                                            _result.chunk_pos_media.size());
+    ret.Set("chunk_pos", chunk_pos);
+    
+    auto chunk_pos_media = Napi::Array::New(env, _result.chunk_pos_media.size());
     for (size_t i = 0; i < _result.chunk_pos_media.size(); i++) {
-      chunk_pos_media.Set(i, _result.chunk_pos_media[i]);
+      chunk_pos_media.Set(i, Napi::Number::New(env, static_cast<double>(_result.chunk_pos_media[i])));
     }
-    result.Set("chunk_pos_media", chunk_pos_media);
+    ret.Set("chunk_pos_media", chunk_pos_media);
   }
-  Napi::Promise::Deferred::Resolve(result);
+  
+  Napi::Promise::Deferred::Resolve(ret);
 }
 
 void TokenizeWorker::OnError(const Napi::Error &err) {
   Napi::Promise::Deferred::Reject(err.Value());
-}
+}

package/src/TokenizeWorker.h

@@ -1,10 +1,19 @@
 #include "common.hpp"
+#include "rn-llama/rn-llama.h"
 #include <vector>
 
+struct TokenizeResult {
+  std::vector<int32_t> tokens;
+  bool has_media;
+  std::vector<std::string> bitmap_hashes;
+  std::vector<size_t> chunk_pos;
+  std::vector<size_t> chunk_pos_media;
+};
+
 class TokenizeWorker : public Napi::AsyncWorker,
                        public Napi::Promise::Deferred {
 public:
-  TokenizeWorker(const Napi::CallbackInfo &info, LlamaSessionPtr &sess,
+  TokenizeWorker(const Napi::CallbackInfo &info, rnllama::llama_rn_context* rn_ctx,
                  std::string text, std::vector<std::string> media_paths);
 
 protected:
@@ -13,8 +22,8 @@ protected:
   void OnError(const Napi::Error &err);
 
 private:
-  LlamaSessionPtr _sess;
+  rnllama::llama_rn_context* _rn_ctx;
   std::string _text;
   std::vector<std::string> _media_paths;
   TokenizeResult _result;
-};
+};