@fugood/llama.node 0.4.4 → 0.4.6

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "0.4.4",
+  "version": "0.4.6",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {

package/src/LlamaCompletionWorker.cpp

@@ -266,6 +266,30 @@ llama_pos processImage(
 
   llama_pos new_n_past = n_past;
 
+  // Adjust n_past to position of the text chunk
+  // TODO: Edit the text chunk to remove the tokens before n_past to speed up
+  // need to update the mtmd api
+  auto adjusted_n_past = -1;
+  for (size_t i = 0; i < chunk_pos.size(); i++) {
+    if (n_past < chunk_pos[i]) {
+      break;
+    }
+    bool is_end = i + 1 == chunk_pos.size();
+    if (
+      chunk_pos[i] < n_past &&
+      (!is_end && chunk_pos[i + 1] > n_past)
+      // is_end & n_past < total_token_count:
+      // don't need to adjust and it will skip eval_chunk_single, let nextToken() to finish the job
+    ) {
+      adjusted_n_past = chunk_pos[i];
+    }
+  }
+  if (adjusted_n_past != -1) {
+    n_past = adjusted_n_past;
+    new_n_past = n_past;
+    fprintf(stdout, "[DEBUG] Adjusted n_past to %d\n", n_past);
+  }
+
   // Compare bitmap hashes, if they are not the same, backtrack n_past to the position of the first mismatch
   auto mtmd_bitmap_past_hashes = sess->mtmd_bitmap_past_hashes_ptr();
   if (mtmd_bitmap_past_hashes->size() > 0) {

package/src/LlamaContext.cpp

@@ -12,6 +12,10 @@
 #include "SaveSessionWorker.h"
 #include "TokenizeWorker.h"
 
+#include <mutex>
+#include <queue>
+#include <atomic>
+
 // Helper function for formatted strings (for console logs)
 template<typename ... Args>
 static std::string format_string(const std::string& format, Args ... args) {
@@ -383,17 +387,60 @@ bool validateModelChatTemplate(const struct llama_model * model, const bool use_
   return common_chat_verify_template(tmpl, use_jinja);
 }
 
-static Napi::FunctionReference _log_callback;
+// Store log messages for processing
+struct LogMessage {
+  std::string level;
+  std::string text;
+};
+
+// Global variables for logging
+static Napi::ThreadSafeFunction g_tsfn;
+static std::atomic<bool> g_logging_enabled{false};
+static std::mutex g_mutex;
+static std::queue<LogMessage> g_message_queue;
+
+// Forward declaration of the cleanup function
+extern "C" void cleanup_logging();
 
 // toggleNativeLog(enable: boolean, callback: (log: string) => void): void
 void LlamaContext::ToggleNativeLog(const Napi::CallbackInfo &info) {
+  Napi::Env env = info.Env();
   bool enable = info[0].ToBoolean().Value();
+  
   if (enable) {
-    _log_callback.Reset(info[1].As<Napi::Function>());
+    if (!info[1].IsFunction()) {
+      Napi::TypeError::New(env, "Callback function required").ThrowAsJavaScriptException();
+      return;
+    }
+    
+    // First clean up existing thread-safe function if any
+    if (g_logging_enabled) {
+      g_tsfn.Release();
+      g_logging_enabled = false;
+    }
+    
+    // Create thread-safe function that can be called from any thread
+    g_tsfn = Napi::ThreadSafeFunction::New(
+      env,
+      info[1].As<Napi::Function>(),
+      "LLAMA Logger",
+      0,
+      1,
+      [](Napi::Env) {
+        // Finalizer callback - nothing needed here
+      }
+    );
+
+    g_logging_enabled = true;
     
-    llama_log_set([](ggml_log_level level, const char * text, void * user_data) {
+    // Set up log callback
+    llama_log_set([](ggml_log_level level, const char* text, void* user_data) {
+      // First call the default logger
       llama_log_callback_default(level, text, user_data);
 
+      if (!g_logging_enabled) return;
+
+      // Determine log level string
       std::string level_str = "";
       if (level == GGML_LOG_LEVEL_ERROR) {
         level_str = "error";
@@ -402,24 +449,32 @@ void LlamaContext::ToggleNativeLog(const Napi::CallbackInfo &info) {
       } else if (level == GGML_LOG_LEVEL_WARN) {
         level_str = "warn";
       }
-
-      if (_log_callback.IsEmpty()) {
-        return;
-      }
-      try {
-        Napi::Env env = _log_callback.Env();
-        Napi::HandleScope scope(env);
-        _log_callback.Call({
-          Napi::String::New(env, level_str),
-          Napi::String::New(env, text)
+      
+      // Create a heap-allocated copy of the data
+      auto* data = new LogMessage{level_str, text};
+
+      // Queue callback to be executed on the JavaScript thread
+      auto status = g_tsfn.BlockingCall(data, [](Napi::Env env, Napi::Function jsCallback, LogMessage* data) {
+        // This code runs on the JavaScript thread
+        jsCallback.Call({
+          Napi::String::New(env, data->level),
+          Napi::String::New(env, data->text)
         });
-      } catch (const std::exception &e) {
-        // printf("Error calling log callback: %s\n", e.what());
+        delete data;
+      });
+
+      // If the call failed (e.g., runtime is shutting down), clean up the data
+      if (status != napi_ok) {
+        delete data;
       }
     }, nullptr);
   } else {
-    _log_callback.Reset();
-    llama_log_set(llama_log_callback_default, nullptr);
+    // Disable logging
+    if (g_logging_enabled) {
+      g_logging_enabled = false;
+      g_tsfn.Release();
+      llama_log_set(llama_log_callback_default, nullptr);
+    }
   }
 }
 
@@ -1004,6 +1059,7 @@ Napi::Value LlamaContext::Release(const Napi::CallbackInfo &info) {
   if (_wip != nullptr) {
     _wip->SetStop();
   }
+  
   if (_sess == nullptr) {
     auto promise = Napi::Promise::Deferred(env);
     promise.Resolve(env.Undefined());
@@ -1020,6 +1076,15 @@ Napi::Value LlamaContext::Release(const Napi::CallbackInfo &info) {
   return worker->Promise();
 }
 
+// Cleanup function for the logging system
+// This is exposed externally for module cleanup
+extern "C" void cleanup_logging() {
+  if (g_logging_enabled) {
+    g_logging_enabled = false;
+    g_tsfn.Release();
+  }
+}
+
 LlamaContext::~LlamaContext() {
   if (_mtmd_ctx != nullptr) {
     mtmd_free(_mtmd_ctx);

package/src/SaveSessionWorker.cpp

@@ -9,15 +9,16 @@ SaveSessionWorker::SaveSessionWorker(const Napi::CallbackInfo &info,
 void SaveSessionWorker::Execute() {
   _sess->get_mutex().lock();
   auto tokens = _sess->tokens_ptr();
+  auto tokens_to_save = std::vector<llama_token>(tokens->begin(), tokens->end());
 
   // Find LLAMA_TOKEN_NULL in the tokens and resize the array to the index of the null token
-  auto null_token_iter = std::find(tokens->begin(), tokens->end(), LLAMA_TOKEN_NULL);
-  if (null_token_iter != tokens->end()) {
-    tokens->resize(std::distance(tokens->begin(), null_token_iter));
+  auto null_token_iter = std::find(tokens_to_save.begin(), tokens_to_save.end(), LLAMA_TOKEN_NULL);
+  if (null_token_iter != tokens_to_save.end()) {
+    tokens_to_save.resize(std::distance(tokens_to_save.begin(), null_token_iter));
   }
 
-  if (!llama_state_save_file(_sess->context(), _path.c_str(), tokens->data(),
-                             tokens->size())) {
+  if (!llama_state_save_file(_sess->context(), _path.c_str(), tokens_to_save.data(),
+                             tokens_to_save.size())) {
     SetError("Failed to save session");
   }
   _sess->get_mutex().unlock();

package/src/addons.cc

@@ -1,8 +1,29 @@
 #include "LlamaContext.h"
 #include <napi.h>
 
+// Forward declaration of our cleanup function
+extern "C" void cleanup_logging();
+
+// Register cleanup function on module unload
+static Napi::Value register_cleanup(const Napi::CallbackInfo& info) {
+  napi_add_env_cleanup_hook(info.Env(), [](void*) {
+    cleanup_logging();
+  }, nullptr);
+  
+  return info.Env().Undefined();
+}
+
 Napi::Object Init(Napi::Env env, Napi::Object exports) {
   LlamaContext::Init(env, exports);
+  
+  // Register our cleanup handler for module unload
+  exports.Set("__registerCleanup", Napi::Function::New(env, register_cleanup));
+  
+  // Also register cleanup directly on module init
+  napi_add_env_cleanup_hook(env, [](void*) {
+    cleanup_logging();
+  }, nullptr);
+  
   return exports;
 }