@fugood/llama.node 0.4.3 → 0.4.5

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "0.4.3",
+  "version": "0.4.5",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {

package/src/LlamaCompletionWorker.cpp

@@ -507,6 +507,8 @@ void LlamaCompletionWorker::Execute() {
     _result.tokens_predicted += 1;
     n_input = 1;
     if (_has_callback) {
+      // TODO: When we got possible stop words (startsWith)
+      // we should avoid calling the callback, wait for the next token
       const char *c_token = strdup(token.c_str());
       _tsfn.BlockingCall(c_token, [](Napi::Env env, Napi::Function jsCallback,
                                      const char *value) {
@@ -518,6 +520,8 @@ void LlamaCompletionWorker::Execute() {
     }
     // is it an end of generation?
     if (llama_vocab_is_eog(vocab, new_token_id)) {
+      _result.stopped_eos = true;
+      // TODO: EOS token should be cut
       break;
     }
     // check for stop words
@@ -525,10 +529,16 @@ void LlamaCompletionWorker::Execute() {
       const size_t stop_pos =
           findStoppingStrings(_result.text, token.size(), _stop_words);
       if (stop_pos != std::string::npos) {
+        _result.stopped_words = true;
+        _result.stopping_word = _result.text.substr(stop_pos, token.size());
+        _result.text = _result.text.substr(0, stop_pos - 1);
         break;
       }
     }
   }
+  if (!_result.stopped_eos && !_result.stopped_words) {
+    _result.stopped_limited = true;
+  }
   const auto t_main_end = ggml_time_us();
   _sess->get_mutex().unlock();
   if (_onComplete) {
@@ -549,6 +559,14 @@ void LlamaCompletionWorker::OnOK() {
              Napi::Boolean::New(env, _result.context_full));
   result.Set("text",
              Napi::String::New(env, _result.text.c_str()));
+  result.Set("stopped_eos",
+             Napi::Boolean::New(env, _result.stopped_eos));
+  result.Set("stopped_words",
+             Napi::Boolean::New(env, _result.stopped_words));
+  result.Set("stopping_word",
+             Napi::String::New(env, _result.stopping_word.c_str()));
+  result.Set("stopped_limited",
+             Napi::Boolean::New(env, _result.stopped_limited));
 
   Napi::Array tool_calls = Napi::Array::New(Napi::AsyncWorker::Env());
   std::string reasoning_content = "";

package/src/LlamaCompletionWorker.h

@@ -57,5 +57,9 @@ private:
     bool truncated = false;
     bool context_full = false;
     std::string text;
+    bool stopped_eos = false;
+    bool stopped_words = false;
+    std::string stopping_word;
+    bool stopped_limited = false;
   } _result;
 };

package/src/LlamaContext.cpp

@@ -12,6 +12,10 @@
 #include "SaveSessionWorker.h"
 #include "TokenizeWorker.h"
 
+#include <mutex>
+#include <queue>
+#include <atomic>
+
 // Helper function for formatted strings (for console logs)
 template<typename ... Args>
 static std::string format_string(const std::string& format, Args ... args) {
@@ -383,17 +387,60 @@ bool validateModelChatTemplate(const struct llama_model * model, const bool use_
   return common_chat_verify_template(tmpl, use_jinja);
 }
 
-static Napi::FunctionReference _log_callback;
+// Store log messages for processing
+struct LogMessage {
+  std::string level;
+  std::string text;
+};
+
+// Global variables for logging
+static Napi::ThreadSafeFunction g_tsfn;
+static std::atomic<bool> g_logging_enabled{false};
+static std::mutex g_mutex;
+static std::queue<LogMessage> g_message_queue;
+
+// Forward declaration of the cleanup function
+extern "C" void cleanup_logging();
 
 // toggleNativeLog(enable: boolean, callback: (log: string) => void): void
 void LlamaContext::ToggleNativeLog(const Napi::CallbackInfo &info) {
+  Napi::Env env = info.Env();
   bool enable = info[0].ToBoolean().Value();
+  
   if (enable) {
-    _log_callback.Reset(info[1].As<Napi::Function>());
+    if (!info[1].IsFunction()) {
+      Napi::TypeError::New(env, "Callback function required").ThrowAsJavaScriptException();
+      return;
+    }
+    
+    // First clean up existing thread-safe function if any
+    if (g_logging_enabled) {
+      g_tsfn.Release();
+      g_logging_enabled = false;
+    }
+    
+    // Create thread-safe function that can be called from any thread
+    g_tsfn = Napi::ThreadSafeFunction::New(
+      env,
+      info[1].As<Napi::Function>(),
+      "LLAMA Logger",
+      0,
+      1,
+      [](Napi::Env) {
+        // Finalizer callback - nothing needed here
+      }
+    );
+
+    g_logging_enabled = true;
     
-    llama_log_set([](ggml_log_level level, const char * text, void * user_data) {
+    // Set up log callback
+    llama_log_set([](ggml_log_level level, const char* text, void* user_data) {
+      // First call the default logger
       llama_log_callback_default(level, text, user_data);
 
+      if (!g_logging_enabled) return;
+
+      // Determine log level string
       std::string level_str = "";
       if (level == GGML_LOG_LEVEL_ERROR) {
         level_str = "error";
@@ -402,24 +449,32 @@ void LlamaContext::ToggleNativeLog(const Napi::CallbackInfo &info) {
       } else if (level == GGML_LOG_LEVEL_WARN) {
         level_str = "warn";
       }
-
-      if (_log_callback.IsEmpty()) {
-        return;
-      }
-      try {
-        Napi::Env env = _log_callback.Env();
-        Napi::HandleScope scope(env);
-        _log_callback.Call({
-          Napi::String::New(env, level_str),
-          Napi::String::New(env, text)
+      
+      // Create a heap-allocated copy of the data
+      auto* data = new LogMessage{level_str, text};
+
+      // Queue callback to be executed on the JavaScript thread
+      auto status = g_tsfn.BlockingCall(data, [](Napi::Env env, Napi::Function jsCallback, LogMessage* data) {
+        // This code runs on the JavaScript thread
+        jsCallback.Call({
+          Napi::String::New(env, data->level),
+          Napi::String::New(env, data->text)
         });
-      } catch (const std::exception &e) {
-        // printf("Error calling log callback: %s\n", e.what());
+        delete data;
+      });
+
+      // If the call failed (e.g., runtime is shutting down), clean up the data
+      if (status != napi_ok) {
+        delete data;
       }
     }, nullptr);
   } else {
-    _log_callback.Reset();
-    llama_log_set(llama_log_callback_default, nullptr);
+    // Disable logging
+    if (g_logging_enabled) {
+      g_logging_enabled = false;
+      g_tsfn.Release();
+      llama_log_set(llama_log_callback_default, nullptr);
+    }
   }
 }
 
@@ -1004,6 +1059,7 @@ Napi::Value LlamaContext::Release(const Napi::CallbackInfo &info) {
   if (_wip != nullptr) {
     _wip->SetStop();
   }
+  
   if (_sess == nullptr) {
     auto promise = Napi::Promise::Deferred(env);
     promise.Resolve(env.Undefined());
@@ -1020,6 +1076,15 @@ Napi::Value LlamaContext::Release(const Napi::CallbackInfo &info) {
   return worker->Promise();
 }
 
+// Cleanup function for the logging system
+// This is exposed externally for module cleanup
+extern "C" void cleanup_logging() {
+  if (g_logging_enabled) {
+    g_logging_enabled = false;
+    g_tsfn.Release();
+  }
+}
+
 LlamaContext::~LlamaContext() {
   if (_mtmd_ctx != nullptr) {
     mtmd_free(_mtmd_ctx);

package/src/addons.cc

@@ -1,8 +1,29 @@
 #include "LlamaContext.h"
 #include <napi.h>
 
+// Forward declaration of our cleanup function
+extern "C" void cleanup_logging();
+
+// Register cleanup function on module unload
+static Napi::Value register_cleanup(const Napi::CallbackInfo& info) {
+  napi_add_env_cleanup_hook(info.Env(), [](void*) {
+    cleanup_logging();
+  }, nullptr);
+  
+  return info.Env().Undefined();
+}
+
 Napi::Object Init(Napi::Env env, Napi::Object exports) {
   LlamaContext::Init(env, exports);
+  
+  // Register our cleanup handler for module unload
+  exports.Set("__registerCleanup", Napi::Function::New(env, register_cleanup));
+  
+  // Also register cleanup directly on module init
+  napi_add_env_cleanup_hook(env, [](void*) {
+    cleanup_logging();
+  }, nullptr);
+  
   return exports;
 }