@fugood/llama.node 0.4.2 → 0.4.4

package/lib/binding.ts

@@ -113,6 +113,8 @@ export type LlamaCompletionOptions = {
 
 export type LlamaCompletionResult = {
   text: string
+  reasoning_content?: string
+  content?: string
   tokens_predicted: number
   tokens_evaluated: number
   truncated: boolean

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "0.4.2",
+  "version": "0.4.4",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {

package/src/LlamaCompletionWorker.cpp

@@ -507,6 +507,8 @@ void LlamaCompletionWorker::Execute() {
     _result.tokens_predicted += 1;
     n_input = 1;
     if (_has_callback) {
+      // TODO: When we got possible stop words (startsWith)
+      // we should avoid calling the callback, wait for the next token
       const char *c_token = strdup(token.c_str());
       _tsfn.BlockingCall(c_token, [](Napi::Env env, Napi::Function jsCallback,
                                      const char *value) {
@@ -518,6 +520,8 @@ void LlamaCompletionWorker::Execute() {
     }
     // is it an end of generation?
     if (llama_vocab_is_eog(vocab, new_token_id)) {
+      _result.stopped_eos = true;
+      // TODO: EOS token should be cut
       break;
     }
     // check for stop words
@@ -525,10 +529,16 @@ void LlamaCompletionWorker::Execute() {
       const size_t stop_pos =
           findStoppingStrings(_result.text, token.size(), _stop_words);
       if (stop_pos != std::string::npos) {
+        _result.stopped_words = true;
+        _result.stopping_word = _result.text.substr(stop_pos, token.size());
+        _result.text = _result.text.substr(0, stop_pos - 1);
         break;
       }
     }
   }
+  if (!_result.stopped_eos && !_result.stopped_words) {
+    _result.stopped_limited = true;
+  }
   const auto t_main_end = ggml_time_us();
   _sess->get_mutex().unlock();
   if (_onComplete) {
@@ -549,18 +559,26 @@ void LlamaCompletionWorker::OnOK() {
              Napi::Boolean::New(env, _result.context_full));
   result.Set("text",
              Napi::String::New(env, _result.text.c_str()));
+  result.Set("stopped_eos",
+             Napi::Boolean::New(env, _result.stopped_eos));
+  result.Set("stopped_words",
+             Napi::Boolean::New(env, _result.stopped_words));
+  result.Set("stopping_word",
+             Napi::String::New(env, _result.stopping_word.c_str()));
+  result.Set("stopped_limited",
+             Napi::Boolean::New(env, _result.stopped_limited));
 
   Napi::Array tool_calls = Napi::Array::New(Napi::AsyncWorker::Env());
-  std::string * reasoning_content = nullptr;
-  std::string * content = nullptr;
+  std::string reasoning_content = "";
+  std::string content;
   if (!_stop) {
     try {
       common_chat_msg message = common_chat_parse(_result.text, static_cast<common_chat_format>(_chat_format));
       if (!message.reasoning_content.empty()) {
-        reasoning_content = &message.reasoning_content;
+        reasoning_content = message.reasoning_content;
       }
       if (!message.content.empty()) {
-        content = &message.content;
+        content = message.content;
       }
       for (size_t i = 0; i < message.tool_calls.size(); i++) {
         const auto &tc = message.tool_calls[i];
@@ -582,11 +600,11 @@ void LlamaCompletionWorker::OnOK() {
   if (tool_calls.Length() > 0) {
     result.Set("tool_calls", tool_calls);
   }
-  if (reasoning_content) {
-    result.Set("reasoning_content", Napi::String::New(env, reasoning_content->c_str()));
+  if (!reasoning_content.empty()) {
+    result.Set("reasoning_content", Napi::String::New(env, reasoning_content.c_str()));
   }
-  if (content) {
-    result.Set("content", Napi::String::New(env, content->c_str()));
+  if (!content.empty()) {
+    result.Set("content", Napi::String::New(env, content.c_str()));
   }
 
   auto ctx = _sess->context();

package/src/LlamaCompletionWorker.h

@@ -57,5 +57,9 @@ private:
     bool truncated = false;
     bool context_full = false;
     std::string text;
+    bool stopped_eos = false;
+    bool stopped_words = false;
+    std::string stopping_word;
+    bool stopped_limited = false;
   } _result;
 };