@fugood/llama.node 0.4.3 → 0.4.4

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "0.4.3",
+  "version": "0.4.4",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {

package/src/LlamaCompletionWorker.cpp

@@ -507,6 +507,8 @@ void LlamaCompletionWorker::Execute() {
     _result.tokens_predicted += 1;
     n_input = 1;
     if (_has_callback) {
+      // TODO: When we got possible stop words (startsWith)
+      // we should avoid calling the callback, wait for the next token
       const char *c_token = strdup(token.c_str());
       _tsfn.BlockingCall(c_token, [](Napi::Env env, Napi::Function jsCallback,
                                      const char *value) {
@@ -518,6 +520,8 @@ void LlamaCompletionWorker::Execute() {
     }
     // is it an end of generation?
     if (llama_vocab_is_eog(vocab, new_token_id)) {
+      _result.stopped_eos = true;
+      // TODO: EOS token should be cut
       break;
     }
     // check for stop words
@@ -525,10 +529,16 @@ void LlamaCompletionWorker::Execute() {
       const size_t stop_pos =
           findStoppingStrings(_result.text, token.size(), _stop_words);
       if (stop_pos != std::string::npos) {
+        _result.stopped_words = true;
+        _result.stopping_word = _result.text.substr(stop_pos, token.size());
+        _result.text = _result.text.substr(0, stop_pos - 1);
         break;
       }
     }
   }
+  if (!_result.stopped_eos && !_result.stopped_words) {
+    _result.stopped_limited = true;
+  }
   const auto t_main_end = ggml_time_us();
   _sess->get_mutex().unlock();
   if (_onComplete) {
@@ -549,6 +559,14 @@ void LlamaCompletionWorker::OnOK() {
              Napi::Boolean::New(env, _result.context_full));
   result.Set("text",
              Napi::String::New(env, _result.text.c_str()));
+  result.Set("stopped_eos",
+             Napi::Boolean::New(env, _result.stopped_eos));
+  result.Set("stopped_words",
+             Napi::Boolean::New(env, _result.stopped_words));
+  result.Set("stopping_word",
+             Napi::String::New(env, _result.stopping_word.c_str()));
+  result.Set("stopped_limited",
+             Napi::Boolean::New(env, _result.stopped_limited));
 
   Napi::Array tool_calls = Napi::Array::New(Napi::AsyncWorker::Env());
   std::string reasoning_content = "";

package/src/LlamaCompletionWorker.h

@@ -57,5 +57,9 @@ private:
     bool truncated = false;
     bool context_full = false;
     std::string text;
+    bool stopped_eos = false;
+    bool stopped_words = false;
+    std::string stopping_word;
+    bool stopped_limited = false;
   } _result;
 };