@fugood/llama.node 0.4.7 → 0.6.0

package/src/LlamaCompletionWorker.cpp

@@ -1,7 +1,6 @@
 #include "LlamaCompletionWorker.h"
 #include "LlamaContext.h"
 
-
 size_t findStoppingStrings(const std::string &text,
                            const size_t last_token_size,
                            const std::vector<std::string> &stop_words) {
@@ -27,12 +26,12 @@ size_t findStoppingStrings(const std::string &text,
 LlamaCompletionWorker::LlamaCompletionWorker(
     const Napi::CallbackInfo &info, LlamaSessionPtr &sess,
     Napi::Function callback, common_params params,
-    std::vector<std::string> stop_words,
-    int32_t chat_format,
-    std::vector<std::string> image_paths)
+    std::vector<std::string> stop_words, int32_t chat_format,
+    const std::vector<std::string> &media_paths,
+    const std::vector<llama_token> &guide_tokens)
     : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess),
       _params(params), _stop_words(stop_words), _chat_format(chat_format),
-      _image_paths(image_paths) {
+      _media_paths(media_paths), _guide_tokens(guide_tokens) {
   if (!callback.IsEmpty()) {
     _tsfn = Napi::ThreadSafeFunction::New(info.Env(), callback,
                                           "LlamaCompletionCallback", 0, 1);
@@ -64,34 +63,29 @@ void LlamaCompletionWorker::Execute() {
   LlamaCppSampling sampling{common_sampler_init(model, _params.sampling),
                             common_sampler_free};
 
-  // Process images if any are provided
-  if (!_image_paths.empty()) {
-    const auto* mtmd_ctx = _sess->get_mtmd_ctx();
-    
+  // Process media if any are provided
+  if (!_media_paths.empty()) {
+    const auto *mtmd_ctx = _sess->get_mtmd_ctx();
+
     if (mtmd_ctx != nullptr) {
-      // Process the images and get the tokens
+      // Process the media and get the tokens
       try {
-        n_cur = process_image_prompt(
-          ctx,
-          mtmd_ctx,
-          _sess,
-          _params,
-          _image_paths
-        );
-      } catch (const std::exception& e) {
+        n_cur = processMediaPrompt(ctx, mtmd_ctx, _sess, _params, _media_paths);
+      } catch (const std::exception &e) {
         SetError(e.what());
         _sess->get_mutex().unlock();
         return;
       }
-      
+
       if (n_cur <= 0) {
-        SetError("Failed to process images");
+        SetError("Failed to process media");
         _sess->get_mutex().unlock();
         return;
       }
 
-      fprintf(stdout, "[DEBUG] Image processing successful, n_cur=%zu, tokens=%zu\n", 
-                       n_cur, _sess->tokens_ptr()->size());
+      fprintf(stdout,
+              "[DEBUG] Media processing successful, n_cur=%zu, tokens=%zu\n",
+              n_cur, _sess->tokens_ptr()->size());
 
       n_input = _sess->tokens_ptr()->size();
       if (n_cur == n_input) {
@@ -105,9 +99,10 @@ void LlamaCompletionWorker::Execute() {
     }
   } else {
     // Text-only path
-    std::vector<llama_token> prompt_tokens = ::common_tokenize(ctx, _params.prompt, add_bos);
+    std::vector<llama_token> prompt_tokens =
+        ::common_tokenize(ctx, _params.prompt, add_bos);
     n_input = prompt_tokens.size();
-    
+
     if (_sess->tokens_ptr()->size() > 0) {
       n_cur = common_tokens_part(*(_sess->tokens_ptr()), prompt_tokens);
       if (n_cur == n_input) {
@@ -132,7 +127,7 @@ void LlamaCompletionWorker::Execute() {
         _result.context_full = true;
         break;
       }
-      
+
       const int n_left = n_cur - n_keep - 1;
       const int n_discard = n_left / 2;
 
@@ -147,21 +142,27 @@ void LlamaCompletionWorker::Execute() {
       n_cur -= n_discard;
       _result.truncated = true;
     }
-    
+
     // For multimodal input, n_past might already be set
     // Only decode text tokens if we have any input left
     if (n_input > 0) {
-      int ret = llama_decode(
-          ctx, llama_batch_get_one(embd->data() + n_cur, n_input));
+      int ret =
+          llama_decode(ctx, llama_batch_get_one(embd->data() + n_cur, n_input));
       if (ret < 0) {
         SetError("Failed to decode token, code: " + std::to_string(ret));
         break;
       }
     }
-    
+
     // sample the next token
-    const llama_token new_token_id =
-        common_sampler_sample(sampling.get(), ctx, -1);
+    llama_token new_token_id = common_sampler_sample(sampling.get(), ctx, -1);
+    if (_next_token_uses_guide_token && !_guide_tokens.empty() &&
+        !llama_vocab_is_control(vocab, new_token_id) &&
+        !llama_vocab_is_eog(vocab, new_token_id)) {
+      new_token_id = _guide_tokens[0];
+      _guide_tokens.erase(_guide_tokens.begin());
+    }
+    _next_token_uses_guide_token = (new_token_id == 198);
     common_sampler_accept(sampling.get(), new_token_id, true);
     // prepare the next batch
     embd->emplace_back(new_token_id);
@@ -214,20 +215,15 @@ void LlamaCompletionWorker::Execute() {
 void LlamaCompletionWorker::OnOK() {
   auto env = Napi::AsyncWorker::Env();
   auto result = Napi::Object::New(env);
-  result.Set("tokens_evaluated", Napi::Number::New(env,
-                                                   _result.tokens_evaluated));
+  result.Set("tokens_evaluated",
+             Napi::Number::New(env, _result.tokens_evaluated));
   result.Set("tokens_predicted", Napi::Number::New(Napi::AsyncWorker::Env(),
                                                    _result.tokens_predicted));
-  result.Set("truncated",
-             Napi::Boolean::New(env, _result.truncated));
-  result.Set("context_full",
-             Napi::Boolean::New(env, _result.context_full));
-  result.Set("text",
-             Napi::String::New(env, _result.text.c_str()));
-  result.Set("stopped_eos",
-             Napi::Boolean::New(env, _result.stopped_eos));
-  result.Set("stopped_words",
-             Napi::Boolean::New(env, _result.stopped_words));
+  result.Set("truncated", Napi::Boolean::New(env, _result.truncated));
+  result.Set("context_full", Napi::Boolean::New(env, _result.context_full));
+  result.Set("text", Napi::String::New(env, _result.text.c_str()));
+  result.Set("stopped_eos", Napi::Boolean::New(env, _result.stopped_eos));
+  result.Set("stopped_words", Napi::Boolean::New(env, _result.stopped_words));
   result.Set("stopping_word",
              Napi::String::New(env, _result.stopping_word.c_str()));
   result.Set("stopped_limited",
@@ -238,7 +234,8 @@ void LlamaCompletionWorker::OnOK() {
   std::string content;
   if (!_stop) {
     try {
-      common_chat_msg message = common_chat_parse(_result.text, static_cast<common_chat_format>(_chat_format));
+      common_chat_msg message = common_chat_parse(
+          _result.text, static_cast<common_chat_format>(_chat_format));
       if (!message.reasoning_content.empty()) {
         reasoning_content = message.reasoning_content;
       }
@@ -266,7 +263,8 @@ void LlamaCompletionWorker::OnOK() {
     result.Set("tool_calls", tool_calls);
   }
   if (!reasoning_content.empty()) {
-    result.Set("reasoning_content", Napi::String::New(env, reasoning_content.c_str()));
+    result.Set("reasoning_content",
+               Napi::String::New(env, reasoning_content.c_str()));
   }
   if (!content.empty()) {
     result.Set("content", Napi::String::New(env, content.c_str()));
@@ -276,17 +274,33 @@ void LlamaCompletionWorker::OnOK() {
   const auto timings_token = llama_perf_context(ctx);
 
   auto timingsResult = Napi::Object::New(Napi::AsyncWorker::Env());
-  timingsResult.Set("prompt_n", Napi::Number::New(Napi::AsyncWorker::Env(), timings_token.n_p_eval));
-  timingsResult.Set("prompt_ms", Napi::Number::New(Napi::AsyncWorker::Env(), timings_token.t_p_eval_ms));
-  timingsResult.Set("prompt_per_token_ms", Napi::Number::New(Napi::AsyncWorker::Env(), timings_token.t_p_eval_ms / timings_token.n_p_eval));
-  timingsResult.Set("prompt_per_second", Napi::Number::New(Napi::AsyncWorker::Env(), 1e3 / timings_token.t_p_eval_ms * timings_token.n_p_eval));
-  timingsResult.Set("predicted_n", Napi::Number::New(Napi::AsyncWorker::Env(), timings_token.n_eval));
-  timingsResult.Set("predicted_ms", Napi::Number::New(Napi::AsyncWorker::Env(), timings_token.t_eval_ms));
-  timingsResult.Set("predicted_per_token_ms", Napi::Number::New(Napi::AsyncWorker::Env(), timings_token.t_eval_ms / timings_token.n_eval));
-  timingsResult.Set("predicted_per_second", Napi::Number::New(Napi::AsyncWorker::Env(), 1e3 / timings_token.t_eval_ms * timings_token.n_eval));
+  timingsResult.Set("prompt_n", Napi::Number::New(Napi::AsyncWorker::Env(),
+                                                  timings_token.n_p_eval));
+  timingsResult.Set("prompt_ms", Napi::Number::New(Napi::AsyncWorker::Env(),
+                                                   timings_token.t_p_eval_ms));
+  timingsResult.Set(
+      "prompt_per_token_ms",
+      Napi::Number::New(Napi::AsyncWorker::Env(),
+                        timings_token.t_p_eval_ms / timings_token.n_p_eval));
+  timingsResult.Set("prompt_per_second",
+                    Napi::Number::New(Napi::AsyncWorker::Env(),
+                                      1e3 / timings_token.t_p_eval_ms *
+                                          timings_token.n_p_eval));
+  timingsResult.Set("predicted_n", Napi::Number::New(Napi::AsyncWorker::Env(),
+                                                     timings_token.n_eval));
+  timingsResult.Set("predicted_ms", Napi::Number::New(Napi::AsyncWorker::Env(),
+                                                      timings_token.t_eval_ms));
+  timingsResult.Set(
+      "predicted_per_token_ms",
+      Napi::Number::New(Napi::AsyncWorker::Env(),
+                        timings_token.t_eval_ms / timings_token.n_eval));
+  timingsResult.Set(
+      "predicted_per_second",
+      Napi::Number::New(Napi::AsyncWorker::Env(),
+                        1e3 / timings_token.t_eval_ms * timings_token.n_eval));
 
   result.Set("timings", timingsResult);
-  
+
   Napi::Promise::Deferred::Resolve(result);
 }
 

package/src/LlamaCompletionWorker.h

@@ -20,19 +20,16 @@ public:
                         Napi::Function callback, common_params params,
                         std::vector<std::string> stop_words,
                         int32_t chat_format,
-                        std::vector<std::string> image_paths = {});
+                        const std::vector<std::string> &media_paths = {},
+                        const std::vector<llama_token> &guide_tokens = {});
 
   ~LlamaCompletionWorker();
 
   Napi::Promise GetPromise() { return Napi::Promise::Deferred::Promise(); }
 
-  void OnComplete(std::function<void()> cb) {
-    _onComplete = cb;
-  }
+  void OnComplete(std::function<void()> cb) { _onComplete = cb; }
 
-  void SetStop() {
-    _stop = true;
-  }
+  void SetStop() { _stop = true; }
 
 protected:
   void Execute() override;
@@ -44,11 +41,13 @@ private:
   common_params _params;
   std::vector<std::string> _stop_words;
   int32_t _chat_format;
-  std::vector<std::string> _image_paths;
+  std::vector<std::string> _media_paths;
+  std::vector<llama_token> _guide_tokens;
   std::function<void()> _onComplete;
   bool _has_callback = false;
   bool _stop = false;
   Napi::ThreadSafeFunction _tsfn;
+  bool _next_token_uses_guide_token = true;
   struct {
     size_t tokens_evaluated = 0;
     size_t tokens_predicted = 0;