@fugood/llama.node 0.4.7 → 0.6.0

package/src/LlamaContext.h

@@ -1,15 +1,22 @@
 #include "common.hpp"
-#include "tools/mtmd/mtmd.h"
 #include "tools/mtmd/clip.h"
+#include "tools/mtmd/mtmd.h"
+#include "tts_utils.h"
 
 class LlamaCompletionWorker;
 
+struct vocoder_context {
+  common_params params;
+  std::shared_ptr<llama_model> model;
+  std::shared_ptr<llama_context> context;
+};
+
 class LlamaContext : public Napi::ObjectWrap<LlamaContext> {
 public:
   LlamaContext(const Napi::CallbackInfo &info);
   ~LlamaContext();
   static void ToggleNativeLog(const Napi::CallbackInfo &info);
-  static Napi::Value ModelInfo(const Napi::CallbackInfo& info);
+  static Napi::Value ModelInfo(const Napi::CallbackInfo &info);
   static void Init(Napi::Env env, Napi::Object &exports);
 
 private:
@@ -27,20 +34,35 @@ private:
   void RemoveLoraAdapters(const Napi::CallbackInfo &info);
   Napi::Value GetLoadedLoraAdapters(const Napi::CallbackInfo &info);
   Napi::Value Release(const Napi::CallbackInfo &info);
-  
+
   // Multimodal methods
   Napi::Value InitMultimodal(const Napi::CallbackInfo &info);
   Napi::Value IsMultimodalEnabled(const Napi::CallbackInfo &info);
+  Napi::Value GetMultimodalSupport(const Napi::CallbackInfo &info);
   void ReleaseMultimodal(const Napi::CallbackInfo &info);
 
+  // TTS methods
+  tts_type getTTSType(Napi::Env env, nlohmann::json speaker = nullptr);
+  Napi::Value InitVocoder(const Napi::CallbackInfo &info);
+  void ReleaseVocoder(const Napi::CallbackInfo &info);
+  Napi::Value IsVocoderEnabled(const Napi::CallbackInfo &info);
+  Napi::Value GetFormattedAudioCompletion(const Napi::CallbackInfo &info);
+  Napi::Value GetAudioCompletionGuideTokens(const Napi::CallbackInfo &info);
+  Napi::Value DecodeAudioTokens(const Napi::CallbackInfo &info);
+
   std::string _info;
   Napi::Object _meta;
   LlamaSessionPtr _sess = nullptr;
   common_chat_templates_ptr _templates;
   std::vector<common_adapter_lora_info> _lora;
   LlamaCompletionWorker *_wip = nullptr;
-  
+
   // Multimodal support
   mtmd_context *_mtmd_ctx = nullptr;
   bool _has_multimodal = false;
+
+  // Vocoder support
+  tts_type _tts_type = UNKNOWN;
+  vocoder_context _vocoder;
+  bool _has_vocoder = false;
 };

package/src/LoadSessionWorker.cpp

@@ -12,8 +12,10 @@ void LoadSessionWorker::Execute() {
   std::vector<llama_token> tokens;
   tokens.reserve(_sess->params().n_ctx);
 
-  // Find LLAMA_TOKEN_NULL in the tokens and resize the array to the index of the null token
-  auto null_token_iter = std::find(tokens.begin(), tokens.end(), LLAMA_TOKEN_NULL);
+  // Find LLAMA_TOKEN_NULL in the tokens and resize the array to the index of
+  // the null token
+  auto null_token_iter =
+      std::find(tokens.begin(), tokens.end(), LLAMA_TOKEN_NULL);
   if (null_token_iter != tokens.end()) {
     tokens.resize(std::distance(tokens.begin(), null_token_iter));
   }

package/src/SaveSessionWorker.cpp

@@ -9,16 +9,20 @@ SaveSessionWorker::SaveSessionWorker(const Napi::CallbackInfo &info,
 void SaveSessionWorker::Execute() {
   _sess->get_mutex().lock();
   auto tokens = _sess->tokens_ptr();
-  auto tokens_to_save = std::vector<llama_token>(tokens->begin(), tokens->end());
+  auto tokens_to_save =
+      std::vector<llama_token>(tokens->begin(), tokens->end());
 
-  // Find LLAMA_TOKEN_NULL in the tokens and resize the array to the index of the null token
-  auto null_token_iter = std::find(tokens_to_save.begin(), tokens_to_save.end(), LLAMA_TOKEN_NULL);
+  // Find LLAMA_TOKEN_NULL in the tokens and resize the array to the index of
+  // the null token
+  auto null_token_iter =
+      std::find(tokens_to_save.begin(), tokens_to_save.end(), LLAMA_TOKEN_NULL);
   if (null_token_iter != tokens_to_save.end()) {
-    tokens_to_save.resize(std::distance(tokens_to_save.begin(), null_token_iter));
+    tokens_to_save.resize(
+        std::distance(tokens_to_save.begin(), null_token_iter));
   }
 
-  if (!llama_state_save_file(_sess->context(), _path.c_str(), tokens_to_save.data(),
-                             tokens_to_save.size())) {
+  if (!llama_state_save_file(_sess->context(), _path.c_str(),
+                             tokens_to_save.data(), tokens_to_save.size())) {
     SetError("Failed to save session");
   }
   _sess->get_mutex().unlock();

package/src/TokenizeWorker.cpp

@@ -2,17 +2,24 @@
 #include "LlamaContext.h"
 
 TokenizeWorker::TokenizeWorker(const Napi::CallbackInfo &info,
-                               LlamaSessionPtr &sess, std::string text, std::vector<std::string> image_paths)
-    : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess), _text(text), _image_paths(image_paths) {}
+                               LlamaSessionPtr &sess, std::string text,
+                               std::vector<std::string> media_paths)
+    : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess), _text(text),
+      _media_paths(media_paths) {}
 
 void TokenizeWorker::Execute() {
   auto mtmd_ctx = _sess->get_mtmd_ctx();
-  if (!_image_paths.empty()) {
-    _result = tokenizeWithImages(mtmd_ctx, _text, _image_paths);
+  if (!_media_paths.empty()) {
+    try {
+      _result = tokenizeWithMedia(mtmd_ctx, _text, _media_paths);
+      mtmd_input_chunks_free(_result.chunks);
+    } catch (const std::exception &e) {
+      SetError(e.what());
+    }
   } else {
     const auto tokens = common_tokenize(_sess->context(), _text, false);
     _result.tokens = tokens;
-    _result.has_image = false;
+    _result.has_media = false;
   }
 }
 
@@ -24,24 +31,26 @@ void TokenizeWorker::OnOK() {
   memcpy(tokens.Data(), _result.tokens.data(),
          _result.tokens.size() * sizeof(llama_token));
   result.Set("tokens", tokens);
-  if (_result.has_image) {
-    result.Set("has_image", _result.has_image);
-
-    auto bitmap_hashes = Napi::Array::New(Napi::AsyncWorker::Env(), _result.bitmap_hashes.size());
+  result.Set("has_media", _result.has_media);
+  if (_result.has_media) {
+    auto bitmap_hashes = Napi::Array::New(Napi::AsyncWorker::Env(),
+                                          _result.bitmap_hashes.size());
     for (size_t i = 0; i < _result.bitmap_hashes.size(); i++) {
       bitmap_hashes.Set(i, _result.bitmap_hashes[i]);
     }
     result.Set("bitmap_hashes", bitmap_hashes);
-    auto chunk_pos = Napi::Array::New(Napi::AsyncWorker::Env(), _result.chunk_pos.size());
+    auto chunk_pos =
+        Napi::Array::New(Napi::AsyncWorker::Env(), _result.chunk_pos.size());
     for (size_t i = 0; i < _result.chunk_pos.size(); i++) {
       chunk_pos.Set(i, _result.chunk_pos[i]);
     }
     result.Set("chunk_pos", chunk_pos);
-    auto chunk_pos_images = Napi::Array::New(Napi::AsyncWorker::Env(), _result.chunk_pos_images.size());
-    for (size_t i = 0; i < _result.chunk_pos_images.size(); i++) {
-      chunk_pos_images.Set(i, _result.chunk_pos_images[i]);
+    auto chunk_pos_media = Napi::Array::New(Napi::AsyncWorker::Env(),
+                                            _result.chunk_pos_media.size());
+    for (size_t i = 0; i < _result.chunk_pos_media.size(); i++) {
+      chunk_pos_media.Set(i, _result.chunk_pos_media[i]);
     }
-    result.Set("chunk_pos_images", chunk_pos_images);
+    result.Set("chunk_pos_media", chunk_pos_media);
   }
   Napi::Promise::Deferred::Resolve(result);
 }

package/src/TokenizeWorker.h

@@ -5,7 +5,7 @@ class TokenizeWorker : public Napi::AsyncWorker,
                        public Napi::Promise::Deferred {
 public:
   TokenizeWorker(const Napi::CallbackInfo &info, LlamaSessionPtr &sess,
-                 std::string text, std::vector<std::string> image_paths);
+                 std::string text, std::vector<std::string> media_paths);
 
 protected:
   void Execute();
@@ -15,6 +15,6 @@ protected:
 private:
   LlamaSessionPtr _sess;
   std::string _text;
-  std::vector<std::string> _image_paths;
+  std::vector<std::string> _media_paths;
   TokenizeResult _result;
 };

package/src/addons.cc

@@ -5,25 +5,22 @@
 extern "C" void cleanup_logging();
 
 // Register cleanup function on module unload
-static Napi::Value register_cleanup(const Napi::CallbackInfo& info) {
-  napi_add_env_cleanup_hook(info.Env(), [](void*) {
-    cleanup_logging();
-  }, nullptr);
-  
+static Napi::Value register_cleanup(const Napi::CallbackInfo &info) {
+  napi_add_env_cleanup_hook(
+      info.Env(), [](void *) { cleanup_logging(); }, nullptr);
+
   return info.Env().Undefined();
 }
 
 Napi::Object Init(Napi::Env env, Napi::Object exports) {
   LlamaContext::Init(env, exports);
-  
+
   // Register our cleanup handler for module unload
   exports.Set("__registerCleanup", Napi::Function::New(env, register_cleanup));
-  
+
   // Also register cleanup directly on module init
-  napi_add_env_cleanup_hook(env, [](void*) {
-    cleanup_logging();
-  }, nullptr);
-  
+  napi_add_env_cleanup_hook(env, [](void *) { cleanup_logging(); }, nullptr);
+
   return exports;
 }