@fugood/llama.node 1.1.10 → 1.2.0-rc.0

package/CMakeLists.txt

@@ -137,10 +137,6 @@ file(
     "src/LlamaCompletionWorker.h"
     "src/LlamaContext.cpp"
     "src/LlamaContext.h"
-    "src/TokenizeWorker.cpp"
-    "src/TokenizeWorker.h"
-    "src/DetokenizeWorker.cpp"
-    "src/DetokenizeWorker.h"
     "src/EmbeddingWorker.cpp"
     "src/EmbeddingWorker.h"
     "src/RerankWorker.cpp"
@@ -149,12 +145,13 @@ file(
     "src/LoadSessionWorker.h"
     "src/SaveSessionWorker.cpp"
     "src/SaveSessionWorker.h"
+    "src/TokenizeWorker.cpp"
+    "src/TokenizeWorker.h"
+    "src/DetokenizeWorker.cpp"
+    "src/DetokenizeWorker.h"
     "src/DecodeAudioTokenWorker.cpp"
     "src/DecodeAudioTokenWorker.h"
-    "src/tts_utils.cpp"
-    "src/tts_utils.h"
-    "src/anyascii.h"
-    "src/anyascii.c"
+    "src/rn-llama/*"
 )
 
 if (NOT MSVC AND CMAKE_SYSTEM_NAME STREQUAL "Windows")

package/lib/binding.ts

@@ -27,7 +27,8 @@ export type LlamaModelOptions = {
   n_ubatch?: number
   n_threads?: number
   n_gpu_layers?: number
-  flash_attn?: boolean
+  flash_attn_type?: 'auto' | 'on' | 'off'
+  flash_attn?: boolean // Deprecated: use flash_attn_type instead
   cache_type_k?:
     | 'f16'
     | 'f32'
@@ -149,6 +150,21 @@ export type LlamaCompletionOptions = {
    * Help prevent hallucinations by forcing the TTS to use the correct words.
    */
   guide_tokens?: number[] | Int32Array
+  /**
+   * Number of top token probabilities to return for each generated token.
+   * When > 0, completion_probabilities will be included in streaming callbacks and final result.
+   */
+  n_probs?: number
+}
+
+export type TokenProbability = {
+  tok_str: string
+  prob: number
+}
+
+export type CompletionProbability = {
+  content: string
+  probs: TokenProbability[]
 }
 
 export type LlamaCompletionResult = {
@@ -162,6 +178,7 @@ export type LlamaCompletionResult = {
   context_full: boolean
   interrupted: boolean
   audio_tokens?: Array<number>
+  completion_probabilities?: CompletionProbability[]
   timings: {
     prompt_n: number
     prompt_ms: number
@@ -180,6 +197,7 @@ export type LlamaCompletionToken = {
   reasoning_content?: string
   tool_calls?: ToolCall[]
   accumulated_text?: string
+  completion_probabilities?: CompletionProbability[]
 }
 
 export type TokenizeResult = {
@@ -308,7 +326,7 @@ export interface LlamaContext {
   stopCompletion(): void
   tokenize(text: string, media_paths?: string[]): Promise<TokenizeResult>
   detokenize(tokens: number[]): Promise<string>
-  embedding(text: string): Promise<EmbeddingResult>
+  embedding(text: string, params?: { embd_normalize?: number }): Promise<EmbeddingResult>
   rerank(query: string, documents: string[], params?: RerankParams): Promise<RerankResult[]>
   saveSession(path: string): Promise<void>
   loadSession(path: string): Promise<void>

package/lib/index.js

@@ -180,8 +180,8 @@ class LlamaContextWrapper {
     detokenize(tokens) {
         return this.ctx.detokenize(tokens);
     }
-    embedding(text) {
-        return this.ctx.embedding(text);
+    embedding(text, params) {
+        return this.ctx.embedding(text, params);
     }
     rerank(query, documents, params) {
         return this.ctx

package/lib/index.ts

@@ -251,8 +251,8 @@ class LlamaContextWrapper {
     return this.ctx.detokenize(tokens)
   }
 
-  embedding(text: string): Promise<EmbeddingResult> {
-    return this.ctx.embedding(text)
+  embedding(text: string, params?: { embd_normalize?: number }): Promise<EmbeddingResult> {
+    return this.ctx.embedding(text, params)
   }
 
   rerank(

package/package.json

@@ -1,11 +1,12 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "1.1.10",
+  "version": "1.2.0-rc.0",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {
-    "bootstrap": "npm install --omit=optional",
+    "copy-rn-llama-source": "node scripts/copy-rn-llama-source.js",
+    "bootstrap": "npm run copy-rn-llama-source && npm install --omit=optional",
     "postinstall": "node scripts/check.js",
     "pretest": "node scripts/download-test-models.js",
     "test": "jest",
@@ -53,6 +54,7 @@
     "scripts/check.js",
     "scripts/llama.cpp.patch",
     "src/*.{cc,c,h,hpp}",
+    "src/rn-llama/*",
     "src/DecodeAudioTokenWorker.cpp",
     "src/DetokenizeWorker.cpp",
     "src/DisposeWorker.cpp",
@@ -62,7 +64,6 @@
     "src/LoadSessionWorker.cpp",
     "src/SaveSessionWorker.cpp",
     "src/TokenizeWorker.cpp",
-    "src/tts_utils.cpp",
     "src/llama.cpp/{common,src,include}/**/*.{h,hpp,cpp,cc,c}",
     "src/llama.cpp/ggml/include/*.h",
     "src/llama.cpp/ggml/src/ggml-cpu/**/*.{h,hpp,cpp,cc,c}",
@@ -71,19 +72,19 @@
     "CMakeLists.txt"
   ],
   "optionalDependencies": {
-    "@fugood/node-llama-linux-x64": "1.1.10",
-    "@fugood/node-llama-linux-x64-vulkan": "1.1.10",
-    "@fugood/node-llama-linux-x64-cuda": "1.1.10",
-    "@fugood/node-llama-linux-arm64": "1.1.10",
-    "@fugood/node-llama-linux-arm64-vulkan": "1.1.10",
-    "@fugood/node-llama-linux-arm64-cuda": "1.1.10",
-    "@fugood/node-llama-win32-x64": "1.1.10",
-    "@fugood/node-llama-win32-x64-vulkan": "1.1.10",
-    "@fugood/node-llama-win32-x64-cuda": "1.1.10",
-    "@fugood/node-llama-win32-arm64": "1.1.10",
-    "@fugood/node-llama-win32-arm64-vulkan": "1.1.10",
-    "@fugood/node-llama-darwin-x64": "1.1.10",
-    "@fugood/node-llama-darwin-arm64": "1.1.10"
+    "@fugood/node-llama-linux-x64": "1.2.0-rc.0",
+    "@fugood/node-llama-linux-x64-vulkan": "1.2.0-rc.0",
+    "@fugood/node-llama-linux-x64-cuda": "1.2.0-rc.0",
+    "@fugood/node-llama-linux-arm64": "1.2.0-rc.0",
+    "@fugood/node-llama-linux-arm64-vulkan": "1.2.0-rc.0",
+    "@fugood/node-llama-linux-arm64-cuda": "1.2.0-rc.0",
+    "@fugood/node-llama-win32-x64": "1.2.0-rc.0",
+    "@fugood/node-llama-win32-x64-vulkan": "1.2.0-rc.0",
+    "@fugood/node-llama-win32-x64-cuda": "1.2.0-rc.0",
+    "@fugood/node-llama-win32-arm64": "1.2.0-rc.0",
+    "@fugood/node-llama-win32-arm64-vulkan": "1.2.0-rc.0",
+    "@fugood/node-llama-darwin-x64": "1.2.0-rc.0",
+    "@fugood/node-llama-darwin-arm64": "1.2.0-rc.0"
   },
   "devDependencies": {
     "@babel/preset-env": "^7.24.4",
@@ -115,6 +116,9 @@
     ],
     "testMatch": [
       "**/*.test.ts"
+    ],
+    "testPathIgnorePatterns": [
+      "<rootDir>/src/llama.rn/"
     ]
   },
   "prettier": {

package/src/DecodeAudioTokenWorker.cpp

@@ -1,40 +1,37 @@
 #include "DecodeAudioTokenWorker.h"
-#include "tts_utils.h"
-#include <vector>
+#include "LlamaContext.h"
 
-DecodeAudioTokenWorker::DecodeAudioTokenWorker(
-    const Napi::CallbackInfo &info, llama_model *model, llama_context *ctx,
-    int n_threads, const std::vector<llama_token> &tokens)
-    : AsyncWorker(info.Env()), Deferred(info.Env()), _model(model), _ctx(ctx),
-      _n_threads(n_threads), _tokens(tokens) {}
+DecodeAudioTokenWorker::DecodeAudioTokenWorker(const Napi::CallbackInfo &info,
+                                               rnllama::llama_rn_context* rn_ctx, std::vector<int32_t> tokens)
+    : AsyncWorker(info.Env()), Deferred(info.Env()), _rn_ctx(rn_ctx), _tokens(tokens) {}
 
 void DecodeAudioTokenWorker::Execute() {
-  const int n_codes = _tokens.size();
-  llama_batch batch = llama_batch_init(n_codes, 0, 1);
-  for (size_t i = 0; i < _tokens.size(); ++i) {
-    common_batch_add(batch, _tokens[i], i, {0}, true);
+  try {
+    if (!_rn_ctx->tts_wrapper) {
+      SetError("Vocoder not initialized");
+      return;
+    }
+    
+    // Convert to llama_token vector - rn-tts handles token adjustment internally
+    std::vector<llama_token> llama_tokens;
+    for (const auto& token : _tokens) {
+      llama_tokens.push_back(token);
+    }
+    
+    // Use the rn-tts API instead of directly accessing the worker
+    _result = _rn_ctx->tts_wrapper->decodeAudioTokens(_rn_ctx, llama_tokens);
+  } catch (const std::exception &e) {
+    SetError(e.what());
   }
-  if (batch.n_tokens != n_codes) {
-    SetError("batch.n_tokens != n_codes");
-    return;
-  }
-  if (llama_encode(_ctx, batch) != 0) {
-    SetError("llama_encode() failed");
-    return;
-  }
-  llama_synchronize(_ctx);
-  const int n_embd = llama_model_n_embd(_model);
-  const float *embd = llama_get_embeddings(_ctx);
-  _result = embd_to_audio(embd, n_codes, n_embd, _n_threads);
 }
 
 void DecodeAudioTokenWorker::OnOK() {
-  auto result =
-      Napi::Float32Array::New(Napi::AsyncWorker::Env(), _result.size());
+  // Create Float32Array and copy the data
+  auto result = Napi::Float32Array::New(Napi::AsyncWorker::Env(), _result.size());
   memcpy(result.Data(), _result.data(), _result.size() * sizeof(float));
   Napi::Promise::Deferred::Resolve(result);
 }
 
 void DecodeAudioTokenWorker::OnError(const Napi::Error &err) {
   Napi::Promise::Deferred::Reject(err.Value());
-}
+}

package/src/DecodeAudioTokenWorker.h

@@ -1,12 +1,12 @@
 #include "common.hpp"
+#include "rn-llama/rn-llama.h"
 #include <vector>
 
 class DecodeAudioTokenWorker : public Napi::AsyncWorker,
                                public Napi::Promise::Deferred {
 public:
-  DecodeAudioTokenWorker(const Napi::CallbackInfo &info, llama_model *model,
-                         llama_context *ctx, int n_threads,
-                         const std::vector<llama_token> &tokens);
+  DecodeAudioTokenWorker(const Napi::CallbackInfo &info, rnllama::llama_rn_context* rn_ctx,
+                         std::vector<int32_t> tokens);
 
 protected:
   void Execute();
@@ -14,9 +14,7 @@ protected:
   void OnError(const Napi::Error &err);
 
 private:
-  llama_model *_model;
-  llama_context *_ctx;
-  int _n_threads;
-  std::vector<llama_token> _tokens;
+  rnllama::llama_rn_context* _rn_ctx;
+  std::vector<int32_t> _tokens;
   std::vector<float> _result;
-};
+};

package/src/DetokenizeWorker.cpp

@@ -2,21 +2,18 @@
 #include "LlamaContext.h"
 
 DetokenizeWorker::DetokenizeWorker(const Napi::CallbackInfo &info,
-                                   LlamaSessionPtr &sess,
-                                   std::vector<llama_token> &tokens)
-    : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess),
-      _tokens(std::move(tokens)) {}
+                                   rnllama::llama_rn_context* rn_ctx, std::vector<int32_t> tokens)
+    : AsyncWorker(info.Env()), Deferred(info.Env()), _rn_ctx(rn_ctx), _tokens(tokens) {}
 
 void DetokenizeWorker::Execute() {
-  const auto text = ::common_detokenize(_sess->context(), _tokens);
+  const auto text = tokens_to_str(_rn_ctx->ctx, _tokens.begin(), _tokens.end());
   _text = std::move(text);
 }
 
 void DetokenizeWorker::OnOK() {
-  Napi::Promise::Deferred::Resolve(
-      Napi::String::New(Napi::AsyncWorker::Env(), _text));
+  Napi::Promise::Deferred::Resolve(Napi::String::New(Napi::AsyncWorker::Env(), _text));
 }
 
 void DetokenizeWorker::OnError(const Napi::Error &err) {
   Napi::Promise::Deferred::Reject(err.Value());
-}
+}

package/src/DetokenizeWorker.h

@@ -1,11 +1,12 @@
 #include "common.hpp"
+#include "rn-llama/rn-llama.h"
 #include <vector>
 
 class DetokenizeWorker : public Napi::AsyncWorker,
                          public Napi::Promise::Deferred {
 public:
-  DetokenizeWorker(const Napi::CallbackInfo &info, LlamaSessionPtr &sess,
-                   std::vector<llama_token> &tokens);
+  DetokenizeWorker(const Napi::CallbackInfo &info, rnllama::llama_rn_context* rn_ctx,
+                   std::vector<int32_t> tokens);
 
 protected:
   void Execute();
@@ -13,7 +14,7 @@ protected:
   void OnError(const Napi::Error &err);
 
 private:
-  LlamaSessionPtr _sess;
-  std::vector<llama_token> _tokens;
+  rnllama::llama_rn_context* _rn_ctx;
+  std::vector<int32_t> _tokens;
   std::string _text;
-};
+};

package/src/DisposeWorker.cpp

@@ -1,10 +1,30 @@
 #include "DisposeWorker.h"
+#include "rn-llama/rn-completion.h"
 
 DisposeWorker::DisposeWorker(const Napi::CallbackInfo &info,
-                             LlamaSessionPtr sess)
-    : AsyncWorker(info.Env()), Deferred(info.Env()), sess_(std::move(sess)) {}
+                             rnllama::llama_rn_context* rn_ctx, rnllama::llama_rn_context** parent_ptr)
+    : AsyncWorker(info.Env()), Deferred(info.Env()), _rn_ctx(rn_ctx), _parent_ptr(parent_ptr) {}
 
-void DisposeWorker::Execute() { sess_->dispose(); }
+void DisposeWorker::Execute() { 
+  if (_rn_ctx) {
+    // Ensure all child contexts are properly cleaned up first
+    try {
+      // Now delete the main context
+      delete _rn_ctx;
+      
+      // Set parent pointer to nullptr to prevent double free
+      if (_parent_ptr) {
+        *_parent_ptr = nullptr;
+      }
+    } catch (const std::exception& e) {
+      SetError(std::string("Error during context disposal: ") + e.what());
+      return;
+    } catch (...) {
+      SetError("Unknown error during context disposal");
+      return;
+    }
+  }
+}
 
 void DisposeWorker::OnOK() { Resolve(AsyncWorker::Env().Undefined()); }
 

package/src/DisposeWorker.h

@@ -1,8 +1,9 @@
 #include "common.hpp"
+#include "rn-llama/rn-llama.h"
 
 class DisposeWorker : public Napi::AsyncWorker, public Napi::Promise::Deferred {
 public:
-  DisposeWorker(const Napi::CallbackInfo &info, LlamaSessionPtr sess);
+  DisposeWorker(const Napi::CallbackInfo &info, rnllama::llama_rn_context* rn_ctx, rnllama::llama_rn_context** parent_ptr);
 
 protected:
   void Execute();
@@ -10,5 +11,6 @@ protected:
   void OnError(const Napi::Error &err);
 
 private:
-  LlamaSessionPtr sess_;
+  rnllama::llama_rn_context* _rn_ctx;
+  rnllama::llama_rn_context** _parent_ptr; // Pointer to the parent's _rn_ctx pointer
 };

package/src/EmbeddingWorker.cpp

@@ -2,46 +2,20 @@
 #include "LlamaContext.h"
 
 EmbeddingWorker::EmbeddingWorker(const Napi::CallbackInfo &info,
-                                 LlamaSessionPtr &sess, std::string text,
+                                 rnllama::llama_rn_context* rn_ctx, std::string text,
                                  common_params &params)
-    : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess), _text(text),
+    : AsyncWorker(info.Env()), Deferred(info.Env()), _rn_ctx(rn_ctx), _text(text),
       _params(params) {}
 
 void EmbeddingWorker::Execute() {
-  llama_memory_clear(llama_get_memory(_sess->context()), true);
-  auto tokens = ::common_tokenize(_sess->context(), _text, true);
-  // add SEP if not present
-  auto vocab = llama_model_get_vocab(_sess->model());
-  if (tokens.empty() || tokens.back() != llama_vocab_sep(vocab)) {
-    tokens.push_back(llama_vocab_sep(vocab));
-  }
-  const int n_embd = llama_model_n_embd(_sess->model());
-  do {
-    auto ctx = _sess->context();
-    int ret =
-        llama_decode(ctx, llama_batch_get_one(tokens.data(), tokens.size()));
-    if (ret < 0) {
-      SetError("Failed to inference, code: " + std::to_string(ret));
-      break;
-    }
+  try {
+    _rn_ctx->params.prompt = _text;
+    _rn_ctx->params.n_predict = 0;
 
-    float *embd;
-    const enum llama_pooling_type pooling_type = llama_pooling_type(ctx);
-    if (pooling_type == LLAMA_POOLING_TYPE_NONE) {
-      embd = llama_get_embeddings(ctx);
-    } else {
-      embd = llama_get_embeddings_seq(ctx, 0);
-    }
-    if (embd == nullptr) {
-      SetError("Failed to get embeddings");
-      break;
-    }
-    _result.embedding.resize(n_embd);
-    std::vector<float> embedding(embd, embd + n_embd), out(embd, embd + n_embd);
-    common_embd_normalize(embedding.data(), out.data(), n_embd,
-                          _params.embd_normalize);
-    memcpy(_result.embedding.data(), out.data(), n_embd * sizeof(float));
-  } while (false);
+    _result.embedding = _rn_ctx->completion->embedding(_params);
+  } catch (const std::exception &e) {
+    SetError(e.what());
+  }
 }
 
 void EmbeddingWorker::OnOK() {

package/src/EmbeddingWorker.h

@@ -1,4 +1,5 @@
 #include "common.hpp"
+#include "rn-llama/rn-llama.h"
 #include <vector>
 
 struct EmbeddingResult {
@@ -8,7 +9,7 @@ struct EmbeddingResult {
 class EmbeddingWorker : public Napi::AsyncWorker,
                         public Napi::Promise::Deferred {
 public:
-  EmbeddingWorker(const Napi::CallbackInfo &info, LlamaSessionPtr &sess,
+  EmbeddingWorker(const Napi::CallbackInfo &info, rnllama::llama_rn_context* rn_ctx,
                   std::string text, common_params &params);
 
 protected:
@@ -17,7 +18,7 @@ protected:
   void OnError(const Napi::Error &err);
 
 private:
-  LlamaSessionPtr _sess;
+  rnllama::llama_rn_context* _rn_ctx;
   std::string _text;
   common_params _params;
   EmbeddingResult _result;