@fugood/llama.node 1.0.0-beta.7 → 1.0.1

package/CMakeLists.txt

@@ -140,6 +140,8 @@ file(
     "src/DetokenizeWorker.h"
     "src/EmbeddingWorker.cpp"
     "src/EmbeddingWorker.h"
+    "src/RerankWorker.cpp"
+    "src/RerankWorker.h"
     "src/LoadSessionWorker.cpp"
     "src/LoadSessionWorker.h"
     "src/SaveSessionWorker.cpp"

package/lib/binding.ts

@@ -159,6 +159,15 @@ export type EmbeddingResult = {
   embedding: Float32Array
 }
 
+export type RerankParams = {
+  normalize?: number
+}
+
+export type RerankResult = {
+  score: number
+  index: number
+}
+
 export interface LlamaContext {
   new (options: LlamaModelOptions): LlamaContext
   getSystemInfo(): string
@@ -182,6 +191,7 @@ export interface LlamaContext {
   tokenize(text: string, media_paths?: string[]): Promise<TokenizeResult>
   detokenize(tokens: number[]): Promise<string>
   embedding(text: string): Promise<EmbeddingResult>
+  rerank(query: string, documents: string[], params?: RerankParams): Promise<RerankResult[]>
   saveSession(path: string): Promise<void>
   loadSession(path: string): Promise<void>
   release(): Promise<void>

package/lib/index.js

@@ -176,6 +176,14 @@ class LlamaContextWrapper {
     embedding(text) {
         return this.ctx.embedding(text);
     }
+    rerank(query, documents, params) {
+        return this.ctx.rerank(query, documents, params).then((results) => {
+            // Sort by score descending and add document text for convenience
+            return results
+                .map((result) => (Object.assign(Object.assign({}, result), { document: documents[result.index] })))
+                .sort((a, b) => b.score - a.score);
+        });
+    }
     saveSession(path) {
         return this.ctx.saveSession(path);
     }

package/lib/index.ts

@@ -9,6 +9,8 @@ import type {
   LlamaCompletionResult,
   TokenizeResult,
   EmbeddingResult,
+  RerankParams,
+  RerankResult,
   CompletionResponseFormat,
 } from './binding'
 
@@ -226,6 +228,18 @@ class LlamaContextWrapper {
     return this.ctx.embedding(text)
   }
 
+  rerank(query: string, documents: string[], params?: RerankParams): Promise<Array<RerankResult & { document: string }>> {
+    return this.ctx.rerank(query, documents, params).then((results: RerankResult[]) => {
+      // Sort by score descending and add document text for convenience
+      return results
+        .map((result: RerankResult) => ({
+          ...result,
+          document: documents[result.index],
+        }))
+        .sort((a: RerankResult & { document: string }, b: RerankResult & { document: string }) => b.score - a.score)
+    })
+  }
+
   saveSession(path: string): Promise<void> {
     return this.ctx.saveSession(path)
   }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "1.0.0-beta.7",
+  "version": "1.0.1",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {
@@ -70,19 +70,19 @@
     "CMakeLists.txt"
   ],
   "optionalDependencies": {
-    "@fugood/node-llama-linux-x64": "1.0.0-beta.7",
-    "@fugood/node-llama-linux-x64-vulkan": "1.0.0-beta.7",
-    "@fugood/node-llama-linux-x64-cuda": "1.0.0-beta.7",
-    "@fugood/node-llama-linux-arm64": "1.0.0-beta.7",
-    "@fugood/node-llama-linux-arm64-vulkan": "1.0.0-beta.7",
-    "@fugood/node-llama-linux-arm64-cuda": "1.0.0-beta.7",
-    "@fugood/node-llama-win32-x64": "1.0.0-beta.7",
-    "@fugood/node-llama-win32-x64-vulkan": "1.0.0-beta.7",
-    "@fugood/node-llama-win32-x64-cuda": "1.0.0-beta.7",
-    "@fugood/node-llama-win32-arm64": "1.0.0-beta.7",
-    "@fugood/node-llama-win32-arm64-vulkan": "1.0.0-beta.7",
-    "@fugood/node-llama-darwin-x64": "1.0.0-beta.7",
-    "@fugood/node-llama-darwin-arm64": "1.0.0-beta.7"
+    "@fugood/node-llama-linux-x64": "1.0.1",
+    "@fugood/node-llama-linux-x64-vulkan": "1.0.1",
+    "@fugood/node-llama-linux-x64-cuda": "1.0.1",
+    "@fugood/node-llama-linux-arm64": "1.0.1",
+    "@fugood/node-llama-linux-arm64-vulkan": "1.0.1",
+    "@fugood/node-llama-linux-arm64-cuda": "1.0.1",
+    "@fugood/node-llama-win32-x64": "1.0.1",
+    "@fugood/node-llama-win32-x64-vulkan": "1.0.1",
+    "@fugood/node-llama-win32-x64-cuda": "1.0.1",
+    "@fugood/node-llama-win32-arm64": "1.0.1",
+    "@fugood/node-llama-win32-arm64-vulkan": "1.0.1",
+    "@fugood/node-llama-darwin-x64": "1.0.1",
+    "@fugood/node-llama-darwin-arm64": "1.0.1"
   },
   "devDependencies": {
     "@babel/preset-env": "^7.24.4",

package/src/LlamaContext.cpp

@@ -3,6 +3,7 @@
 #include "DetokenizeWorker.h"
 #include "DisposeWorker.h"
 #include "EmbeddingWorker.h"
+#include "RerankWorker.h"
 #include "LlamaCompletionWorker.h"
 #include "LoadSessionWorker.h"
 #include "SaveSessionWorker.h"
@@ -110,6 +111,8 @@ void LlamaContext::Init(Napi::Env env, Napi::Object &exports) {
            static_cast<napi_property_attributes>(napi_enumerable)),
        InstanceMethod<&LlamaContext::Embedding>(
            "embedding", static_cast<napi_property_attributes>(napi_enumerable)),
+       InstanceMethod<&LlamaContext::Rerank>(
+           "rerank", static_cast<napi_property_attributes>(napi_enumerable)),
        InstanceMethod<&LlamaContext::SaveSession>(
            "saveSession",
            static_cast<napi_property_attributes>(napi_enumerable)),
@@ -583,7 +586,7 @@ Napi::Value LlamaContext::GetFormattedChat(const Napi::CallbackInfo &info) {
                 : "{}";
       }
     }
-    auto tools_str = params.Has("tools")
+    auto tools_str = !is_nil(params.Get("tools"))
                          ? json_stringify(params.Get("tools").As<Napi::Array>())
                          : "";
     auto parallel_tool_calls =
@@ -591,9 +594,15 @@ Napi::Value LlamaContext::GetFormattedChat(const Napi::CallbackInfo &info) {
     auto tool_choice = get_option<std::string>(params, "tool_choice", "");
     auto enable_thinking = get_option<bool>(params, "enable_thinking", false);
 
-    auto chatParams = getFormattedChatWithJinja(
-        _sess, _templates, messages, chat_template, json_schema_str, tools_str,
-        parallel_tool_calls, tool_choice, enable_thinking);
+    common_chat_params chatParams;
+    try {
+      chatParams = getFormattedChatWithJinja(
+          _sess, _templates, messages, chat_template, json_schema_str, tools_str,
+          parallel_tool_calls, tool_choice, enable_thinking);
+    } catch (const std::exception &e) {
+      Napi::Error::New(env, e.what()).ThrowAsJavaScriptException();
+      return env.Undefined();
+    }
 
     Napi::Object result = Napi::Object::New(env);
     result.Set("prompt", chatParams.prompt);
@@ -790,7 +799,7 @@ Napi::Value LlamaContext::Completion(const Napi::CallbackInfo &info) {
     auto jinja = get_option<bool>(options, "jinja", false);
     if (jinja) {
       auto tools_str =
-          options.Has("tools")
+          !is_nil(options.Get("tools"))
               ? json_stringify(options.Get("tools").As<Napi::Array>())
               : "";
       auto parallel_tool_calls =
@@ -799,9 +808,16 @@ Napi::Value LlamaContext::Completion(const Napi::CallbackInfo &info) {
           get_option<std::string>(options, "tool_choice", "none");
       auto enable_thinking = get_option<bool>(options, "enable_thinking", true);
 
-      auto chatParams = getFormattedChatWithJinja(
-          _sess, _templates, json_stringify(messages), chat_template,
-          json_schema_str, tools_str, parallel_tool_calls, tool_choice, enable_thinking);
+      common_chat_params chatParams;
+      
+      try {
+        chatParams = getFormattedChatWithJinja(
+            _sess, _templates, json_stringify(messages), chat_template,
+            json_schema_str, tools_str, parallel_tool_calls, tool_choice, enable_thinking);
+      } catch (const std::exception &e) {
+        Napi::Error::New(env, e.what()).ThrowAsJavaScriptException();
+        return env.Undefined();
+      }
 
       params.prompt = chatParams.prompt;
 
@@ -982,6 +998,40 @@ Napi::Value LlamaContext::Embedding(const Napi::CallbackInfo &info) {
   return worker->Promise();
 }
 
+// rerank(query: string, documents: string[], params?: object): Promise<RerankResult[]>
+Napi::Value LlamaContext::Rerank(const Napi::CallbackInfo &info) {
+  Napi::Env env = info.Env();
+  if (info.Length() < 2 || !info[0].IsString() || !info[1].IsArray()) {
+    Napi::TypeError::New(env, "Query string and documents array expected").ThrowAsJavaScriptException();
+  }
+  if (_sess == nullptr) {
+    Napi::TypeError::New(env, "Context is disposed")
+        .ThrowAsJavaScriptException();
+  }
+  
+  auto query = info[0].ToString().Utf8Value();
+  auto documents_array = info[1].As<Napi::Array>();
+  
+  // Convert documents array to vector
+  std::vector<std::string> documents;
+  for (size_t i = 0; i < documents_array.Length(); i++) {
+    documents.push_back(documents_array.Get(i).ToString().Utf8Value());
+  }
+  
+  auto options = Napi::Object::New(env);
+  if (info.Length() >= 3 && info[2].IsObject()) {
+    options = info[2].As<Napi::Object>();
+  }
+
+  common_params rerankParams;
+  rerankParams.embedding = true;
+  rerankParams.embd_normalize = get_option<int32_t>(options, "normalize", -1);
+  
+  auto *worker = new RerankWorker(info, _sess, query, documents, rerankParams);
+  worker->Queue();
+  return worker->Promise();
+}
+
 // saveSession(path: string): Promise<void> throws error
 Napi::Value LlamaContext::SaveSession(const Napi::CallbackInfo &info) {
   Napi::Env env = info.Env();

package/src/LlamaContext.h

@@ -28,6 +28,7 @@ private:
   Napi::Value Tokenize(const Napi::CallbackInfo &info);
   Napi::Value Detokenize(const Napi::CallbackInfo &info);
   Napi::Value Embedding(const Napi::CallbackInfo &info);
+  Napi::Value Rerank(const Napi::CallbackInfo &info);
   Napi::Value SaveSession(const Napi::CallbackInfo &info);
   Napi::Value LoadSession(const Napi::CallbackInfo &info);
   void ApplyLoraAdapters(const Napi::CallbackInfo &info);

package/src/RerankWorker.h

@@ -0,0 +1,26 @@
+#include "common.hpp"
+#include <vector>
+
+struct RerankResult {
+  std::vector<float> scores;
+};
+
+class RerankWorker : public Napi::AsyncWorker,
+                     public Napi::Promise::Deferred {
+public:
+  RerankWorker(const Napi::CallbackInfo &info, LlamaSessionPtr &sess,
+               std::string query, std::vector<std::string> documents, 
+               common_params &params);
+
+protected:
+  void Execute();
+  void OnOK();
+  void OnError(const Napi::Error &err);
+
+private:
+  LlamaSessionPtr _sess;
+  std::string _query;
+  std::vector<std::string> _documents;
+  common_params _params;
+  RerankResult _result;
+}; 

package/src/llama.cpp/CMakeLists.txt

@@ -95,7 +95,7 @@ endif()
 if (NOT DEFINED LLAMA_BUILD_COMMIT)
     set(LLAMA_BUILD_COMMIT        ${BUILD_COMMIT})
 endif()
-set(LLAMA_INSTALL_VERSION 0.0.${BUILD_NUMBER})
+set(LLAMA_INSTALL_VERSION 0.0.${LLAMA_BUILD_NUMBER})
 
 # override ggml options
 set(GGML_ALL_WARNINGS   ${LLAMA_ALL_WARNINGS})

package/src/llama.cpp/common/json-schema-to-grammar.cpp

@@ -41,49 +41,6 @@ static std::string build_repetition(const std::string & item_rule, int min_items
     return result;
 }
 
-/* Minimalistic replacement for std::string_view, which is only available from C++17 onwards */
-class string_view {
-    const std::string & _str;
-    const size_t _start;
-    const size_t _end;
-public:
-    string_view(const std::string & str, size_t start = 0, size_t end  = std::string::npos) : _str(str), _start(start), _end(end == std::string::npos ? str.length() : end) {}
-
-    size_t size() const {
-        return _end - _start;
-    }
-
-    size_t length() const {
-        return size();
-    }
-
-    operator std::string() const {
-        return str();
-    }
-
-    std::string str() const {
-        return _str.substr(_start, _end - _start);
-    }
-
-    string_view substr(size_t pos, size_t len = std::string::npos) const {
-        return string_view(_str, _start + pos, len == std::string::npos ? _end : _start + pos + len);
-    }
-
-    char operator[](size_t pos) const {
-        auto index = _start + pos;
-        if (index >= _end) {
-            throw std::out_of_range("string_view index out of range");
-        }
-        return _str[_start + pos];
-    }
-
-    bool operator==(const string_view & other) const {
-        std::string this_str = *this;
-        std::string other_str = other;
-        return this_str == other_str;
-    }
-};
-
 static void _build_min_max_int(int min_value, int max_value, std::stringstream & out, int decimals_left = 16, bool top_level = true) {
     auto has_min = min_value != std::numeric_limits<int>::min();
     auto has_max = max_value != std::numeric_limits<int>::max();
@@ -112,14 +69,14 @@ static void _build_min_max_int(int min_value, int max_value, std::stringstream &
         }
         out << "}";
     };
-    std::function<void(const string_view &, const string_view &)> uniform_range =
-        [&](const string_view & from, const string_view & to) {
+    std::function<void(const std::string_view &, const std::string_view &)> uniform_range =
+        [&](const std::string_view & from, const std::string_view & to) {
             size_t i = 0;
             while (i < from.length() && i < to.length() && from[i] == to[i]) {
                 i++;
             }
             if (i > 0) {
-                out << "\"" << from.substr(0, i).str() << "\"";
+                out << "\"" << from.substr(0, i) << "\"";
             }
             if (i < from.length() && i < to.length()) {
                 if (i > 0) {

package/src/llama.cpp/ggml/CMakeLists.txt

@@ -131,6 +131,7 @@ option(GGML_RVV              "ggml: enable rvv"              ON)
 option(GGML_RV_ZFH           "ggml: enable riscv zfh"        OFF)
 option(GGML_XTHEADVECTOR     "ggml: enable xtheadvector"     OFF)
 option(GGML_VXE              "ggml: enable vxe"              ON)
+option(GGML_NNPA             "ggml: enable nnpa"             ON)
 
 option(GGML_CPU_ALL_VARIANTS "ggml: build all variants of the CPU backend (requires GGML_BACKEND_DL)" OFF)
 set(GGML_CPU_ARM_ARCH        "" CACHE STRING "ggml: CPU architecture for ARM")

package/src/llama.cpp/ggml/include/ggml-cpu.h

@@ -101,6 +101,7 @@ extern "C" {
     GGML_BACKEND_API int ggml_cpu_has_riscv_v    (void);
     GGML_BACKEND_API int ggml_cpu_has_vsx        (void);
     GGML_BACKEND_API int ggml_cpu_has_vxe        (void);
+    GGML_BACKEND_API int ggml_cpu_has_nnpa       (void);
     GGML_BACKEND_API int ggml_cpu_has_wasm_simd  (void);
     GGML_BACKEND_API int ggml_cpu_has_llamafile  (void);
 

package/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt

@@ -448,6 +448,7 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
 
         # TODO: Separation to determine activation of VX/VXE/VXE2
         if (${S390X_M} MATCHES "8561|8562")
+            set(GGML_NNPA OFF)
             message(STATUS "z15 target")
             list(APPEND ARCH_FLAGS -march=z15)
         elseif (${S390X_M} MATCHES "3931")
@@ -464,7 +465,14 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
         endif()
 
         if (GGML_VXE)
+            message(STATUS "VX/VXE/VXE2 enabled")
             list(APPEND ARCH_FLAGS -mvx -mzvector)
+            list(APPEND ARCH_DEFINITIONS GGML_VXE)
+        endif()
+
+        if (GGML_NNPA)
+            message(STATUS "NNPA enabled")
+            list(APPEND ARCH_DEFINITIONS GGML_NNPA)
         endif()
     elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "wasm")
         message(STATUS "Wasm detected")

package/src/llama.cpp/ggml/src/ggml-cpu/amx/mmq.cpp

@@ -8,6 +8,7 @@
 #include "mmq.h"
 #include "ggml-impl.h"
 #include "ggml-cpu-impl.h"
+#include "simd-mappings.h"
 #include "quants.h"
 #include "ggml-quants.h"
 #include <algorithm>
@@ -453,7 +454,7 @@ void quantize_row_q8_K_vnni(const float * RESTRICT x, void * RESTRICT vy, int64_
 
         // Quantize these floats
         const float iscale = 127.f / amax;
-        y[i].d = GGML_FP32_TO_FP16(1 / iscale);
+        y[i].d = GGML_CPU_FP32_TO_FP16(1 / iscale);
         const float id = ( amax != 0.0f ) ? iscale : 0.f;
         const __m512 vscale = _mm512_set1_ps(id);
 
@@ -1090,7 +1091,7 @@ struct acc_C<block_q8_0, block_q4_0, is_acc> {
         const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)((const char *)packed_B + offset)));
 
         for (int m = 0; m < nr; ++m) {
-            const __m512 vd1 = _mm512_set1_ps(GGML_FP16_TO_FP32(A[m * lda].d));
+            const __m512 vd1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[m * lda].d));
             const __m512 vtile = _mm512_cvtepi32_ps(_mm512_loadu_si512(tile + m * TILE_N));
 
             __m512 vsum;
@@ -1113,8 +1114,8 @@ struct acc_C<block_q8_1, block_q4_1, is_acc> {
         const __m512 vm0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)((const char *)packed_B + offset + TILE_N * sizeof(ggml_half))));
 
         for (int m = 0; m < nr; ++m) {
-            const __m512 vd1 = _mm512_set1_ps(GGML_FP16_TO_FP32(A[m * lda].d));
-            const __m512 vs1 = _mm512_set1_ps(GGML_FP16_TO_FP32(A[m * lda].s));
+            const __m512 vd1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[m * lda].d));
+            const __m512 vs1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[m * lda].s));
             const __m512 vtile = _mm512_cvtepi32_ps(_mm512_loadu_si512(tile + m * TILE_N));
 
             __m512 vsum;
@@ -1137,7 +1138,7 @@ struct acc_C<block_q8_0, block_q8_0, is_acc> {
         const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)((const char *)packed_B + offset)));
 
         for (int m = 0; m < nr; ++m) {
-            const __m512 vd1 = _mm512_set1_ps(GGML_FP16_TO_FP32(A[m * lda].d));
+            const __m512 vd1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[m * lda].d));
             const __m512 vtile = _mm512_cvtepi32_ps(_mm512_loadu_si512(tile + m * TILE_N));
 
             __m512 vsum;
@@ -1437,7 +1438,7 @@ struct tinygemm_kernel_vnni<block_q8_0, block_q4_0, float, BLOCK_M, BLOCK_N, BLO
                     va[k] = _mm512_set1_epi32(a_ptr[k]);
                     vcomp = _mm512_dpbusd_epi32(vcomp, off, va[k]);
                 }
-                vd1 = _mm512_set1_ps(GGML_FP16_TO_FP32(A[0 * KB + i].d));
+                vd1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[0 * KB + i].d));
             }
 
             // load b
@@ -1498,8 +1499,8 @@ struct tinygemm_kernel_vnni<block_q8_1, block_q4_1, float, 1, BLOCK_N, BLOCK_K>
                 for (int k = 0; k < 8; ++k) {
                     va[k] = _mm512_set1_epi32(a_ptr[k]);
                 }
-                vd1 = _mm512_set1_ps(GGML_FP16_TO_FP32(A[0 * KB + i].d));
-                vs1 = _mm512_set1_ps(GGML_FP16_TO_FP32(A[0 * KB + i].s));
+                vd1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[0 * KB + i].d));
+                vs1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[0 * KB + i].s));
             }
 
             // load b
@@ -1571,7 +1572,7 @@ struct tinygemm_kernel_vnni<block_q8_0, block_q8_0, float, BLOCK_M, BLOCK_N, BLO
                     va[k] = _mm512_set1_epi32(a_ptr[k]);
                     va[k] = _mm512_add_epi8(va[k], off);
                 }
-                vd1 = _mm512_set1_ps(GGML_FP16_TO_FP32(A[0 * KB + i].d));
+                vd1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[0 * KB + i].d));
             }
 
             // load b