@fugood/llama.node 0.3.11 → 0.3.13

package/lib/binding.ts

@@ -8,6 +8,7 @@ export type ChatMessage = {
 export type LlamaModelOptions = {
   model: string
   chat_template?: string
+  reasoning_format?: string
   embedding?: boolean
   embd_normalize?: number
   pooling_type?: 'none' | 'mean' | 'cls' | 'last' | 'rank'

package/lib/index.js

@@ -23,15 +23,39 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.toggleNativeLog = exports.loadLlamaModelInfo = exports.initLlama = exports.loadModel = void 0;
+exports.loadLlamaModelInfo = exports.initLlama = exports.loadModel = exports.toggleNativeLog = void 0;
 exports.addNativeLogListener = addNativeLogListener;
 const binding_1 = require("./binding");
 __exportStar(require("./binding"), exports);
 const mods = {};
+const logListeners = [];
+const logCallback = (level, text) => {
+    logListeners.forEach((listener) => listener(level, text));
+};
+let logEnabled = false;
+const refreshNativeLogSetup = () => {
+    Object.entries(mods).forEach(([, mod]) => {
+        mod.LlamaContext.toggleNativeLog(logEnabled, logCallback);
+    });
+};
+const toggleNativeLog = (enable) => __awaiter(void 0, void 0, void 0, function* () {
+    logEnabled = enable;
+    refreshNativeLogSetup();
+});
+exports.toggleNativeLog = toggleNativeLog;
+function addNativeLogListener(listener) {
+    logListeners.push(listener);
+    return {
+        remove: () => {
+            logListeners.splice(logListeners.indexOf(listener), 1);
+        },
+    };
+}
 const loadModel = (options) => __awaiter(void 0, void 0, void 0, function* () {
     var _a, _b;
     const variant = (_a = options.lib_variant) !== null && _a !== void 0 ? _a : 'default';
     (_b = mods[variant]) !== null && _b !== void 0 ? _b : (mods[variant] = yield (0, binding_1.loadModule)(options.lib_variant));
+    refreshNativeLogSetup();
     return new mods[variant].LlamaContext(options);
 });
 exports.loadModel = loadModel;
@@ -47,25 +71,7 @@ const loadLlamaModelInfo = (path) => __awaiter(void 0, void 0, void 0, function*
     var _a;
     const variant = 'default';
     (_a = mods[variant]) !== null && _a !== void 0 ? _a : (mods[variant] = yield (0, binding_1.loadModule)(variant));
+    refreshNativeLogSetup();
     return mods[variant].LlamaContext.loadModelInfo(path, modelInfoSkip);
 });
 exports.loadLlamaModelInfo = loadLlamaModelInfo;
-const logListeners = [];
-const logCallback = (level, text) => {
-    logListeners.forEach((listener) => listener(level, text));
-};
-const toggleNativeLog = (enable, options) => __awaiter(void 0, void 0, void 0, function* () {
-    var _a, _b;
-    const v = (_a = options === null || options === void 0 ? void 0 : options.variant) !== null && _a !== void 0 ? _a : 'default';
-    (_b = mods[v]) !== null && _b !== void 0 ? _b : (mods[v] = yield (0, binding_1.loadModule)(v));
-    return mods[v].LlamaContext.toggleNativeLog(enable, logCallback);
-});
-exports.toggleNativeLog = toggleNativeLog;
-function addNativeLogListener(listener) {
-    logListeners.push(listener);
-    return {
-        remove: () => {
-            logListeners.splice(logListeners.indexOf(listener), 1);
-        },
-    };
-}

package/lib/index.ts

@@ -9,11 +9,42 @@ export interface LlamaModelOptionsExtended extends LlamaModelOptions {
 
 const mods: { [key: string]: Module } = {}
 
+const logListeners: Array<(level: string, text: string) => void> = []
+
+const logCallback = (level: string, text: string) => {
+  logListeners.forEach((listener) => listener(level, text))
+}
+
+let logEnabled = false
+
+const refreshNativeLogSetup = () => {
+  Object.entries(mods).forEach(([, mod]) => {
+    mod.LlamaContext.toggleNativeLog(logEnabled, logCallback)
+  })
+}
+
+export const toggleNativeLog = async (enable: boolean) => {
+  logEnabled = enable
+  refreshNativeLogSetup()
+}
+
+export function addNativeLogListener(
+  listener: (level: string, text: string) => void,
+): { remove: () => void } {
+  logListeners.push(listener)
+  return {
+    remove: () => {
+      logListeners.splice(logListeners.indexOf(listener), 1)
+    },
+  }
+}
+
 export const loadModel = async (
   options: LlamaModelOptionsExtended,
 ): Promise<LlamaContext> => {
   const variant = options.lib_variant ?? 'default'
   mods[variant] ??= await loadModule(options.lib_variant)
+  refreshNativeLogSetup()
   return new mods[variant].LlamaContext(options)
 }
 
@@ -30,33 +61,6 @@ const modelInfoSkip = [
 export const loadLlamaModelInfo = async (path: string): Promise<Object> => {
   const variant = 'default'
   mods[variant] ??= await loadModule(variant)
+  refreshNativeLogSetup()
   return mods[variant].LlamaContext.loadModelInfo(path, modelInfoSkip)
 }
-
-const logListeners: Array<(level: string, text: string) => void> = []
-
-const logCallback = (level: string, text: string) => {
-  logListeners.forEach((listener) => listener(level, text))
-}
-
-export const toggleNativeLog = async (
-  enable: boolean,
-  options?: {
-    variant?: LibVariant
-  },
-) => {
-  const v = options?.variant ?? 'default'
-  mods[v] ??= await loadModule(v)
-  return mods[v].LlamaContext.toggleNativeLog(enable, logCallback)
-}
-
-export function addNativeLogListener(
-  listener: (level: string, text: string) => void,
-): { remove: () => void } {
-  logListeners.push(listener)
-  return {
-    remove: () => {
-      logListeners.splice(logListeners.indexOf(listener), 1)
-    },
-  }
-}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "0.3.11",
+  "version": "0.3.13",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {

package/src/LlamaCompletionWorker.cpp

@@ -165,9 +165,17 @@ void LlamaCompletionWorker::OnOK() {
              Napi::String::New(env, _result.text.c_str()));
 
   Napi::Array tool_calls = Napi::Array::New(Napi::AsyncWorker::Env());
+  std::string * reasoning_content = nullptr;
+  std::string * content = nullptr;
   if (!_stop) {
     try {
       common_chat_msg message = common_chat_parse(_result.text, static_cast<common_chat_format>(_chat_format));
+      if (!message.reasoning_content.empty()) {
+        reasoning_content = &message.reasoning_content;
+      }
+      if (!message.content.empty()) {
+        content = &message.content;
+      }
       for (size_t i = 0; i < message.tool_calls.size(); i++) {
         const auto &tc = message.tool_calls[i];
         Napi::Object tool_call = Napi::Object::New(env);
@@ -188,6 +196,12 @@ void LlamaCompletionWorker::OnOK() {
   if (tool_calls.Length() > 0) {
     result.Set("tool_calls", tool_calls);
   }
+  if (reasoning_content) {
+    result.Set("reasoning_content", Napi::String::New(env, reasoning_content->c_str()));
+  }
+  if (content) {
+    result.Set("content", Napi::String::New(env, content->c_str()));
+  }
 
   auto ctx = _sess->context();
   const auto timings_token = llama_perf_context(ctx);

package/src/LlamaContext.cpp

@@ -185,6 +185,13 @@ LlamaContext::LlamaContext(const Napi::CallbackInfo &info)
 
   params.chat_template = get_option<std::string>(options, "chat_template", "");
 
+  std::string reasoning_format = get_option<std::string>(options, "reasoning_format", "none");
+  if (reasoning_format == "deepseek") {
+    params.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
+  } else {
+    params.reasoning_format = COMMON_REASONING_FORMAT_NONE;
+  }
+
   params.n_ctx = get_option<int32_t>(options, "n_ctx", 512);
   params.n_batch = get_option<int32_t>(options, "n_batch", 2048);
   params.n_ubatch = get_option<int32_t>(options, "n_ubatch", 512);
@@ -377,7 +384,7 @@ Napi::Value LlamaContext::GetModelInfo(const Napi::CallbackInfo &info) {
 }
 
 common_chat_params getFormattedChatWithJinja(
-  const struct llama_model * model,
+  const std::shared_ptr<LlamaSession> &sess,
   const common_chat_templates &templates,
   const std::string &messages,
   const std::string &chat_template,
@@ -399,11 +406,12 @@ common_chat_params getFormattedChatWithJinja(
   if (!json_schema.empty()) {
       inputs.json_schema = json::parse(json_schema);
   }
+  inputs.extract_reasoning = sess->params().reasoning_format != COMMON_REASONING_FORMAT_NONE;
   inputs.stream = true;
 
   // If chat_template is provided, create new one and use it (probably slow)
   if (!chat_template.empty()) {
-      auto tmp = common_chat_templates_from_model(model, chat_template);
+      auto tmp = common_chat_templates_from_model(sess->model(), chat_template);
       const common_chat_template* template_ptr = useTools && tmp.template_tool_use ? tmp.template_tool_use.get() : tmp.template_default.get();
       if (inputs.parallel_tool_calls && !template_ptr->original_caps().supports_parallel_tool_calls) {
           inputs.parallel_tool_calls = false;
@@ -493,7 +501,7 @@ Napi::Value LlamaContext::GetFormattedChat(const Napi::CallbackInfo &info) {
     auto parallel_tool_calls = get_option<bool>(params, "parallel_tool_calls", false);
     auto tool_choice = get_option<std::string>(params, "tool_choice", "");
 
-    auto chatParams = getFormattedChatWithJinja(_sess->model(), _templates, messages, chat_template, json_schema_str, tools_str, parallel_tool_calls, tool_choice);
+    auto chatParams = getFormattedChatWithJinja(_sess, _templates, messages, chat_template, json_schema_str, tools_str, parallel_tool_calls, tool_choice);
     
     Napi::Object result = Napi::Object::New(env);
     result.Set("prompt", chatParams.prompt.get<std::string>());
@@ -598,7 +606,7 @@ Napi::Value LlamaContext::Completion(const Napi::CallbackInfo &info) {
       auto tool_choice = get_option<std::string>(options, "tool_choice", "none");
 
       auto chatParams = getFormattedChatWithJinja(
-        _sess->model(),
+        _sess,
         _templates,
         json_stringify(messages),
         chat_template,
@@ -685,6 +693,7 @@ Napi::Value LlamaContext::Completion(const Napi::CallbackInfo &info) {
   params.sampling.dry_base = get_option<float>(options, "dry_base", 2);
   params.sampling.dry_allowed_length = get_option<float>(options, "dry_allowed_length", -1);
   params.sampling.dry_penalty_last_n = get_option<float>(options, "dry_penalty_last_n", 0);
+  params.sampling.top_n_sigma = get_option<float>(options, "top_n_sigma", -1.0f);
   params.sampling.ignore_eos = get_option<bool>(options, "ignore_eos", false);
   params.n_keep = get_option<int32_t>(options, "n_keep", 0);
   params.sampling.seed = get_option<int32_t>(options, "seed", LLAMA_DEFAULT_SEED);

package/src/llama.cpp/.github/workflows/build.yml

@@ -129,7 +129,7 @@ jobs:
         run: |
           sysctl -a
           # Metal is disabled due to intermittent failures with Github runners not having a GPU:
-          # https://github.com/ggerganov/llama.cpp/actions/runs/8635935781/job/23674807267#step:5:2313
+          # https://github.com/ggml-org/llama.cpp/actions/runs/8635935781/job/23674807267#step:5:2313
           cmake -B build \
             -DCMAKE_BUILD_RPATH="@loader_path" \
             -DLLAMA_FATAL_WARNINGS=ON \
@@ -374,6 +374,8 @@ jobs:
       - name: Clone
         id: checkout
         uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: ccache
         uses: hendrikmuhs/ccache-action@v1.2.16
@@ -401,7 +403,35 @@ jobs:
         run: |
           cd build
           # This is using llvmpipe and runs slower than other backends
-          ctest -L main --verbose --timeout 1800
+          ctest -L main --verbose --timeout 2700
+
+      - name: Determine tag name
+        id: tag
+        shell: bash
+        run: |
+          BUILD_NUMBER="$(git rev-list --count HEAD)"
+          SHORT_HASH="$(git rev-parse --short=7 HEAD)"
+          if [[ "${{ env.BRANCH_NAME }}" == "master" ]]; then
+            echo "name=b${BUILD_NUMBER}" >> $GITHUB_OUTPUT
+          else
+            SAFE_NAME=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-')
+            echo "name=${SAFE_NAME}-b${BUILD_NUMBER}-${SHORT_HASH}" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Pack artifacts
+        id: pack_artifacts
+        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
+        run: |
+          cp LICENSE ./build/bin/
+          cp examples/run/linenoise.cpp/LICENSE ./build/bin/LICENSE.linenoise.cpp
+          zip -r llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.zip ./build/bin/*
+
+      - name: Upload artifacts
+        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
+        uses: actions/upload-artifact@v4
+        with:
+          path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.zip
+          name: llama-bin-ubuntu-vulkan-x64.zip
 
   ubuntu-22-cmake-hip:
     runs-on: ubuntu-22.04
@@ -443,7 +473,7 @@ jobs:
 
   ubuntu-22-cmake-musa:
     runs-on: ubuntu-22.04
-    container: mthreads/musa:rc3.1.0-devel-ubuntu22.04
+    container: mthreads/musa:rc3.1.1-devel-ubuntu22.04
 
     steps:
       - name: Clone
@@ -1345,8 +1375,10 @@ jobs:
 
     needs:
       - ubuntu-cpu-cmake
+      - ubuntu-22-cmake-vulkan
       - windows-latest-cmake
       - windows-2019-cmake-cuda
+      - windows-latest-cmake-sycl
       - windows-latest-cmake-hip-release
       - macOS-latest-cmake-arm64
       - macOS-latest-cmake-x64

package/src/llama.cpp/.github/workflows/docker.yml

@@ -51,6 +51,8 @@ jobs:
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
+        with:
+          image: tonistiigi/binfmt:qemu-v7.0.0-28
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3

package/src/llama.cpp/.github/workflows/labeler.yml

@@ -11,7 +11,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
       with:
-        repository: "ggerganov/llama.cpp"
+        repository: "ggml-org/llama.cpp"
     - uses: actions/labeler@v5
       with:
         configuration-path: '.github/labeler.yml'

package/src/llama.cpp/common/CMakeLists.txt

@@ -96,6 +96,22 @@ if (LLAMA_LLGUIDANCE)
     include(ExternalProject)
     set(LLGUIDANCE_SRC ${CMAKE_BINARY_DIR}/llguidance/source)
     set(LLGUIDANCE_PATH ${LLGUIDANCE_SRC}/target/release)
+
+    # Set the correct library file extension based on platform
+    if (WIN32)
+        set(LLGUIDANCE_LIB_NAME "llguidance.lib")
+        # Add Windows-specific libraries
+        set(LLGUIDANCE_PLATFORM_LIBS
+            ws2_32    # Windows Sockets API
+            userenv   # For GetUserProfileDirectoryW
+            ntdll     # For NT functions
+            bcrypt    # For BCryptGenRandom
+        )
+    else()
+        set(LLGUIDANCE_LIB_NAME "libllguidance.a")
+        set(LLGUIDANCE_PLATFORM_LIBS "")
+    endif()
+
     ExternalProject_Add(llguidance_ext
         GIT_REPOSITORY https://github.com/guidance-ai/llguidance
         # v0.6.12:
@@ -106,17 +122,18 @@ if (LLAMA_LLGUIDANCE)
         CONFIGURE_COMMAND ""
         BUILD_COMMAND cargo build --release
         INSTALL_COMMAND ""
-        BUILD_BYPRODUCTS ${LLGUIDANCE_PATH}/libllguidance.a ${LLGUIDANCE_PATH}/llguidance.h
+        BUILD_BYPRODUCTS ${LLGUIDANCE_PATH}/${LLGUIDANCE_LIB_NAME} ${LLGUIDANCE_PATH}/llguidance.h
         UPDATE_COMMAND ""
     )
     target_compile_definitions(${TARGET} PUBLIC LLAMA_USE_LLGUIDANCE)
 
     add_library(llguidance STATIC IMPORTED)
-    set_target_properties(llguidance PROPERTIES IMPORTED_LOCATION ${LLGUIDANCE_PATH}/libllguidance.a)
+    set_target_properties(llguidance PROPERTIES IMPORTED_LOCATION ${LLGUIDANCE_PATH}/${LLGUIDANCE_LIB_NAME})
     add_dependencies(llguidance llguidance_ext)
 
     target_include_directories(${TARGET} PRIVATE ${LLGUIDANCE_PATH})
-    set(LLAMA_COMMON_EXTRA_LIBS ${LLAMA_COMMON_EXTRA_LIBS} llguidance)
+    # Add platform libraries to the main target
+    set(LLAMA_COMMON_EXTRA_LIBS ${LLAMA_COMMON_EXTRA_LIBS} llguidance ${LLGUIDANCE_PLATFORM_LIBS})
 endif ()
 
 target_include_directories(${TARGET} PUBLIC .)