@fugood/llama.node 1.0.0-beta.6 → 1.0.0

package/CMakeLists.txt

@@ -140,6 +140,8 @@ file(
     "src/DetokenizeWorker.h"
     "src/EmbeddingWorker.cpp"
     "src/EmbeddingWorker.h"
+    "src/RerankWorker.cpp"
+    "src/RerankWorker.h"
     "src/LoadSessionWorker.cpp"
     "src/LoadSessionWorker.h"
     "src/SaveSessionWorker.cpp"

package/lib/binding.ts

@@ -79,6 +79,8 @@ export type LlamaCompletionOptions = {
   tools?: object
   parallel_tool_calls?: boolean
   tool_choice?: string
+  enable_thinking?: boolean
+  thinking_forced_open?: boolean
   prompt?: string
   temperature?: number
   top_k?: number
@@ -157,6 +159,15 @@ export type EmbeddingResult = {
   embedding: Float32Array
 }
 
+export type RerankParams = {
+  normalize?: number
+}
+
+export type RerankResult = {
+  score: number
+  index: number
+}
+
 export interface LlamaContext {
   new (options: LlamaModelOptions): LlamaContext
   getSystemInfo(): string
@@ -180,6 +191,7 @@ export interface LlamaContext {
   tokenize(text: string, media_paths?: string[]): Promise<TokenizeResult>
   detokenize(tokens: number[]): Promise<string>
   embedding(text: string): Promise<EmbeddingResult>
+  rerank(query: string, documents: string[], params?: RerankParams): Promise<RerankResult[]>
   saveSession(path: string): Promise<void>
   loadSession(path: string): Promise<void>
   release(): Promise<void>

package/lib/index.js

@@ -131,6 +131,7 @@ class LlamaContextWrapper {
         };
     }
     getFormattedChat(messages, template, params) {
+        var _a;
         const { messages: chat, has_media, media_paths, } = this._formatMediaChat(messages);
         const useJinja = this.isJinjaSupported() && (params === null || params === void 0 ? void 0 : params.jinja);
         let tmpl;
@@ -143,6 +144,7 @@ class LlamaContextWrapper {
             tools: params === null || params === void 0 ? void 0 : params.tools,
             parallel_tool_calls: params === null || params === void 0 ? void 0 : params.parallel_tool_calls,
             tool_choice: params === null || params === void 0 ? void 0 : params.tool_choice,
+            enable_thinking: (_a = params === null || params === void 0 ? void 0 : params.enable_thinking) !== null && _a !== void 0 ? _a : true,
         });
         if (!useJinja) {
             return {
@@ -174,6 +176,14 @@ class LlamaContextWrapper {
     embedding(text) {
         return this.ctx.embedding(text);
     }
+    rerank(query, documents, params) {
+        return this.ctx.rerank(query, documents, params).then((results) => {
+            // Sort by score descending and add document text for convenience
+            return results
+                .map((result) => (Object.assign(Object.assign({}, result), { document: documents[result.index] })))
+                .sort((a, b) => b.score - a.score);
+        });
+    }
     saveSession(path) {
         return this.ctx.saveSession(path);
     }

package/lib/index.ts

@@ -9,6 +9,8 @@ import type {
   LlamaCompletionResult,
   TokenizeResult,
   EmbeddingResult,
+  RerankParams,
+  RerankResult,
   CompletionResponseFormat,
 } from './binding'
 
@@ -158,7 +160,8 @@ class LlamaContextWrapper {
       response_format?: CompletionResponseFormat
       tools?: object
       parallel_tool_calls?: object
-      tool_choice?: string
+      tool_choice?: string,
+      enable_thinking?: boolean,
     },
   ): FormattedChatResult {
     const {
@@ -178,6 +181,7 @@ class LlamaContextWrapper {
       tools: params?.tools,
       parallel_tool_calls: params?.parallel_tool_calls,
       tool_choice: params?.tool_choice,
+      enable_thinking: params?.enable_thinking ?? true,
     })
 
     if (!useJinja) {
@@ -224,6 +228,18 @@ class LlamaContextWrapper {
     return this.ctx.embedding(text)
   }
 
+  rerank(query: string, documents: string[], params?: RerankParams): Promise<Array<RerankResult & { document: string }>> {
+    return this.ctx.rerank(query, documents, params).then((results: RerankResult[]) => {
+      // Sort by score descending and add document text for convenience
+      return results
+        .map((result: RerankResult) => ({
+          ...result,
+          document: documents[result.index],
+        }))
+        .sort((a: RerankResult & { document: string }, b: RerankResult & { document: string }) => b.score - a.score)
+    })
+  }
+
   saveSession(path: string): Promise<void> {
     return this.ctx.saveSession(path)
   }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "1.0.0-beta.6",
+  "version": "1.0.0",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {
@@ -70,19 +70,19 @@
     "CMakeLists.txt"
   ],
   "optionalDependencies": {
-    "@fugood/node-llama-linux-x64": "1.0.0-beta.6",
-    "@fugood/node-llama-linux-x64-vulkan": "1.0.0-beta.6",
-    "@fugood/node-llama-linux-x64-cuda": "1.0.0-beta.6",
-    "@fugood/node-llama-linux-arm64": "1.0.0-beta.6",
-    "@fugood/node-llama-linux-arm64-vulkan": "1.0.0-beta.6",
-    "@fugood/node-llama-linux-arm64-cuda": "1.0.0-beta.6",
-    "@fugood/node-llama-win32-x64": "1.0.0-beta.6",
-    "@fugood/node-llama-win32-x64-vulkan": "1.0.0-beta.6",
-    "@fugood/node-llama-win32-x64-cuda": "1.0.0-beta.6",
-    "@fugood/node-llama-win32-arm64": "1.0.0-beta.6",
-    "@fugood/node-llama-win32-arm64-vulkan": "1.0.0-beta.6",
-    "@fugood/node-llama-darwin-x64": "1.0.0-beta.6",
-    "@fugood/node-llama-darwin-arm64": "1.0.0-beta.6"
+    "@fugood/node-llama-linux-x64": "1.0.0",
+    "@fugood/node-llama-linux-x64-vulkan": "1.0.0",
+    "@fugood/node-llama-linux-x64-cuda": "1.0.0",
+    "@fugood/node-llama-linux-arm64": "1.0.0",
+    "@fugood/node-llama-linux-arm64-vulkan": "1.0.0",
+    "@fugood/node-llama-linux-arm64-cuda": "1.0.0",
+    "@fugood/node-llama-win32-x64": "1.0.0",
+    "@fugood/node-llama-win32-x64-vulkan": "1.0.0",
+    "@fugood/node-llama-win32-x64-cuda": "1.0.0",
+    "@fugood/node-llama-win32-arm64": "1.0.0",
+    "@fugood/node-llama-win32-arm64-vulkan": "1.0.0",
+    "@fugood/node-llama-darwin-x64": "1.0.0",
+    "@fugood/node-llama-darwin-arm64": "1.0.0"
   },
   "devDependencies": {
     "@babel/preset-env": "^7.24.4",

package/src/EmbeddingWorker.cpp

@@ -8,7 +8,7 @@ EmbeddingWorker::EmbeddingWorker(const Napi::CallbackInfo &info,
       _params(params) {}
 
 void EmbeddingWorker::Execute() {
-  llama_kv_self_clear(_sess->context());
+  llama_memory_clear(llama_get_memory(_sess->context()), true);
   auto tokens = ::common_tokenize(_sess->context(), _text, true);
   // add SEP if not present
   auto vocab = llama_model_get_vocab(_sess->model());

package/src/LlamaCompletionWorker.cpp

@@ -29,11 +29,13 @@ LlamaCompletionWorker::LlamaCompletionWorker(
     common_params params,
     std::vector<std::string> stop_words,
     int32_t chat_format,
+    bool thinking_forced_open,
     std::string reasoning_format,
     const std::vector<std::string> &media_paths,
     const std::vector<llama_token> &guide_tokens)
     : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess),
       _params(params), _stop_words(stop_words), _chat_format(chat_format),
+      _thinking_forced_open(thinking_forced_open),
       _reasoning_format(reasoning_format),
       _media_paths(media_paths), _guide_tokens(guide_tokens) {
   if (!callback.IsEmpty()) {
@@ -113,7 +115,7 @@ void LlamaCompletionWorker::Execute() {
         --n_cur;
       }
       n_input -= n_cur;
-      llama_kv_self_seq_rm(ctx, 0, n_cur, -1);
+      llama_memory_seq_rm(llama_get_memory(ctx), 0, n_cur, -1);
     }
     // Set the tokens
     _sess->set_tokens(std::move(prompt_tokens));
@@ -135,8 +137,9 @@ void LlamaCompletionWorker::Execute() {
       const int n_left = n_cur - n_keep - 1;
       const int n_discard = n_left / 2;
 
-      llama_kv_self_seq_rm(ctx, 0, n_keep + 1, n_keep + n_discard + 1);
-      llama_kv_self_seq_add(ctx, 0, n_keep + 1 + n_discard, n_cur, -n_discard);
+      auto mem = llama_get_memory(ctx);
+      llama_memory_seq_rm(mem, 0, n_keep + 1, n_keep + n_discard + 1);
+      llama_memory_seq_add(mem, 0, n_keep + 1 + n_discard, n_cur, -n_discard);
 
       // shift the tokens
       embd->insert(embd->begin() + n_keep + 1,
@@ -240,6 +243,7 @@ void LlamaCompletionWorker::OnOK() {
     try {
       common_chat_syntax chat_syntax;
       chat_syntax.format = static_cast<common_chat_format>(_chat_format);
+      chat_syntax.thinking_forced_open = _thinking_forced_open;
 
       if (_reasoning_format == "deepseek") {
           chat_syntax.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;

package/src/LlamaCompletionWorker.h

@@ -20,6 +20,7 @@ public:
                         Napi::Function callback, common_params params,
                         std::vector<std::string> stop_words,
                         int32_t chat_format,
+                        bool thinking_forced_open,
                         std::string reasoning_format,
                         const std::vector<std::string> &media_paths = {},
                         const std::vector<llama_token> &guide_tokens = {});
@@ -42,6 +43,7 @@ private:
   common_params _params;
   std::vector<std::string> _stop_words;
   int32_t _chat_format;
+  bool _thinking_forced_open;
   std::string _reasoning_format;
   std::vector<std::string> _media_paths;
   std::vector<llama_token> _guide_tokens;

package/src/LlamaContext.cpp

@@ -3,6 +3,7 @@
 #include "DetokenizeWorker.h"
 #include "DisposeWorker.h"
 #include "EmbeddingWorker.h"
+#include "RerankWorker.h"
 #include "LlamaCompletionWorker.h"
 #include "LoadSessionWorker.h"
 #include "SaveSessionWorker.h"
@@ -110,6 +111,8 @@ void LlamaContext::Init(Napi::Env env, Napi::Object &exports) {
            static_cast<napi_property_attributes>(napi_enumerable)),
        InstanceMethod<&LlamaContext::Embedding>(
            "embedding", static_cast<napi_property_attributes>(napi_enumerable)),
+       InstanceMethod<&LlamaContext::Rerank>(
+           "rerank", static_cast<napi_property_attributes>(napi_enumerable)),
        InstanceMethod<&LlamaContext::SaveSession>(
            "saveSession",
            static_cast<napi_property_attributes>(napi_enumerable)),
@@ -499,7 +502,9 @@ common_chat_params getFormattedChatWithJinja(
     const common_chat_templates_ptr &templates, const std::string &messages,
     const std::string &chat_template, const std::string &json_schema,
     const std::string &tools, const bool &parallel_tool_calls,
-    const std::string &tool_choice) {
+    const std::string &tool_choice,
+    const bool &enable_thinking
+) {
   common_chat_templates_inputs inputs;
   inputs.messages = common_chat_msgs_parse_oaicompat(json::parse(messages));
   auto useTools = !tools.empty();
@@ -513,6 +518,7 @@ common_chat_params getFormattedChatWithJinja(
   if (!json_schema.empty()) {
     inputs.json_schema = json::parse(json_schema);
   }
+  inputs.enable_thinking = enable_thinking;
 
   // If chat_template is provided, create new one and use it (probably slow)
   if (!chat_template.empty()) {
@@ -586,12 +592,11 @@ Napi::Value LlamaContext::GetFormattedChat(const Napi::CallbackInfo &info) {
     auto parallel_tool_calls =
         get_option<bool>(params, "parallel_tool_calls", false);
     auto tool_choice = get_option<std::string>(params, "tool_choice", "");
+    auto enable_thinking = get_option<bool>(params, "enable_thinking", false);
 
     auto chatParams = getFormattedChatWithJinja(
         _sess, _templates, messages, chat_template, json_schema_str, tools_str,
-        parallel_tool_calls, tool_choice);
-
-    console_log(env, std::string("format: ") + std::to_string(chatParams.format));
+        parallel_tool_calls, tool_choice, enable_thinking);
 
     Napi::Object result = Napi::Object::New(env);
     result.Set("prompt", chatParams.prompt);
@@ -612,6 +617,7 @@ Napi::Value LlamaContext::GetFormattedChat(const Napi::CallbackInfo &info) {
       grammar_triggers.Set(i, triggerObj);
     }
     result.Set("grammar_triggers", grammar_triggers);
+    result.Set("thinking_forced_open", chatParams.thinking_forced_open);
     // preserved_tokens: string[]
     Napi::Array preserved_tokens = Napi::Array::New(env);
     for (size_t i = 0; i < chatParams.preserved_tokens.size(); i++) {
@@ -685,6 +691,7 @@ Napi::Value LlamaContext::Completion(const Napi::CallbackInfo &info) {
   }
 
   int32_t chat_format = get_option<int32_t>(options, "chat_format", 0);
+  bool thinking_forced_open = get_option<bool>(options, "thinking_forced_open", false);
   std::string reasoning_format = get_option<std::string>(options, "reasoning_format", "none");
 
   common_params params = _sess->params();
@@ -793,14 +800,16 @@ Napi::Value LlamaContext::Completion(const Napi::CallbackInfo &info) {
           get_option<bool>(options, "parallel_tool_calls", false);
       auto tool_choice =
           get_option<std::string>(options, "tool_choice", "none");
+      auto enable_thinking = get_option<bool>(options, "enable_thinking", true);
 
       auto chatParams = getFormattedChatWithJinja(
           _sess, _templates, json_stringify(messages), chat_template,
-          json_schema_str, tools_str, parallel_tool_calls, tool_choice);
+          json_schema_str, tools_str, parallel_tool_calls, tool_choice, enable_thinking);
 
       params.prompt = chatParams.prompt;
 
       chat_format = chatParams.format;
+      thinking_forced_open = chatParams.thinking_forced_open;
 
       for (const auto &token : chatParams.preserved_tokens) {
         auto ids =
@@ -895,7 +904,7 @@ Napi::Value LlamaContext::Completion(const Napi::CallbackInfo &info) {
 
   auto *worker =
       new LlamaCompletionWorker(info, _sess, callback, params, stop_words,
-                                chat_format, reasoning_format, media_paths, guide_tokens);
+                                chat_format, thinking_forced_open, reasoning_format, media_paths, guide_tokens);
   worker->Queue();
   _wip = worker;
   worker->OnComplete([this]() { _wip = nullptr; });
@@ -976,6 +985,40 @@ Napi::Value LlamaContext::Embedding(const Napi::CallbackInfo &info) {
   return worker->Promise();
 }
 
+// rerank(query: string, documents: string[], params?: object): Promise<RerankResult[]>
+Napi::Value LlamaContext::Rerank(const Napi::CallbackInfo &info) {
+  Napi::Env env = info.Env();
+  if (info.Length() < 2 || !info[0].IsString() || !info[1].IsArray()) {
+    Napi::TypeError::New(env, "Query string and documents array expected").ThrowAsJavaScriptException();
+  }
+  if (_sess == nullptr) {
+    Napi::TypeError::New(env, "Context is disposed")
+        .ThrowAsJavaScriptException();
+  }
+  
+  auto query = info[0].ToString().Utf8Value();
+  auto documents_array = info[1].As<Napi::Array>();
+  
+  // Convert documents array to vector
+  std::vector<std::string> documents;
+  for (size_t i = 0; i < documents_array.Length(); i++) {
+    documents.push_back(documents_array.Get(i).ToString().Utf8Value());
+  }
+  
+  auto options = Napi::Object::New(env);
+  if (info.Length() >= 3 && info[2].IsObject()) {
+    options = info[2].As<Napi::Object>();
+  }
+
+  common_params rerankParams;
+  rerankParams.embedding = true;
+  rerankParams.embd_normalize = get_option<int32_t>(options, "normalize", -1);
+  
+  auto *worker = new RerankWorker(info, _sess, query, documents, rerankParams);
+  worker->Queue();
+  return worker->Promise();
+}
+
 // saveSession(path: string): Promise<void> throws error
 Napi::Value LlamaContext::SaveSession(const Napi::CallbackInfo &info) {
   Napi::Env env = info.Env();

package/src/LlamaContext.h

@@ -28,6 +28,7 @@ private:
   Napi::Value Tokenize(const Napi::CallbackInfo &info);
   Napi::Value Detokenize(const Napi::CallbackInfo &info);
   Napi::Value Embedding(const Napi::CallbackInfo &info);
+  Napi::Value Rerank(const Napi::CallbackInfo &info);
   Napi::Value SaveSession(const Napi::CallbackInfo &info);
   Napi::Value LoadSession(const Napi::CallbackInfo &info);
   void ApplyLoraAdapters(const Napi::CallbackInfo &info);

package/src/RerankWorker.h

@@ -0,0 +1,26 @@
+#include "common.hpp"
+#include <vector>
+
+struct RerankResult {
+  std::vector<float> scores;
+};
+
+class RerankWorker : public Napi::AsyncWorker,
+                     public Napi::Promise::Deferred {
+public:
+  RerankWorker(const Napi::CallbackInfo &info, LlamaSessionPtr &sess,
+               std::string query, std::vector<std::string> documents, 
+               common_params &params);
+
+protected:
+  void Execute();
+  void OnOK();
+  void OnError(const Napi::Error &err);
+
+private:
+  LlamaSessionPtr _sess;
+  std::string _query;
+  std::vector<std::string> _documents;
+  common_params _params;
+  RerankResult _result;
+}; 

package/src/common.hpp

@@ -461,7 +461,7 @@ processMediaPrompt(llama_context *ctx, mtmd_context *mtmd_ctx,
   }
 
   // Clear all KV cache entries after position n_past
-  llama_kv_self_seq_rm(ctx, 0, n_past, -1);
+  llama_memory_seq_rm(llama_get_memory(ctx), 0, n_past, -1);
 
   size_t num_chunks = mtmd_input_chunks_size(chunks);
 

package/src/llama.cpp/CMakeLists.txt

@@ -95,7 +95,7 @@ endif()
 if (NOT DEFINED LLAMA_BUILD_COMMIT)
     set(LLAMA_BUILD_COMMIT        ${BUILD_COMMIT})
 endif()
-set(LLAMA_INSTALL_VERSION 0.0.${BUILD_NUMBER})
+set(LLAMA_INSTALL_VERSION 0.0.${LLAMA_BUILD_NUMBER})
 
 # override ggml options
 set(GGML_ALL_WARNINGS   ${LLAMA_ALL_WARNINGS})

package/src/llama.cpp/common/json-schema-to-grammar.cpp

@@ -41,49 +41,6 @@ static std::string build_repetition(const std::string & item_rule, int min_items
     return result;
 }
 
-/* Minimalistic replacement for std::string_view, which is only available from C++17 onwards */
-class string_view {
-    const std::string & _str;
-    const size_t _start;
-    const size_t _end;
-public:
-    string_view(const std::string & str, size_t start = 0, size_t end  = std::string::npos) : _str(str), _start(start), _end(end == std::string::npos ? str.length() : end) {}
-
-    size_t size() const {
-        return _end - _start;
-    }
-
-    size_t length() const {
-        return size();
-    }
-
-    operator std::string() const {
-        return str();
-    }
-
-    std::string str() const {
-        return _str.substr(_start, _end - _start);
-    }
-
-    string_view substr(size_t pos, size_t len = std::string::npos) const {
-        return string_view(_str, _start + pos, len == std::string::npos ? _end : _start + pos + len);
-    }
-
-    char operator[](size_t pos) const {
-        auto index = _start + pos;
-        if (index >= _end) {
-            throw std::out_of_range("string_view index out of range");
-        }
-        return _str[_start + pos];
-    }
-
-    bool operator==(const string_view & other) const {
-        std::string this_str = *this;
-        std::string other_str = other;
-        return this_str == other_str;
-    }
-};
-
 static void _build_min_max_int(int min_value, int max_value, std::stringstream & out, int decimals_left = 16, bool top_level = true) {
     auto has_min = min_value != std::numeric_limits<int>::min();
     auto has_max = max_value != std::numeric_limits<int>::max();
@@ -112,14 +69,14 @@ static void _build_min_max_int(int min_value, int max_value, std::stringstream &
         }
         out << "}";
     };
-    std::function<void(const string_view &, const string_view &)> uniform_range =
-        [&](const string_view & from, const string_view & to) {
+    std::function<void(const std::string_view &, const std::string_view &)> uniform_range =
+        [&](const std::string_view & from, const std::string_view & to) {
             size_t i = 0;
             while (i < from.length() && i < to.length() && from[i] == to[i]) {
                 i++;
             }
             if (i > 0) {
-                out << "\"" << from.substr(0, i).str() << "\"";
+                out << "\"" << from.substr(0, i) << "\"";
             }
             if (i < from.length() && i < to.length()) {
                 if (i > 0) {

package/src/llama.cpp/ggml/CMakeLists.txt

@@ -131,6 +131,7 @@ option(GGML_RVV              "ggml: enable rvv"              ON)
 option(GGML_RV_ZFH           "ggml: enable riscv zfh"        OFF)
 option(GGML_XTHEADVECTOR     "ggml: enable xtheadvector"     OFF)
 option(GGML_VXE              "ggml: enable vxe"              ON)
+option(GGML_NNPA             "ggml: enable nnpa"             ON)
 
 option(GGML_CPU_ALL_VARIANTS "ggml: build all variants of the CPU backend (requires GGML_BACKEND_DL)" OFF)
 set(GGML_CPU_ARM_ARCH        "" CACHE STRING "ggml: CPU architecture for ARM")

package/src/llama.cpp/ggml/include/ggml-cpu.h

@@ -101,6 +101,7 @@ extern "C" {
     GGML_BACKEND_API int ggml_cpu_has_riscv_v    (void);
     GGML_BACKEND_API int ggml_cpu_has_vsx        (void);
     GGML_BACKEND_API int ggml_cpu_has_vxe        (void);
+    GGML_BACKEND_API int ggml_cpu_has_nnpa       (void);
     GGML_BACKEND_API int ggml_cpu_has_wasm_simd  (void);
     GGML_BACKEND_API int ggml_cpu_has_llamafile  (void);
 

package/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt

@@ -448,6 +448,7 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
 
         # TODO: Separation to determine activation of VX/VXE/VXE2
         if (${S390X_M} MATCHES "8561|8562")
+            set(GGML_NNPA OFF)
             message(STATUS "z15 target")
             list(APPEND ARCH_FLAGS -march=z15)
         elseif (${S390X_M} MATCHES "3931")
@@ -464,7 +465,14 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
         endif()
 
         if (GGML_VXE)
+            message(STATUS "VX/VXE/VXE2 enabled")
             list(APPEND ARCH_FLAGS -mvx -mzvector)
+            list(APPEND ARCH_DEFINITIONS GGML_VXE)
+        endif()
+
+        if (GGML_NNPA)
+            message(STATUS "NNPA enabled")
+            list(APPEND ARCH_DEFINITIONS GGML_NNPA)
         endif()
     elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "wasm")
         message(STATUS "Wasm detected")

package/src/llama.cpp/ggml/src/ggml-cpu/amx/mmq.cpp

@@ -8,6 +8,7 @@
 #include "mmq.h"
 #include "ggml-impl.h"
 #include "ggml-cpu-impl.h"
+#include "simd-mappings.h"
 #include "quants.h"
 #include "ggml-quants.h"
 #include <algorithm>
@@ -453,7 +454,7 @@ void quantize_row_q8_K_vnni(const float * RESTRICT x, void * RESTRICT vy, int64_
 
         // Quantize these floats
         const float iscale = 127.f / amax;
-        y[i].d = GGML_FP32_TO_FP16(1 / iscale);
+        y[i].d = GGML_CPU_FP32_TO_FP16(1 / iscale);
         const float id = ( amax != 0.0f ) ? iscale : 0.f;
         const __m512 vscale = _mm512_set1_ps(id);
 
@@ -1090,7 +1091,7 @@ struct acc_C<block_q8_0, block_q4_0, is_acc> {
         const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)((const char *)packed_B + offset)));
 
         for (int m = 0; m < nr; ++m) {
-            const __m512 vd1 = _mm512_set1_ps(GGML_FP16_TO_FP32(A[m * lda].d));
+            const __m512 vd1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[m * lda].d));
             const __m512 vtile = _mm512_cvtepi32_ps(_mm512_loadu_si512(tile + m * TILE_N));
 
             __m512 vsum;
@@ -1113,8 +1114,8 @@ struct acc_C<block_q8_1, block_q4_1, is_acc> {
         const __m512 vm0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)((const char *)packed_B + offset + TILE_N * sizeof(ggml_half))));
 
         for (int m = 0; m < nr; ++m) {
-            const __m512 vd1 = _mm512_set1_ps(GGML_FP16_TO_FP32(A[m * lda].d));
-            const __m512 vs1 = _mm512_set1_ps(GGML_FP16_TO_FP32(A[m * lda].s));
+            const __m512 vd1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[m * lda].d));
+            const __m512 vs1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[m * lda].s));
             const __m512 vtile = _mm512_cvtepi32_ps(_mm512_loadu_si512(tile + m * TILE_N));
 
             __m512 vsum;
@@ -1137,7 +1138,7 @@ struct acc_C<block_q8_0, block_q8_0, is_acc> {
         const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)((const char *)packed_B + offset)));
 
         for (int m = 0; m < nr; ++m) {
-            const __m512 vd1 = _mm512_set1_ps(GGML_FP16_TO_FP32(A[m * lda].d));
+            const __m512 vd1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[m * lda].d));
             const __m512 vtile = _mm512_cvtepi32_ps(_mm512_loadu_si512(tile + m * TILE_N));
 
             __m512 vsum;
@@ -1437,7 +1438,7 @@ struct tinygemm_kernel_vnni<block_q8_0, block_q4_0, float, BLOCK_M, BLOCK_N, BLO
                     va[k] = _mm512_set1_epi32(a_ptr[k]);
                     vcomp = _mm512_dpbusd_epi32(vcomp, off, va[k]);
                 }
-                vd1 = _mm512_set1_ps(GGML_FP16_TO_FP32(A[0 * KB + i].d));
+                vd1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[0 * KB + i].d));
             }
 
             // load b
@@ -1498,8 +1499,8 @@ struct tinygemm_kernel_vnni<block_q8_1, block_q4_1, float, 1, BLOCK_N, BLOCK_K>
                 for (int k = 0; k < 8; ++k) {
                     va[k] = _mm512_set1_epi32(a_ptr[k]);
                 }
-                vd1 = _mm512_set1_ps(GGML_FP16_TO_FP32(A[0 * KB + i].d));
-                vs1 = _mm512_set1_ps(GGML_FP16_TO_FP32(A[0 * KB + i].s));
+                vd1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[0 * KB + i].d));
+                vs1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[0 * KB + i].s));
             }
 
             // load b
@@ -1571,7 +1572,7 @@ struct tinygemm_kernel_vnni<block_q8_0, block_q8_0, float, BLOCK_M, BLOCK_N, BLO
                     va[k] = _mm512_set1_epi32(a_ptr[k]);
                     va[k] = _mm512_add_epi8(va[k], off);
                 }
-                vd1 = _mm512_set1_ps(GGML_FP16_TO_FP32(A[0 * KB + i].d));
+                vd1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[0 * KB + i].d));
             }
 
             // load b