cui-llama.rn 1.3.3 → 1.3.5

package/cpp/ggml-backend-reg.cpp

@@ -2,8 +2,13 @@
 #include "ggml-backend.h"
 #include "ggml-impl.h"
 #include <algorithm>
+#include <codecvt>
 #include <cstring>
+#include <filesystem>
+#include <locale>
+#include <memory>
 #include <string>
+#include <type_traits>
 #include <vector>
 
 #ifdef _WIN32
@@ -41,6 +46,10 @@
 #include "ggml-vulkan.h"
 #endif
 
+#ifdef LM_GGML_USE_OPENCL
+#include "ggml-opencl.h"
+#endif
+
 #ifdef LM_GGML_USE_BLAS
 #include "ggml-blas.h"
 #endif
@@ -57,9 +66,86 @@
 #include "ggml-kompute.h"
 #endif
 
+// disable C++17 deprecation warning for std::codecvt_utf8
+#if defined(__clang__)
+#    pragma clang diagnostic push
+#    pragma clang diagnostic ignored "-Wdeprecated-declarations"
+#endif
+
+static std::wstring utf8_to_utf16(const std::string & str) {
+    std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
+    return converter.from_bytes(str);
+}
+
+static std::string utf16_to_utf8(const std::wstring & str) {
+    std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
+    return converter.to_bytes(str);
+}
+
+#if defined(__clang__)
+#    pragma clang diagnostic pop
+#endif
+
+#ifdef _WIN32
+
+using dl_handle = std::remove_pointer_t<HMODULE>;
+
+struct dl_handle_deleter {
+    void operator()(HMODULE handle) {
+        FreeLibrary(handle);
+    }
+};
+
+static dl_handle * dl_load_library(const std::wstring & path) {
+    // suppress error dialogs for missing DLLs
+    DWORD old_mode = SetErrorMode(SEM_FAILCRITICALERRORS);
+    SetErrorMode(old_mode | SEM_FAILCRITICALERRORS);
+
+    HMODULE handle = LoadLibraryW(path.c_str());
+
+    SetErrorMode(old_mode);
+
+    return handle;
+}
+
+static void * dl_get_sym(dl_handle * handle, const char * name) {
+    DWORD old_mode = SetErrorMode(SEM_FAILCRITICALERRORS);
+    SetErrorMode(old_mode | SEM_FAILCRITICALERRORS);
+
+    void * p = (void *) GetProcAddress(handle, name);
+
+    SetErrorMode(old_mode);
+
+    return p;
+}
+
+#else
+
+using dl_handle = void;
+
+struct dl_handle_deleter {
+    void operator()(void * handle) {
+        dlclose(handle);
+    }
+};
+
+static void * dl_load_library(const std::wstring & path) {
+    dl_handle * handle = dlopen(utf16_to_utf8(path).c_str(), RTLD_NOW | RTLD_LOCAL);
+
+    return handle;
+}
+
+static void * dl_get_sym(dl_handle * handle, const char * name) {
+    return dlsym(handle, name);
+}
+
+#endif
+
+using dl_handle_ptr = std::unique_ptr<dl_handle, dl_handle_deleter>;
+
 struct lm_ggml_backend_reg_entry {
     lm_ggml_backend_reg_t reg;
-    void * handle;
+    dl_handle_ptr handle;
 };
 
 struct lm_ggml_backend_registry {
@@ -79,6 +165,9 @@ struct lm_ggml_backend_registry {
 #ifdef LM_GGML_USE_VULKAN
         register_backend(lm_ggml_backend_vk_reg());
 #endif
+#ifdef LM_GGML_USE_OPENCL
+        register_backend(lm_ggml_backend_opencl_reg());
+#endif
 #ifdef LM_GGML_USE_CANN
         register_backend(lm_ggml_backend_cann_reg());
 #endif
@@ -97,13 +186,16 @@ struct lm_ggml_backend_registry {
     }
 
     ~lm_ggml_backend_registry() {
-        while (!backends.empty()) {
-            // use silent since the log system may have been destroyed at this point
-            unload_backend(backends.back().reg, true);
+        // FIXME: backends cannot be safely unloaded without a function to destroy all the backend resources,
+        // since backend threads may still be running and accessing resources from the dynamic library
+        for (auto & entry : backends) {
+            if (entry.handle) {
+                entry.handle.release(); // NOLINT
+            }
         }
     }
 
-    void register_backend(lm_ggml_backend_reg_t reg, void * handle = nullptr) {
+    void register_backend(lm_ggml_backend_reg_t reg, dl_handle_ptr handle = nullptr) {
         if (!reg) {
             return;
         }
@@ -112,7 +204,7 @@ struct lm_ggml_backend_registry {
         LM_GGML_LOG_DEBUG("%s: registered backend %s (%zu devices)\n",
             __func__, lm_ggml_backend_reg_name(reg), lm_ggml_backend_reg_dev_count(reg));
 #endif
-        backends.push_back({ reg, handle });
+        backends.push_back({ reg, std::move(handle) });
         for (size_t i = 0; i < lm_ggml_backend_reg_dev_count(reg); i++) {
             register_device(lm_ggml_backend_reg_dev_get(reg, i));
         }
@@ -125,80 +217,54 @@ struct lm_ggml_backend_registry {
         devices.push_back(device);
     }
 
-    lm_ggml_backend_reg_t load_backend(const char * path, bool silent) {
-#ifdef _WIN32
-        // suppress error dialogs for missing DLLs
-        DWORD old_mode = SetErrorMode(SEM_FAILCRITICALERRORS);
-        SetErrorMode(old_mode | SEM_FAILCRITICALERRORS);
-
-        HMODULE handle = LoadLibraryA(path);
-
+    lm_ggml_backend_reg_t load_backend(const std::wstring & path, bool silent) {
+        dl_handle_ptr handle { dl_load_library(path) };
         if (!handle) {
             if (!silent) {
-                LM_GGML_LOG_ERROR("%s: failed to load %s: %lu\n", __func__, path, GetLastError());
+                LM_GGML_LOG_ERROR("%s: failed to load %s\n", __func__, utf16_to_utf8(path).c_str());
             }
-            SetErrorMode(old_mode);
             return nullptr;
         }
 
-        lm_ggml_backend_init_t backend_init = (lm_ggml_backend_init_t) GetProcAddress(handle, "lm_ggml_backend_init");
-
-        SetErrorMode(old_mode);
-
-        if (!backend_init) {
+        auto score_fn = (lm_ggml_backend_score_t) dl_get_sym(handle.get(), "lm_ggml_backend_score");
+        if (score_fn && score_fn() == 0) {
             if (!silent) {
-                LM_GGML_LOG_ERROR("%s: failed to find lm_ggml_backend_init in %s: %lu\n", __func__, path, GetLastError());
+                LM_GGML_LOG_INFO("%s: backend %s is not supported on this system\n", __func__, utf16_to_utf8(path).c_str());
             }
-            FreeLibrary(handle);
             return nullptr;
         }
-#else
-        void * handle = dlopen(path, RTLD_NOW | RTLD_LOCAL);
-
-        if (!handle) {
-            if (!silent) {
-                LM_GGML_LOG_ERROR("%s: failed to load %s: %s\n", __func__, path, dlerror());
-            }
-            return nullptr;
-        }
-
-        auto * backend_init = (lm_ggml_backend_init_t) dlsym(handle, "lm_ggml_backend_init");
 
-        if (!backend_init) {
+        auto backend_init_fn = (lm_ggml_backend_init_t) dl_get_sym(handle.get(), "lm_ggml_backend_init");
+        if (!backend_init_fn) {
             if (!silent) {
-                LM_GGML_LOG_ERROR("%s: failed to find lm_ggml_backend_init in %s: %s\n", __func__, path, dlerror());
+                LM_GGML_LOG_ERROR("%s: failed to find lm_ggml_backend_init in %s\n", __func__, utf16_to_utf8(path).c_str());
             }
-            dlclose(handle);
             return nullptr;
         }
-#endif
-        lm_ggml_backend_reg_t reg = backend_init();
 
+        lm_ggml_backend_reg_t reg = backend_init_fn();
         if (!reg || reg->api_version != LM_GGML_BACKEND_API_VERSION) {
             if (!silent) {
                 if (!reg) {
-                    LM_GGML_LOG_ERROR("%s: failed to initialize backend from %s: lm_ggml_backend_init returned NULL\n", __func__, path);
+                    LM_GGML_LOG_ERROR("%s: failed to initialize backend from %s: lm_ggml_backend_init returned NULL\n", __func__, utf16_to_utf8(path).c_str());
                 } else {
                     LM_GGML_LOG_ERROR("%s: failed to initialize backend from %s: incompatible API version (backend: %d, current: %d)\n",
-                                   __func__, path, reg->api_version, LM_GGML_BACKEND_API_VERSION);
+                        __func__, utf16_to_utf8(path).c_str(), reg->api_version, LM_GGML_BACKEND_API_VERSION);
                 }
             }
-#ifdef _WIN32
-            FreeLibrary(handle);
-#else
-            dlclose(handle);
-#endif
             return nullptr;
         }
 
-        LM_GGML_LOG_INFO("%s: loaded %s backend from %s\n", __func__, lm_ggml_backend_reg_name(reg), path);
-        register_backend(reg, handle);
+        LM_GGML_LOG_INFO("%s: loaded %s backend from %s\n", __func__, lm_ggml_backend_reg_name(reg), utf16_to_utf8(path).c_str());
+
+        register_backend(reg, std::move(handle));
+
         return reg;
     }
 
     void unload_backend(lm_ggml_backend_reg_t reg, bool silent) {
         auto it = std::find_if(backends.begin(), backends.end(),
-                                [reg](lm_ggml_backend_reg_entry entry) { return entry.reg == reg; });
+                               [reg](const lm_ggml_backend_reg_entry & entry) { return entry.reg == reg; });
 
         if (it == backends.end()) {
             if (!silent) {
@@ -217,15 +283,6 @@ struct lm_ggml_backend_registry {
                             [reg](lm_ggml_backend_dev_t dev) { return lm_ggml_backend_dev_backend_reg(dev) == reg; }),
             devices.end());
 
-        // unload library
-        if (it->handle) {
-#ifdef _WIN32
-            FreeLibrary((HMODULE) it->handle);
-#else
-            dlclose(it->handle);
-#endif
-        }
-
         // remove backend
         backends.erase(it);
     }
@@ -334,19 +391,14 @@ lm_ggml_backend_t lm_ggml_backend_init_best(void) {
 
 // Dynamic loading
 lm_ggml_backend_reg_t lm_ggml_backend_load(const char * path) {
-    return get_reg().load_backend(path, false);
+    return get_reg().load_backend(utf8_to_utf16(path), false);
 }
 
 void lm_ggml_backend_unload(lm_ggml_backend_reg_t reg) {
     get_reg().unload_backend(reg, true);
 }
 
-void lm_ggml_backend_load_all() {
-    std::vector<std::string> search_prefix;
-
-    // add the executable directory to the search path
-    // FIXME: this is convenient for development, but it should probably be disabled in production
-
+static std::wstring get_executable_path() {
 #if defined(__APPLE__)
     // get executable path
     std::vector<char> path;
@@ -364,13 +416,17 @@ void lm_ggml_backend_load_all() {
     if (last_slash != std::string::npos) {
         base_path = base_path.substr(0, last_slash);
     }
-    search_prefix.push_back(base_path + "/");
-#elif defined(__linux__)
+    return utf8_to_utf16(base_path + "/");
+#elif defined(__linux__) || defined(__FreeBSD__)
     std::string base_path = ".";
     std::vector<char> path(1024);
     while (true) {
         // get executable path
+#    if defined(__linux__)
         ssize_t len = readlink("/proc/self/exe", path.data(), path.size());
+#    elif defined(__FreeBSD__)
+        ssize_t len = readlink("/proc/curproc/file", path.data(), path.size());
+#    endif
         if (len == -1) {
             break;
         }
@@ -386,38 +442,136 @@ void lm_ggml_backend_load_all() {
         path.resize(path.size() * 2);
     }
 
-    search_prefix.push_back(base_path + "/");
+    return utf8_to_utf16(base_path + "/");
+#elif defined(_WIN32)
+    std::vector<wchar_t> path(MAX_PATH);
+    DWORD len = GetModuleFileNameW(NULL, path.data(), path.size());
+    if (len == 0) {
+        return {};
+    }
+    std::wstring base_path(path.data(), len);
+    // remove executable name
+    auto last_slash = base_path.find_last_of('\\');
+    if (last_slash != std::string::npos) {
+        base_path = base_path.substr(0, last_slash);
+    }
+    return base_path + L"\\";
+#else
+    return {};
 #endif
+}
 
-    auto & reg = get_reg();
+static std::wstring backend_filename_prefix() {
+#ifdef _WIN32
+    return L"ggml-";
+#else
+    return L"libggml-";
+#endif
+}
 
-    auto try_load = [&](const std::string & name) {
-        std::string os_name;
+static std::wstring backend_filename_suffix() {
 #ifdef _WIN32
-        os_name = "ggml-" + name + ".dll";
+    return L".dll";
 #else
-        os_name = "libggml-" + name + ".so";
+    return L".so";
 #endif
-        if (reg.load_backend(os_name.c_str(), true)) {
-            return;
+}
+
+static std::wstring path_separator() {
+#ifdef _WIN32
+    return L"\\";
+#else
+    return L"/";
+#endif
+}
+
+static lm_ggml_backend_reg_t lm_ggml_backend_load_best(const char * name, bool silent, const char * user_search_path) {
+    // enumerate all the files that match [lib]ggml-name-*.[so|dll] in the search paths
+     // TODO: search system paths
+    std::wstring file_prefix = backend_filename_prefix() + utf8_to_utf16(name) + L"-";
+    std::vector<std::wstring> search_paths;
+    if (user_search_path == nullptr) {
+        search_paths.push_back(L"." + path_separator());
+        search_paths.push_back(get_executable_path());
+    } else {
+        search_paths.push_back(utf8_to_utf16(user_search_path) + path_separator());
+    }
+
+    int best_score = 0;
+    std::wstring best_path;
+
+    namespace fs = std::filesystem;
+    for (const auto & search_path : search_paths) {
+        if (!fs::exists(search_path)) {
+            continue;
         }
-        for (const auto & prefix : search_prefix) {
-            if (reg.load_backend((prefix + os_name).c_str(), true)) {
-                return;
+        fs::directory_iterator dir_it(search_path, fs::directory_options::skip_permission_denied);
+        for (const auto & entry : dir_it) {
+            if (entry.is_regular_file()) {
+                std::wstring filename = entry.path().filename().wstring();
+                std::wstring ext = entry.path().extension().wstring();
+                if (filename.find(file_prefix) == 0 && ext == backend_filename_suffix()) {
+                    dl_handle_ptr handle { dl_load_library(entry.path().wstring()) };
+                    if (!handle && !silent) {
+                        LM_GGML_LOG_ERROR("%s: failed to load %s\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str());
+                    }
+                    if (handle) {
+                        auto score_fn = (lm_ggml_backend_score_t) dl_get_sym(handle.get(), "lm_ggml_backend_score");
+                        if (score_fn) {
+                            int s = score_fn();
+#ifndef NDEBUG
+                            LM_GGML_LOG_DEBUG("%s: %s score: %d\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str(), s);
+#endif
+                            if (s > best_score) {
+                                best_score = s;
+                                best_path = entry.path().wstring();
+                            }
+                        } else {
+                            if (!silent) {
+                                LM_GGML_LOG_INFO("%s: failed to find lm_ggml_backend_score in %s\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str());
+                            }
+                        }
+                    }
+                }
             }
         }
-    };
-
-    try_load("amx");
-    try_load("blas");
-    try_load("cann");
-    try_load("cuda");
-    try_load("hip");
-    try_load("kompute");
-    try_load("metal");
-    try_load("rpc");
-    try_load("sycl");
-    try_load("vulkan");
-    try_load("musa");
-    try_load("cpu");
+    }
+
+    if (best_score == 0) {
+        // try to load the base backend
+        for (const auto & search_path : search_paths) {
+            std::wstring path = search_path + backend_filename_prefix() + utf8_to_utf16(name) + backend_filename_suffix();
+            if (fs::exists(path)) {
+                return get_reg().load_backend(path, silent);
+            }
+        }
+        return nullptr;
+    }
+
+    return get_reg().load_backend(best_path, silent);
+}
+
+void lm_ggml_backend_load_all() {
+    lm_ggml_backend_load_all_from_path(nullptr);
+}
+
+void lm_ggml_backend_load_all_from_path(const char * dir_path) {
+#ifdef NDEBUG
+    bool silent = true;
+#else
+    bool silent = false;
+#endif
+
+    lm_ggml_backend_load_best("blas", silent, dir_path);
+    lm_ggml_backend_load_best("cann", silent, dir_path);
+    lm_ggml_backend_load_best("cuda", silent, dir_path);
+    lm_ggml_backend_load_best("hip", silent, dir_path);
+    lm_ggml_backend_load_best("kompute", silent, dir_path);
+    lm_ggml_backend_load_best("metal", silent, dir_path);
+    lm_ggml_backend_load_best("rpc", silent, dir_path);
+    lm_ggml_backend_load_best("sycl", silent, dir_path);
+    lm_ggml_backend_load_best("vulkan", silent, dir_path);
+    lm_ggml_backend_load_best("opencl", silent, dir_path);
+    lm_ggml_backend_load_best("musa", silent, dir_path);
+    lm_ggml_backend_load_best("cpu", silent, dir_path);
 }

package/cpp/ggml-backend.h

@@ -228,6 +228,7 @@ extern "C" {
     LM_GGML_API void               lm_ggml_backend_unload(lm_ggml_backend_reg_t reg);
     // Load all known backends from dynamic libraries
     LM_GGML_API void               lm_ggml_backend_load_all(void);
+    LM_GGML_API void               lm_ggml_backend_load_all_from_path(const char * dir_path);
 
     //
     // Backend scheduler

package/cpp/ggml-common.h

@@ -6,7 +6,20 @@
 typedef uint16_t lm_ggml_half;
 typedef uint32_t lm_ggml_half2;
 
-#define LM_GGML_COMMON_AGGR
+#define LM_GGML_COMMON_AGGR_U
+#define LM_GGML_COMMON_AGGR_S
+
+#define LM_GGML_COMMON_DECL
+#elif defined(LM_GGML_COMMON_DECL_CPP)
+#include <cstdint>
+
+typedef uint16_t lm_ggml_half;
+typedef uint32_t lm_ggml_half2;
+
+// std-c++ allow anonymous unions but some compiler warn on it
+#define LM_GGML_COMMON_AGGR_U data
+// std-c++ do not allow it.
+#define LM_GGML_COMMON_AGGR_S data
 
 #define LM_GGML_COMMON_DECL
 #elif defined(LM_GGML_COMMON_DECL_METAL)
@@ -15,7 +28,8 @@ typedef uint32_t lm_ggml_half2;
 typedef half  lm_ggml_half;
 typedef half2 lm_ggml_half2;
 
-#define LM_GGML_COMMON_AGGR
+#define LM_GGML_COMMON_AGGR_U
+#define LM_GGML_COMMON_AGGR_S
 
 #define LM_GGML_COMMON_DECL
 #elif defined(LM_GGML_COMMON_DECL_CUDA)
@@ -29,7 +43,8 @@ typedef half2 lm_ggml_half2;
 typedef half  lm_ggml_half;
 typedef half2 lm_ggml_half2;
 
-#define LM_GGML_COMMON_AGGR data
+#define LM_GGML_COMMON_AGGR_U
+#define LM_GGML_COMMON_AGGR_S data
 
 #define LM_GGML_COMMON_DECL
 #elif defined(LM_GGML_COMMON_DECL_HIP)
@@ -39,7 +54,8 @@ typedef half2 lm_ggml_half2;
 typedef half  lm_ggml_half;
 typedef half2 lm_ggml_half2;
 
-#define LM_GGML_COMMON_AGGR data
+#define LM_GGML_COMMON_AGGR_U
+#define LM_GGML_COMMON_AGGR_S data
 
 #define LM_GGML_COMMON_DECL
 #elif defined(LM_GGML_COMMON_DECL_SYCL)
@@ -49,7 +65,8 @@ typedef half2 lm_ggml_half2;
 typedef sycl::half  lm_ggml_half;
 typedef sycl::half2 lm_ggml_half2;
 
-#define LM_GGML_COMMON_AGGR data
+#define LM_GGML_COMMON_AGGR_U
+#define LM_GGML_COMMON_AGGR_S data
 
 #define LM_GGML_COMMON_DECL
 #endif
@@ -154,9 +171,9 @@ typedef struct {
         struct {
             lm_ggml_half d; // delta
             lm_ggml_half m; // min
-        } LM_GGML_COMMON_AGGR;
+        } LM_GGML_COMMON_AGGR_S;
         lm_ggml_half2 dm;
-    };
+    } LM_GGML_COMMON_AGGR_U;
     uint8_t qs[QK4_1 / 2]; // nibbles / quants
 } block_q4_1;
 static_assert(sizeof(block_q4_1) == 2 * sizeof(lm_ggml_half) + QK4_1 / 2, "wrong q4_1 block size/padding");
@@ -175,9 +192,9 @@ typedef struct {
         struct {
             lm_ggml_half d; // delta
             lm_ggml_half m; // min
-        } LM_GGML_COMMON_AGGR;
+        } LM_GGML_COMMON_AGGR_S;
         lm_ggml_half2 dm;
-    };
+    } LM_GGML_COMMON_AGGR_U;
     uint8_t qh[4];         // 5-th bit of quants
     uint8_t qs[QK5_1 / 2]; // nibbles / quants
 } block_q5_1;
@@ -196,37 +213,13 @@ typedef struct {
         struct {
             lm_ggml_half d; // delta
             lm_ggml_half s; // d * sum(qs[i])
-        } LM_GGML_COMMON_AGGR;
+        } LM_GGML_COMMON_AGGR_S;
         lm_ggml_half2 ds;
-    };
+    } LM_GGML_COMMON_AGGR_U;
     int8_t qs[QK8_1]; // quants
 } block_q8_1;
 static_assert(sizeof(block_q8_1) == 2*sizeof(lm_ggml_half) + QK8_1, "wrong q8_1 block size/padding");
 
-typedef struct {
-    lm_ggml_half d[4];        // deltas for 4 q4_0 blocks
-    uint8_t qs[QK4_0 * 2]; // nibbles / quants for 4 q4_0 blocks
-} block_q4_0x4;
-static_assert(sizeof(block_q4_0x4) == 4 * sizeof(lm_ggml_half) + QK4_0 * 2, "wrong q4_0x4 block size/padding");
-
-typedef struct {
-    lm_ggml_half d[8];        // deltas for 8 q4_0 blocks
-    uint8_t qs[QK4_0 * 4]; // nibbles / quants for 8 q4_0 blocks
-} block_q4_0x8;
-static_assert(sizeof(block_q4_0x8) == 8 * sizeof(lm_ggml_half) + QK4_0 * 4, "wrong q4_0x8 block size/padding");
-
-typedef struct {
-    lm_ggml_half d[4];        // deltas for 4 q8_0 blocks
-    int8_t qs[QK8_0 * 4];  // quants for 4 q8_0 blocks
-} block_q8_0x4;
-static_assert(sizeof(block_q8_0x4) == 4 * sizeof(lm_ggml_half) + QK8_0 * 4, "wrong q8_0x4 block size/padding");
-
-typedef struct {
-    lm_ggml_half d[8];        // deltas for 8 q8_0 blocks
-    int8_t qs[QK8_0 * 8];  // quants for 8 q8_0 blocks
-} block_q8_0x8;
-static_assert(sizeof(block_q8_0x8) == 8 * sizeof(lm_ggml_half) + QK8_0 * 8, "wrong q8_0x8 block size/padding");
-
 //
 // Ternary quantization
 //
@@ -261,9 +254,9 @@ typedef struct {
         struct {
             lm_ggml_half d;    // super-block scale for quantized scales
             lm_ggml_half dmin; // super-block scale for quantized mins
-        } LM_GGML_COMMON_AGGR;
+        } LM_GGML_COMMON_AGGR_S;
         lm_ggml_half2 dm;
-    };
+    } LM_GGML_COMMON_AGGR_U;
 } block_q2_K;
 static_assert(sizeof(block_q2_K) == 2*sizeof(lm_ggml_half) + QK_K/16 + QK_K/4, "wrong q2_K block size/padding");
 
@@ -288,9 +281,9 @@ typedef struct {
         struct {
             lm_ggml_half d;    // super-block scale for quantized scales
             lm_ggml_half dmin; // super-block scale for quantized mins
-        } LM_GGML_COMMON_AGGR;
+        } LM_GGML_COMMON_AGGR_S;
         lm_ggml_half2 dm;
-    };
+    } LM_GGML_COMMON_AGGR_U;
     uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits
     uint8_t qs[QK_K/2];           // 4--bit quants
 } block_q4_K;
@@ -305,9 +298,9 @@ typedef struct {
         struct {
             lm_ggml_half d;    // super-block scale for quantized scales
             lm_ggml_half dmin; // super-block scale for quantized mins
-        } LM_GGML_COMMON_AGGR;
+        } LM_GGML_COMMON_AGGR_S;
         lm_ggml_half2 dm;
-    };
+    } LM_GGML_COMMON_AGGR_U;
     uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits
     uint8_t qh[QK_K/8];           // quants, high bit
     uint8_t qs[QK_K/2];           // quants, low 4 bits
@@ -418,12 +411,6 @@ typedef struct {
 } block_iq4_xs;
 static_assert(sizeof(block_iq4_xs) == sizeof(lm_ggml_half) + sizeof(uint16_t) + QK_K/64 + QK_K/2, "wrong iq4_xs block size/padding");
 
-typedef struct {
-    lm_ggml_half d[4];        // deltas for 4 iq4_nl blocks
-    uint8_t qs[QK4_NL * 2];// nibbles / quants for 4 iq4_nl blocks
-} block_iq4_nlx4;
-static_assert(sizeof(block_iq4_nlx4) == 4 * sizeof(lm_ggml_half) + QK4_NL * 2, "wrong iq4_nlx4 block size/padding");
-
 #endif // LM_GGML_COMMON_DECL
 #endif // LM_GGML_COMMON_DECL
 
@@ -437,6 +424,13 @@ static_assert(sizeof(block_iq4_nlx4) == 4 * sizeof(lm_ggml_half) + QK4_NL * 2, "
 #define LM_GGML_TABLE_BEGIN(type, name, size) static const type name[size] = {
 #define LM_GGML_TABLE_END() };
 
+#define LM_GGML_COMMON_IMPL
+#elif defined(LM_GGML_COMMON_IMPL_CPP)
+#include <cstdint>
+
+#define LM_GGML_TABLE_BEGIN(type, name, size) static const type name[size] = {
+#define LM_GGML_TABLE_END() };
+
 #define LM_GGML_COMMON_IMPL
 #elif defined(LM_GGML_COMMON_IMPL_METAL)
 #include <metal_stdlib>
@@ -479,7 +473,7 @@ LM_GGML_TABLE_BEGIN(uint8_t, ksigns_iq2xs, 128)
     240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255,
 LM_GGML_TABLE_END()
 
-//#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics
+//#if __CUDA_ARCH__ >= LM_GGML_CUDA_CC_DP4A // lowest compute capability for integer intrinsics
 LM_GGML_TABLE_BEGIN(uint64_t, ksigns64, 128)
     0x0000000000000000, 0xff000000000000ff, 0xff0000000000ff00, 0x000000000000ffff,
     0xff00000000ff0000, 0x0000000000ff00ff, 0x0000000000ffff00, 0xff00000000ffffff,