cui-llama.rn 1.2.6 → 1.3.3

package/cpp/ggml-aarch64.h

@@ -1,9 +1,5 @@
-// SPDX-FileCopyrightText: Copyright 2024 Arm Ltd.
 #pragma once
 
-#define LM_GGML_COMMON_DECL_C
-#include "ggml-common.h"
-
 #include "ggml.h"
 
 // GGML internal header
@@ -12,27 +8,11 @@
 extern "C" {
 #endif
 
-// Quantization
-void quantize_q8_0_4x4(const float * LM_GGML_RESTRICT x, void * LM_GGML_RESTRICT y, int64_t k);
-void quantize_q8_0_4x8(const float * LM_GGML_RESTRICT x, void * LM_GGML_RESTRICT y, int64_t k);
-
-void quantize_mat_q8_0(const float * LM_GGML_RESTRICT x, void * LM_GGML_RESTRICT y, int64_t nrows, int64_t n_per_row, int64_t blck_size_interleave);
-
 // Quantization utilizing an importance matrix (a.k.a. "Activation aWare Quantization")
 size_t quantize_q4_0_4x4(const float * LM_GGML_RESTRICT src, void * LM_GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
 size_t quantize_q4_0_4x8(const float * LM_GGML_RESTRICT src, void * LM_GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
 size_t quantize_q4_0_8x8(const float * LM_GGML_RESTRICT src, void * LM_GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
 
-// GEMV
-void lm_ggml_gemv_q4_0_4x4_q8_0(int n, float * LM_GGML_RESTRICT s, size_t bs, const void * LM_GGML_RESTRICT vx, const void * LM_GGML_RESTRICT vy, int nr, int nc);
-void lm_ggml_gemv_q4_0_4x8_q8_0(int n, float * LM_GGML_RESTRICT s, size_t bs, const void * LM_GGML_RESTRICT vx, const void * LM_GGML_RESTRICT vy, int nr, int nc);
-void lm_ggml_gemv_q4_0_8x8_q8_0(int n, float * LM_GGML_RESTRICT s, size_t bs, const void * LM_GGML_RESTRICT vx, const void * LM_GGML_RESTRICT vy, int nr, int nc);
-
-// GEMM
-void lm_ggml_gemm_q4_0_4x4_q8_0(int n, float * LM_GGML_RESTRICT s, size_t bs, const void * LM_GGML_RESTRICT vx, const void * LM_GGML_RESTRICT vy, int nr, int nc);
-void lm_ggml_gemm_q4_0_4x8_q8_0(int n, float * LM_GGML_RESTRICT s, size_t bs, const void * LM_GGML_RESTRICT vx, const void * LM_GGML_RESTRICT vy, int nr, int nc);
-void lm_ggml_gemm_q4_0_8x8_q8_0(int n, float * LM_GGML_RESTRICT s, size_t bs, const void * LM_GGML_RESTRICT vx, const void * LM_GGML_RESTRICT vy, int nr, int nc);
-
 #ifdef __cplusplus
 }
 #endif

package/cpp/ggml-alloc.c

@@ -466,18 +466,12 @@ static bool lm_ggml_gallocr_is_own(lm_ggml_gallocr_t galloc, struct lm_ggml_tens
     return lm_ggml_gallocr_hash_get(galloc, t)->allocated;
 }
 
-static void lm_ggml_gallocr_set_node_offset(lm_ggml_gallocr_t galloc, struct lm_ggml_tensor * node, int buffer_id, size_t offset) {
-    struct hash_node * hn = lm_ggml_gallocr_hash_get(galloc, node);
-    hn->buffer_id = buffer_id;
-    hn->offset = offset;
-    hn->allocated = true;
-}
-
 static bool lm_ggml_gallocr_is_allocated(lm_ggml_gallocr_t galloc, struct lm_ggml_tensor * t) {
     return t->data != NULL || lm_ggml_gallocr_hash_get(galloc, t)->allocated;
 }
 
 static void lm_ggml_gallocr_allocate_node(lm_ggml_gallocr_t galloc, struct lm_ggml_tensor * node, int buffer_id) {
+    LM_GGML_ASSERT(buffer_id >= 0);
     struct hash_node * hn = lm_ggml_gallocr_hash_get(galloc, node);
 
     if (!lm_ggml_gallocr_is_allocated(galloc, node) && !lm_ggml_is_view(node)) {
@@ -816,7 +810,11 @@ static void lm_ggml_gallocr_init_tensor(lm_ggml_gallocr_t galloc, struct lm_ggml
 }
 
 static bool lm_ggml_gallocr_node_needs_realloc(lm_ggml_gallocr_t galloc, struct lm_ggml_tensor * node, struct tensor_alloc * talloc) {
-    size_t node_size = (node->data || node->view_src) ? 0 : lm_ggml_backend_buft_get_alloc_size(galloc->bufts[talloc->buffer_id], node);
+    size_t node_size = 0;
+    if (!node->data && !node->view_src) {
+        LM_GGML_ASSERT(talloc->buffer_id >= 0); // prevent segfault when misusing the API
+        node_size = lm_ggml_backend_buft_get_alloc_size(galloc->bufts[talloc->buffer_id], node);
+    }
     return talloc->size_max >= node_size;
 }
 

package/cpp/ggml-backend-impl.h

@@ -8,6 +8,8 @@
 extern "C" {
 #endif
 
+    #define LM_GGML_BACKEND_API_VERSION 1
+
     //
     // Backend buffer type
     //
@@ -63,20 +65,20 @@ extern "C" {
         enum lm_ggml_backend_buffer_usage usage;
     };
 
-    lm_ggml_backend_buffer_t lm_ggml_backend_buffer_init(
+    LM_GGML_API lm_ggml_backend_buffer_t lm_ggml_backend_buffer_init(
                    lm_ggml_backend_buffer_type_t buft,
             struct lm_ggml_backend_buffer_i      iface,
                    void *                     context,
                    size_t                     size);
 
     // do not use directly, use lm_ggml_backend_tensor_copy instead
-    bool lm_ggml_backend_buffer_copy_tensor(const struct lm_ggml_tensor * src, struct lm_ggml_tensor * dst);
+    LM_GGML_API bool lm_ggml_backend_buffer_copy_tensor(const struct lm_ggml_tensor * src, struct lm_ggml_tensor * dst);
 
     // multi-buffer
     // buffer that contains a collection of buffers
-    lm_ggml_backend_buffer_t lm_ggml_backend_multi_buffer_alloc_buffer(lm_ggml_backend_buffer_t * buffers, size_t n_buffers);
-    bool                  lm_ggml_backend_buffer_is_multi_buffer(lm_ggml_backend_buffer_t buffer);
-    void                  lm_ggml_backend_multi_buffer_set_usage(lm_ggml_backend_buffer_t buffer, enum lm_ggml_backend_buffer_usage usage);
+    LM_GGML_API lm_ggml_backend_buffer_t lm_ggml_backend_multi_buffer_alloc_buffer(lm_ggml_backend_buffer_t * buffers, size_t n_buffers);
+    LM_GGML_API bool                  lm_ggml_backend_buffer_is_multi_buffer(lm_ggml_backend_buffer_t buffer);
+    LM_GGML_API void                  lm_ggml_backend_multi_buffer_set_usage(lm_ggml_backend_buffer_t buffer, enum lm_ggml_backend_buffer_usage usage);
 
     //
     // Backend (stream)
@@ -199,17 +201,37 @@ extern "C" {
     };
 
     struct lm_ggml_backend_reg {
-        // int api_version; // TODO: for dynamic loading
+        int api_version; // initialize to LM_GGML_BACKEND_API_VERSION
         struct lm_ggml_backend_reg_i iface;
         void * context;
     };
 
-
     // Internal backend registry API
-    void lm_ggml_backend_register(lm_ggml_backend_reg_t reg);
-    void lm_ggml_backend_device_register(lm_ggml_backend_dev_t device);
-    // TODO: backends can be loaded as a dynamic library, in which case it needs to export this function
-    // typedef lm_ggml_backend_register_t * (*lm_ggml_backend_init)(void);
+    LM_GGML_API void lm_ggml_backend_register(lm_ggml_backend_reg_t reg);
+    LM_GGML_API void lm_ggml_backend_device_register(lm_ggml_backend_dev_t device);
+
+    // Add backend dynamic loading support to the backend
+    typedef lm_ggml_backend_reg_t (*lm_ggml_backend_init_t)(void);
+
+    #ifdef LM_GGML_BACKEND_DL
+        #ifdef __cplusplus
+        #    define LM_GGML_BACKEND_DL_IMPL(reg_fn)                                 \
+                extern "C" {                                                     \
+                    LM_GGML_BACKEND_API lm_ggml_backend_reg_t lm_ggml_backend_init(void); \
+                }                                                                \
+                lm_ggml_backend_reg_t lm_ggml_backend_init(void) {                     \
+                    return reg_fn();                                             \
+                }
+        #else
+        #    define LM_GGML_BACKEND_DL_IMPL(reg_fn)                             \
+                LM_GGML_BACKEND_API lm_ggml_backend_reg_t lm_ggml_backend_init(void); \
+                lm_ggml_backend_reg_t lm_ggml_backend_init(void) {                 \
+                    return reg_fn();                                         \
+                }
+        #endif
+    #else
+    #    define LM_GGML_BACKEND_DL_IMPL(reg_fn)
+    #endif
 
 #ifdef  __cplusplus
 }

package/cpp/ggml-backend-reg.cpp

@@ -0,0 +1,423 @@
+#include "ggml-backend-impl.h"
+#include "ggml-backend.h"
+#include "ggml-impl.h"
+#include <algorithm>
+#include <cstring>
+#include <string>
+#include <vector>
+
+#ifdef _WIN32
+#    define WIN32_LEAN_AND_MEAN
+#    ifndef NOMINMAX
+#        define NOMINMAX
+#    endif
+#    include <windows.h>
+#elif defined(__APPLE__)
+#    include <mach-o/dyld.h>
+#    include <dlfcn.h>
+#else
+#    include <dlfcn.h>
+#    include <unistd.h>
+#endif
+
+// Backend registry
+#ifdef LM_GGML_USE_CPU
+#include "ggml-cpu.h"
+#endif
+
+#ifdef LM_GGML_USE_CUDA
+#include "ggml-cuda.h"
+#endif
+
+#ifdef LM_GGML_USE_METAL
+#include "ggml-metal.h"
+#endif
+
+#ifdef LM_GGML_USE_SYCL
+#include "ggml-sycl.h"
+#endif
+
+#ifdef LM_GGML_USE_VULKAN
+#include "ggml-vulkan.h"
+#endif
+
+#ifdef LM_GGML_USE_BLAS
+#include "ggml-blas.h"
+#endif
+
+#ifdef LM_GGML_USE_RPC
+#include "ggml-rpc.h"
+#endif
+
+#ifdef LM_GGML_USE_CANN
+#include "ggml-cann.h"
+#endif
+
+#ifdef LM_GGML_USE_KOMPUTE
+#include "ggml-kompute.h"
+#endif
+
+struct lm_ggml_backend_reg_entry {
+    lm_ggml_backend_reg_t reg;
+    void * handle;
+};
+
+struct lm_ggml_backend_registry {
+    std::vector<lm_ggml_backend_reg_entry> backends;
+    std::vector<lm_ggml_backend_dev_t> devices;
+
+    lm_ggml_backend_registry() {
+#ifdef LM_GGML_USE_CUDA
+        register_backend(lm_ggml_backend_cuda_reg());
+#endif
+#ifdef LM_GGML_USE_METAL
+        register_backend(lm_ggml_backend_metal_reg());
+#endif
+#ifdef LM_GGML_USE_SYCL
+        register_backend(lm_ggml_backend_sycl_reg());
+#endif
+#ifdef LM_GGML_USE_VULKAN
+        register_backend(lm_ggml_backend_vk_reg());
+#endif
+#ifdef LM_GGML_USE_CANN
+        register_backend(lm_ggml_backend_cann_reg());
+#endif
+#ifdef LM_GGML_USE_BLAS
+        register_backend(lm_ggml_backend_blas_reg());
+#endif
+#ifdef LM_GGML_USE_RPC
+        register_backend(lm_ggml_backend_rpc_reg());
+#endif
+#ifdef LM_GGML_USE_KOMPUTE
+        register_backend(lm_ggml_backend_kompute_reg());
+#endif
+#ifdef LM_GGML_USE_CPU
+        register_backend(lm_ggml_backend_cpu_reg());
+#endif
+    }
+
+    ~lm_ggml_backend_registry() {
+        while (!backends.empty()) {
+            // use silent since the log system may have been destroyed at this point
+            unload_backend(backends.back().reg, true);
+        }
+    }
+
+    void register_backend(lm_ggml_backend_reg_t reg, void * handle = nullptr) {
+        if (!reg) {
+            return;
+        }
+
+#ifndef NDEBUG
+        LM_GGML_LOG_DEBUG("%s: registered backend %s (%zu devices)\n",
+            __func__, lm_ggml_backend_reg_name(reg), lm_ggml_backend_reg_dev_count(reg));
+#endif
+        backends.push_back({ reg, handle });
+        for (size_t i = 0; i < lm_ggml_backend_reg_dev_count(reg); i++) {
+            register_device(lm_ggml_backend_reg_dev_get(reg, i));
+        }
+    }
+
+    void register_device(lm_ggml_backend_dev_t device) {
+#ifndef NDEBUG
+        LM_GGML_LOG_DEBUG("%s: registered device %s (%s)\n", __func__, lm_ggml_backend_dev_name(device), lm_ggml_backend_dev_description(device));
+#endif
+        devices.push_back(device);
+    }
+
+    lm_ggml_backend_reg_t load_backend(const char * path, bool silent) {
+#ifdef _WIN32
+        // suppress error dialogs for missing DLLs
+        DWORD old_mode = SetErrorMode(SEM_FAILCRITICALERRORS);
+        SetErrorMode(old_mode | SEM_FAILCRITICALERRORS);
+
+        HMODULE handle = LoadLibraryA(path);
+
+        if (!handle) {
+            if (!silent) {
+                LM_GGML_LOG_ERROR("%s: failed to load %s: %lu\n", __func__, path, GetLastError());
+            }
+            SetErrorMode(old_mode);
+            return nullptr;
+        }
+
+        lm_ggml_backend_init_t backend_init = (lm_ggml_backend_init_t) GetProcAddress(handle, "lm_ggml_backend_init");
+
+        SetErrorMode(old_mode);
+
+        if (!backend_init) {
+            if (!silent) {
+                LM_GGML_LOG_ERROR("%s: failed to find lm_ggml_backend_init in %s: %lu\n", __func__, path, GetLastError());
+            }
+            FreeLibrary(handle);
+            return nullptr;
+        }
+#else
+        void * handle = dlopen(path, RTLD_NOW | RTLD_LOCAL);
+
+        if (!handle) {
+            if (!silent) {
+                LM_GGML_LOG_ERROR("%s: failed to load %s: %s\n", __func__, path, dlerror());
+            }
+            return nullptr;
+        }
+
+        auto * backend_init = (lm_ggml_backend_init_t) dlsym(handle, "lm_ggml_backend_init");
+
+        if (!backend_init) {
+            if (!silent) {
+                LM_GGML_LOG_ERROR("%s: failed to find lm_ggml_backend_init in %s: %s\n", __func__, path, dlerror());
+            }
+            dlclose(handle);
+            return nullptr;
+        }
+#endif
+        lm_ggml_backend_reg_t reg = backend_init();
+
+        if (!reg || reg->api_version != LM_GGML_BACKEND_API_VERSION) {
+            if (!silent) {
+                if (!reg) {
+                    LM_GGML_LOG_ERROR("%s: failed to initialize backend from %s: lm_ggml_backend_init returned NULL\n", __func__, path);
+                } else {
+                    LM_GGML_LOG_ERROR("%s: failed to initialize backend from %s: incompatible API version (backend: %d, current: %d)\n",
+                                   __func__, path, reg->api_version, LM_GGML_BACKEND_API_VERSION);
+                }
+            }
+#ifdef _WIN32
+            FreeLibrary(handle);
+#else
+            dlclose(handle);
+#endif
+            return nullptr;
+        }
+
+        LM_GGML_LOG_INFO("%s: loaded %s backend from %s\n", __func__, lm_ggml_backend_reg_name(reg), path);
+        register_backend(reg, handle);
+        return reg;
+    }
+
+    void unload_backend(lm_ggml_backend_reg_t reg, bool silent) {
+        auto it = std::find_if(backends.begin(), backends.end(),
+                                [reg](lm_ggml_backend_reg_entry entry) { return entry.reg == reg; });
+
+        if (it == backends.end()) {
+            if (!silent) {
+                LM_GGML_LOG_ERROR("%s: backend not found\n", __func__);
+            }
+            return;
+        }
+
+        if (!silent) {
+            LM_GGML_LOG_DEBUG("%s: unloading %s backend\n", __func__, lm_ggml_backend_reg_name(reg));
+        }
+
+        // remove devices
+        devices.erase(
+            std::remove_if(devices.begin(), devices.end(),
+                            [reg](lm_ggml_backend_dev_t dev) { return lm_ggml_backend_dev_backend_reg(dev) == reg; }),
+            devices.end());
+
+        // unload library
+        if (it->handle) {
+#ifdef _WIN32
+            FreeLibrary((HMODULE) it->handle);
+#else
+            dlclose(it->handle);
+#endif
+        }
+
+        // remove backend
+        backends.erase(it);
+    }
+};
+
+static lm_ggml_backend_registry & get_reg() {
+    static lm_ggml_backend_registry reg;
+    return reg;
+}
+
+// Internal API
+void lm_ggml_backend_register(lm_ggml_backend_reg_t reg) {
+    get_reg().register_backend(reg);
+}
+
+void lm_ggml_backend_device_register(lm_ggml_backend_dev_t device) {
+    get_reg().register_device(device);
+}
+
+// Backend (reg) enumeration
+static bool striequals(const char * a, const char * b) {
+    for (; *a && *b; a++, b++) {
+        if (std::tolower(*a) != std::tolower(*b)) {
+            return false;
+        }
+    }
+    return *a == *b;
+}
+
+size_t lm_ggml_backend_reg_count() {
+    return get_reg().backends.size();
+}
+
+lm_ggml_backend_reg_t lm_ggml_backend_reg_get(size_t index) {
+    LM_GGML_ASSERT(index < lm_ggml_backend_reg_count());
+    return get_reg().backends[index].reg;
+}
+
+lm_ggml_backend_reg_t lm_ggml_backend_reg_by_name(const char * name) {
+    for (size_t i = 0; i < lm_ggml_backend_reg_count(); i++) {
+        lm_ggml_backend_reg_t reg = lm_ggml_backend_reg_get(i);
+        if (striequals(lm_ggml_backend_reg_name(reg), name)) {
+            return reg;
+        }
+    }
+    return nullptr;
+}
+
+// Device enumeration
+size_t lm_ggml_backend_dev_count() {
+    return get_reg().devices.size();
+}
+
+lm_ggml_backend_dev_t lm_ggml_backend_dev_get(size_t index) {
+    LM_GGML_ASSERT(index < lm_ggml_backend_dev_count());
+    return get_reg().devices[index];
+}
+
+lm_ggml_backend_dev_t lm_ggml_backend_dev_by_name(const char * name) {
+    for (size_t i = 0; i < lm_ggml_backend_dev_count(); i++) {
+        lm_ggml_backend_dev_t dev = lm_ggml_backend_dev_get(i);
+        if (striequals(lm_ggml_backend_dev_name(dev), name)) {
+            return dev;
+        }
+    }
+    return nullptr;
+}
+
+lm_ggml_backend_dev_t lm_ggml_backend_dev_by_type(enum lm_ggml_backend_dev_type type) {
+    for (size_t i = 0; i < lm_ggml_backend_dev_count(); i++) {
+        lm_ggml_backend_dev_t dev = lm_ggml_backend_dev_get(i);
+        if (lm_ggml_backend_dev_type(dev) == type) {
+            return dev;
+        }
+    }
+    return nullptr;
+}
+
+// Convenience functions
+lm_ggml_backend_t lm_ggml_backend_init_by_name(const char * name, const char * params) {
+    lm_ggml_backend_dev_t dev = lm_ggml_backend_dev_by_name(name);
+    if (!dev) {
+        return nullptr;
+    }
+    return lm_ggml_backend_dev_init(dev, params);
+}
+
+lm_ggml_backend_t lm_ggml_backend_init_by_type(enum lm_ggml_backend_dev_type type, const char * params) {
+    lm_ggml_backend_dev_t dev = lm_ggml_backend_dev_by_type(type);
+    if (!dev) {
+        return nullptr;
+    }
+    return lm_ggml_backend_dev_init(dev, params);
+}
+
+lm_ggml_backend_t lm_ggml_backend_init_best(void) {
+    lm_ggml_backend_dev_t dev = lm_ggml_backend_dev_by_type(LM_GGML_BACKEND_DEVICE_TYPE_GPU);
+    if (!dev) {
+        dev = lm_ggml_backend_dev_by_type(LM_GGML_BACKEND_DEVICE_TYPE_CPU);
+    }
+    if (!dev) {
+        return nullptr;
+    }
+    return lm_ggml_backend_dev_init(dev, nullptr);
+}
+
+// Dynamic loading
+lm_ggml_backend_reg_t lm_ggml_backend_load(const char * path) {
+    return get_reg().load_backend(path, false);
+}
+
+void lm_ggml_backend_unload(lm_ggml_backend_reg_t reg) {
+    get_reg().unload_backend(reg, true);
+}
+
+void lm_ggml_backend_load_all() {
+    std::vector<std::string> search_prefix;
+
+    // add the executable directory to the search path
+    // FIXME: this is convenient for development, but it should probably be disabled in production
+
+#if defined(__APPLE__)
+    // get executable path
+    std::vector<char> path;
+    uint32_t size;
+    while (true) {
+        size = path.size();
+        if (_NSGetExecutablePath(path.data(), &size) == 0) {
+            break;
+        }
+        path.resize(size);
+    }
+    std::string base_path(path.data(), size);
+    // remove executable name
+    auto last_slash = base_path.find_last_of('/');
+    if (last_slash != std::string::npos) {
+        base_path = base_path.substr(0, last_slash);
+    }
+    search_prefix.push_back(base_path + "/");
+#elif defined(__linux__)
+    std::string base_path = ".";
+    std::vector<char> path(1024);
+    while (true) {
+        // get executable path
+        ssize_t len = readlink("/proc/self/exe", path.data(), path.size());
+        if (len == -1) {
+            break;
+        }
+        if (len < (ssize_t) path.size()) {
+            base_path = std::string(path.data(), len);
+            // remove executable name
+            auto last_slash = base_path.find_last_of('/');
+            if (last_slash != std::string::npos) {
+                base_path = base_path.substr(0, last_slash);
+            }
+            break;
+        }
+        path.resize(path.size() * 2);
+    }
+
+    search_prefix.push_back(base_path + "/");
+#endif
+
+    auto & reg = get_reg();
+
+    auto try_load = [&](const std::string & name) {
+        std::string os_name;
+#ifdef _WIN32
+        os_name = "ggml-" + name + ".dll";
+#else
+        os_name = "libggml-" + name + ".so";
+#endif
+        if (reg.load_backend(os_name.c_str(), true)) {
+            return;
+        }
+        for (const auto & prefix : search_prefix) {
+            if (reg.load_backend((prefix + os_name).c_str(), true)) {
+                return;
+            }
+        }
+    };
+
+    try_load("amx");
+    try_load("blas");
+    try_load("cann");
+    try_load("cuda");
+    try_load("hip");
+    try_load("kompute");
+    try_load("metal");
+    try_load("rpc");
+    try_load("sycl");
+    try_load("vulkan");
+    try_load("musa");
+    try_load("cpu");
+}