cui-llama.rn 1.2.6 → 1.3.3

package/cpp/ggml-backend.cpp

@@ -252,6 +252,7 @@ void lm_ggml_backend_tensor_get_async(lm_ggml_backend_t backend, const struct lm
 }
 
 void lm_ggml_backend_tensor_set(struct lm_ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
+    LM_GGML_ASSERT(tensor);
     lm_ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
 
     if (size == 0) {
@@ -266,6 +267,7 @@ void lm_ggml_backend_tensor_set(struct lm_ggml_tensor * tensor, const void * dat
 }
 
 void lm_ggml_backend_tensor_get(const struct lm_ggml_tensor * tensor, void * data, size_t offset, size_t size) {
+    LM_GGML_ASSERT(tensor);
     lm_ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
 
     if (size == 0) {
@@ -279,7 +281,7 @@ void lm_ggml_backend_tensor_get(const struct lm_ggml_tensor * tensor, void * dat
     buf->iface.get_tensor(buf, tensor, data, offset, size);
 }
 
-LM_GGML_API void lm_ggml_backend_tensor_memset(struct lm_ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
+void lm_ggml_backend_tensor_memset(struct lm_ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
     lm_ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
 
     if (size == 0) {
@@ -525,197 +527,6 @@ void * lm_ggml_backend_reg_get_proc_address(lm_ggml_backend_reg_t reg, const cha
     return reg->iface.get_proc_address(reg, name);
 }
 
-// Backend registry
-
-#ifdef LM_GGML_USE_CUDA
-#include "ggml-cuda.h"
-#endif
-
-#ifdef LM_GGML_USE_METAL
-#include "ggml-metal.h"
-#endif
-
-#ifdef LM_GGML_USE_SYCL
-#include "ggml-sycl.h"
-#endif
-
-#ifdef LM_GGML_USE_VULKAN
-#include "ggml-vulkan.h"
-#endif
-
-#ifdef LM_GGML_USE_BLAS
-#include "ggml-blas.h"
-#endif
-
-#ifdef LM_GGML_USE_RPC
-#include "ggml-rpc.h"
-#endif
-
-#ifndef __AMX_INT8__
-#undef LM_GGML_USE_AMX
-#endif
-
-#ifdef LM_GGML_USE_AMX
-#  include "ggml-amx.h"
-#endif
-
-#ifdef LM_GGML_USE_CANN
-#include "ggml-cann.h"
-#endif
-
-#ifdef LM_GGML_USE_KOMPUTE
-#include "ggml-kompute.h"
-#endif
-
-#include "ggml-cpu.h"
-
-struct lm_ggml_backend_registry {
-    std::vector<lm_ggml_backend_reg_t> backends;
-    std::vector<lm_ggml_backend_dev_t> devices;
-
-    lm_ggml_backend_registry() {
-#ifdef LM_GGML_USE_CUDA
-        register_backend(lm_ggml_backend_cuda_reg());
-#endif
-#ifdef LM_GGML_USE_METAL
-        register_backend(lm_ggml_backend_metal_reg());
-#endif
-#ifdef LM_GGML_USE_SYCL
-        register_backend(lm_ggml_backend_sycl_reg());
-#endif
-#ifdef LM_GGML_USE_VULKAN
-        register_backend(lm_ggml_backend_vk_reg());
-#endif
-#ifdef LM_GGML_USE_CANN
-        register_backend(lm_ggml_backend_cann_reg());
-#endif
-#ifdef LM_GGML_USE_BLAS
-        register_backend(lm_ggml_backend_blas_reg());
-#endif
-#ifdef LM_GGML_USE_RPC
-        register_backend(lm_ggml_backend_rpc_reg());
-#endif
-#ifdef LM_GGML_USE_AMX
-        register_backend(lm_ggml_backend_amx_reg());
-#endif
-#ifdef LM_GGML_USE_KOMPUTE
-        register_backend(lm_ggml_backend_kompute_reg());
-#endif
-
-        register_backend(lm_ggml_backend_cpu_reg());
-    }
-
-    void register_backend(lm_ggml_backend_reg_t reg) {
-#ifndef NDEBUG
-        LM_GGML_LOG_DEBUG("%s: registered backend %s (%zu devices)\n",
-            __func__, lm_ggml_backend_reg_name(reg), lm_ggml_backend_reg_dev_count(reg));
-#endif
-        backends.push_back(reg);
-        for (size_t i = 0; i < lm_ggml_backend_reg_dev_count(reg); i++) {
-            register_device(lm_ggml_backend_reg_dev_get(reg, i));
-        }
-    }
-
-    void register_device(lm_ggml_backend_dev_t device) {
-#ifndef NDEBUG
-        LM_GGML_LOG_DEBUG("%s: registered device %s (%s)\n", __func__, lm_ggml_backend_dev_name(device), lm_ggml_backend_dev_description(device));
-#endif
-        devices.push_back(device);
-    }
-};
-
-static lm_ggml_backend_registry & get_reg() {
-    static lm_ggml_backend_registry reg;
-    return reg;
-}
-
-// Internal API
-void lm_ggml_backend_register(lm_ggml_backend_reg_t reg) {
-    get_reg().register_backend(reg);
-}
-
-void lm_ggml_backend_device_register(lm_ggml_backend_dev_t device) {
-    get_reg().register_device(device);
-}
-
-// Backend (reg) enumeration
-size_t lm_ggml_backend_reg_count() {
-    return get_reg().backends.size();
-}
-
-lm_ggml_backend_reg_t lm_ggml_backend_reg_get(size_t index) {
-    LM_GGML_ASSERT(index < lm_ggml_backend_reg_count());
-    return get_reg().backends[index];
-}
-
-lm_ggml_backend_reg_t lm_ggml_backend_reg_by_name(const char * name) {
-    for (size_t i = 0; i < lm_ggml_backend_reg_count(); i++) {
-        lm_ggml_backend_reg_t reg = lm_ggml_backend_reg_get(i);
-        if (strcmp(lm_ggml_backend_reg_name(reg), name) == 0) {
-            return reg;
-        }
-    }
-    return NULL;
-}
-
-// Device enumeration
-size_t lm_ggml_backend_dev_count() {
-    return get_reg().devices.size();
-}
-
-lm_ggml_backend_dev_t lm_ggml_backend_dev_get(size_t index) {
-    LM_GGML_ASSERT(index < lm_ggml_backend_dev_count());
-    return get_reg().devices[index];
-}
-
-lm_ggml_backend_dev_t lm_ggml_backend_dev_by_name(const char * name) {
-    for (size_t i = 0; i < lm_ggml_backend_dev_count(); i++) {
-        lm_ggml_backend_dev_t dev = lm_ggml_backend_dev_get(i);
-        if (strcmp(lm_ggml_backend_dev_name(dev), name) == 0) {
-            return dev;
-        }
-    }
-    return NULL;
-}
-
-lm_ggml_backend_dev_t lm_ggml_backend_dev_by_type(enum lm_ggml_backend_dev_type type) {
-    for (size_t i = 0; i < lm_ggml_backend_dev_count(); i++) {
-        lm_ggml_backend_dev_t dev = lm_ggml_backend_dev_get(i);
-        if (lm_ggml_backend_dev_type(dev) == type) {
-            return dev;
-        }
-    }
-    return NULL;
-}
-
-// Convenience functions
-lm_ggml_backend_t lm_ggml_backend_init_by_name(const char * name, const char * params) {
-    lm_ggml_backend_dev_t dev = lm_ggml_backend_dev_by_name(name);
-    if (!dev) {
-        return NULL;
-    }
-    return lm_ggml_backend_dev_init(dev, params);
-}
-
-lm_ggml_backend_t lm_ggml_backend_init_by_type(enum lm_ggml_backend_dev_type type, const char * params) {
-    lm_ggml_backend_dev_t dev = lm_ggml_backend_dev_by_type(type);
-    if (!dev) {
-        return NULL;
-    }
-    return lm_ggml_backend_dev_init(dev, params);
-}
-
-lm_ggml_backend_t lm_ggml_backend_init_best(void) {
-    lm_ggml_backend_dev_t dev = lm_ggml_backend_dev_by_type(LM_GGML_BACKEND_DEVICE_TYPE_GPU);
-    if (!dev) {
-        dev = lm_ggml_backend_dev_by_type(LM_GGML_BACKEND_DEVICE_TYPE_CPU);
-    }
-    if (!dev) {
-        return NULL;
-    }
-    return lm_ggml_backend_dev_init(dev, NULL);
-}
-
 // multi-buffer buffer
 
 struct lm_ggml_backend_multi_buffer_context {
@@ -880,7 +691,7 @@ static int lm_ggml_backend_sched_backend_id(lm_ggml_backend_sched_t sched, lm_gg
 }
 
 static int lm_ggml_backend_sched_backend_from_buffer(lm_ggml_backend_sched_t sched, const struct lm_ggml_tensor * tensor, const struct lm_ggml_tensor * op) {
-    lm_ggml_backend_buffer_t buffer = tensor->buffer;
+    lm_ggml_backend_buffer_t buffer = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
     if (buffer == NULL) {
         return -1;
     }
@@ -913,8 +724,6 @@ static char causes[LM_GGML_DEFAULT_GRAPH_SIZE*16 + LM_GGML_SCHED_MAX_SPLITS_DEBU
 
 // returns the backend that should be used for the node based on the current locations
 static int lm_ggml_backend_sched_backend_id_from_cur(lm_ggml_backend_sched_t sched, struct lm_ggml_tensor * tensor) {
-    // TODO: use supports_op to check if the backend supports the op
-
     // assign pre-allocated nodes to their backend
     int cur_backend_id = lm_ggml_backend_sched_backend_from_buffer(sched, tensor, tensor);
     if (cur_backend_id != -1) {
@@ -933,7 +742,8 @@ static int lm_ggml_backend_sched_backend_id_from_cur(lm_ggml_backend_sched_t sch
 
     if (tensor->buffer || (tensor->view_src && tensor->view_src->buffer)) {
         // since the tensor is pre-allocated, it cannot be moved to another backend
-        LM_GGML_ABORT("pre-allocated tensor in a backend that cannot run the operation");
+        lm_ggml_backend_buffer_t buffer = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
+        LM_GGML_ABORT("pre-allocated tensor (%s) in a buffer (%s) that cannot run the operation (%s)", tensor->name, lm_ggml_backend_buffer_name(buffer), lm_ggml_op_name(tensor->op));
     }
 
     // graph input
@@ -1640,7 +1450,7 @@ lm_ggml_backend_sched_t lm_ggml_backend_sched_new(
         bool parallel) {
     LM_GGML_ASSERT(n_backends > 0);
     LM_GGML_ASSERT(n_backends <= LM_GGML_SCHED_MAX_BACKENDS);
-    LM_GGML_ASSERT(lm_ggml_backend_is_cpu(backends[n_backends - 1])); // last backend must be CPU
+    LM_GGML_ASSERT(lm_ggml_backend_dev_type(lm_ggml_backend_get_device(backends[n_backends - 1])) == LM_GGML_BACKEND_DEVICE_TYPE_CPU);
 
     struct lm_ggml_backend_sched * sched = (lm_ggml_backend_sched *) calloc(1, sizeof(struct lm_ggml_backend_sched));
 
@@ -1729,12 +1539,13 @@ bool lm_ggml_backend_sched_reserve(lm_ggml_backend_sched_t sched, struct lm_ggml
 
     lm_ggml_backend_sched_split_graph(sched, measure_graph);
 
+    lm_ggml_backend_sched_synchronize(sched);
+
     if (!lm_ggml_gallocr_reserve_n(sched->galloc, &sched->graph, sched->node_backend_ids, sched->leaf_backend_ids)) {
         return false;
     }
 
     lm_ggml_backend_sched_reset(sched);
-    lm_ggml_backend_sched_synchronize(sched);
 
     return true;
 }
@@ -2036,17 +1847,6 @@ bool lm_ggml_backend_compare_graph_backend(lm_ggml_backend_t backend1, lm_ggml_b
     return true;
 }
 
-
-
-#include "ggml-backend.h"
-#include "ggml-backend-impl.h"
-#include "ggml-cpu.h"
-#include "ggml-impl.h"
-#include <cctype>
-#include <string>
-
-// ggml-backend interface
-
 // CPU backend - buffer
 
 static void * lm_ggml_backend_cpu_buffer_get_base(lm_ggml_backend_buffer_t buffer) {
@@ -2120,7 +1920,9 @@ static const struct lm_ggml_backend_buffer_i lm_ggml_backend_cpu_buffer_from_ptr
     /* .reset           = */ NULL,
 };
 
-// CPU backend - buffer type
+// CPU backend buffer type
+
+// this buffer type is defined here to make it available to all backends
 
 static const char * lm_ggml_backend_cpu_buffer_type_get_name(lm_ggml_backend_buffer_type_t buft) {
     return "CPU";
@@ -2161,7 +1963,7 @@ lm_ggml_backend_buffer_type_t lm_ggml_backend_cpu_buffer_type(void) {
             /* .get_alloc_size   = */ NULL, // defaults to lm_ggml_nbytes
             /* .is_host          = */ lm_ggml_backend_cpu_buffer_type_is_host,
         },
-        /* .device  = */ lm_ggml_backend_reg_dev_get(lm_ggml_backend_cpu_reg(), 0),
+        /* .device  = */ NULL, // FIXME lm_ggml_backend_reg_dev_get(lm_ggml_backend_cpu_reg(), 0),
         /* .context = */ NULL,
     };
 
@@ -2184,478 +1986,14 @@ static lm_ggml_backend_buffer_type_t lm_ggml_backend_cpu_buffer_from_ptr_type(vo
             /* .get_alloc_size   = */ NULL, // defaults to lm_ggml_nbytes
             /* .is_host          = */ lm_ggml_backend_cpu_buffer_type_is_host,
         },
-        /* .device  = */ lm_ggml_backend_reg_dev_get(lm_ggml_backend_cpu_reg(), 0),
+        /* .device  = */ NULL, // FIXME lm_ggml_backend_reg_dev_get(lm_ggml_backend_cpu_reg(), 0),
         /* .context = */ NULL,
     };
 
     return &lm_ggml_backend_cpu_buffer_type;
 }
 
-#ifdef LM_GGML_USE_CPU_HBM
-
-// buffer type HBM
-
-#include <hbwmalloc.h>
-
-static const char * lm_ggml_backend_cpu_hbm_buffer_type_get_name(lm_ggml_backend_buffer_type_t buft) {
-    return "CPU_HBM";
-
-    LM_GGML_UNUSED(buft);
-}
-
-static void lm_ggml_backend_cpu_hbm_buffer_free_buffer(lm_ggml_backend_buffer_t buffer) {
-    hbw_free(buffer->context);
-}
-
-static lm_ggml_backend_buffer_t lm_ggml_backend_cpu_hbm_buffer_type_alloc_buffer(lm_ggml_backend_buffer_type_t buft, size_t size) {
-    void * ptr;
-    int result = hbw_posix_memalign(&ptr, lm_ggml_backend_cpu_buffer_type_get_alignment(buft), size);
-    if (result != 0) {
-        LM_GGML_LOG_ERROR("failed to allocate HBM buffer of size %zu\n", size);
-        return NULL;
-    }
-
-    lm_ggml_backend_buffer_t buffer = lm_ggml_backend_cpu_buffer_from_ptr(ptr, size);
-    buffer->buft = buft;
-    buffer->iface.free_buffer = lm_ggml_backend_cpu_hbm_buffer_free_buffer;
-
-    return buffer;
-}
-
-lm_ggml_backend_buffer_type_t lm_ggml_backend_cpu_hbm_buffer_type(void) {
-    static struct lm_ggml_backend_buffer_type lm_ggml_backend_cpu_buffer_type_hbm = {
-        /* .iface    = */ {
-            /* .get_name         = */ lm_ggml_backend_cpu_hbm_buffer_type_get_name,
-            /* .alloc_buffer     = */ lm_ggml_backend_cpu_hbm_buffer_type_alloc_buffer,
-            /* .get_alignment    = */ lm_ggml_backend_cpu_buffer_type_get_alignment,
-            /* .get_max_size     = */ NULL, // defaults to SIZE_MAX
-            /* .get_alloc_size   = */ NULL, // defaults to lm_ggml_nbytes
-            /* .is_host          = */ lm_ggml_backend_cpu_buffer_type_is_host,
-        },
-        /* .context  = */ NULL,
-    };
-
-    return &lm_ggml_backend_cpu_buffer_type_hbm;
-}
-#endif
-
-static lm_ggml_backend_buffer_type_t * lm_ggml_backend_cpu_get_extra_bufts(lm_ggml_backend_dev_t device) {
-    static lm_ggml_backend_buffer_type_t bufts[] = {
-#ifdef LM_GGML_USE_CPU_HBM
-        lm_ggml_backend_cpu_hbm_buffer_type(),
-#endif
-        NULL
-    };
-
-    return bufts;
-
-    LM_GGML_UNUSED(device);
-}
-
-// CPU backend - backend (stream)
-
-struct lm_ggml_backend_cpu_context {
-    int                 n_threads;
-    lm_ggml_threadpool_t   threadpool;
-
-    uint8_t *           work_data;
-    size_t              work_size;
-
-    lm_ggml_abort_callback abort_callback;
-    void *              abort_callback_data;
-};
-
-static const char * lm_ggml_backend_cpu_get_name(lm_ggml_backend_t backend) {
-    return "CPU";
-
-    LM_GGML_UNUSED(backend);
-}
-
-static void lm_ggml_backend_cpu_free(lm_ggml_backend_t backend) {
-    struct lm_ggml_backend_cpu_context * cpu_ctx = (struct lm_ggml_backend_cpu_context *)backend->context;
-    delete[] cpu_ctx->work_data;
-    delete cpu_ctx;
-    delete backend;
-}
-
-struct lm_ggml_backend_plan_cpu {
-    struct lm_ggml_cplan cplan;
-    struct lm_ggml_cgraph cgraph;
-};
-
-static lm_ggml_backend_graph_plan_t lm_ggml_backend_cpu_graph_plan_create(lm_ggml_backend_t backend, const struct lm_ggml_cgraph * cgraph) {
-    struct lm_ggml_backend_cpu_context * cpu_ctx = (struct lm_ggml_backend_cpu_context *)backend->context;
-
-    struct lm_ggml_backend_plan_cpu * cpu_plan = new lm_ggml_backend_plan_cpu;
-
-    cpu_plan->cplan = lm_ggml_graph_plan(cgraph, cpu_ctx->n_threads, cpu_ctx->threadpool);
-    cpu_plan->cgraph = *cgraph; // FIXME: deep copy
-
-    if (cpu_plan->cplan.work_size > 0) {
-        cpu_plan->cplan.work_data = new uint8_t[cpu_plan->cplan.work_size];
-        if (cpu_plan->cplan.work_data == NULL) {
-            delete cpu_plan;
-            return NULL;
-        }
-    }
-
-    cpu_plan->cplan.abort_callback      = cpu_ctx->abort_callback;
-    cpu_plan->cplan.abort_callback_data = cpu_ctx->abort_callback_data;
-
-    return cpu_plan;
-}
-
-static void lm_ggml_backend_cpu_graph_plan_free(lm_ggml_backend_t backend, lm_ggml_backend_graph_plan_t plan) {
-    struct lm_ggml_backend_plan_cpu * cpu_plan = (struct lm_ggml_backend_plan_cpu *)plan;
-
-    delete[] cpu_plan->cplan.work_data;
-    delete cpu_plan;
-
-    LM_GGML_UNUSED(backend);
-}
-
-static enum lm_ggml_status lm_ggml_backend_cpu_graph_plan_compute(lm_ggml_backend_t backend, lm_ggml_backend_graph_plan_t plan) {
-    struct lm_ggml_backend_plan_cpu * cpu_plan = (struct lm_ggml_backend_plan_cpu *)plan;
-
-    return lm_ggml_graph_compute(&cpu_plan->cgraph, &cpu_plan->cplan);
-
-    LM_GGML_UNUSED(backend);
-}
-
-static enum lm_ggml_status lm_ggml_backend_cpu_graph_compute(lm_ggml_backend_t backend, struct lm_ggml_cgraph * cgraph) {
-    struct lm_ggml_backend_cpu_context * cpu_ctx = (struct lm_ggml_backend_cpu_context *)backend->context;
-
-    struct lm_ggml_cplan cplan = lm_ggml_graph_plan(cgraph, cpu_ctx->n_threads, cpu_ctx->threadpool);
-
-    if (cpu_ctx->work_size < cplan.work_size) {
-        delete[] cpu_ctx->work_data;
-        cpu_ctx->work_data = new uint8_t[cplan.work_size];
-        if (cpu_ctx->work_data == NULL) {
-            cpu_ctx->work_size = 0;
-            return LM_GGML_STATUS_ALLOC_FAILED;
-        }
-        cpu_ctx->work_size = cplan.work_size;
-    }
-    cplan.work_data = (uint8_t *)cpu_ctx->work_data;
-
-    cplan.abort_callback      = cpu_ctx->abort_callback;
-    cplan.abort_callback_data = cpu_ctx->abort_callback_data;
-
-    return lm_ggml_graph_compute(cgraph, &cplan);
-}
-
-static const struct lm_ggml_backend_i lm_ggml_backend_cpu_i = {
-    /* .get_name                = */ lm_ggml_backend_cpu_get_name,
-    /* .free                    = */ lm_ggml_backend_cpu_free,
-    /* .set_tensor_async        = */ NULL,
-    /* .get_tensor_async        = */ NULL,
-    /* .cpy_tensor_async        = */ NULL,
-    /* .synchronize             = */ NULL,
-    /* .graph_plan_create       = */ lm_ggml_backend_cpu_graph_plan_create,
-    /* .graph_plan_free         = */ lm_ggml_backend_cpu_graph_plan_free,
-    /* .graph_plan_update       = */ NULL,
-    /* .graph_plan_compute      = */ lm_ggml_backend_cpu_graph_plan_compute,
-    /* .graph_compute           = */ lm_ggml_backend_cpu_graph_compute,
-    /* .event_record            = */ NULL,
-    /* .event_wait              = */ NULL,
-};
-
-static lm_ggml_guid_t lm_ggml_backend_cpu_guid(void) {
-    static lm_ggml_guid guid = { 0xaa, 0x67, 0xc7, 0x43, 0x96, 0xe6, 0xa3, 0x8a, 0xe3, 0xaf, 0xea, 0x92, 0x36, 0xbc, 0xfc, 0x89 };
-    return &guid;
-}
-
-lm_ggml_backend_t lm_ggml_backend_cpu_init(void) {
-    // initialize CPU backend now to avoid slowing the first graph computation
-    lm_ggml_cpu_init();
-
-    struct lm_ggml_backend_cpu_context * ctx = new lm_ggml_backend_cpu_context;
-    if (ctx == NULL) {
-        return NULL;
-    }
-
-    ctx->n_threads           = LM_GGML_DEFAULT_N_THREADS;
-    ctx->threadpool          = NULL;
-    ctx->work_data           = NULL;
-    ctx->work_size           = 0;
-    ctx->abort_callback      = NULL;
-    ctx->abort_callback_data = NULL;
-
-    lm_ggml_backend_t cpu_backend = new lm_ggml_backend {
-        /* .guid      = */ lm_ggml_backend_cpu_guid(),
-        /* .interface = */ lm_ggml_backend_cpu_i,
-        /* .device    = */ lm_ggml_backend_reg_dev_get(lm_ggml_backend_cpu_reg(), 0),
-        /* .context   = */ ctx,
-    };
-
-    if (cpu_backend == NULL) {
-        delete ctx;
-        return NULL;
-    }
-
-    return cpu_backend;
-}
-
-bool lm_ggml_backend_is_cpu(lm_ggml_backend_t backend) {
-    return backend != NULL && lm_ggml_guid_matches(backend->guid, lm_ggml_backend_cpu_guid());
-}
-
-void lm_ggml_backend_cpu_set_n_threads(lm_ggml_backend_t backend_cpu, int n_threads) {
-    LM_GGML_ASSERT(lm_ggml_backend_is_cpu(backend_cpu));
-
-    struct lm_ggml_backend_cpu_context * ctx = (struct lm_ggml_backend_cpu_context *)backend_cpu->context;
-    ctx->n_threads = n_threads;
-}
-
-void lm_ggml_backend_cpu_set_threadpool(lm_ggml_backend_t backend_cpu, lm_ggml_threadpool_t threadpool) {
-    LM_GGML_ASSERT(lm_ggml_backend_is_cpu(backend_cpu));
-
-    struct lm_ggml_backend_cpu_context * ctx = (struct lm_ggml_backend_cpu_context *)backend_cpu->context;
-
-    if (ctx->threadpool && ctx->threadpool != threadpool) {
-        // already had a different threadpool, pause/suspend it before switching
-        lm_ggml_threadpool_pause(ctx->threadpool);
-    }
-    ctx->threadpool = threadpool;
-}
-
-void lm_ggml_backend_cpu_set_abort_callback(lm_ggml_backend_t backend_cpu, lm_ggml_abort_callback abort_callback, void * abort_callback_data) {
-    LM_GGML_ASSERT(lm_ggml_backend_is_cpu(backend_cpu));
-
-    struct lm_ggml_backend_cpu_context * ctx = (struct lm_ggml_backend_cpu_context *)backend_cpu->context;
-    ctx->abort_callback = abort_callback;
-    ctx->abort_callback_data = abort_callback_data;
-}
-
 lm_ggml_backend_buffer_t lm_ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size) {
     LM_GGML_ASSERT((uintptr_t)ptr % TENSOR_ALIGNMENT == 0 && "buffer pointer must be aligned");
     return lm_ggml_backend_buffer_init(lm_ggml_backend_cpu_buffer_from_ptr_type(), lm_ggml_backend_cpu_buffer_from_ptr_i, ptr, size);
 }
-
-// CPU backend - device
-
-struct lm_ggml_backend_cpu_device_context {
-    std::string description = "CPU";
-
-    lm_ggml_backend_cpu_device_context() {
-#ifdef __APPLE__
-        size_t len = 0;
-        if (!sysctlbyname("machdep.cpu.brand_string", NULL, &len, NULL, 0)) {
-            description.resize(len);
-            sysctlbyname("machdep.cpu.brand_string", &description[0], &len, NULL, 0); // NOLINT
-        }
-#elif defined(__linux__)
-        FILE * f = fopen("/proc/cpuinfo", "r");
-        if (f) {
-            char buf[1024];
-            while (fgets(buf, sizeof(buf), f)) {
-                if (strncmp(buf, "model name", 10) == 0) {
-                    char * p = strchr(buf, ':');
-                    if (p) {
-                        p++;
-                        while (std::isspace(*p)) {
-                            p++;
-                        }
-                        while (std::isspace(p[strlen(p) - 1])) {
-                            p[strlen(p) - 1] = '\0';
-                        }
-                        description = p;
-                        break;
-                    }
-                }
-            }
-            fclose(f);
-        }
-#elif defined(_WIN32)
-        HKEY hKey;
-        if (RegOpenKeyEx(HKEY_LOCAL_MACHINE,
-                        TEXT("HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0"),
-                        0,
-                        KEY_READ,
-                        &hKey) == ERROR_SUCCESS) {
-            DWORD cpu_brand_size = 0;
-            if (RegQueryValueExA(hKey,
-                                TEXT("ProcessorNameString"),
-                                NULL,
-                                NULL,
-                                NULL,
-                                &cpu_brand_size) == ERROR_SUCCESS) {
-                description.resize(cpu_brand_size);
-                if (RegQueryValueExA(hKey,
-                                    TEXT("ProcessorNameString"),
-                                    NULL,
-                                    NULL,
-                                    (LPBYTE)&description[0], // NOLINT
-                                    &cpu_brand_size) == ERROR_SUCCESS) {
-                    if (description.find('\0') != std::string::npos) {
-                        description.resize(description.find('\0'));
-                    }
-                }
-            }
-            RegCloseKey(hKey);
-        }
-#endif
-    }
-};
-
-static const char * lm_ggml_backend_cpu_device_get_name(lm_ggml_backend_dev_t dev) {
-    return "CPU";
-
-    LM_GGML_UNUSED(dev);
-}
-
-static const char * lm_ggml_backend_cpu_device_get_description(lm_ggml_backend_dev_t dev) {
-    struct lm_ggml_backend_cpu_device_context * ctx = (struct lm_ggml_backend_cpu_device_context *)dev->context;
-
-    return ctx->description.c_str();
-}
-
-static void lm_ggml_backend_cpu_device_get_memory(lm_ggml_backend_dev_t dev, size_t * free, size_t * total) {
-    // TODO
-    *free = 0;
-    *total = 0;
-
-    LM_GGML_UNUSED(dev);
-}
-
-static enum lm_ggml_backend_dev_type lm_ggml_backend_cpu_device_get_type(lm_ggml_backend_dev_t dev) {
-    return LM_GGML_BACKEND_DEVICE_TYPE_CPU;
-
-    LM_GGML_UNUSED(dev);
-}
-
-static void lm_ggml_backend_cpu_device_get_props(lm_ggml_backend_dev_t dev, struct lm_ggml_backend_dev_props * props) {
-    props->name        = lm_ggml_backend_cpu_device_get_name(dev);
-    props->description = lm_ggml_backend_cpu_device_get_description(dev);
-    props->type        = lm_ggml_backend_cpu_device_get_type(dev);
-    lm_ggml_backend_cpu_device_get_memory(dev, &props->memory_free, &props->memory_total);
-    props->caps = {
-        /* .async                 = */ false,
-        /* .host_buffer           = */ false,
-        /* .buffer_from_host_ptr  = */ true,
-        /* .events                = */ false,
-    };
-}
-
-static lm_ggml_backend_t lm_ggml_backend_cpu_device_init_backend(lm_ggml_backend_dev_t dev, const char * params) {
-    return lm_ggml_backend_cpu_init();
-
-    LM_GGML_UNUSED(dev);
-    LM_GGML_UNUSED(params);
-}
-
-static lm_ggml_backend_buffer_type_t lm_ggml_backend_cpu_device_get_buffer_type(lm_ggml_backend_dev_t dev) {
-    return lm_ggml_backend_cpu_buffer_type();
-
-    LM_GGML_UNUSED(dev);
-}
-
-static lm_ggml_backend_buffer_t lm_ggml_backend_cpu_device_buffer_from_host_ptr(lm_ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) {
-    return lm_ggml_backend_cpu_buffer_from_ptr(ptr, size);
-
-    LM_GGML_UNUSED(dev);
-    LM_GGML_UNUSED(max_tensor_size);
-}
-
-static bool lm_ggml_backend_cpu_device_supports_op(lm_ggml_backend_dev_t dev, const struct lm_ggml_tensor * op) {
-    switch (op->op) {
-        case LM_GGML_OP_CPY:
-            return
-                op->type != LM_GGML_TYPE_IQ2_XXS &&
-                op->type != LM_GGML_TYPE_IQ2_XS  &&
-                op->type != LM_GGML_TYPE_IQ1_S   &&
-                op->type != LM_GGML_TYPE_IQ1_M; // missing type_traits.from_float
-        case LM_GGML_OP_MUL_MAT:
-            return op->src[1]->type == LM_GGML_TYPE_F32;// FIXME || op->src[1]->type == lm_ggml_get_type_traits(op->src[0]->type)->vec_dot_type;
-        case LM_GGML_OP_ROPE_BACK:
-            return op->src[2] == NULL && (op->op_params[2] & 4) == 0;
-        case LM_GGML_OP_IM2COL_BACK:
-            return op->src[0]->type == LM_GGML_TYPE_F32 && op->src[1]->type == LM_GGML_TYPE_F32;
-        case LM_GGML_OP_OUT_PROD:
-            return (op->src[0]->type == LM_GGML_TYPE_F32 || lm_ggml_is_quantized(op->src[0]->type)) && op->src[1]->type == LM_GGML_TYPE_F32;
-        default:
-            return true;
-    }
-
-    LM_GGML_UNUSED(dev);
-}
-
-static bool lm_ggml_backend_cpu_device_supports_buft(lm_ggml_backend_dev_t dev, lm_ggml_backend_buffer_type_t buft) {
-    return lm_ggml_backend_buft_is_host(buft);
-
-    LM_GGML_UNUSED(dev);
-}
-
-static const struct lm_ggml_backend_device_i lm_ggml_backend_cpu_device_i = {
-    /* .get_name             = */ lm_ggml_backend_cpu_device_get_name,
-    /* .get_description      = */ lm_ggml_backend_cpu_device_get_description,
-    /* .get_memory           = */ lm_ggml_backend_cpu_device_get_memory,
-    /* .get_type             = */ lm_ggml_backend_cpu_device_get_type,
-    /* .get_props            = */ lm_ggml_backend_cpu_device_get_props,
-    /* .init_backend         = */ lm_ggml_backend_cpu_device_init_backend,
-    /* .get_buffer_type      = */ lm_ggml_backend_cpu_device_get_buffer_type,
-    /* .get_host_buffer_type = */ NULL,
-    /* .buffer_from_host_ptr = */ lm_ggml_backend_cpu_device_buffer_from_host_ptr,
-    /* .supports_op          = */ lm_ggml_backend_cpu_device_supports_op,
-    /* .supports_buft        = */ lm_ggml_backend_cpu_device_supports_buft,
-    /* .offload_op           = */ NULL,
-    /* .event_new            = */ NULL,
-    /* .event_free           = */ NULL,
-    /* .event_synchronize    = */ NULL,
-};
-
-// CPU backend - backend (reg)
-
-static const char * lm_ggml_backend_cpu_reg_get_name(lm_ggml_backend_reg_t reg) {
-    return "CPU";
-
-    LM_GGML_UNUSED(reg);
-}
-
-static size_t lm_ggml_backend_cpu_reg_get_device_count(lm_ggml_backend_reg_t reg) {
-    return 1;
-
-    LM_GGML_UNUSED(reg);
-}
-
-static lm_ggml_backend_dev_t lm_ggml_backend_cpu_reg_get_device(lm_ggml_backend_reg_t reg, size_t index) {
-    LM_GGML_ASSERT(index == 0);
-
-    static lm_ggml_backend_cpu_device_context ctx;
-    static lm_ggml_backend_device lm_ggml_backend_cpu_device = {
-        /* .iface   = */ lm_ggml_backend_cpu_device_i,
-        /* .reg     = */ reg,
-        /* .context = */ &ctx,
-    };
-
-    return &lm_ggml_backend_cpu_device;
-}
-
-static void * lm_ggml_backend_cpu_get_proc_address(lm_ggml_backend_reg_t reg, const char * name) {
-    if (strcmp(name, "lm_ggml_backend_set_n_threads") == 0) {
-        return (void *)lm_ggml_backend_cpu_set_n_threads;
-    }
-    if (strcmp(name, "lm_ggml_backend_dev_get_extra_bufts") == 0) {
-        return (void *)lm_ggml_backend_cpu_get_extra_bufts;
-    }
-
-    return NULL;
-
-    LM_GGML_UNUSED(reg);
-}
-
-static const struct lm_ggml_backend_reg_i lm_ggml_backend_cpu_reg_i = {
-    /* .get_name         = */ lm_ggml_backend_cpu_reg_get_name,
-    /* .get_device_count = */ lm_ggml_backend_cpu_reg_get_device_count,
-    /* .get_device       = */ lm_ggml_backend_cpu_reg_get_device,
-    /* .get_proc_address = */ lm_ggml_backend_cpu_get_proc_address,
-};
-
-lm_ggml_backend_reg_t lm_ggml_backend_cpu_reg(void) {
-    static struct lm_ggml_backend_reg lm_ggml_backend_cpu_reg = {
-        /* .iface   = */ lm_ggml_backend_cpu_reg_i,
-        /* .context = */ NULL,
-    };
-
-    return &lm_ggml_backend_cpu_reg;
-}