cui-llama.rn 1.2.4 → 1.3.0

package/cpp/ggml-backend.h

@@ -1,326 +1,328 @@
-#pragma once
-
-#include "ggml.h"
-#include "ggml-alloc.h"
-
-#ifdef  __cplusplus
-extern "C" {
-#endif
-
-    typedef struct lm_ggml_backend_buffer_type * lm_ggml_backend_buffer_type_t;
-    typedef struct lm_ggml_backend_buffer * lm_ggml_backend_buffer_t;
-    typedef struct lm_ggml_backend_event * lm_ggml_backend_event_t;
-    typedef struct lm_ggml_backend * lm_ggml_backend_t;
-    typedef void * lm_ggml_backend_graph_plan_t;
-    typedef struct lm_ggml_backend_reg * lm_ggml_backend_reg_t;
-    typedef struct lm_ggml_backend_device * lm_ggml_backend_dev_t;
-
-
-    //
-    // Backend buffer type
-    //
-
-    LM_GGML_API const char *          lm_ggml_backend_buft_name          (lm_ggml_backend_buffer_type_t buft);
-    LM_GGML_API lm_ggml_backend_buffer_t lm_ggml_backend_buft_alloc_buffer  (lm_ggml_backend_buffer_type_t buft, size_t size);
-    LM_GGML_API size_t                lm_ggml_backend_buft_get_alignment (lm_ggml_backend_buffer_type_t buft);
-    LM_GGML_API size_t                lm_ggml_backend_buft_get_max_size  (lm_ggml_backend_buffer_type_t buft);
-    LM_GGML_API size_t                lm_ggml_backend_buft_get_alloc_size(lm_ggml_backend_buffer_type_t buft, struct lm_ggml_tensor * tensor);
-    LM_GGML_API bool                  lm_ggml_backend_buft_is_host       (lm_ggml_backend_buffer_type_t buft);
-    LM_GGML_API lm_ggml_backend_dev_t    lm_ggml_backend_buft_get_device    (lm_ggml_backend_buffer_type_t buft);
-
-    //
-    // Backend buffer
-    //
-
-    enum lm_ggml_backend_buffer_usage {
-        LM_GGML_BACKEND_BUFFER_USAGE_ANY = 0,
-        LM_GGML_BACKEND_BUFFER_USAGE_WEIGHTS = 1,
-        LM_GGML_BACKEND_BUFFER_USAGE_COMPUTE = 2,
-    };
-
-    LM_GGML_API const char *                   lm_ggml_backend_buffer_name          (lm_ggml_backend_buffer_t buffer);
-    LM_GGML_API void                           lm_ggml_backend_buffer_free          (lm_ggml_backend_buffer_t buffer);
-    LM_GGML_API void *                         lm_ggml_backend_buffer_get_base      (lm_ggml_backend_buffer_t buffer);
-    LM_GGML_API size_t                         lm_ggml_backend_buffer_get_size      (lm_ggml_backend_buffer_t buffer);
-    LM_GGML_API void                           lm_ggml_backend_buffer_init_tensor   (lm_ggml_backend_buffer_t buffer, struct lm_ggml_tensor * tensor);
-    LM_GGML_API size_t                         lm_ggml_backend_buffer_get_alignment (lm_ggml_backend_buffer_t buffer);
-    LM_GGML_API size_t                         lm_ggml_backend_buffer_get_max_size  (lm_ggml_backend_buffer_t buffer);
-    LM_GGML_API size_t                         lm_ggml_backend_buffer_get_alloc_size(lm_ggml_backend_buffer_t buffer, struct lm_ggml_tensor * tensor);
-    LM_GGML_API void                           lm_ggml_backend_buffer_clear         (lm_ggml_backend_buffer_t buffer, uint8_t value);
-    LM_GGML_API bool                           lm_ggml_backend_buffer_is_host       (lm_ggml_backend_buffer_t buffer);
-    LM_GGML_API void                           lm_ggml_backend_buffer_set_usage     (lm_ggml_backend_buffer_t buffer, enum lm_ggml_backend_buffer_usage usage);
-    LM_GGML_API enum lm_ggml_backend_buffer_usage lm_ggml_backend_buffer_get_usage     (lm_ggml_backend_buffer_t buffer);
-    LM_GGML_API lm_ggml_backend_buffer_type_t     lm_ggml_backend_buffer_get_type      (lm_ggml_backend_buffer_t buffer);
-    LM_GGML_API void                           lm_ggml_backend_buffer_reset         (lm_ggml_backend_buffer_t buffer);
-
-    // tensor copy between different backends
-    LM_GGML_API void lm_ggml_backend_tensor_copy(struct lm_ggml_tensor * src, struct lm_ggml_tensor * dst);
-
-    //
-    // Backend (stream)
-    //
-
-    LM_GGML_API lm_ggml_guid_t  lm_ggml_backend_guid(lm_ggml_backend_t backend);
-    LM_GGML_API const char * lm_ggml_backend_name(lm_ggml_backend_t backend);
-    LM_GGML_API void         lm_ggml_backend_free(lm_ggml_backend_t backend);
-
-    LM_GGML_API lm_ggml_backend_buffer_type_t lm_ggml_backend_get_default_buffer_type(lm_ggml_backend_t backend);
-    LM_GGML_API lm_ggml_backend_buffer_t      lm_ggml_backend_alloc_buffer(lm_ggml_backend_t backend, size_t size);
-    LM_GGML_API size_t                     lm_ggml_backend_get_alignment(lm_ggml_backend_t backend);
-    LM_GGML_API size_t                     lm_ggml_backend_get_max_size(lm_ggml_backend_t backend);
-
-    LM_GGML_API void lm_ggml_backend_tensor_set_async(lm_ggml_backend_t backend,       struct lm_ggml_tensor * tensor, const void * data, size_t offset, size_t size);
-    LM_GGML_API void lm_ggml_backend_tensor_get_async(lm_ggml_backend_t backend, const struct lm_ggml_tensor * tensor,       void * data, size_t offset, size_t size);
-
-    // "offset" refers to the offset of the tensor data for setting/getting data
-    LM_GGML_API void lm_ggml_backend_tensor_set(      struct lm_ggml_tensor * tensor, const void * data, size_t offset, size_t size);
-    LM_GGML_API void lm_ggml_backend_tensor_get(const struct lm_ggml_tensor * tensor,       void * data, size_t offset, size_t size);
-    LM_GGML_API void lm_ggml_backend_tensor_memset(   struct lm_ggml_tensor * tensor,     uint8_t value, size_t offset, size_t size);
-
-    LM_GGML_API void lm_ggml_backend_synchronize(lm_ggml_backend_t backend);
-
-    LM_GGML_API lm_ggml_backend_graph_plan_t lm_ggml_backend_graph_plan_create(lm_ggml_backend_t backend, struct lm_ggml_cgraph * cgraph);
-    LM_GGML_API void                      lm_ggml_backend_graph_plan_free  (lm_ggml_backend_t backend, lm_ggml_backend_graph_plan_t plan);
-
-    LM_GGML_API enum lm_ggml_status lm_ggml_backend_graph_plan_compute (lm_ggml_backend_t backend, lm_ggml_backend_graph_plan_t plan);
-    LM_GGML_API enum lm_ggml_status lm_ggml_backend_graph_compute      (lm_ggml_backend_t backend, struct lm_ggml_cgraph * cgraph);
-    LM_GGML_API enum lm_ggml_status lm_ggml_backend_graph_compute_async(lm_ggml_backend_t backend, struct lm_ggml_cgraph * cgraph);
-
-    // NOTE: will be removed, use device version instead
-    LM_GGML_API bool lm_ggml_backend_supports_op(lm_ggml_backend_t backend, const struct lm_ggml_tensor * op);
-    LM_GGML_API bool lm_ggml_backend_supports_buft(lm_ggml_backend_t backend, lm_ggml_backend_buffer_type_t buft);
-    LM_GGML_API bool lm_ggml_backend_offload_op(lm_ggml_backend_t backend, const struct lm_ggml_tensor * op);
-
-    // asynchronous copy
-    // the copy is performed after all the currently queued operations in backend_src
-    // backend_dst will wait for the copy to complete before performing other operations
-    // automatic fallback to sync copy if async is not supported
-    LM_GGML_API void lm_ggml_backend_tensor_copy_async(lm_ggml_backend_t backend_src, lm_ggml_backend_t backend_dst, struct lm_ggml_tensor * src, struct lm_ggml_tensor * dst);
-
-    LM_GGML_API lm_ggml_backend_dev_t lm_ggml_backend_get_device(lm_ggml_backend_t backend);
-
-    //
-    // Events
-    //
-
-    LM_GGML_API lm_ggml_backend_event_t lm_ggml_backend_event_new(lm_ggml_backend_dev_t device);
-    LM_GGML_API void                 lm_ggml_backend_event_free(lm_ggml_backend_event_t event);
-    LM_GGML_API void                 lm_ggml_backend_event_record(lm_ggml_backend_event_t event, lm_ggml_backend_t backend);
-    LM_GGML_API void                 lm_ggml_backend_event_synchronize(lm_ggml_backend_event_t event);
-    LM_GGML_API void                 lm_ggml_backend_event_wait(lm_ggml_backend_t backend, lm_ggml_backend_event_t event);
-
-    //
-    // Backend device
-    //
-
-    enum lm_ggml_backend_dev_type {
-        LM_GGML_BACKEND_DEVICE_TYPE_CPU,
-        LM_GGML_BACKEND_DEVICE_TYPE_GPU,
-        // devices with full capabilities (excludes backends such as BLAS that only support matrix multiplication)
-        LM_GGML_BACKEND_DEVICE_TYPE_CPU_FULL,
-        LM_GGML_BACKEND_DEVICE_TYPE_GPU_FULL
-    };
-
-    // functionality supported by the device
-    struct lm_ggml_backend_dev_caps {
-        // asynchronous operations
-        bool async;
-        // pinned host buffer
-        bool host_buffer;
-        // creating buffers from host ptr
-        bool buffer_from_host_ptr;
-        // event synchronization
-        bool events;
-    };
-
-    // all the device properties
-    struct lm_ggml_backend_dev_props {
-        const char * name;
-        const char * description;
-        size_t memory_free;
-        size_t memory_total;
-        enum lm_ggml_backend_dev_type type;
-        struct lm_ggml_backend_dev_caps caps;
-    };
-
-    LM_GGML_API const char *                  lm_ggml_backend_dev_name(lm_ggml_backend_dev_t device);
-    LM_GGML_API const char *                  lm_ggml_backend_dev_description(lm_ggml_backend_dev_t device);
-    LM_GGML_API void                          lm_ggml_backend_dev_memory(lm_ggml_backend_dev_t device, size_t * free, size_t * total);
-    LM_GGML_API enum lm_ggml_backend_dev_type    lm_ggml_backend_dev_type(lm_ggml_backend_dev_t device);
-    LM_GGML_API void                          lm_ggml_backend_dev_get_props(lm_ggml_backend_dev_t device, struct lm_ggml_backend_dev_props * props);
-    LM_GGML_API lm_ggml_backend_reg_t            lm_ggml_backend_dev_backend_reg(lm_ggml_backend_dev_t device);
-    LM_GGML_API lm_ggml_backend_t                lm_ggml_backend_dev_init(lm_ggml_backend_dev_t device, const char * params);
-    LM_GGML_API lm_ggml_backend_buffer_type_t    lm_ggml_backend_dev_buffer_type(lm_ggml_backend_dev_t device);
-    LM_GGML_API lm_ggml_backend_buffer_type_t    lm_ggml_backend_dev_host_buffer_type(lm_ggml_backend_dev_t device);
-    LM_GGML_API lm_ggml_backend_buffer_t         lm_ggml_backend_dev_buffer_from_host_ptr(lm_ggml_backend_dev_t device, void * ptr, size_t size, size_t max_tensor_size);
-
-    LM_GGML_API bool                          lm_ggml_backend_dev_supports_op(lm_ggml_backend_dev_t device, const struct lm_ggml_tensor * op);
-    LM_GGML_API bool                          lm_ggml_backend_dev_supports_buft(lm_ggml_backend_dev_t device, lm_ggml_backend_buffer_type_t buft);
-    LM_GGML_API bool                          lm_ggml_backend_dev_offload_op(lm_ggml_backend_dev_t device, const struct lm_ggml_tensor * op);
-
-    //
-    // Backend (reg)
-    //
-
-    LM_GGML_API const char *       lm_ggml_backend_reg_name(lm_ggml_backend_reg_t reg);
-    LM_GGML_API size_t             lm_ggml_backend_reg_dev_count(lm_ggml_backend_reg_t reg);
-    LM_GGML_API lm_ggml_backend_dev_t lm_ggml_backend_reg_dev_get(lm_ggml_backend_reg_t reg, size_t index);
-    LM_GGML_API void *             lm_ggml_backend_reg_get_proc_address(lm_ggml_backend_reg_t reg, const char * name);
-
-
-    // Functions that may be obtained using lm_ggml_backend_reg_get_proc_address
-    typedef lm_ggml_backend_buffer_type_t (*lm_ggml_backend_split_buffer_type_t)(const float *);
-    typedef void (*lm_ggml_backend_set_n_threads_t)(lm_ggml_backend_t, int);
-
-    //
-    // Backend registry
-    //
-
-    // Backend (reg) enumeration
-    LM_GGML_API size_t             lm_ggml_backend_reg_count(void);
-    LM_GGML_API lm_ggml_backend_reg_t lm_ggml_backend_reg_get(size_t index);
-    LM_GGML_API lm_ggml_backend_reg_t lm_ggml_backend_reg_by_name(const char * name);
-
-    // Device enumeration
-    LM_GGML_API size_t             lm_ggml_backend_dev_count(void);
-    LM_GGML_API lm_ggml_backend_dev_t lm_ggml_backend_dev_get(size_t index);
-    LM_GGML_API lm_ggml_backend_dev_t lm_ggml_backend_dev_by_name(const char * name);
-    LM_GGML_API lm_ggml_backend_dev_t lm_ggml_backend_dev_by_type(enum lm_ggml_backend_dev_type type);
-
-    // Direct backend (stream) initialization
-    // = lm_ggml_backend_dev_init(lm_ggml_backend_dev_by_name(name), params)
-    LM_GGML_API lm_ggml_backend_t lm_ggml_backend_init_by_name(const char * name, const char * params);
-    // = lm_ggml_backend_dev_init(lm_ggml_backend_dev_by_type(type), params)
-    LM_GGML_API lm_ggml_backend_t lm_ggml_backend_init_by_type(enum lm_ggml_backend_dev_type type, const char * params);
-    // = lm_ggml_backend_dev_init(lm_ggml_backend_dev_by_type(GPU_FULL) OR lm_ggml_backend_dev_by_type(CPU_FULL), NULL)
-    LM_GGML_API lm_ggml_backend_t lm_ggml_backend_init_best(void);
-
-    //
-    // Backend scheduler
-    //
-
-    // The backend scheduler allows for multiple backend devices to be used together
-    // Handles compute buffer allocation, assignment of tensors to backends, and copying of tensors between backends
-    // The backends are selected based on:
-    // - the backend that supports the operation
-    // - the location of the pre-allocated tensors (e.g. the weights)
-    /*
-      Example usage:
-
-        // operations that use tensors allocated in a buffer with USAGE_WEIGHTS will be assigned
-        // preferrably to run on the same backend as the buffer
-        lm_ggml_backend_buffer_set_usage(buf_weights, LM_GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
-
-        sched = lm_ggml_backend_sched_new({backend_gpu, backend_gpu2, backend_cpu}, NULL, num_backends, LM_GGML_DEFAULT_GRAPH_SIZE, false);
-
-        // initialize buffers from a max size graph (optional)
-        reserve_graph = build_graph(sched, max_batch_size);
-
-        // manually assign nodes to a backend (optional, should not be needed in most cases)
-        struct lm_ggml_tensor * node = lm_ggml_mul_mat(ctx, ...);
-        lm_ggml_backend_sched_set_tensor_backend(sched, node, backend_gpu);
-
-        lm_ggml_backend_sched_reserve(sched, reserve_graph);
-
-        // compute
-        graph = build_graph(sched);
-        lm_ggml_backend_sched_graph_compute(sched, graph);
-
-        // if there are graph inputs:
-        lm_ggml_backend_sched_reset(sched);
-        lm_ggml_backend_sched_alloc_graph(sched, graph);
-        lm_ggml_backend_tensor_set(input_tensor, ...);
-        lm_ggml_backend_sched_graph_compute(sched, graph);
-    }
-    */
-
-    typedef struct lm_ggml_backend_sched * lm_ggml_backend_sched_t;
-
-    // Evaluation callback for each node in the graph (set with lm_ggml_backend_sched_set_eval_callback)
-    // when ask == true, the scheduler wants to know if the user wants to observe this node
-    // this allows the scheduler to batch nodes together in order to evaluate them in a single call
-    //
-    // when ask == false, the scheduler is passing the node tensor to the user for observation
-    // if the user returns false, the scheduler will cancel the graph compute
-    //
-    typedef bool (*lm_ggml_backend_sched_eval_callback)(struct lm_ggml_tensor * t, bool ask, void * user_data);
-
-    // Initialize a backend scheduler
-    LM_GGML_API lm_ggml_backend_sched_t lm_ggml_backend_sched_new(lm_ggml_backend_t * backends, lm_ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel);
-    LM_GGML_API void                 lm_ggml_backend_sched_free(lm_ggml_backend_sched_t sched);
-
-    // Initialize backend buffers from a measure graph
-    LM_GGML_API bool                 lm_ggml_backend_sched_reserve(lm_ggml_backend_sched_t sched, struct lm_ggml_cgraph * measure_graph); // returns success
-
-    LM_GGML_API int                  lm_ggml_backend_sched_get_n_backends(lm_ggml_backend_sched_t sched);
-    LM_GGML_API lm_ggml_backend_t       lm_ggml_backend_sched_get_backend(lm_ggml_backend_sched_t sched, int i);
-
-    // Get the number of splits of the last graph
-    LM_GGML_API int                  lm_ggml_backend_sched_get_n_splits(lm_ggml_backend_sched_t sched);
-    LM_GGML_API int                  lm_ggml_backend_sched_get_n_copies(lm_ggml_backend_sched_t sched);
-
-    LM_GGML_API size_t               lm_ggml_backend_sched_get_buffer_size(lm_ggml_backend_sched_t sched, lm_ggml_backend_t backend);
-
-    LM_GGML_API void                 lm_ggml_backend_sched_set_tensor_backend(lm_ggml_backend_sched_t sched, struct lm_ggml_tensor * node, lm_ggml_backend_t backend);
-    LM_GGML_API lm_ggml_backend_t       lm_ggml_backend_sched_get_tensor_backend(lm_ggml_backend_sched_t sched, struct lm_ggml_tensor * node);
-
-    // Allocate and compute graph on the backend scheduler
-    LM_GGML_API bool                 lm_ggml_backend_sched_alloc_graph(lm_ggml_backend_sched_t sched, struct lm_ggml_cgraph * graph); // returns success
-    LM_GGML_API enum lm_ggml_status     lm_ggml_backend_sched_graph_compute(lm_ggml_backend_sched_t sched, struct lm_ggml_cgraph * graph);
-    LM_GGML_API enum lm_ggml_status     lm_ggml_backend_sched_graph_compute_async(lm_ggml_backend_sched_t sched, struct lm_ggml_cgraph * graph);
-    LM_GGML_API void                 lm_ggml_backend_sched_synchronize(lm_ggml_backend_sched_t sched);
-
-    // Reset all assignments and allocators - must be called before changing the node backends
-    LM_GGML_API void                 lm_ggml_backend_sched_reset(lm_ggml_backend_sched_t sched);
-
-    // Set a callback to be called for each resulting node during graph compute
-    LM_GGML_API void                 lm_ggml_backend_sched_set_eval_callback(lm_ggml_backend_sched_t sched, lm_ggml_backend_sched_eval_callback callback, void * user_data);
-
-    //
-    // Utils
-    //
-
-    struct lm_ggml_backend_graph_copy {
-        lm_ggml_backend_buffer_t buffer;
-        struct lm_ggml_context * ctx_allocated;
-        struct lm_ggml_context * ctx_unallocated;
-        struct lm_ggml_cgraph * graph;
-    };
-
-    // Copy a graph to a different backend
-    LM_GGML_API struct lm_ggml_backend_graph_copy lm_ggml_backend_graph_copy(lm_ggml_backend_t backend, struct lm_ggml_cgraph * graph);
-    LM_GGML_API void                           lm_ggml_backend_graph_copy_free(struct lm_ggml_backend_graph_copy copy);
-
-    typedef bool (*lm_ggml_backend_eval_callback)(int node_index, struct lm_ggml_tensor * t1, struct lm_ggml_tensor * t2, void * user_data);
-
-    // Compare the output of two backends
-    LM_GGML_API bool lm_ggml_backend_compare_graph_backend(lm_ggml_backend_t backend1, lm_ggml_backend_t backend2, struct lm_ggml_cgraph * graph, lm_ggml_backend_eval_callback callback, void * user_data);
-
-    // Tensor initialization
-    LM_GGML_API void lm_ggml_backend_tensor_alloc(lm_ggml_backend_buffer_t buffer, struct lm_ggml_tensor * tensor, void * addr);
-    LM_GGML_API void lm_ggml_backend_view_init(struct lm_ggml_tensor * tensor);
-
-    //
-    // CPU backend
-    //
-
-    LM_GGML_API lm_ggml_backend_t lm_ggml_backend_cpu_init(void);
-
-    LM_GGML_API bool lm_ggml_backend_is_cpu                (lm_ggml_backend_t backend);
-    LM_GGML_API void lm_ggml_backend_cpu_set_n_threads     (lm_ggml_backend_t backend_cpu, int n_threads);
-    LM_GGML_API void lm_ggml_backend_cpu_set_threadpool    (lm_ggml_backend_t backend_cpu, lm_ggml_threadpool_t threadpool);
-    LM_GGML_API void lm_ggml_backend_cpu_set_abort_callback(lm_ggml_backend_t backend_cpu, lm_ggml_abort_callback abort_callback, void * abort_callback_data);
-
-    // Create a backend buffer from an existing pointer
-    LM_GGML_API lm_ggml_backend_buffer_t      lm_ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size);
-    LM_GGML_API lm_ggml_backend_buffer_type_t lm_ggml_backend_cpu_buffer_type(void);
-
-    LM_GGML_API lm_ggml_backend_reg_t lm_ggml_backend_cpu_reg(void);
-
-#ifdef LM_GGML_USE_CPU_HBM
-    LM_GGML_API lm_ggml_backend_buffer_type_t lm_ggml_backend_cpu_hbm_buffer_type(void);
-#endif
-
-#ifdef  __cplusplus
-}
-#endif
+#pragma once
+
+#include "ggml.h"
+#include "ggml-alloc.h"
+
+#ifdef LM_GGML_BACKEND_SHARED
+#    if defined(_WIN32) && !defined(__MINGW32__)
+#        ifdef LM_GGML_BACKEND_BUILD
+#            define LM_GGML_BACKEND_API __declspec(dllexport) extern
+#        else
+#            define LM_GGML_BACKEND_API __declspec(dllimport) extern
+#        endif
+#    else
+#        define LM_GGML_BACKEND_API __attribute__ ((visibility ("default"))) extern
+#    endif
+#else
+#    define LM_GGML_BACKEND_API extern
+#endif
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+    typedef struct lm_ggml_backend_buffer_type * lm_ggml_backend_buffer_type_t;
+    typedef struct lm_ggml_backend_buffer * lm_ggml_backend_buffer_t;
+    typedef struct lm_ggml_backend_event * lm_ggml_backend_event_t;
+    typedef struct lm_ggml_backend * lm_ggml_backend_t;
+    typedef void * lm_ggml_backend_graph_plan_t;
+    typedef struct lm_ggml_backend_reg * lm_ggml_backend_reg_t;
+    typedef struct lm_ggml_backend_device * lm_ggml_backend_dev_t;
+
+
+    //
+    // Backend buffer type
+    //
+
+    LM_GGML_API const char *          lm_ggml_backend_buft_name          (lm_ggml_backend_buffer_type_t buft);
+    LM_GGML_API lm_ggml_backend_buffer_t lm_ggml_backend_buft_alloc_buffer  (lm_ggml_backend_buffer_type_t buft, size_t size);
+    LM_GGML_API size_t                lm_ggml_backend_buft_get_alignment (lm_ggml_backend_buffer_type_t buft);
+    LM_GGML_API size_t                lm_ggml_backend_buft_get_max_size  (lm_ggml_backend_buffer_type_t buft);
+    LM_GGML_API size_t                lm_ggml_backend_buft_get_alloc_size(lm_ggml_backend_buffer_type_t buft, struct lm_ggml_tensor * tensor);
+    LM_GGML_API bool                  lm_ggml_backend_buft_is_host       (lm_ggml_backend_buffer_type_t buft);
+    LM_GGML_API lm_ggml_backend_dev_t    lm_ggml_backend_buft_get_device    (lm_ggml_backend_buffer_type_t buft);
+
+    //
+    // Backend buffer
+    //
+
+    enum lm_ggml_backend_buffer_usage {
+        LM_GGML_BACKEND_BUFFER_USAGE_ANY = 0,
+        LM_GGML_BACKEND_BUFFER_USAGE_WEIGHTS = 1,
+        LM_GGML_BACKEND_BUFFER_USAGE_COMPUTE = 2,
+    };
+
+    LM_GGML_API const char *                   lm_ggml_backend_buffer_name          (lm_ggml_backend_buffer_t buffer);
+    LM_GGML_API void                           lm_ggml_backend_buffer_free          (lm_ggml_backend_buffer_t buffer);
+    LM_GGML_API void *                         lm_ggml_backend_buffer_get_base      (lm_ggml_backend_buffer_t buffer);
+    LM_GGML_API size_t                         lm_ggml_backend_buffer_get_size      (lm_ggml_backend_buffer_t buffer);
+    LM_GGML_API void                           lm_ggml_backend_buffer_init_tensor   (lm_ggml_backend_buffer_t buffer, struct lm_ggml_tensor * tensor);
+    LM_GGML_API size_t                         lm_ggml_backend_buffer_get_alignment (lm_ggml_backend_buffer_t buffer);
+    LM_GGML_API size_t                         lm_ggml_backend_buffer_get_max_size  (lm_ggml_backend_buffer_t buffer);
+    LM_GGML_API size_t                         lm_ggml_backend_buffer_get_alloc_size(lm_ggml_backend_buffer_t buffer, struct lm_ggml_tensor * tensor);
+    LM_GGML_API void                           lm_ggml_backend_buffer_clear         (lm_ggml_backend_buffer_t buffer, uint8_t value);
+    LM_GGML_API bool                           lm_ggml_backend_buffer_is_host       (lm_ggml_backend_buffer_t buffer);
+    LM_GGML_API void                           lm_ggml_backend_buffer_set_usage     (lm_ggml_backend_buffer_t buffer, enum lm_ggml_backend_buffer_usage usage);
+    LM_GGML_API enum lm_ggml_backend_buffer_usage lm_ggml_backend_buffer_get_usage     (lm_ggml_backend_buffer_t buffer);
+    LM_GGML_API lm_ggml_backend_buffer_type_t     lm_ggml_backend_buffer_get_type      (lm_ggml_backend_buffer_t buffer);
+    LM_GGML_API void                           lm_ggml_backend_buffer_reset         (lm_ggml_backend_buffer_t buffer);
+
+    // tensor copy between different backends
+    LM_GGML_API void lm_ggml_backend_tensor_copy(struct lm_ggml_tensor * src, struct lm_ggml_tensor * dst);
+
+    //
+    // Backend (stream)
+    //
+
+    LM_GGML_API lm_ggml_guid_t  lm_ggml_backend_guid(lm_ggml_backend_t backend);
+    LM_GGML_API const char * lm_ggml_backend_name(lm_ggml_backend_t backend);
+    LM_GGML_API void         lm_ggml_backend_free(lm_ggml_backend_t backend);
+
+    LM_GGML_API lm_ggml_backend_buffer_type_t lm_ggml_backend_get_default_buffer_type(lm_ggml_backend_t backend);
+    LM_GGML_API lm_ggml_backend_buffer_t      lm_ggml_backend_alloc_buffer(lm_ggml_backend_t backend, size_t size);
+    LM_GGML_API size_t                     lm_ggml_backend_get_alignment(lm_ggml_backend_t backend);
+    LM_GGML_API size_t                     lm_ggml_backend_get_max_size(lm_ggml_backend_t backend);
+
+    LM_GGML_API void lm_ggml_backend_tensor_set_async(lm_ggml_backend_t backend,       struct lm_ggml_tensor * tensor, const void * data, size_t offset, size_t size);
+    LM_GGML_API void lm_ggml_backend_tensor_get_async(lm_ggml_backend_t backend, const struct lm_ggml_tensor * tensor,       void * data, size_t offset, size_t size);
+
+    // "offset" refers to the offset of the tensor data for setting/getting data
+    LM_GGML_API void lm_ggml_backend_tensor_set(      struct lm_ggml_tensor * tensor, const void * data, size_t offset, size_t size);
+    LM_GGML_API void lm_ggml_backend_tensor_get(const struct lm_ggml_tensor * tensor,       void * data, size_t offset, size_t size);
+    LM_GGML_API void lm_ggml_backend_tensor_memset(   struct lm_ggml_tensor * tensor,     uint8_t value, size_t offset, size_t size);
+
+    LM_GGML_API void lm_ggml_backend_synchronize(lm_ggml_backend_t backend);
+
+    LM_GGML_API lm_ggml_backend_graph_plan_t lm_ggml_backend_graph_plan_create(lm_ggml_backend_t backend, struct lm_ggml_cgraph * cgraph);
+    LM_GGML_API void                      lm_ggml_backend_graph_plan_free  (lm_ggml_backend_t backend, lm_ggml_backend_graph_plan_t plan);
+
+    LM_GGML_API enum lm_ggml_status lm_ggml_backend_graph_plan_compute (lm_ggml_backend_t backend, lm_ggml_backend_graph_plan_t plan);
+    LM_GGML_API enum lm_ggml_status lm_ggml_backend_graph_compute      (lm_ggml_backend_t backend, struct lm_ggml_cgraph * cgraph);
+    LM_GGML_API enum lm_ggml_status lm_ggml_backend_graph_compute_async(lm_ggml_backend_t backend, struct lm_ggml_cgraph * cgraph);
+
+    // NOTE: will be removed, use device version instead
+    LM_GGML_API bool lm_ggml_backend_supports_op(lm_ggml_backend_t backend, const struct lm_ggml_tensor * op);
+    LM_GGML_API bool lm_ggml_backend_supports_buft(lm_ggml_backend_t backend, lm_ggml_backend_buffer_type_t buft);
+    LM_GGML_API bool lm_ggml_backend_offload_op(lm_ggml_backend_t backend, const struct lm_ggml_tensor * op);
+
+    // asynchronous copy
+    // the copy is performed after all the currently queued operations in backend_src
+    // backend_dst will wait for the copy to complete before performing other operations
+    // automatic fallback to sync copy if async is not supported
+    LM_GGML_API void lm_ggml_backend_tensor_copy_async(lm_ggml_backend_t backend_src, lm_ggml_backend_t backend_dst, struct lm_ggml_tensor * src, struct lm_ggml_tensor * dst);
+
+    LM_GGML_API lm_ggml_backend_dev_t lm_ggml_backend_get_device(lm_ggml_backend_t backend);
+
+    //
+    // Events
+    //
+
+    LM_GGML_API lm_ggml_backend_event_t lm_ggml_backend_event_new(lm_ggml_backend_dev_t device);
+    LM_GGML_API void                 lm_ggml_backend_event_free(lm_ggml_backend_event_t event);
+    LM_GGML_API void                 lm_ggml_backend_event_record(lm_ggml_backend_event_t event, lm_ggml_backend_t backend);
+    LM_GGML_API void                 lm_ggml_backend_event_synchronize(lm_ggml_backend_event_t event);
+    LM_GGML_API void                 lm_ggml_backend_event_wait(lm_ggml_backend_t backend, lm_ggml_backend_event_t event);
+
+    //
+    // Backend device
+    //
+
+    enum lm_ggml_backend_dev_type {
+        // CPU device using system memory
+        LM_GGML_BACKEND_DEVICE_TYPE_CPU,
+        // GPU device using dedicated memory
+        LM_GGML_BACKEND_DEVICE_TYPE_GPU,
+        // accelerator devices intended to be used together with the CPU backend (e.g. BLAS or AMX)
+        LM_GGML_BACKEND_DEVICE_TYPE_ACCEL
+    };
+
+    // functionality supported by the device
+    struct lm_ggml_backend_dev_caps {
+        // asynchronous operations
+        bool async;
+        // pinned host buffer
+        bool host_buffer;
+        // creating buffers from host ptr
+        bool buffer_from_host_ptr;
+        // event synchronization
+        bool events;
+    };
+
+    // all the device properties
+    struct lm_ggml_backend_dev_props {
+        const char * name;
+        const char * description;
+        size_t memory_free;
+        size_t memory_total;
+        enum lm_ggml_backend_dev_type type;
+        struct lm_ggml_backend_dev_caps caps;
+    };
+
+    LM_GGML_API const char *                  lm_ggml_backend_dev_name(lm_ggml_backend_dev_t device);
+    LM_GGML_API const char *                  lm_ggml_backend_dev_description(lm_ggml_backend_dev_t device);
+    LM_GGML_API void                          lm_ggml_backend_dev_memory(lm_ggml_backend_dev_t device, size_t * free, size_t * total);
+    LM_GGML_API enum lm_ggml_backend_dev_type    lm_ggml_backend_dev_type(lm_ggml_backend_dev_t device);
+    LM_GGML_API void                          lm_ggml_backend_dev_get_props(lm_ggml_backend_dev_t device, struct lm_ggml_backend_dev_props * props);
+    LM_GGML_API lm_ggml_backend_reg_t            lm_ggml_backend_dev_backend_reg(lm_ggml_backend_dev_t device);
+    LM_GGML_API lm_ggml_backend_t                lm_ggml_backend_dev_init(lm_ggml_backend_dev_t device, const char * params);
+    LM_GGML_API lm_ggml_backend_buffer_type_t    lm_ggml_backend_dev_buffer_type(lm_ggml_backend_dev_t device);
+    LM_GGML_API lm_ggml_backend_buffer_type_t    lm_ggml_backend_dev_host_buffer_type(lm_ggml_backend_dev_t device);
+    LM_GGML_API lm_ggml_backend_buffer_t         lm_ggml_backend_dev_buffer_from_host_ptr(lm_ggml_backend_dev_t device, void * ptr, size_t size, size_t max_tensor_size);
+
+    LM_GGML_API bool                          lm_ggml_backend_dev_supports_op(lm_ggml_backend_dev_t device, const struct lm_ggml_tensor * op);
+    LM_GGML_API bool                          lm_ggml_backend_dev_supports_buft(lm_ggml_backend_dev_t device, lm_ggml_backend_buffer_type_t buft);
+    LM_GGML_API bool                          lm_ggml_backend_dev_offload_op(lm_ggml_backend_dev_t device, const struct lm_ggml_tensor * op);
+
+    //
+    // Backend (reg)
+    //
+
+    LM_GGML_API const char *       lm_ggml_backend_reg_name(lm_ggml_backend_reg_t reg);
+    LM_GGML_API size_t             lm_ggml_backend_reg_dev_count(lm_ggml_backend_reg_t reg);
+    LM_GGML_API lm_ggml_backend_dev_t lm_ggml_backend_reg_dev_get(lm_ggml_backend_reg_t reg, size_t index);
+    LM_GGML_API void *             lm_ggml_backend_reg_get_proc_address(lm_ggml_backend_reg_t reg, const char * name);
+
+    // Common functions that may be obtained using lm_ggml_backend_reg_get_proc_address
+
+    // Split buffer type for tensor parallelism
+    typedef lm_ggml_backend_buffer_type_t   (*lm_ggml_backend_split_buffer_type_t)(int main_device, const float * tensor_split);
+    // Set the number of threads for the backend
+    typedef void                         (*lm_ggml_backend_set_n_threads_t)(lm_ggml_backend_t backend, int n_threads);
+    // Get additional buffer types provided by the device (returns a NULL-terminated array)
+    typedef lm_ggml_backend_buffer_type_t * (*lm_ggml_backend_dev_get_extra_bufts_t)(lm_ggml_backend_dev_t device);
+
+    //
+    // Backend registry
+    //
+
+    // Backend (reg) enumeration
+    LM_GGML_API size_t             lm_ggml_backend_reg_count(void);
+    LM_GGML_API lm_ggml_backend_reg_t lm_ggml_backend_reg_get(size_t index);
+    LM_GGML_API lm_ggml_backend_reg_t lm_ggml_backend_reg_by_name(const char * name);
+
+    // Device enumeration
+    LM_GGML_API size_t             lm_ggml_backend_dev_count(void);
+    LM_GGML_API lm_ggml_backend_dev_t lm_ggml_backend_dev_get(size_t index);
+    LM_GGML_API lm_ggml_backend_dev_t lm_ggml_backend_dev_by_name(const char * name);
+    LM_GGML_API lm_ggml_backend_dev_t lm_ggml_backend_dev_by_type(enum lm_ggml_backend_dev_type type);
+
+    // Direct backend (stream) initialization
+    // = lm_ggml_backend_dev_init(lm_ggml_backend_dev_by_name(name), params)
+    LM_GGML_API lm_ggml_backend_t lm_ggml_backend_init_by_name(const char * name, const char * params);
+    // = lm_ggml_backend_dev_init(lm_ggml_backend_dev_by_type(type), params)
+    LM_GGML_API lm_ggml_backend_t lm_ggml_backend_init_by_type(enum lm_ggml_backend_dev_type type, const char * params);
+    // = lm_ggml_backend_dev_init(lm_ggml_backend_dev_by_type(GPU) OR lm_ggml_backend_dev_by_type(CPU), NULL)
+    LM_GGML_API lm_ggml_backend_t lm_ggml_backend_init_best(void);
+
+    //
+    // Backend scheduler
+    //
+
+    // The backend scheduler allows for multiple backend devices to be used together
+    // Handles compute buffer allocation, assignment of tensors to backends, and copying of tensors between backends
+    // The backends are selected based on:
+    // - the backend that supports the operation
+    // - the location of the pre-allocated tensors (e.g. the weights)
+    /*
+      Example usage:
+
+        // operations that use tensors allocated in a buffer with USAGE_WEIGHTS will be assigned
+        // preferrably to run on the same backend as the buffer
+        lm_ggml_backend_buffer_set_usage(buf_weights, LM_GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
+
+        sched = lm_ggml_backend_sched_new({backend_gpu, backend_gpu2, backend_cpu}, NULL, num_backends, LM_GGML_DEFAULT_GRAPH_SIZE, false);
+
+        // initialize buffers from a max size graph (optional)
+        reserve_graph = build_graph(sched, max_batch_size);
+
+        // manually assign nodes to a backend (optional, should not be needed in most cases)
+        struct lm_ggml_tensor * node = lm_ggml_mul_mat(ctx, ...);
+        lm_ggml_backend_sched_set_tensor_backend(sched, node, backend_gpu);
+
+        lm_ggml_backend_sched_reserve(sched, reserve_graph);
+
+        // compute
+        graph = build_graph(sched);
+        lm_ggml_backend_sched_graph_compute(sched, graph);
+
+        // if there are graph inputs:
+        lm_ggml_backend_sched_reset(sched);
+        lm_ggml_backend_sched_alloc_graph(sched, graph);
+        lm_ggml_backend_tensor_set(input_tensor, ...);
+        lm_ggml_backend_sched_graph_compute(sched, graph);
+    }
+    */
+
+    typedef struct lm_ggml_backend_sched * lm_ggml_backend_sched_t;
+
+    // Evaluation callback for each node in the graph (set with lm_ggml_backend_sched_set_eval_callback)
+    // when ask == true, the scheduler wants to know if the user wants to observe this node
+    // this allows the scheduler to batch nodes together in order to evaluate them in a single call
+    //
+    // when ask == false, the scheduler is passing the node tensor to the user for observation
+    // if the user returns false, the scheduler will cancel the graph compute
+    //
+    typedef bool (*lm_ggml_backend_sched_eval_callback)(struct lm_ggml_tensor * t, bool ask, void * user_data);
+
+    // Initialize a backend scheduler
+    LM_GGML_API lm_ggml_backend_sched_t lm_ggml_backend_sched_new(lm_ggml_backend_t * backends, lm_ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel);
+    LM_GGML_API void                 lm_ggml_backend_sched_free(lm_ggml_backend_sched_t sched);
+
+    // Initialize backend buffers from a measure graph
+    LM_GGML_API bool                 lm_ggml_backend_sched_reserve(lm_ggml_backend_sched_t sched, struct lm_ggml_cgraph * measure_graph); // returns success
+
+    LM_GGML_API int                  lm_ggml_backend_sched_get_n_backends(lm_ggml_backend_sched_t sched);
+    LM_GGML_API lm_ggml_backend_t       lm_ggml_backend_sched_get_backend(lm_ggml_backend_sched_t sched, int i);
+
+    // Get the number of splits of the last graph
+    LM_GGML_API int                  lm_ggml_backend_sched_get_n_splits(lm_ggml_backend_sched_t sched);
+    LM_GGML_API int                  lm_ggml_backend_sched_get_n_copies(lm_ggml_backend_sched_t sched);
+
+    LM_GGML_API size_t               lm_ggml_backend_sched_get_buffer_size(lm_ggml_backend_sched_t sched, lm_ggml_backend_t backend);
+
+    LM_GGML_API void                 lm_ggml_backend_sched_set_tensor_backend(lm_ggml_backend_sched_t sched, struct lm_ggml_tensor * node, lm_ggml_backend_t backend);
+    LM_GGML_API lm_ggml_backend_t       lm_ggml_backend_sched_get_tensor_backend(lm_ggml_backend_sched_t sched, struct lm_ggml_tensor * node);
+
+    // Allocate and compute graph on the backend scheduler
+    LM_GGML_API bool                 lm_ggml_backend_sched_alloc_graph(lm_ggml_backend_sched_t sched, struct lm_ggml_cgraph * graph); // returns success
+    LM_GGML_API enum lm_ggml_status     lm_ggml_backend_sched_graph_compute(lm_ggml_backend_sched_t sched, struct lm_ggml_cgraph * graph);
+    LM_GGML_API enum lm_ggml_status     lm_ggml_backend_sched_graph_compute_async(lm_ggml_backend_sched_t sched, struct lm_ggml_cgraph * graph);
+    LM_GGML_API void                 lm_ggml_backend_sched_synchronize(lm_ggml_backend_sched_t sched);
+
+    // Reset all assignments and allocators - must be called before changing the node backends
+    LM_GGML_API void                 lm_ggml_backend_sched_reset(lm_ggml_backend_sched_t sched);
+
+    // Set a callback to be called for each resulting node during graph compute
+    LM_GGML_API void                 lm_ggml_backend_sched_set_eval_callback(lm_ggml_backend_sched_t sched, lm_ggml_backend_sched_eval_callback callback, void * user_data);
+
+    //
+    // Utils
+    //
+
+    struct lm_ggml_backend_graph_copy {
+        lm_ggml_backend_buffer_t buffer;
+        struct lm_ggml_context * ctx_allocated;
+        struct lm_ggml_context * ctx_unallocated;
+        struct lm_ggml_cgraph * graph;
+    };
+
+    // Copy a graph to a different backend
+    LM_GGML_API struct lm_ggml_backend_graph_copy lm_ggml_backend_graph_copy(lm_ggml_backend_t backend, struct lm_ggml_cgraph * graph);
+    LM_GGML_API void                           lm_ggml_backend_graph_copy_free(struct lm_ggml_backend_graph_copy copy);
+
+    typedef bool (*lm_ggml_backend_eval_callback)(int node_index, struct lm_ggml_tensor * t1, struct lm_ggml_tensor * t2, void * user_data);
+
+    // Compare the output of two backends
+    LM_GGML_API bool lm_ggml_backend_compare_graph_backend(lm_ggml_backend_t backend1, lm_ggml_backend_t backend2, struct lm_ggml_cgraph * graph, lm_ggml_backend_eval_callback callback, void * user_data);
+
+    // Tensor initialization
+    LM_GGML_API void lm_ggml_backend_tensor_alloc(lm_ggml_backend_buffer_t buffer, struct lm_ggml_tensor * tensor, void * addr);
+    LM_GGML_API void lm_ggml_backend_view_init(struct lm_ggml_tensor * tensor);
+
+    // CPU buffer types are always available
+    LM_GGML_API lm_ggml_backend_buffer_t      lm_ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size);
+    LM_GGML_API lm_ggml_backend_buffer_type_t lm_ggml_backend_cpu_buffer_type(void);
+
+#ifdef  __cplusplus
+}
+#endif