cui-llama.rn 1.2.0 → 1.2.2

package/README.md

@@ -12,6 +12,8 @@ The following features have been added for Android:
 - tokenizeSync: non-blocking, synchronous tokenizer function
 - Context Shift taken from [kobold.cpp](https://github.com/LostRuins/koboldcpp)
 - XTC sampling
+- Progress callback
+- Retrieving CPU Features to check for i8mm and dotprod flags
 
 Original repo README.md below.
 

package/android/src/main/CMakeLists.txt

@@ -16,7 +16,7 @@ set(
 
     ${RNLLAMA_LIB_DIR}/ggml-aarch64.c
     ${RNLLAMA_LIB_DIR}/ggml-alloc.c
-    ${RNLLAMA_LIB_DIR}/ggml-backend.c
+    ${RNLLAMA_LIB_DIR}/ggml-backend.cpp
     ${RNLLAMA_LIB_DIR}/ggml.c
     ${RNLLAMA_LIB_DIR}/ggml-quants.c
     ${RNLLAMA_LIB_DIR}/common.cpp
@@ -55,7 +55,7 @@ function(build_library target_name cpu_flags)
     # NOTE: If you want to debug the native code, you can uncomment if and endif
     # Note that it will be extremely slow
     # if (NOT ${CMAKE_BUILD_TYPE} STREQUAL "Debug")
-        target_compile_options(${target_name} PRIVATE -O3 -DNDEBUG)
+        target_compile_options(${target_name} PRIVATE -O3 -DNDEBUG -DRNLLAMA_USE_FD_FILE)
         target_compile_options(${target_name} PRIVATE -fvisibility=hidden -fvisibility-inlines-hidden)
         target_compile_options(${target_name} PRIVATE -ffunction-sections -fdata-sections)
 

package/android/src/main/java/com/rnllama/LlamaContext.java

@@ -10,6 +10,8 @@ import com.facebook.react.modules.core.DeviceEventManagerModule;
 
 import android.util.Log;
 import android.os.Build;
+import android.os.ParcelFileDescriptor;
+import android.net.Uri;
 import android.content.res.AssetManager;
 
 import java.lang.StringBuilder;
@@ -17,6 +19,7 @@ import java.io.BufferedReader;
 import java.io.FileReader;
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.FileInputStream;
 
 public class LlamaContext {
@@ -31,11 +34,17 @@ public class LlamaContext {
 
   private byte[] ggufHeader = {0x47, 0x47, 0x55, 0x46};
 
-  private boolean isGGUF(final String filepath) {
+  private boolean isGGUF(final String filepath, final ReactApplicationContext reactContext) {
     byte[] fileHeader = new byte[4];
-    FileInputStream fis = null;
+    InputStream fis = null;
     try {
-      fis = new FileInputStream(filepath);
+      if (filepath.startsWith("content")) {
+        fis = reactContext.getApplicationContext().getContentResolver().openInputStream(Uri.parse(filepath));
+      } else {
+        fis = new FileInputStream(filepath);
+      }
+      
+
       int bytesRead = fis.read(fileHeader);
       if(bytesRead < 4) {
         return false;
@@ -52,7 +61,7 @@ public class LlamaContext {
           try {
               fis.close();
           } catch (Exception e) {
-              Log.d(NAME, "Closing FileInputStream failed.");
+              Log.d(NAME, "Closing InputStream failed.");
           }
       }
     }
@@ -65,16 +74,29 @@ public class LlamaContext {
     if (!params.hasKey("model")) {
       throw new IllegalArgumentException("Missing required parameter: model");
     }
-    // Check if file has GGUF magic numbers
-    if(!isGGUF(params.getString("model"))) {
+
+    String modelName = params.getString("model");
+
+    if(!isGGUF(modelName, reactContext)) {
       throw new IllegalArgumentException("File is not in GGUF format");
     }
 
+    if ( modelName.startsWith("content://")) {
+    Uri uri = Uri.parse(modelName);
+      try {
+        ParcelFileDescriptor pfd = reactContext.getApplicationContext().getContentResolver().openFileDescriptor(uri, "r");
+        modelName = "" + pfd.getFd();
+      } catch (Exception e) {
+        Log.e(NAME, "Failed to convert to FD!");
+      }
+    }
+    
+    // Check if file has GGUF magic numbers
     this.id = id;
     eventEmitter = reactContext.getJSModule(DeviceEventManagerModule.RCTDeviceEventEmitter.class);
     this.context = initContext(
       // String model,
-      params.getString("model"),
+      modelName,
       // boolean embedding,
       params.hasKey("embedding") ? params.getBoolean("embedding") : false,
       // int n_ctx,
@@ -325,7 +347,7 @@ public class LlamaContext {
     }
   }
 
-  private static boolean isArm64V8a() {
+  public static boolean isArm64V8a() {
     return Build.SUPPORTED_ABIS[0].equals("arm64-v8a");
   }
 
@@ -333,7 +355,7 @@ public class LlamaContext {
     return Build.SUPPORTED_ABIS[0].equals("x86_64");
   }
 
-  private static String getCpuFeatures() {
+  public static String getCpuFeatures() {
     File file = new File("/proc/cpuinfo");
     StringBuilder stringBuilder = new StringBuilder();
     try {

package/android/src/main/java/com/rnllama/RNLlama.java

@@ -294,7 +294,46 @@ public class RNLlama implements LifecycleEventListener {
     return context.tokenize(text);
   }
 
+  public void getCpuFeatures(Promise promise) {
+    AsyncTask task = new AsyncTask<Void, Void, WritableMap>() {
+      private Exception exception;
+      @Override
+      protected WritableMap doInBackground(Void... voids) {
+        try {
+          WritableMap result = Arguments.createMap();
+          boolean isV8 = LlamaContext.isArm64V8a();
+          result.putBoolean("armv8", isV8);
+          
+          if(isV8) {
+            String cpuFeatures = LlamaContext.getCpuFeatures();
+            boolean hasDotProd = cpuFeatures.contains("dotprod") || cpuFeatures.contains("asimddp");
+            boolean hasInt8Matmul = cpuFeatures.contains("i8mm");
+            result.putBoolean("i8mm", hasInt8Matmul);
+            result.putBoolean("dotprod", hasDotProd);
+          } else {
+            result.putBoolean("i8mm", false);
+            result.putBoolean("dotprod", false);
+          }
+          return result;
+        } catch (Exception e) {
+          exception = e;
+          return null;
+        }
+      }
 
+      @Override
+      protected void onPostExecute(WritableMap result) {
+        if (exception != null) {
+          promise.reject(exception);
+          return;
+        }
+        promise.resolve(result);
+        tasks.remove(this);
+      }
+    }.executeOnExecutor(AsyncTask.THREAD_POOL_EXECUTOR);
+    tasks.put(task, "getCPUFeatures");
+  }
+  
   public void detokenize(double id, final ReadableArray tokens, final Promise promise) {
     final int contextId = (int) id;
     AsyncTask task = new AsyncTask<Void, Void, String>() {

package/android/src/newarch/java/com/rnllama/RNLlamaModule.java

@@ -78,6 +78,11 @@ public class RNLlamaModule extends NativeRNLlamaSpec {
     return rnllama.tokenizeSync(id, text);
   }
 
+  @ReactMethod
+  public void getCpuFeatures(final Promise promise) {
+    rnllama.getCpuFeatures(promise);
+  }
+
   @ReactMethod
   public void detokenize(double id, final ReadableArray tokens, final Promise promise) {
     rnllama.detokenize(id, tokens, promise);

package/android/src/oldarch/java/com/rnllama/RNLlamaModule.java

@@ -79,6 +79,11 @@ public class RNLlamaModule extends ReactContextBaseJavaModule {
     return rnllama.tokenizeSync(id, text);
   }
 
+  @ReactMethod
+  public void getCpuFeatures(final Promise promise) {
+    rnllama.getCpuFeatures(promise);
+  }
+
   @ReactMethod
   public void detokenize(double id, final ReadableArray tokens, final Promise promise) {  
     rnllama.detokenize(id, tokens, promise);

package/cpp/common.cpp

@@ -844,6 +844,31 @@ struct llama_init_result llama_init_from_gpt_params(gpt_params & params) {
         return iparams;
     }
 
+    if (params.reranking) {
+        bool ok = true;
+
+        if (llama_token_bos(model) == LLAMA_TOKEN_NULL) {
+            LOG_WRN("%s: warning: model does not have a  BOS token, reranking will not work\n", __func__);
+            ok = false;
+        }
+
+        if (llama_token_eos(model) == LLAMA_TOKEN_NULL) {
+            LOG_WRN("%s: warning: model does not have an EOS token, reranking will not work\n", __func__);
+            ok = false;
+        }
+
+        if (llama_token_sep(model) == LLAMA_TOKEN_NULL) {
+            LOG_WRN("%s: warning: model does not have a  SEP token, reranking will not work\n", __func__);
+            ok = false;
+        }
+
+        if (!ok) {
+            llama_free_model(model);
+
+            return iparams;
+        }
+    }
+
     auto cparams = llama_context_params_from_gpt_params(params);
 
     llama_context * lctx = llama_new_context_with_model(model, cparams);
@@ -861,6 +886,7 @@ struct llama_init_result llama_init_from_gpt_params(gpt_params & params) {
         if (cvec.n_embd == -1) {
             llama_free(lctx);
             llama_free_model(model);
+
             return iparams;
         }
 
@@ -873,6 +899,7 @@ struct llama_init_result llama_init_from_gpt_params(gpt_params & params) {
         if (err) {
             llama_free(lctx);
             llama_free_model(model);
+
             return iparams;
         }
     }
@@ -895,7 +922,7 @@ struct llama_init_result llama_init_from_gpt_params(gpt_params & params) {
         llama_lora_adapters_apply(lctx, iparams.lora_adapters);
     }
 
-    if (params.sparams.ignore_eos && llama_token_eos(model) == -1) {
+    if (params.sparams.ignore_eos && llama_token_eos(model) == LLAMA_TOKEN_NULL) {
         LOG_WRN("%s: warning: model does not have an EOS token, ignoring --ignore-eos\n", __func__);
         params.sparams.ignore_eos = false;
     }
@@ -936,6 +963,7 @@ struct llama_init_result llama_init_from_gpt_params(gpt_params & params) {
 
     iparams.model   = model;
     iparams.context = lctx;
+
     return iparams;
 }
 
@@ -1033,6 +1061,11 @@ struct llama_context_params llama_context_params_from_gpt_params(const gpt_param
     cparams.flash_attn        = params.flash_attn;
     cparams.no_perf           = params.no_perf;
 
+    if (params.reranking) {
+        cparams.embeddings    = true;
+        cparams.pooling_type  = LLAMA_POOLING_TYPE_RANK;
+    }
+
     cparams.type_k = kv_cache_type_from_str(params.cache_type_k);
     cparams.type_v = kv_cache_type_from_str(params.cache_type_v);
 
@@ -1442,6 +1475,8 @@ void llama_batch_add(
                           llama_pos   pos,
     const std::vector<llama_seq_id> & seq_ids,
                                bool   logits) {
+    LM_GGML_ASSERT(batch.seq_id[batch.n_tokens] && "llama_batch size exceeded");
+
     batch.token   [batch.n_tokens] = id;
     batch.pos     [batch.n_tokens] = pos;
     batch.n_seq_id[batch.n_tokens] = seq_ids.size();

package/cpp/common.h

@@ -290,6 +290,7 @@ struct gpt_params {
     int32_t embd_normalize = 2;     // normalisation for embendings (-1=none, 0=max absolute int16, 1=taxicab, 2=euclidean, >2=p-norm)
     std::string embd_out   = "";    // empty = default, "array" = [[],[]...], "json" = openai style, "json+" = same "json" + cosine similarity matrix
     std::string embd_sep   = "\n";  // separator of embendings
+    bool reranking         = false; // enable reranking support on server
 
     // server params
     int32_t port           = 8080;         // server listens on this network port
@@ -308,7 +309,10 @@ struct gpt_params {
     std::string ssl_file_key  = "";                                                                         // NOLINT
     std::string ssl_file_cert = "";                                                                         // NOLINT
 
-    bool endpoint_slots   = true;
+    // "advanced" endpoints are disabled by default for better security
+    bool webui            = true;
+    bool endpoint_slots   = false;
+    bool endpoint_props   = false; // only control POST requests, not GET
     bool endpoint_metrics = false;
 
     bool log_json = false;

package/cpp/ggml-aarch64.c

@@ -598,15 +598,6 @@ size_t quantize_q4_0_8x8(const float * restrict src, void * restrict dst, int64_
     return quantize_q4_0_nr_bl(src, dst, nrow, n_per_row, 8, 8);
 }
 
-// Return the number of byte lanes in the SVE vector if SVE is supported; otherwise, returns 0 if SVE is not supported.
-static int sve_lane_count(void) {
-#if defined(__ARM_FEATURE_SVE)
-    return lm_ggml_sve_cnt_b;
-#else
-    return 0;
-#endif
-}
-
 void lm_ggml_gemv_q4_0_4x4_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, const void * restrict vy, int nr, int nc) {
     const int qk = QK8_0;
     const int nb = n / qk;
@@ -843,7 +834,7 @@ void lm_ggml_gemv_q4_0_8x8_q8_0(int n, float * restrict s, size_t bs, const void
 
 #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__)
 #if defined(__ARM_FEATURE_SVE)
-    if (lm_ggml_cpu_has_sve() && sve_lane_count() == QK8_0) {
+    if (lm_ggml_cpu_has_sve() && lm_ggml_cpu_get_sve_cnt() == QK8_0) {
         const void * b_ptr = vx;
         const void * a_ptr = vy;
         float * res_ptr = s;
@@ -2020,7 +2011,7 @@ void lm_ggml_gemm_q4_0_8x8_q8_0(int n, float * restrict s, size_t bs, const void
 
 #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__)
 #if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8)
-    if (lm_ggml_cpu_has_sve() && lm_ggml_cpu_has_matmul_int8() && sve_lane_count() == QK8_0) {
+    if (lm_ggml_cpu_has_sve() && lm_ggml_cpu_has_matmul_int8() && lm_ggml_cpu_get_sve_cnt() == QK8_0) {
         const void * b_ptr = vx;
         const void * a_ptr = vy;
         float * res_ptr = s;

package/cpp/ggml-alloc.h

@@ -24,7 +24,7 @@ LM_GGML_API void                lm_ggml_tallocr_alloc(struct lm_ggml_tallocr * t
 // Graph allocator
 /*
   Example usage:
-    lm_ggml_gallocr_t galloc = lm_ggml_gallocr_new(lm_ggml_bacckend_cpu_buffer_type());
+    lm_ggml_gallocr_t galloc = lm_ggml_gallocr_new(lm_ggml_backend_cpu_buffer_type());
 
     // optional: create a worst-case graph and reserve the buffers to avoid reallocations
     lm_ggml_gallocr_reserve(galloc, build_graph(max_batch));

package/cpp/ggml-backend-impl.h

@@ -9,145 +9,218 @@ extern "C" {
 #endif
 
     //
-    // Backend buffer
+    // Backend buffer type
     //
 
-    // buffer type
-    typedef void * lm_ggml_backend_buffer_type_context_t;
-
     struct lm_ggml_backend_buffer_type_i {
-        const char *          (*LM_GGML_CALL get_name)        (lm_ggml_backend_buffer_type_t buft);
+        const char *          (*get_name)      (lm_ggml_backend_buffer_type_t buft);
         // allocate a buffer of this type
-        lm_ggml_backend_buffer_t (*LM_GGML_CALL alloc_buffer)    (lm_ggml_backend_buffer_type_t buft, size_t size);
+        lm_ggml_backend_buffer_t (*alloc_buffer)  (lm_ggml_backend_buffer_type_t buft, size_t size);
         // tensor alignment
-        size_t                (*LM_GGML_CALL get_alignment)   (lm_ggml_backend_buffer_type_t buft);
-        // max buffer size that can be allocated
-        size_t                (*LM_GGML_CALL get_max_size)    (lm_ggml_backend_buffer_type_t buft);
-        // data size needed to allocate the tensor, including padding
-        size_t                (*LM_GGML_CALL get_alloc_size)  (lm_ggml_backend_buffer_type_t buft, const struct lm_ggml_tensor * tensor);
-        // check if tensor data is in host memory
-        bool                  (*LM_GGML_CALL is_host)         (lm_ggml_backend_buffer_type_t buft);
+        size_t                (*get_alignment) (lm_ggml_backend_buffer_type_t buft);
+        // (optional) max buffer size that can be allocated (defaults to SIZE_MAX)
+        size_t                (*get_max_size)  (lm_ggml_backend_buffer_type_t buft);
+        // (optional) data size needed to allocate the tensor, including padding (defaults to lm_ggml_nbytes)
+        size_t                (*get_alloc_size)(lm_ggml_backend_buffer_type_t buft, const struct lm_ggml_tensor * tensor);
+        // (optional) check if tensor data is in host memory (defaults to false)
+        bool                  (*is_host)       (lm_ggml_backend_buffer_type_t buft);
     };
 
     struct lm_ggml_backend_buffer_type {
         struct lm_ggml_backend_buffer_type_i  iface;
-        lm_ggml_backend_buffer_type_context_t context;
+        lm_ggml_backend_dev_t device;
+        void * context;
     };
 
-    // buffer
-    typedef void * lm_ggml_backend_buffer_context_t;
+    //
+    // Backend buffer
+    //
 
     struct lm_ggml_backend_buffer_i {
-        const char * (*LM_GGML_CALL get_name)      (lm_ggml_backend_buffer_t buffer);
-        void         (*LM_GGML_CALL free_buffer)   (lm_ggml_backend_buffer_t buffer);
-        void *       (*LM_GGML_CALL get_base)      (lm_ggml_backend_buffer_t buffer);
-        void         (*LM_GGML_CALL init_tensor)   (lm_ggml_backend_buffer_t buffer, struct lm_ggml_tensor * tensor);
-        void         (*LM_GGML_CALL memset_tensor) (lm_ggml_backend_buffer_t buffer,       struct lm_ggml_tensor * tensor,     uint8_t value, size_t offset, size_t size);
-        void         (*LM_GGML_CALL set_tensor)    (lm_ggml_backend_buffer_t buffer,       struct lm_ggml_tensor * tensor, const void * data, size_t offset, size_t size);
-        void         (*LM_GGML_CALL get_tensor)    (lm_ggml_backend_buffer_t buffer, const struct lm_ggml_tensor * tensor,       void * data, size_t offset, size_t size);
-        bool         (*LM_GGML_CALL cpy_tensor)    (lm_ggml_backend_buffer_t buffer, const struct lm_ggml_tensor * src, struct lm_ggml_tensor * dst); // dst is in the buffer, src may be in any buffer
-        void         (*LM_GGML_CALL clear)         (lm_ggml_backend_buffer_t buffer, uint8_t value);
-        void         (*LM_GGML_CALL reset)         (lm_ggml_backend_buffer_t buffer); // reset any internal state due to tensor initialization, such as tensor extras
+        const char * (*get_name)     (lm_ggml_backend_buffer_t buffer);
+        // (optional) free the buffer
+        void         (*free_buffer)  (lm_ggml_backend_buffer_t buffer);
+        // base address of the buffer
+        void *       (*get_base)     (lm_ggml_backend_buffer_t buffer);
+        // (optional) initialize a tensor in the buffer (eg. add tensor extras)
+        void         (*init_tensor)  (lm_ggml_backend_buffer_t buffer, struct lm_ggml_tensor * tensor);
+        // tensor data access
+        void         (*memset_tensor)(lm_ggml_backend_buffer_t buffer,       struct lm_ggml_tensor * tensor,     uint8_t value, size_t offset, size_t size);
+        void         (*set_tensor)   (lm_ggml_backend_buffer_t buffer,       struct lm_ggml_tensor * tensor, const void * data, size_t offset, size_t size);
+        void         (*get_tensor)   (lm_ggml_backend_buffer_t buffer, const struct lm_ggml_tensor * tensor,       void * data, size_t offset, size_t size);
+        // (optional) tensor copy: dst is in the buffer, src may be in any buffer, including buffers from a different backend (return false if not supported)
+        bool         (*cpy_tensor)   (lm_ggml_backend_buffer_t buffer, const struct lm_ggml_tensor * src, struct lm_ggml_tensor * dst);
+        // clear the entire buffer
+        void         (*clear)        (lm_ggml_backend_buffer_t buffer, uint8_t value);
+        // (optional) reset any internal state due to tensor initialization, such as tensor extras
+        void         (*reset)        (lm_ggml_backend_buffer_t buffer);
     };
 
     struct lm_ggml_backend_buffer {
         struct lm_ggml_backend_buffer_i  iface;
         lm_ggml_backend_buffer_type_t    buft;
-        lm_ggml_backend_buffer_context_t context;
+        void * context;
         size_t size;
         enum lm_ggml_backend_buffer_usage usage;
     };
 
-    LM_GGML_CALL lm_ggml_backend_buffer_t lm_ggml_backend_buffer_init(
-                   lm_ggml_backend_buffer_type_t      buft,
-            struct lm_ggml_backend_buffer_i           iface,
-                   lm_ggml_backend_buffer_context_t   context,
-                   size_t                          size);
+    lm_ggml_backend_buffer_t lm_ggml_backend_buffer_init(
+                   lm_ggml_backend_buffer_type_t buft,
+            struct lm_ggml_backend_buffer_i      iface,
+                   void *                     context,
+                   size_t                     size);
 
     // do not use directly, use lm_ggml_backend_tensor_copy instead
     bool lm_ggml_backend_buffer_copy_tensor(const struct lm_ggml_tensor * src, struct lm_ggml_tensor * dst);
 
+    // multi-buffer
     // buffer that contains a collection of buffers
-    LM_GGML_CALL lm_ggml_backend_buffer_t lm_ggml_backend_multi_buffer_alloc_buffer(lm_ggml_backend_buffer_t * buffers, size_t n_buffers);
-    LM_GGML_CALL bool                  lm_ggml_backend_buffer_is_multi_buffer(lm_ggml_backend_buffer_t buffer);
-    LM_GGML_CALL void                  lm_ggml_backend_multi_buffer_set_usage(lm_ggml_backend_buffer_t buffer, enum lm_ggml_backend_buffer_usage usage);
+    lm_ggml_backend_buffer_t lm_ggml_backend_multi_buffer_alloc_buffer(lm_ggml_backend_buffer_t * buffers, size_t n_buffers);
+    bool                  lm_ggml_backend_buffer_is_multi_buffer(lm_ggml_backend_buffer_t buffer);
+    void                  lm_ggml_backend_multi_buffer_set_usage(lm_ggml_backend_buffer_t buffer, enum lm_ggml_backend_buffer_usage usage);
 
     //
-    // Backend
+    // Backend (stream)
     //
 
-    typedef void * lm_ggml_backend_context_t;
-
     struct lm_ggml_backend_i {
-        const char * (*LM_GGML_CALL get_name)(lm_ggml_backend_t backend);
+        const char * (*get_name)(lm_ggml_backend_t backend);
 
-        void (*LM_GGML_CALL free)(lm_ggml_backend_t backend);
+        void (*free)(lm_ggml_backend_t backend);
 
+        // Will be moved to the device interface
         // buffer allocation
-        lm_ggml_backend_buffer_type_t (*LM_GGML_CALL get_default_buffer_type)(lm_ggml_backend_t backend);
+        lm_ggml_backend_buffer_type_t (*get_default_buffer_type)(lm_ggml_backend_t backend);
 
         // (optional) asynchronous tensor data access
-        void (*LM_GGML_CALL set_tensor_async)(lm_ggml_backend_t backend,       struct lm_ggml_tensor * tensor, const void * data, size_t offset, size_t size);
-        void (*LM_GGML_CALL get_tensor_async)(lm_ggml_backend_t backend, const struct lm_ggml_tensor * tensor,       void * data, size_t offset, size_t size);
-        bool (*LM_GGML_CALL cpy_tensor_async)(lm_ggml_backend_t backend_src, lm_ggml_backend_t backend_dst, const struct lm_ggml_tensor * src, struct lm_ggml_tensor * dst);
+        void (*set_tensor_async)(lm_ggml_backend_t backend,       struct lm_ggml_tensor * tensor, const void * data, size_t offset, size_t size);
+        void (*get_tensor_async)(lm_ggml_backend_t backend, const struct lm_ggml_tensor * tensor,       void * data, size_t offset, size_t size);
+        bool (*cpy_tensor_async)(lm_ggml_backend_t backend_src, lm_ggml_backend_t backend_dst, const struct lm_ggml_tensor * src, struct lm_ggml_tensor * dst);
 
         // (optional) complete all pending operations
-        void (*LM_GGML_CALL synchronize)(lm_ggml_backend_t backend);
+        void (*synchronize)(lm_ggml_backend_t backend);
 
-        // compute graph with a plan (not used currently)
-        // create a new plan for a graph
-        lm_ggml_backend_graph_plan_t (*LM_GGML_CALL graph_plan_create) (lm_ggml_backend_t backend, const struct lm_ggml_cgraph * cgraph);
-        void                      (*LM_GGML_CALL graph_plan_free)   (lm_ggml_backend_t backend, lm_ggml_backend_graph_plan_t plan);
+        // (optional) compute graph with a plan (not used currently)
+        lm_ggml_backend_graph_plan_t (*graph_plan_create) (lm_ggml_backend_t backend, const struct lm_ggml_cgraph * cgraph);
+        void                      (*graph_plan_free)   (lm_ggml_backend_t backend, lm_ggml_backend_graph_plan_t plan);
         // update the plan with a new graph - this should be faster than creating a new plan when the graph has the same topology
-        void                      (*LM_GGML_CALL graph_plan_update) (lm_ggml_backend_t backend, lm_ggml_backend_graph_plan_t plan, const struct lm_ggml_cgraph * cgraph);
+        void                      (*graph_plan_update) (lm_ggml_backend_t backend, lm_ggml_backend_graph_plan_t plan, const struct lm_ggml_cgraph * cgraph);
         // compute the graph with the plan
-        enum lm_ggml_status          (*LM_GGML_CALL graph_plan_compute)(lm_ggml_backend_t backend, lm_ggml_backend_graph_plan_t plan);
-
-        // compute graph without a plan (async)
-        enum lm_ggml_status (*LM_GGML_CALL graph_compute)     (lm_ggml_backend_t backend, struct lm_ggml_cgraph * cgraph);
-
-        // check if the backend can compute an operation
-        bool (*LM_GGML_CALL supports_op)(lm_ggml_backend_t backend, const struct lm_ggml_tensor * op);
+        enum lm_ggml_status          (*graph_plan_compute)(lm_ggml_backend_t backend, lm_ggml_backend_graph_plan_t plan);
 
-        // check if the backend can use tensors allocated in a buffer type
-        bool (*LM_GGML_CALL supports_buft)(lm_ggml_backend_t backend, lm_ggml_backend_buffer_type_t buft);
+        // compute graph (always async if supported by the backend)
+        enum lm_ggml_status          (*graph_compute)     (lm_ggml_backend_t backend, struct lm_ggml_cgraph * cgraph);
 
-        // check if the backend wants to run an operation, even if the weights are allocated in a CPU buffer
-        // these should be expensive operations with large batch sizes that may benefit from running on this backend
-        // even if the weight has to be copied from the CPU temporarily
-        bool (*LM_GGML_CALL offload_op)(lm_ggml_backend_t backend, const struct lm_ggml_tensor * op);
+        // IMPORTANT: these functions have been moved to the device interface and will be removed from the backend interface
+        //            new backends should implement the device interface instead
+        // These functions are being moved to the device interface
+        bool (*supports_op)  (lm_ggml_backend_t backend, const struct lm_ggml_tensor * op);
+        bool (*supports_buft)(lm_ggml_backend_t backend, lm_ggml_backend_buffer_type_t buft);
+        bool (*offload_op)   (lm_ggml_backend_t backend, const struct lm_ggml_tensor * op);
 
         // (optional) event synchronization
-        // create a new event that can record events on this backend instance
-        lm_ggml_backend_event_t (*LM_GGML_CALL event_new)         (lm_ggml_backend_t backend);
-        void                 (*LM_GGML_CALL event_free)        (lm_ggml_backend_event_t event);
-        // record an event on the backend instance that created it
-        void                 (*LM_GGML_CALL event_record)      (lm_ggml_backend_event_t event);
-        // wait for an event on on a different backend instance
-        void                 (*LM_GGML_CALL event_wait)        (lm_ggml_backend_t backend, lm_ggml_backend_event_t event);
-        // block until an event is recorded
-        void                 (*LM_GGML_CALL event_synchronize) (lm_ggml_backend_event_t event);
+        // record an event on this stream
+        void (*event_record)(lm_ggml_backend_t backend, lm_ggml_backend_event_t event);
+        // wait for an event on on a different stream
+        void (*event_wait)  (lm_ggml_backend_t backend, lm_ggml_backend_event_t event);
     };
 
     struct lm_ggml_backend {
         lm_ggml_guid_t guid;
-
         struct lm_ggml_backend_i iface;
-        lm_ggml_backend_context_t context;
+        lm_ggml_backend_dev_t device;
+        void * context;
     };
 
     struct lm_ggml_backend_event {
-        lm_ggml_backend_t backend;
+        struct lm_ggml_backend_device * device;
+        void * context;
+    };
+
+    //
+    // Backend device
+    //
+
+    // Note: if additional properties are needed, we should add a struct with all of them
+    //       the current functions to obtain the properties can remain, since they are more convenient for often used properties
+    struct lm_ggml_backend_device_i {
+        // device name: short identifier for this device, such as "CPU" or "CUDA0"
+        const char * (*get_name)(lm_ggml_backend_dev_t dev);
+
+        // device description: short informative description of the device, could be the model name
+        const char * (*get_description)(lm_ggml_backend_dev_t dev);
+
+        // device memory in bytes
+        void         (*get_memory)(lm_ggml_backend_dev_t dev, size_t * free, size_t * total);
+
+        // device type
+        enum lm_ggml_backend_dev_type (*get_type)(lm_ggml_backend_dev_t dev);
+
+        // device properties
+        void (*get_props)(lm_ggml_backend_dev_t dev, struct lm_ggml_backend_dev_props * props);
+
+        // backend (stream) initialization
+        lm_ggml_backend_t (*init_backend)(lm_ggml_backend_dev_t dev, const char * params);
+
+        // preferred buffer type
+        lm_ggml_backend_buffer_type_t (*get_buffer_type)(lm_ggml_backend_dev_t dev);
+
+        // (optional) host buffer type (in system memory, typically this is a pinned memory buffer for faster transfers between host and device)
+        lm_ggml_backend_buffer_type_t (*get_host_buffer_type)(lm_ggml_backend_dev_t dev);
+
+        // (optional) buffer from pointer: create a buffer from a host pointer (useful for memory mapped models and importing data from other libraries)
+        lm_ggml_backend_buffer_t (*buffer_from_host_ptr)(lm_ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size);
+
+        // check if the backend can compute an operation
+        bool (*supports_op)(lm_ggml_backend_dev_t dev, const struct lm_ggml_tensor * op);
+
+        // check if the backend can use tensors allocated in a buffer type
+        bool (*supports_buft)(lm_ggml_backend_dev_t dev, lm_ggml_backend_buffer_type_t buft);
+
+        // (optional) check if the backend wants to run an operation, even if the weights are allocated in an incompatible buffer
+        // these should be expensive operations that may benefit from running on this backend instead of the CPU backend
+        bool (*offload_op)(lm_ggml_backend_dev_t dev, const struct lm_ggml_tensor * op);
+
+        // (optional) event synchronization
+        lm_ggml_backend_event_t (*event_new)         (lm_ggml_backend_dev_t dev);
+        void                 (*event_free)        (lm_ggml_backend_dev_t dev, lm_ggml_backend_event_t event);
+        void                 (*event_synchronize) (lm_ggml_backend_dev_t dev, lm_ggml_backend_event_t event);
+    };
+
+    struct lm_ggml_backend_device {
+        struct lm_ggml_backend_device_i iface;
+        lm_ggml_backend_reg_t reg;
         void * context;
     };
 
     //
-    // Backend registry
+    // Backend (reg)
     //
 
-    typedef lm_ggml_backend_t (*LM_GGML_CALL lm_ggml_backend_init_fn)(const char * params, void * user_data);
+    struct lm_ggml_backend_reg_i {
+        const char * (*get_name)(lm_ggml_backend_reg_t reg);
+
+        // enumerate available devices
+        size_t             (*get_device_count)(lm_ggml_backend_reg_t reg);
+        lm_ggml_backend_dev_t (*get_device)(lm_ggml_backend_reg_t reg, size_t index);
+
+        // (optional) get a pointer to a function in the backend
+        // backends can add custom functions that are not part of the standard ggml-backend interface
+        void * (*get_proc_address)(lm_ggml_backend_reg_t reg, const char * name);
+    };
+
+    struct lm_ggml_backend_reg {
+        // int api_version; // TODO: for dynamic loading
+        struct lm_ggml_backend_reg_i iface;
+        void * context;
+    };
+
 
-    LM_GGML_CALL void lm_ggml_backend_register(const char * name, lm_ggml_backend_init_fn init_fn, lm_ggml_backend_buffer_type_t default_buffer_type, void * user_data);
+    // Internal backend registry API
+    void lm_ggml_backend_register(lm_ggml_backend_reg_t reg);
+    void lm_ggml_backend_device_register(lm_ggml_backend_dev_t device);
+    // TODO: backends can be loaded as a dynamic library, in which case it needs to export this function
+    // typedef lm_ggml_backend_register_t * (*lm_ggml_backend_init)(void);
 
 #ifdef  __cplusplus
 }