cui-llama.rn 1.1.6 → 1.2.0

package/cpp/ggml-alloc.c

@@ -294,6 +294,12 @@ static void lm_ggml_dyn_tallocr_reset(struct lm_ggml_dyn_tallocr * alloc) {
     alloc->free_blocks[0].offset = 0;
     alloc->free_blocks[0].size = SIZE_MAX/2; // restrict maximum size of a measure allocator to half size_t max to avoid overflows
     alloc->max_size = 0;
+
+#ifdef LM_GGML_ALLOCATOR_DEBUG
+    for (int i = 0; i < 1024; i++) {
+        alloc->allocated_tensors[i].tensor = NULL;
+    }
+#endif
 }
 
 static struct lm_ggml_dyn_tallocr * lm_ggml_dyn_tallocr_new(size_t alignment) {

package/cpp/ggml-backend-impl.h

@@ -38,15 +38,16 @@ extern "C" {
     typedef void * lm_ggml_backend_buffer_context_t;
 
     struct lm_ggml_backend_buffer_i {
-        const char * (*LM_GGML_CALL get_name)   (lm_ggml_backend_buffer_t buffer);
-        void         (*LM_GGML_CALL free_buffer)(lm_ggml_backend_buffer_t buffer);
-        void *       (*LM_GGML_CALL get_base)   (lm_ggml_backend_buffer_t buffer);
-        void         (*LM_GGML_CALL init_tensor)(lm_ggml_backend_buffer_t buffer, struct lm_ggml_tensor * tensor);
-        void         (*LM_GGML_CALL set_tensor) (lm_ggml_backend_buffer_t buffer,       struct lm_ggml_tensor * tensor, const void * data, size_t offset, size_t size);
-        void         (*LM_GGML_CALL get_tensor) (lm_ggml_backend_buffer_t buffer, const struct lm_ggml_tensor * tensor,       void * data, size_t offset, size_t size);
-        bool         (*LM_GGML_CALL cpy_tensor) (lm_ggml_backend_buffer_t buffer, const struct lm_ggml_tensor * src, struct lm_ggml_tensor * dst); // dst is in the buffer, src may be in any buffer
-        void         (*LM_GGML_CALL clear)      (lm_ggml_backend_buffer_t buffer, uint8_t value);
-        void         (*LM_GGML_CALL reset)      (lm_ggml_backend_buffer_t buffer); // reset any internal state due to tensor initialization, such as tensor extras
+        const char * (*LM_GGML_CALL get_name)      (lm_ggml_backend_buffer_t buffer);
+        void         (*LM_GGML_CALL free_buffer)   (lm_ggml_backend_buffer_t buffer);
+        void *       (*LM_GGML_CALL get_base)      (lm_ggml_backend_buffer_t buffer);
+        void         (*LM_GGML_CALL init_tensor)   (lm_ggml_backend_buffer_t buffer, struct lm_ggml_tensor * tensor);
+        void         (*LM_GGML_CALL memset_tensor) (lm_ggml_backend_buffer_t buffer,       struct lm_ggml_tensor * tensor,     uint8_t value, size_t offset, size_t size);
+        void         (*LM_GGML_CALL set_tensor)    (lm_ggml_backend_buffer_t buffer,       struct lm_ggml_tensor * tensor, const void * data, size_t offset, size_t size);
+        void         (*LM_GGML_CALL get_tensor)    (lm_ggml_backend_buffer_t buffer, const struct lm_ggml_tensor * tensor,       void * data, size_t offset, size_t size);
+        bool         (*LM_GGML_CALL cpy_tensor)    (lm_ggml_backend_buffer_t buffer, const struct lm_ggml_tensor * src, struct lm_ggml_tensor * dst); // dst is in the buffer, src may be in any buffer
+        void         (*LM_GGML_CALL clear)         (lm_ggml_backend_buffer_t buffer, uint8_t value);
+        void         (*LM_GGML_CALL reset)         (lm_ggml_backend_buffer_t buffer); // reset any internal state due to tensor initialization, such as tensor extras
     };
 
     struct lm_ggml_backend_buffer {

package/cpp/ggml-backend.c

@@ -246,6 +246,22 @@ LM_GGML_CALL void lm_ggml_backend_tensor_get(const struct lm_ggml_tensor * tenso
     buf->iface.get_tensor(buf, tensor, data, offset, size);
 }
 
+LM_GGML_API LM_GGML_CALL void lm_ggml_backend_tensor_memset(struct lm_ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
+    lm_ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
+
+    LM_GGML_ASSERT(buf != NULL && "tensor buffer not set");
+    LM_GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
+    LM_GGML_ASSERT(offset + size <= lm_ggml_nbytes(tensor) && "tensor write out of bounds");
+
+    if (!size) {
+        return;
+    }
+
+    LM_GGML_ASSERT(buf->iface.memset_tensor != NULL && "memset not supported by backend buffer");
+
+    buf->iface.memset_tensor(buf, tensor, value, offset, size);
+}
+
 void lm_ggml_backend_synchronize(lm_ggml_backend_t backend) {
     if (backend->iface.synchronize == NULL) {
         return;
@@ -569,6 +585,12 @@ LM_GGML_CALL static void lm_ggml_backend_cpu_buffer_free_buffer(lm_ggml_backend_
     free(buffer->context);
 }
 
+LM_GGML_CALL static void lm_ggml_backend_cpu_buffer_memset_tensor(lm_ggml_backend_buffer_t buffer, struct lm_ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
+    memset((char *)tensor->data + offset, value, size);
+
+    LM_GGML_UNUSED(buffer);
+}
+
 LM_GGML_CALL static void lm_ggml_backend_cpu_buffer_set_tensor(lm_ggml_backend_buffer_t buffer, struct lm_ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
     memcpy((char *)tensor->data + offset, data, size);
 
@@ -600,6 +622,7 @@ static struct lm_ggml_backend_buffer_i cpu_backend_buffer_i = {
     /* .free_buffer     = */ lm_ggml_backend_cpu_buffer_free_buffer,
     /* .get_base        = */ lm_ggml_backend_cpu_buffer_get_base,
     /* .init_tensor     = */ NULL, // no initialization required
+    /* .memset_tensor   = */ lm_ggml_backend_cpu_buffer_memset_tensor,
     /* .set_tensor      = */ lm_ggml_backend_cpu_buffer_set_tensor,
     /* .get_tensor      = */ lm_ggml_backend_cpu_buffer_get_tensor,
     /* .cpy_tensor      = */ lm_ggml_backend_cpu_buffer_cpy_tensor,
@@ -613,6 +636,7 @@ static struct lm_ggml_backend_buffer_i cpu_backend_buffer_i_from_ptr = {
     /* .free_buffer     = */ NULL, // ptr is not owned by the buffer, so it does not need to be freed
     /* .get_base        = */ lm_ggml_backend_cpu_buffer_get_base,
     /* .init_tensor     = */ NULL, // no initialization required
+    /* .memset_tensor   = */ lm_ggml_backend_cpu_buffer_memset_tensor,
     /* .set_tensor      = */ lm_ggml_backend_cpu_buffer_set_tensor,
     /* .get_tensor      = */ lm_ggml_backend_cpu_buffer_get_tensor,
     /* .cpy_tensor      = */ lm_ggml_backend_cpu_buffer_cpy_tensor,
@@ -980,6 +1004,7 @@ static struct lm_ggml_backend_buffer_i lm_ggml_backend_multi_buffer_context_inte
         /* .free_buffer     = */ lm_ggml_backend_multi_buffer_free_buffer,
         /* .get_base        = */ NULL,
         /* .init_tensor     = */ NULL,
+        /* .memset_tensor   = */ NULL,
         /* .set_tensor      = */ NULL,
         /* .get_tensor      = */ NULL,
         /* .cpy_tensor      = */ NULL,

package/cpp/ggml-backend.h

@@ -66,6 +66,7 @@ extern "C" {
     // "offset" refers to the offset of the tensor data for setting/getting data
     LM_GGML_API LM_GGML_CALL void lm_ggml_backend_tensor_set(      struct lm_ggml_tensor * tensor, const void * data, size_t offset, size_t size);
     LM_GGML_API LM_GGML_CALL void lm_ggml_backend_tensor_get(const struct lm_ggml_tensor * tensor,       void * data, size_t offset, size_t size);
+    LM_GGML_API LM_GGML_CALL void lm_ggml_backend_tensor_memset(   struct lm_ggml_tensor * tensor,     uint8_t value, size_t offset, size_t size);
 
     LM_GGML_API void lm_ggml_backend_synchronize(lm_ggml_backend_t backend);
 
@@ -122,7 +123,7 @@ extern "C" {
     // The backend registry is a registry of all the available backends, and allows initializing backends in a generic way
 
     LM_GGML_API size_t                     lm_ggml_backend_reg_get_count(void);
-    LM_GGML_API size_t                     lm_ggml_backend_reg_find_by_name(const char * name);
+    LM_GGML_API size_t                     lm_ggml_backend_reg_find_by_name(const char * name); // returns index of backend with name, or SIZE_MAX if not found
     LM_GGML_API lm_ggml_backend_t             lm_ggml_backend_reg_init_backend_from_str(const char * backend_str); // str is backend_name:params (params is optional)
     LM_GGML_API const char *               lm_ggml_backend_reg_get_name(size_t i);
     LM_GGML_API lm_ggml_backend_t             lm_ggml_backend_reg_init_backend(size_t i, const char * params); // params is backend-specific