cui-llama.rn 1.2.2 → 1.2.4

package/cpp/ggml-backend.cpp

@@ -329,7 +329,6 @@ bool lm_ggml_backend_supports_buft(lm_ggml_backend_t backend, lm_ggml_backend_bu
     if (backend->device) {
         return lm_ggml_backend_dev_supports_buft(backend->device, buft);
     }
-
     return backend->iface.supports_buft(backend, buft);
 }
 
@@ -379,7 +378,7 @@ void lm_ggml_backend_tensor_copy(struct lm_ggml_tensor * src, struct lm_ggml_ten
         lm_ggml_backend_tensor_get(src, dst->data, 0, lm_ggml_nbytes(src));
     } else if (!lm_ggml_backend_buffer_copy_tensor(src, dst)) {
 #ifndef NDEBUG
-        fprintf(stderr, "%s: warning: slow copy from %s to %s\n", __func__, lm_ggml_backend_buffer_name(src->buffer), lm_ggml_backend_buffer_name(dst->buffer));
+        LM_GGML_LOG_DEBUG("%s: warning: slow copy from %s to %s\n", __func__, lm_ggml_backend_buffer_name(src->buffer), lm_ggml_backend_buffer_name(dst->buffer));
 #endif
         size_t nbytes = lm_ggml_nbytes(src);
         void * data = malloc(nbytes);
@@ -538,10 +537,30 @@ void * lm_ggml_backend_reg_get_proc_address(lm_ggml_backend_reg_t reg, const cha
 #include "ggml-metal.h"
 #endif
 
+#ifdef LM_GGML_USE_SYCL
+#include "ggml-sycl.h"
+#endif
+
+#ifdef LM_GGML_USE_VULKAN
+#include "ggml-vulkan.h"
+#endif
+
 #ifdef LM_GGML_USE_BLAS
 #include "ggml-blas.h"
 #endif
 
+#ifdef LM_GGML_USE_RPC
+#include "ggml-rpc.h"
+#endif
+
+#ifndef __AMX_INT8__
+#undef LM_GGML_USE_AMX
+#endif
+
+#ifdef LM_GGML_USE_AMX
+#  include "ggml-amx.h"
+#endif
+
 struct lm_ggml_backend_registry {
     std::vector<lm_ggml_backend_reg_t> backends;
     std::vector<lm_ggml_backend_dev_t> devices;
@@ -553,18 +572,30 @@ struct lm_ggml_backend_registry {
 #ifdef LM_GGML_USE_METAL
         register_backend(lm_ggml_backend_metal_reg());
 #endif
+#ifdef LM_GGML_USE_SYCL
+        register_backend(lm_ggml_backend_sycl_reg());
+#endif
+#ifdef LM_GGML_USE_VULKAN
+        register_backend(lm_ggml_backend_vk_reg());
+#endif
 #ifdef LM_GGML_USE_BLAS
         register_backend(lm_ggml_backend_blas_reg());
 #endif
+#ifdef LM_GGML_USE_RPC
+        register_backend(lm_ggml_backend_rpc_reg());
+#endif
+#ifdef LM_GGML_USE_AMX
+        register_backend(lm_ggml_backend_amx_reg());
+#endif
 
-        // TODO: sycl, vulkan, kompute, cann
+        // TODO: kompute, cann
 
         register_backend(lm_ggml_backend_cpu_reg());
     }
 
     void register_backend(lm_ggml_backend_reg_t reg) {
 #ifndef NDEBUG
-        fprintf(stderr, "%s: registered backend %s (%zu devices)\n",
+        LM_GGML_LOG_DEBUG("%s: registered backend %s (%zu devices)\n",
             __func__, lm_ggml_backend_reg_name(reg), lm_ggml_backend_reg_dev_count(reg));
 #endif
         backends.push_back(reg);
@@ -575,7 +606,7 @@ struct lm_ggml_backend_registry {
 
     void register_device(lm_ggml_backend_dev_t device) {
 #ifndef NDEBUG
-        fprintf(stderr, "%s: registered device %s (%s)\n", __func__, lm_ggml_backend_dev_name(device), lm_ggml_backend_dev_description(device));
+        LM_GGML_LOG_DEBUG("%s: registered device %s (%s)\n", __func__, lm_ggml_backend_dev_name(device), lm_ggml_backend_dev_description(device));
 #endif
         devices.push_back(device);
     }
@@ -675,8 +706,6 @@ lm_ggml_backend_t lm_ggml_backend_init_best(void) {
 
 // backend CPU
 
-static const size_t TENSOR_ALIGNMENT = 32; // required for mmap as gguf only guarantees 32-byte alignment
-
 static const char * lm_ggml_backend_cpu_buffer_get_name(lm_ggml_backend_buffer_t buffer) {
     return "CPU";
 
@@ -695,7 +724,7 @@ static void * lm_ggml_backend_cpu_buffer_get_base(lm_ggml_backend_buffer_t buffe
 }
 
 static void lm_ggml_backend_cpu_buffer_free_buffer(lm_ggml_backend_buffer_t buffer) {
-    free(buffer->context);
+    lm_ggml_aligned_free(buffer->context, buffer->size);
 }
 
 static void lm_ggml_backend_cpu_buffer_memset_tensor(lm_ggml_backend_buffer_t buffer, struct lm_ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
@@ -763,14 +792,19 @@ static const char * lm_ggml_backend_cpu_buffer_type_get_name(lm_ggml_backend_buf
 }
 
 static lm_ggml_backend_buffer_t lm_ggml_backend_cpu_buffer_type_alloc_buffer(lm_ggml_backend_buffer_type_t buft, size_t size) {
-    size += TENSOR_ALIGNMENT;   // malloc may return an address that is not aligned
-    void * data = malloc(size); // TODO: use LM_GGML_ALIGNED_MALLOC (move to ggml-impl.h)
+    auto alloc_size = size;
+    if (alloc_size == 0) {
+        alloc_size = 1;
+    }
+
+    void * data = lm_ggml_aligned_malloc(alloc_size);
+
     if (data == NULL) {
-        fprintf(stderr, "%s: failed to allocate buffer of size %zu\n", __func__, size);
+        LM_GGML_LOG_ERROR("%s: failed to allocate buffer of size %zu\n", __func__, alloc_size);
         return NULL;
     }
 
-    return lm_ggml_backend_buffer_init(buft, lm_ggml_backend_cpu_buffer_i, data, size);
+    return lm_ggml_backend_buffer_init(buft, lm_ggml_backend_cpu_buffer_i, data, alloc_size);
 }
 
 static size_t lm_ggml_backend_cpu_buffer_type_get_alignment(lm_ggml_backend_buffer_type_t buft) {
@@ -829,7 +863,7 @@ static lm_ggml_backend_buffer_t lm_ggml_backend_cpu_hbm_buffer_type_alloc_buffer
     void * ptr;
     int result = hbw_posix_memalign(&ptr, lm_ggml_backend_cpu_buffer_type_get_alignment(buft), size);
     if (result != 0) {
-        fprintf(stderr, "failed to allocate HBM buffer of size %zu\n", size);
+        LM_GGML_LOG_ERROR("failed to allocate HBM buffer of size %zu\n", size);
         return NULL;
     }
 
@@ -1452,7 +1486,7 @@ static int lm_ggml_backend_sched_backend_from_buffer(lm_ggml_backend_sched_t sch
     }
 
 #ifndef NDEBUG
-    fprintf(stderr, "%s: warning: no backend supports op %s with a weight with buffer type %s used in tensor %s, the weight will need to be copied\n",
+    LM_GGML_LOG_DEBUG("%s: warning: no backend supports op %s with a weight with buffer type %s used in tensor %s, the weight will need to be copied\n",
         __func__, lm_ggml_op_desc(tensor), lm_ggml_backend_buffer_name(buffer), tensor->name);
 #endif
 
@@ -1541,13 +1575,13 @@ static void lm_ggml_backend_sched_print_assignments(lm_ggml_backend_sched_t sche
     for (int i = 0; i < graph->n_nodes; i++) {
         if (cur_split < sched->n_splits && i == sched->splits[cur_split].i_start) {
             lm_ggml_backend_t split_backend = sched->backends[sched->splits[cur_split].backend_id];
-            fprintf(stderr, "\n## SPLIT #%d: %s # %d inputs: ", cur_split, lm_ggml_backend_name(split_backend),
+            LM_GGML_LOG_DEBUG("\n## SPLIT #%d: %s # %d inputs: ", cur_split, lm_ggml_backend_name(split_backend),
                 sched->splits[cur_split].n_inputs);
             for (int j = 0; j < sched->splits[cur_split].n_inputs; j++) {
-                fprintf(stderr, "[%s (%5.5s)] ", sched->splits[cur_split].inputs[j]->name,
+                LM_GGML_LOG_DEBUG("[%s (%5.5s)] ", sched->splits[cur_split].inputs[j]->name,
                     fmt_size(lm_ggml_nbytes(sched->splits[cur_split].inputs[j])));
             }
-            fprintf(stderr, "\n");
+            LM_GGML_LOG_DEBUG("\n");
             cur_split++;
         }
         struct lm_ggml_tensor * node = graph->nodes[i];
@@ -1555,7 +1589,7 @@ static void lm_ggml_backend_sched_print_assignments(lm_ggml_backend_sched_t sche
             continue;
         }
         lm_ggml_backend_t tensor_backend = lm_ggml_backend_sched_get_tensor_backend(sched, node);
-        fprintf(stderr, "node #%3d (%10.10s): %20.20s (%5.5s) [%5.5s %8.8s]:", i, lm_ggml_op_name(node->op), node->name,
+        LM_GGML_LOG_DEBUG("node #%3d (%10.10s): %20.20s (%5.5s) [%5.5s %8.8s]:", i, lm_ggml_op_name(node->op), node->name,
             fmt_size(lm_ggml_nbytes(node)), tensor_backend ? lm_ggml_backend_name(tensor_backend) : "NULL", GET_CAUSE(node));
         for (int j = 0; j < LM_GGML_MAX_SRC; j++) {
             struct lm_ggml_tensor * src = node->src[j];
@@ -1563,10 +1597,10 @@ static void lm_ggml_backend_sched_print_assignments(lm_ggml_backend_sched_t sche
                 continue;
             }
             lm_ggml_backend_t src_backend = lm_ggml_backend_sched_get_tensor_backend(sched, src);
-            fprintf(stderr, " %20.20s (%5.5s) [%5.5s %8.8s]", src->name,
+            LM_GGML_LOG_DEBUG(" %20.20s (%5.5s) [%5.5s %8.8s]", src->name,
                 fmt_size(lm_ggml_nbytes(src)), src_backend ? lm_ggml_backend_name(src_backend) : "NULL", GET_CAUSE(src));
         }
-        fprintf(stderr, "\n");
+        LM_GGML_LOG_DEBUG("\n");
     }
 }
 
@@ -2080,11 +2114,11 @@ static bool lm_ggml_backend_sched_alloc_splits(lm_ggml_backend_sched_t sched) {
         // the re-allocation may cause the split inputs to be moved to a different address
         lm_ggml_backend_sched_synchronize(sched);
 #ifndef NDEBUG
-        fprintf(stderr, "%s: failed to allocate graph, reserving (backend_ids_changed = %d)\n", __func__, backend_ids_changed);
+        LM_GGML_LOG_DEBUG("%s: failed to allocate graph, reserving (backend_ids_changed = %d)\n", __func__, backend_ids_changed);
 #endif
         lm_ggml_gallocr_reserve_n(sched->galloc, &sched->graph, sched->node_backend_ids, sched->leaf_backend_ids);
         if (!lm_ggml_gallocr_alloc_graph(sched->galloc, &sched->graph)) {
-            fprintf(stderr, "%s: failed to allocate graph\n", __func__);
+            LM_GGML_LOG_ERROR("%s: failed to allocate graph\n", __func__);
             return false;
         }
     }
@@ -2227,6 +2261,7 @@ lm_ggml_backend_sched_t lm_ggml_backend_sched_new(
         sched->backends[b] = backends[b];
         sched->bufts[b] = bufts ? bufts[b] : lm_ggml_backend_get_default_buffer_type(backends[b]);
         LM_GGML_ASSERT(lm_ggml_backend_supports_buft(backends[b], sched->bufts[b]));
+
         if (sched->n_copies > 1) {
             for (int c = 0; c < sched->n_copies; c++) {
                 sched->events[b][c] = lm_ggml_backend_event_new(backends[b]->device);
@@ -2478,7 +2513,7 @@ struct lm_ggml_backend_graph_copy lm_ggml_backend_graph_copy(lm_ggml_backend_t b
     struct lm_ggml_context * ctx_unallocated = lm_ggml_init(params);
 
     if (ctx_allocated == NULL || ctx_unallocated == NULL) {
-        fprintf(stderr, "failed to allocate context for graph copy\n");
+        LM_GGML_LOG_ERROR("%s: failed to allocate context for graph copy\n", __func__);
         lm_ggml_hash_set_free(&hash_set);
         free(node_copies);
         free(node_init);
@@ -2501,7 +2536,7 @@ struct lm_ggml_backend_graph_copy lm_ggml_backend_graph_copy(lm_ggml_backend_t b
     // allocate nodes
     lm_ggml_backend_buffer_t buffer = lm_ggml_backend_alloc_ctx_tensors(ctx_allocated, backend);
     if (buffer == NULL) {
-        fprintf(stderr, "failed to allocate buffer for graph copy\n");
+        LM_GGML_LOG_ERROR("%s: failed to allocate buffer for graph copy\n", __func__);
         lm_ggml_hash_set_free(&hash_set);
         free(node_copies);
         free(node_init);

package/cpp/ggml-impl.h

@@ -19,6 +19,9 @@ extern "C" {
 #define MIN(a, b) ((a) < (b) ? (a) : (b))
 #define MAX(a, b) ((a) > (b) ? (a) : (b))
 
+// required for mmap as gguf only guarantees 32-byte alignment
+#define TENSOR_ALIGNMENT 32
+
 // static_assert should be a #define, but if it's not,
 // fall back to the _Static_assert C11 keyword.
 // if C99 - static_assert is noop
@@ -196,6 +199,11 @@ struct lm_ggml_cgraph {
 
 struct lm_ggml_cgraph lm_ggml_graph_view(struct lm_ggml_cgraph * cgraph, int i0, int i1);
 
+// Memory allocation
+
+void * lm_ggml_aligned_malloc(size_t size);
+void lm_ggml_aligned_free(void * ptr, size_t size);
+
 #ifdef __cplusplus
 }
 #endif

package/cpp/ggml.c

@@ -35,10 +35,6 @@
 #include <omp.h>
 #endif
 
-#ifdef LM_GGML_USE_METAL
-#include <unistd.h>
-#endif
-
 #if defined(__ARM_FEATURE_SVE) || defined(__ARM_FEATURE_MATMUL_INT8)
 #undef LM_GGML_USE_LLAMAFILE
 #endif
@@ -189,6 +185,8 @@ typedef pthread_t lm_ggml_thread_t;
 #endif
 
 #if defined(__APPLE__)
+#include <unistd.h>
+#include <mach/mach.h>
 #include <TargetConditionals.h>
 #endif
 
@@ -326,8 +324,9 @@ struct lm_ggml_logger_state {
 static struct lm_ggml_logger_state g_logger_state = {lm_ggml_log_callback_default, NULL};
 
 static void lm_ggml_log_internal_v(enum lm_ggml_log_level level, const char * format, va_list args) {
-    if (format == NULL)
+    if (format == NULL) {
         return;
+    }
     va_list args_copy;
     va_copy(args_copy, args);
     char buffer[128];
@@ -386,22 +385,40 @@ void lm_ggml_log_callback_default(enum lm_ggml_log_level level, const char * tex
 //#define LM_GGML_SOFT_MAX_ACCELERATE
 #endif
 
+
+void * lm_ggml_aligned_malloc(size_t size) {
 #if defined(_MSC_VER) || defined(__MINGW32__)
-#define LM_GGML_ALIGNED_MALLOC(size) _aligned_malloc(size, LM_GGML_MEM_ALIGN)
-#define LM_GGML_ALIGNED_FREE(ptr)    _aligned_free(ptr)
+    return _aligned_malloc(size, TENSOR_ALIGNMENT);
 #else
-inline static void * lm_ggml_aligned_malloc(size_t size) {
     if (size == 0) {
         LM_GGML_LOG_WARN("Behavior may be unexpected when allocating 0 bytes for lm_ggml_aligned_malloc!\n");
         return NULL;
     }
     void * aligned_memory = NULL;
 #ifdef LM_GGML_USE_CPU_HBM
-    int result = hbw_posix_memalign(&aligned_memory, 16, size);
+    int result = hbw_posix_memalign(&aligned_memory, TENSOR_ALIGNMENT, size);
+#elif TARGET_OS_OSX
+    kern_return_t alloc_status = vm_allocate((vm_map_t) mach_task_self(), (vm_address_t *) &aligned_memory, size, VM_FLAGS_ANYWHERE);
+    int result = EFAULT;
+    switch (alloc_status) {
+        case KERN_SUCCESS:
+            result = 0;
+            break;
+        case KERN_INVALID_ADDRESS:
+            result = EINVAL;
+            break;
+        case KERN_NO_SPACE:
+            result = ENOMEM;
+            break;
+        default:
+            result = EFAULT;
+            break;
+    }
 #elif LM_GGML_USE_METAL
-    int result = posix_memalign(&aligned_memory, sysconf(_SC_PAGESIZE), size);
+    const long page_size = sysconf(_SC_PAGESIZE);
+    int result = posix_memalign(&aligned_memory, MAX(TENSOR_ALIGNMENT, page_size), size);
 #else
-    int result = posix_memalign(&aligned_memory, LM_GGML_MEM_ALIGN, size);
+    int result = posix_memalign(&aligned_memory, TENSOR_ALIGNMENT, size);
 #endif
     if (result != 0) {
         // Handle allocation failure
@@ -419,14 +436,26 @@ inline static void * lm_ggml_aligned_malloc(size_t size) {
         return NULL;
     }
     return aligned_memory;
+#endif
 }
-#define LM_GGML_ALIGNED_MALLOC(size) lm_ggml_aligned_malloc(size)
-#ifdef LM_GGML_USE_CPU_HBM
-#define LM_GGML_ALIGNED_FREE(ptr)    if(NULL != ptr) hbw_free(ptr)
+
+void lm_ggml_aligned_free(void * ptr, size_t size) {
+    LM_GGML_UNUSED(size);
+#if defined(_MSC_VER) || defined(__MINGW32__)
+    _aligned_free(ptr);
+#elif LM_GGML_USE_CPU_HBM
+    if (ptr != NULL) {
+        hbw_free(ptr);
+    }
+#elif TARGET_OS_OSX
+    if (ptr != NULL) {
+        vm_deallocate((vm_map_t)mach_task_self(), (vm_address_t)ptr, size);
+    }
 #else
-#define LM_GGML_ALIGNED_FREE(ptr)    free(ptr)
-#endif
+    free(ptr);
 #endif
+}
+
 
 inline static void * lm_ggml_malloc(size_t size) {
     if (size == 0) {
@@ -3882,7 +3911,7 @@ struct lm_ggml_context * lm_ggml_init(struct lm_ggml_init_params params) {
 
     *ctx = (struct lm_ggml_context) {
         /*.mem_size           =*/ mem_size,
-        /*.mem_buffer         =*/ params.mem_buffer ? params.mem_buffer : LM_GGML_ALIGNED_MALLOC(mem_size),
+        /*.mem_buffer         =*/ params.mem_buffer ? params.mem_buffer : lm_ggml_aligned_malloc(mem_size),
         /*.mem_buffer_owned   =*/ params.mem_buffer ? false : true,
         /*.no_alloc           =*/ params.no_alloc,
         /*.no_alloc_save      =*/ params.no_alloc,
@@ -3922,7 +3951,7 @@ void lm_ggml_free(struct lm_ggml_context * ctx) {
                     __func__, i, lm_ggml_used_mem(ctx));
 
             if (ctx->mem_buffer_owned) {
-                LM_GGML_ALIGNED_FREE(ctx->mem_buffer);
+                lm_ggml_aligned_free(ctx->mem_buffer, ctx->mem_size);
             }
 
             found = true;
@@ -15708,6 +15737,9 @@ static void lm_ggml_compute_forward_flash_attn_ext_f16(
     lm_ggml_vec_dot_t    const kq_vec_dot     = type_traits[k->type].vec_dot;
     lm_ggml_to_float_t   const v_to_float     = type_traits[v->type].to_float;
 
+    LM_GGML_ASSERT(q_to_vec_dot && "fattn: unsupported K-type");
+    LM_GGML_ASSERT(v_to_float   && "fattn: unsupported V-type");
+
     // loop over n_batch and n_head
     for (int ir = ir0; ir < ir1; ++ir) {
         // q indices
@@ -19621,9 +19653,10 @@ static void lm_ggml_thread_cpumask_next(const bool * global_mask, bool * local_m
 void lm_ggml_threadpool_free(struct lm_ggml_threadpool* threadpool) {
     if (!threadpool) return;
 
+    const int n_threads = threadpool->n_threads_max;
+
 #ifndef LM_GGML_USE_OPENMP
     struct lm_ggml_compute_state* workers = threadpool->workers;
-    const int n_threads = threadpool->n_threads_max;
 
     lm_ggml_mutex_lock(&threadpool->mutex);
 
@@ -19643,8 +19676,9 @@ void lm_ggml_threadpool_free(struct lm_ggml_threadpool* threadpool) {
     lm_ggml_cond_destroy(&threadpool->cond);
 #endif // LM_GGML_USE_OPENMP
 
-    LM_GGML_ALIGNED_FREE(threadpool->workers);
-    LM_GGML_ALIGNED_FREE(threadpool);
+    const size_t workers_size = sizeof(struct lm_ggml_compute_state) * n_threads;
+    lm_ggml_aligned_free(threadpool->workers, workers_size);
+    lm_ggml_aligned_free(threadpool, sizeof(struct lm_ggml_threadpool));
 }
 
 #ifndef LM_GGML_USE_OPENMP
@@ -20076,7 +20110,7 @@ static struct lm_ggml_threadpool * lm_ggml_threadpool_new_impl(
                 struct lm_ggml_cplan * cplan) {
 
     struct lm_ggml_threadpool * threadpool =
-        LM_GGML_ALIGNED_MALLOC(sizeof(struct lm_ggml_threadpool));
+        lm_ggml_aligned_malloc(sizeof(struct lm_ggml_threadpool));
     {
         threadpool->cgraph           = cgraph;
         threadpool->cplan            = cplan;
@@ -20097,7 +20131,7 @@ static struct lm_ggml_threadpool * lm_ggml_threadpool_new_impl(
 
     // Allocate and init workers state
     const size_t workers_size = sizeof(struct lm_ggml_compute_state) * tpp->n_threads;
-    struct lm_ggml_compute_state * workers = LM_GGML_ALIGNED_MALLOC(workers_size);
+    struct lm_ggml_compute_state * workers = lm_ggml_aligned_malloc(workers_size);
 
     memset(workers, 0, workers_size);
     for (int j = 0; j < tpp->n_threads; j++) {
@@ -23235,6 +23269,14 @@ int lm_ggml_cpu_has_avx512_bf16(void) {
 #endif
 }
 
+int lm_ggml_cpu_has_amx_int8(void) {
+#if defined(__AMX_INT8__)
+    return 1;
+#else
+    return 0;
+#endif
+}
+
 int lm_ggml_cpu_has_fma(void) {
 #if defined(__FMA__)
     return 1;

package/cpp/ggml.h

@@ -2489,6 +2489,7 @@ extern "C" {
     LM_GGML_API int lm_ggml_cpu_has_avx512_vbmi(void);
     LM_GGML_API int lm_ggml_cpu_has_avx512_vnni(void);
     LM_GGML_API int lm_ggml_cpu_has_avx512_bf16(void);
+    LM_GGML_API int lm_ggml_cpu_has_amx_int8   (void);
     LM_GGML_API int lm_ggml_cpu_has_fma        (void);
     LM_GGML_API int lm_ggml_cpu_has_neon       (void);
     LM_GGML_API int lm_ggml_cpu_has_sve        (void);

package/cpp/json-schema-to-grammar.cpp

@@ -611,7 +611,7 @@ private:
             }
             return join_seq();
         };
-        return _add_rule(name, "\"\\\"\" " + to_rule(transform()) + " \"\\\"\" space");
+        return _add_rule(name, "\"\\\"\" (" + to_rule(transform()) + ") \"\\\"\" space");
     }
 
     /*