cui-llama.rn 1.2.3 → 1.2.6

package/cpp/ggml.h

@@ -218,7 +218,6 @@
 
 #define LM_GGML_MAX_DIMS           4
 #define LM_GGML_MAX_PARAMS         2048
-#define LM_GGML_MAX_CONTEXTS       64
 #define LM_GGML_MAX_SRC            10
 #define LM_GGML_MAX_N_THREADS      512
 #define LM_GGML_MAX_OP_PARAMS      64
@@ -560,10 +559,10 @@ extern "C" {
 
     enum lm_ggml_log_level {
         LM_GGML_LOG_LEVEL_NONE  = 0,
-        LM_GGML_LOG_LEVEL_INFO  = 1,
-        LM_GGML_LOG_LEVEL_WARN  = 2,
-        LM_GGML_LOG_LEVEL_ERROR = 3,
-        LM_GGML_LOG_LEVEL_DEBUG = 4,
+        LM_GGML_LOG_LEVEL_DEBUG = 1,
+        LM_GGML_LOG_LEVEL_INFO  = 2,
+        LM_GGML_LOG_LEVEL_WARN  = 3,
+        LM_GGML_LOG_LEVEL_ERROR = 4,
         LM_GGML_LOG_LEVEL_CONT  = 5, // continue previous log
     };
 
@@ -575,6 +574,13 @@ extern "C" {
         LM_GGML_TENSOR_FLAG_LOSS   =  8, // ...defines loss for numerical optimization (multiple loss tensors add up)
     };
 
+    struct lm_ggml_init_params {
+        // memory pool
+        size_t mem_size;   // bytes
+        void * mem_buffer; // if NULL, memory will be allocated internally
+        bool   no_alloc;   // don't allocate memory for the tensor data
+    };
+
     // n-dimensional tensor
     struct lm_ggml_tensor {
         enum lm_ggml_type type;
@@ -620,66 +626,6 @@ extern "C" {
     // If it returns true, the computation is aborted
     typedef bool (*lm_ggml_abort_callback)(void * data);
 
-    // Scheduling priorities
-    enum lm_ggml_sched_priority {
-        LM_GGML_SCHED_PRIO_NORMAL,
-        LM_GGML_SCHED_PRIO_MEDIUM,
-        LM_GGML_SCHED_PRIO_HIGH,
-        LM_GGML_SCHED_PRIO_REALTIME
-    };
-
-    // Threadpool params
-    // Use lm_ggml_threadpool_params_default() or lm_ggml_threadpool_params_init() to populate the defaults
-    struct lm_ggml_threadpool_params {
-        bool                cpumask[LM_GGML_MAX_N_THREADS]; // mask of cpu cores (all-zeros means use default affinity settings)
-        int                 n_threads;                   // number of threads
-        enum lm_ggml_sched_priority prio;                   // thread priority
-        uint32_t            poll;                        // polling level (0 - no polling, 100 - aggressive polling)
-        bool                strict_cpu;                  // strict cpu placement
-        bool                paused;                      // start in paused state
-    };
-
-    struct lm_ggml_threadpool;     // forward declaration, see ggml.c
-
-    typedef struct lm_ggml_threadpool * lm_ggml_threadpool_t;
-
-    // the compute plan that needs to be prepared for lm_ggml_graph_compute()
-    // since https://github.com/ggerganov/ggml/issues/287
-    struct lm_ggml_cplan {
-        size_t    work_size; // size of work buffer, calculated by `lm_ggml_graph_plan()`
-        uint8_t * work_data; // work buffer, to be allocated by caller before calling to `lm_ggml_graph_compute()`
-
-        int n_threads;
-        struct lm_ggml_threadpool * threadpool;
-
-        // abort lm_ggml_graph_compute when true
-        lm_ggml_abort_callback abort_callback;
-        void *              abort_callback_data;
-    };
-
-    // scratch buffer
-    struct lm_ggml_scratch {
-        size_t offs;
-        size_t size;
-        void * data;
-    };
-
-    struct lm_ggml_init_params {
-        // memory pool
-        size_t mem_size;   // bytes
-        void * mem_buffer; // if NULL, memory will be allocated internally
-        bool   no_alloc;   // don't allocate memory for the tensor data
-    };
-
-    // numa strategies
-    enum lm_ggml_numa_strategy {
-        LM_GGML_NUMA_STRATEGY_DISABLED   = 0,
-        LM_GGML_NUMA_STRATEGY_DISTRIBUTE = 1,
-        LM_GGML_NUMA_STRATEGY_ISOLATE    = 2,
-        LM_GGML_NUMA_STRATEGY_NUMACTL    = 3,
-        LM_GGML_NUMA_STRATEGY_MIRROR     = 4,
-        LM_GGML_NUMA_STRATEGY_COUNT
-    };
 
     //
     // GUID
@@ -702,9 +648,6 @@ extern "C" {
     // accepts a UTF-8 path, even on Windows
     LM_GGML_API FILE *  lm_ggml_fopen(const char * fname, const char * mode);
 
-    LM_GGML_API void    lm_ggml_numa_init(enum lm_ggml_numa_strategy numa); // call once for better performance on NUMA systems
-    LM_GGML_API bool    lm_ggml_is_numa(void); // true if init detected that system has >1 NUMA node
-
     LM_GGML_API void    lm_ggml_print_object (const struct lm_ggml_object * obj);
     LM_GGML_API void    lm_ggml_print_objects(const struct lm_ggml_context * ctx);
 
@@ -761,12 +704,12 @@ extern "C" {
 
     // main
 
-    LM_GGML_API struct lm_ggml_context * lm_ggml_init(struct lm_ggml_init_params params);
-    LM_GGML_API void                  lm_ggml_free(struct lm_ggml_context * ctx);
+    LM_GGML_API struct lm_ggml_context * lm_ggml_init (struct lm_ggml_init_params params);
+    LM_GGML_API void                  lm_ggml_reset(struct lm_ggml_context * ctx);
+    LM_GGML_API void                  lm_ggml_free (struct lm_ggml_context * ctx);
 
     LM_GGML_API size_t  lm_ggml_used_mem(const struct lm_ggml_context * ctx);
 
-    LM_GGML_API size_t  lm_ggml_set_scratch (struct lm_ggml_context * ctx, struct lm_ggml_scratch scratch);
     LM_GGML_API bool    lm_ggml_get_no_alloc(struct lm_ggml_context * ctx);
     LM_GGML_API void    lm_ggml_set_no_alloc(struct lm_ggml_context * ctx, bool no_alloc);
 
@@ -806,8 +749,7 @@ extern "C" {
             int64_t ne2,
             int64_t ne3);
 
-    LM_GGML_API struct lm_ggml_tensor * lm_ggml_new_i32(struct lm_ggml_context * ctx, int32_t value);
-    LM_GGML_API struct lm_ggml_tensor * lm_ggml_new_f32(struct lm_ggml_context * ctx, float value);
+    LM_GGML_API void * lm_ggml_new_buffer(struct lm_ggml_context * ctx, size_t nbytes);
 
     LM_GGML_API struct lm_ggml_tensor * lm_ggml_dup_tensor (struct lm_ggml_context * ctx, const struct lm_ggml_tensor * src);
     LM_GGML_API struct lm_ggml_tensor * lm_ggml_view_tensor(struct lm_ggml_context * ctx, struct lm_ggml_tensor * src);
@@ -817,35 +759,25 @@ extern "C" {
     LM_GGML_API struct lm_ggml_tensor * lm_ggml_get_next_tensor (const struct lm_ggml_context * ctx, struct lm_ggml_tensor * tensor);
     LM_GGML_API struct lm_ggml_tensor * lm_ggml_get_tensor(struct lm_ggml_context * ctx, const char * name);
 
-    LM_GGML_API struct lm_ggml_tensor * lm_ggml_set_zero(struct lm_ggml_tensor * tensor);
-    LM_GGML_API struct lm_ggml_tensor * lm_ggml_set_i32 (struct lm_ggml_tensor * tensor, int32_t value);
-    LM_GGML_API struct lm_ggml_tensor * lm_ggml_set_f32 (struct lm_ggml_tensor * tensor, float value);
-
     // Converts a flat index into coordinates
-    LM_GGML_API void    lm_ggml_unravel_index(const struct lm_ggml_tensor * tensor, int64_t i, int64_t * i0, int64_t * i1, int64_t * i2, int64_t * i3);
-
-    LM_GGML_API int32_t lm_ggml_get_i32_1d(const struct lm_ggml_tensor * tensor, int i);
-    LM_GGML_API void    lm_ggml_set_i32_1d(const struct lm_ggml_tensor * tensor, int i, int32_t value);
+    LM_GGML_API void lm_ggml_unravel_index(const struct lm_ggml_tensor * tensor, int64_t i, int64_t * i0, int64_t * i1, int64_t * i2, int64_t * i3);
 
-    LM_GGML_API int32_t lm_ggml_get_i32_nd(const struct lm_ggml_tensor * tensor, int i0, int i1, int i2, int i3);
-    LM_GGML_API void    lm_ggml_set_i32_nd(const struct lm_ggml_tensor * tensor, int i0, int i1, int i2, int i3, int32_t value);
-
-    LM_GGML_API float   lm_ggml_get_f32_1d(const struct lm_ggml_tensor * tensor, int i);
-    LM_GGML_API void    lm_ggml_set_f32_1d(const struct lm_ggml_tensor * tensor, int i, float value);
-
-    LM_GGML_API float   lm_ggml_get_f32_nd(const struct lm_ggml_tensor * tensor, int i0, int i1, int i2, int i3);
-    LM_GGML_API void    lm_ggml_set_f32_nd(const struct lm_ggml_tensor * tensor, int i0, int i1, int i2, int i3, float value);
+    LM_GGML_API enum lm_ggml_unary_op lm_ggml_get_unary_op(const struct lm_ggml_tensor * tensor);
 
     LM_GGML_API void *  lm_ggml_get_data    (const struct lm_ggml_tensor * tensor);
     LM_GGML_API float * lm_ggml_get_data_f32(const struct lm_ggml_tensor * tensor);
 
-    LM_GGML_API enum lm_ggml_unary_op lm_ggml_get_unary_op(const struct lm_ggml_tensor * tensor);
-
     LM_GGML_API const char *         lm_ggml_get_name   (const struct lm_ggml_tensor * tensor);
     LM_GGML_API struct lm_ggml_tensor * lm_ggml_set_name   (      struct lm_ggml_tensor * tensor, const char * name);
     LM_GGML_ATTRIBUTE_FORMAT(2, 3)
     LM_GGML_API struct lm_ggml_tensor * lm_ggml_format_name(      struct lm_ggml_tensor * tensor, const char * fmt, ...);
 
+    // Tensor flags
+    LM_GGML_API void lm_ggml_set_input(struct lm_ggml_tensor * tensor);
+    LM_GGML_API void lm_ggml_set_output(struct lm_ggml_tensor * tensor);
+    LM_GGML_API void lm_ggml_set_param(struct lm_ggml_context * ctx, struct lm_ggml_tensor * tensor);
+    LM_GGML_API void lm_ggml_set_loss(struct lm_ggml_tensor * tensor);
+
     //
     // operations on tensors with backpropagation
     //
@@ -2061,9 +1993,6 @@ extern "C" {
     // automatic differentiation
     //
 
-    LM_GGML_API void lm_ggml_set_param(struct lm_ggml_context * ctx, struct lm_ggml_tensor * tensor);
-    LM_GGML_API void lm_ggml_set_loss(struct lm_ggml_tensor * tensor);
-
     LM_GGML_API void lm_ggml_build_forward_expand (struct lm_ggml_cgraph * cgraph, struct lm_ggml_tensor * tensor);
     LM_GGML_API void lm_ggml_build_backward_expand(struct lm_ggml_context * ctx, struct lm_ggml_cgraph * gf, struct lm_ggml_cgraph * gb, bool accumulate);
 
@@ -2095,27 +2024,6 @@ extern "C" {
     LM_GGML_API size_t lm_ggml_graph_overhead(void);
     LM_GGML_API size_t lm_ggml_graph_overhead_custom(size_t size, bool grads);
 
-    LM_GGML_API struct lm_ggml_threadpool_params lm_ggml_threadpool_params_default(int n_threads);
-    LM_GGML_API void                          lm_ggml_threadpool_params_init   (struct lm_ggml_threadpool_params * p, int n_threads);
-    LM_GGML_API bool                          lm_ggml_threadpool_params_match  (const struct lm_ggml_threadpool_params * p0, const struct lm_ggml_threadpool_params * p1);
-    LM_GGML_API struct lm_ggml_threadpool *      lm_ggml_threadpool_new          (struct lm_ggml_threadpool_params  * params);
-    LM_GGML_API void                          lm_ggml_threadpool_free         (struct lm_ggml_threadpool * threadpool);
-    LM_GGML_API int                           lm_ggml_threadpool_get_n_threads(struct lm_ggml_threadpool * threadpool);
-    LM_GGML_API void                          lm_ggml_threadpool_pause        (struct lm_ggml_threadpool * threadpool);
-    LM_GGML_API void                          lm_ggml_threadpool_resume       (struct lm_ggml_threadpool * threadpool);
-
-    // lm_ggml_graph_plan() has to be called before lm_ggml_graph_compute()
-    // when plan.work_size > 0, caller must allocate memory for plan.work_data
-    LM_GGML_API struct lm_ggml_cplan lm_ggml_graph_plan(
-                  const struct lm_ggml_cgraph * cgraph,
-                                       int   n_threads, /* = LM_GGML_DEFAULT_N_THREADS */
-                    struct lm_ggml_threadpool * threadpool /* = NULL */ );
-    LM_GGML_API enum lm_ggml_status  lm_ggml_graph_compute(struct lm_ggml_cgraph * cgraph, struct lm_ggml_cplan * cplan);
-
-    // same as lm_ggml_graph_compute() but the work data is allocated as a part of the context
-    // note: the drawback of this API is that you must have ensured that the context has enough memory for the work data
-    LM_GGML_API enum lm_ggml_status  lm_ggml_graph_compute_with_ctx(struct lm_ggml_context * ctx, struct lm_ggml_cgraph * cgraph, int n_threads);
-
     LM_GGML_API struct lm_ggml_tensor * lm_ggml_graph_get_tensor(struct lm_ggml_cgraph * cgraph, const char * name);
 
     LM_GGML_API void                 lm_ggml_graph_export(const struct lm_ggml_cgraph * cgraph, const char * fname);
@@ -2286,6 +2194,8 @@ extern "C" {
         } lbfgs;
     };
 
+    LM_GGML_API struct lm_ggml_tensor * lm_ggml_set_zero(struct lm_ggml_tensor * tensor);
+
     LM_GGML_API struct lm_ggml_opt_params lm_ggml_opt_default_params(enum lm_ggml_opt_type type);
 
     // optimize the function defined by the tensor f
@@ -2317,12 +2227,6 @@ extern "C" {
             lm_ggml_opt_callback callback,
             void * callback_data);
 
-    //
-    // tensor flags
-    //
-    LM_GGML_API void lm_ggml_set_input(struct lm_ggml_tensor * tensor);
-    LM_GGML_API void lm_ggml_set_output(struct lm_ggml_tensor * tensor);
-
     //
     // quantization
     //
@@ -2489,9 +2393,8 @@ extern "C" {
     LM_GGML_API int lm_ggml_cpu_has_avx512_vbmi(void);
     LM_GGML_API int lm_ggml_cpu_has_avx512_vnni(void);
     LM_GGML_API int lm_ggml_cpu_has_avx512_bf16(void);
+    LM_GGML_API int lm_ggml_cpu_has_amx_int8   (void);
     LM_GGML_API int lm_ggml_cpu_has_fma        (void);
-    LM_GGML_API int lm_ggml_cpu_has_neon       (void);
-    LM_GGML_API int lm_ggml_cpu_has_sve        (void);
     LM_GGML_API int lm_ggml_cpu_has_arm_fma    (void);
     LM_GGML_API int lm_ggml_cpu_has_metal      (void);
     LM_GGML_API int lm_ggml_cpu_has_f16c       (void);
@@ -2508,17 +2411,9 @@ extern "C" {
     LM_GGML_API int lm_ggml_cpu_has_sycl       (void);
     LM_GGML_API int lm_ggml_cpu_has_rpc        (void);
     LM_GGML_API int lm_ggml_cpu_has_vsx        (void);
-    LM_GGML_API int lm_ggml_cpu_has_matmul_int8(void);
     LM_GGML_API int lm_ggml_cpu_has_cann       (void);
     LM_GGML_API int lm_ggml_cpu_has_llamafile  (void);
 
-    // get the sve vector length in bytes
-    LM_GGML_API int lm_ggml_cpu_get_sve_cnt(void);
-
-    //
-    // Internal types and functions exposed for tests and benchmarks
-    //
-
 #ifdef  __cplusplus
 // restrict not standard in C++
 #define LM_GGML_RESTRICT
@@ -2527,14 +2422,6 @@ extern "C" {
 #endif
     typedef void (*lm_ggml_to_float_t)  (const void  * LM_GGML_RESTRICT x, float * LM_GGML_RESTRICT y, int64_t k);
     typedef void (*lm_ggml_from_float_t)(const float * LM_GGML_RESTRICT x, void  * LM_GGML_RESTRICT y, int64_t k);
-    typedef void (*lm_ggml_from_float_to_mat_t)
-                                     (const float * LM_GGML_RESTRICT x, void * LM_GGML_RESTRICT y, int64_t nr, int64_t k, int64_t bs);
-    typedef void (*lm_ggml_vec_dot_t)  (int n, float * LM_GGML_RESTRICT s, size_t bs, const void * LM_GGML_RESTRICT x, size_t bx,
-                                       const void * LM_GGML_RESTRICT y, size_t by, int nrc);
-    typedef void (*lm_ggml_gemv_t)     (int n, float * LM_GGML_RESTRICT s, size_t bs, const void * LM_GGML_RESTRICT x,
-                                       const void * LM_GGML_RESTRICT y, int nr, int nc);
-    typedef void (*lm_ggml_gemm_t)     (int n, float * LM_GGML_RESTRICT s, size_t bs, const void * LM_GGML_RESTRICT x,
-                                       const void * LM_GGML_RESTRICT y, int nr, int nc);
 
     struct lm_ggml_type_traits {
         const char             * type_name;
@@ -2545,13 +2432,6 @@ extern "C" {
         lm_ggml_to_float_t          to_float;
         lm_ggml_from_float_t        from_float;
         lm_ggml_from_float_t        from_float_ref;
-        lm_ggml_from_float_to_mat_t from_float_to_mat;
-        lm_ggml_vec_dot_t           vec_dot;
-        enum lm_ggml_type           vec_dot_type;
-        int64_t                  nrows; // number of rows to process simultaneously
-        int64_t                  ncols; // number of columns to process simultaneously
-        lm_ggml_gemv_t              gemv;
-        lm_ggml_gemm_t              gemm;
     };
 
     LM_GGML_API const struct lm_ggml_type_traits * lm_ggml_get_type_traits(enum lm_ggml_type type);

package/cpp/json-schema-to-grammar.cpp

@@ -611,7 +611,7 @@ private:
             }
             return join_seq();
         };
-        return _add_rule(name, "\"\\\"\" " + to_rule(transform()) + " \"\\\"\" space");
+        return _add_rule(name, "\"\\\"\" (" + to_rule(transform()) + ") \"\\\"\" space");
     }
 
     /*