cui-llama.rn 1.4.4 → 1.5.0

package/cpp/unary-ops.cpp

@@ -0,0 +1,186 @@
+#include "unary-ops.h"
+
+static inline float op_abs(float x) {
+    return fabsf(x);
+}
+
+static inline float op_sgn(float x) {
+    return (x > 0.f) ? 1.f : ((x < 0.f) ? -1.f : 0.f);
+}
+
+static inline float op_neg(float x) {
+    return -x;
+}
+
+static inline float op_step(float x) {
+    return (x > 0.f) ? 1.f : 0.f;
+}
+
+static inline float op_tanh(float x) {
+    return tanhf(x);
+}
+
+static inline float op_elu(float x) {
+    return (x > 0.f) ? x : expm1f(x);
+}
+
+static inline float op_relu(float x) {
+    return (x > 0.f) ? x : 0.f;
+}
+
+static inline float op_sigmoid(float x) {
+    return 1.f / (1.f + expf(-x));
+}
+
+static inline float op_hardsigmoid(float x) {
+    return fminf(1.0f, fmaxf(0.0f, (x + 3.0f) / 6.0f));
+}
+
+static inline float op_exp(float x) {
+    return expf(x);
+}
+
+static inline float op_hardswish(float x) {
+    return x * fminf(1.0f, fmaxf(0.0f, (x + 3.0f) / 6.0f));
+}
+
+static inline float op_sqr(float x) {
+    return x * x;
+}
+
+static inline float op_sqrt(float x) {
+    return sqrtf(x);
+}
+
+static inline float op_sin(float x) {
+    return sinf(x);
+}
+
+static inline float op_cos(float x) {
+    return cosf(x);
+}
+
+static inline float op_log(float x) {
+    return logf(x);
+}
+
+template <float (*op)(float), typename src0_t, typename dst_t>
+static inline void vec_unary_op(int64_t n, dst_t * y, const src0_t * x) {
+    constexpr auto src0_to_f32 = type_conversion_table<src0_t>::to_f32;
+    constexpr auto f32_to_dst  = type_conversion_table<dst_t >::from_f32;
+
+    for (int i = 0; i < n; i++) {
+        y[i] = f32_to_dst(op(src0_to_f32(x[i])));
+    }
+}
+
+template <float (*op)(float), typename src0_t, typename dst_t>
+static void apply_unary_op(const lm_ggml_compute_params * params, lm_ggml_tensor * dst) {
+    const lm_ggml_tensor * src0 = dst->src[0];
+
+    LM_GGML_ASSERT(lm_ggml_is_contiguous_1(src0) && lm_ggml_is_contiguous_1(dst) && lm_ggml_are_same_shape(src0, dst));
+
+    LM_GGML_TENSOR_UNARY_OP_LOCALS
+
+    LM_GGML_ASSERT( nb0 == sizeof(dst_t));
+    LM_GGML_ASSERT(nb00 == sizeof(src0_t));
+
+    const auto [ir0, ir1] = get_thread_range(params, src0);
+
+    for (int64_t ir = ir0; ir < ir1; ++ir) {
+        const int64_t i03 = ir/(ne02*ne01);
+        const int64_t i02 = (ir - i03*ne02*ne01)/ne01;
+        const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);
+
+        dst_t        * dst_ptr  = (dst_t  *)       ((char *)       dst->data  + i03*nb3  + i02*nb2  + i01*nb1 );
+        const src0_t * src0_ptr = (const src0_t *) ((const char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);
+
+        vec_unary_op<op>(ne0, dst_ptr, src0_ptr);
+    }
+}
+
+// TODO: Use the 'traits' lookup table (for type conversion fns), instead of a mass of 'if' conditions with long templates
+template <float (*op)(float)>
+static void unary_op(const lm_ggml_compute_params * params, lm_ggml_tensor * dst) {
+    const lm_ggml_tensor * src0 = dst->src[0];
+
+    /*  */ if (src0->type == LM_GGML_TYPE_F32  && dst->type == LM_GGML_TYPE_F32) { // all f32
+        apply_unary_op<op, float, float>(params, dst);
+    } else if (src0->type == LM_GGML_TYPE_F16  && dst->type == LM_GGML_TYPE_F16) { // all f16
+        apply_unary_op<op, lm_ggml_fp16_t, lm_ggml_fp16_t>(params, dst);
+    } else if (src0->type == LM_GGML_TYPE_BF16 && dst->type == LM_GGML_TYPE_BF16) { // all bf16
+        apply_unary_op<op, lm_ggml_bf16_t, lm_ggml_bf16_t>(params, dst);
+    } else if (src0->type == LM_GGML_TYPE_BF16 && dst->type == LM_GGML_TYPE_F32) {
+        apply_unary_op<op, lm_ggml_bf16_t, float>(params, dst);
+    } else if (src0->type == LM_GGML_TYPE_F16  && dst->type == LM_GGML_TYPE_F32) {
+        apply_unary_op<op, lm_ggml_fp16_t, float>(params, dst);
+    } else {
+        fprintf(stderr, "%s: unsupported types: dst: %s, src0: %s\n", __func__,
+            lm_ggml_type_name(dst->type), lm_ggml_type_name(src0->type));
+        LM_GGML_ABORT("fatal error");
+    }
+}
+
+void lm_ggml_compute_forward_abs(const lm_ggml_compute_params * params, lm_ggml_tensor * dst) {
+    unary_op<op_abs>(params, dst);
+}
+
+void lm_ggml_compute_forward_sgn(const lm_ggml_compute_params * params, lm_ggml_tensor * dst) {
+    unary_op<op_sgn>(params, dst);
+}
+
+void lm_ggml_compute_forward_neg(const lm_ggml_compute_params * params, lm_ggml_tensor * dst) {
+    unary_op<op_neg>(params, dst);
+}
+
+void lm_ggml_compute_forward_step(const lm_ggml_compute_params * params, lm_ggml_tensor * dst) {
+    unary_op<op_step>(params, dst);
+}
+
+void lm_ggml_compute_forward_tanh(const lm_ggml_compute_params * params, lm_ggml_tensor * dst) {
+    unary_op<op_tanh>(params, dst);
+}
+
+void lm_ggml_compute_forward_elu(const lm_ggml_compute_params * params, lm_ggml_tensor * dst) {
+    unary_op<op_elu>(params, dst);
+}
+
+void lm_ggml_compute_forward_relu(const lm_ggml_compute_params * params, lm_ggml_tensor * dst) {
+    unary_op<op_relu>(params, dst);
+}
+
+void lm_ggml_compute_forward_sigmoid(const lm_ggml_compute_params * params, lm_ggml_tensor * dst) {
+    unary_op<op_sigmoid>(params, dst);
+}
+
+void lm_ggml_compute_forward_hardsigmoid(const lm_ggml_compute_params * params, lm_ggml_tensor * dst) {
+    unary_op<op_hardsigmoid>(params, dst);
+}
+
+void lm_ggml_compute_forward_exp(const lm_ggml_compute_params * params, lm_ggml_tensor * dst) {
+    unary_op<op_exp>(params, dst);
+}
+
+void lm_ggml_compute_forward_hardswish(const lm_ggml_compute_params * params, lm_ggml_tensor * dst) {
+    unary_op<op_hardswish>(params, dst);
+}
+
+void lm_ggml_compute_forward_sqr(const lm_ggml_compute_params * params, lm_ggml_tensor * dst) {
+    unary_op<op_sqr>(params, dst);
+}
+
+void lm_ggml_compute_forward_sqrt(const lm_ggml_compute_params * params, lm_ggml_tensor * dst) {
+    unary_op<op_sqrt>(params, dst);
+}
+
+void lm_ggml_compute_forward_sin(const lm_ggml_compute_params * params, lm_ggml_tensor * dst) {
+    unary_op<op_sin>(params, dst);
+}
+
+void lm_ggml_compute_forward_cos(const lm_ggml_compute_params * params, lm_ggml_tensor * dst) {
+    unary_op<op_cos>(params, dst);
+}
+
+void lm_ggml_compute_forward_log(const lm_ggml_compute_params * params, lm_ggml_tensor * dst) {
+    unary_op<op_log>(params, dst);
+}

package/cpp/unary-ops.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#include "cpu-common.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void lm_ggml_compute_forward_abs(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
+void lm_ggml_compute_forward_sgn(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
+void lm_ggml_compute_forward_neg(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
+void lm_ggml_compute_forward_step(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
+void lm_ggml_compute_forward_tanh(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
+void lm_ggml_compute_forward_elu(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
+void lm_ggml_compute_forward_relu(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
+void lm_ggml_compute_forward_sigmoid(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
+void lm_ggml_compute_forward_hardsigmoid(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
+void lm_ggml_compute_forward_exp(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
+void lm_ggml_compute_forward_hardswish(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
+void lm_ggml_compute_forward_sqr(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
+void lm_ggml_compute_forward_sqrt(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
+void lm_ggml_compute_forward_sin(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
+void lm_ggml_compute_forward_cos(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
+void lm_ggml_compute_forward_log(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
+
+#ifdef __cplusplus
+}
+#endif

package/cpp/vec.cpp

@@ -0,0 +1,258 @@
+#include "vec.h"
+
+#include <cassert>
+
+#if defined(_MSC_VER)
+// disable "possible loss of data" to avoid hundreds of casts
+// we should just be careful :)
+#pragma warning(disable: 4244 4267)
+#endif
+
+// precomputed gelu table for f16 (128 KB)
+lm_ggml_fp16_t lm_ggml_table_gelu_f16[1 << 16];
+
+// precomputed quick gelu table for f16 (128 KB)
+lm_ggml_fp16_t lm_ggml_table_gelu_quick_f16[1 << 16];
+
+void lm_ggml_vec_dot_f32(int n, float * LM_GGML_RESTRICT s, size_t bs, const float * LM_GGML_RESTRICT x, size_t bx, const float * LM_GGML_RESTRICT y, size_t by, int nrc) {
+   assert(nrc == 1);
+   LM_GGML_UNUSED(nrc);
+   LM_GGML_UNUSED(bx);
+   LM_GGML_UNUSED(by);
+   LM_GGML_UNUSED(bs);
+
+#if defined(LM_GGML_SIMD)
+    float sumf = 0.0f;
+    const int np = (n & ~(LM_GGML_F32_STEP - 1));
+
+    LM_GGML_F32_VEC sum[LM_GGML_F32_ARR] = { LM_GGML_F32_VEC_ZERO };
+
+    LM_GGML_F32_VEC ax[LM_GGML_F32_ARR];
+    LM_GGML_F32_VEC ay[LM_GGML_F32_ARR];
+
+    for (int i = 0; i < np; i += LM_GGML_F32_STEP) {
+        for (int j = 0; j < LM_GGML_F32_ARR; j++) {
+            ax[j] = LM_GGML_F32_VEC_LOAD(x + i + j*LM_GGML_F32_EPR);
+            ay[j] = LM_GGML_F32_VEC_LOAD(y + i + j*LM_GGML_F32_EPR);
+
+            sum[j] = LM_GGML_F32_VEC_FMA(sum[j], ax[j], ay[j]);
+        }
+    }
+
+    // reduce sum0..sum3 to sum0
+    LM_GGML_F32_VEC_REDUCE(sumf, sum);
+
+    // leftovers
+    for (int i = np; i < n; ++i) {
+        sumf += x[i]*y[i];
+    }
+#else
+    // scalar
+    lm_ggml_float sumf = 0.0;
+    for (int i = 0; i < n; ++i) {
+        sumf += (lm_ggml_float)(x[i]*y[i]);
+    }
+#endif
+
+    *s = sumf;
+}
+
+void lm_ggml_vec_dot_bf16(int n, float * LM_GGML_RESTRICT s, size_t bs, lm_ggml_bf16_t * LM_GGML_RESTRICT x, size_t bx, lm_ggml_bf16_t * LM_GGML_RESTRICT y, size_t by, int nrc) {
+    assert(nrc == 1);
+    LM_GGML_UNUSED(nrc);
+    LM_GGML_UNUSED(bx);
+    LM_GGML_UNUSED(by);
+    LM_GGML_UNUSED(bs);
+    int i = 0;
+    lm_ggml_float sumf = 0;
+
+#if defined(__AVX512BF16__)
+    __m512 c1 = _mm512_setzero_ps();
+    __m512 c2 = _mm512_setzero_ps();
+    for (; i + 64 <= n; i += 64) {
+        c1 = _mm512_dpbf16_ps(c1, m512bh(_mm512_loadu_si512((x + i))),
+                             m512bh(_mm512_loadu_si512((y + i))));
+        c2 = _mm512_dpbf16_ps(c2, m512bh(_mm512_loadu_si512((x + i + 32))),
+                             m512bh(_mm512_loadu_si512((y + i + 32))));
+    }
+    sumf += (lm_ggml_float)_mm512_reduce_add_ps(c1);
+    sumf += (lm_ggml_float)_mm512_reduce_add_ps(c2);
+
+#elif defined(__AVX512F__)
+#define LOAD(p) _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i *)(p))), 16))
+    __m512 c1 = _mm512_setzero_ps();
+    __m512 c2 = _mm512_setzero_ps();
+    for (; i + 32 <= n; i += 32) {
+        c1 = _mm512_add_ps(_mm512_mul_ps(LOAD(x + i), LOAD(y + i)), c1);
+        c2 = _mm512_add_ps(_mm512_mul_ps(LOAD(x + i + 16), LOAD(y + i + 16)), c2);
+    }
+    sumf += (lm_ggml_float)_mm512_reduce_add_ps(c1);
+    sumf += (lm_ggml_float)_mm512_reduce_add_ps(c2);
+
+#undef LOAD
+#elif defined(__AVX2__) || defined(__AVX__)
+#if defined(__AVX2__)
+#define LOAD(p) _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)(p))), 16))
+#else
+#define LOAD(p) _mm256_castsi256_ps(_mm256_insertf128_si256(_mm256_castsi128_si256(_mm_slli_epi32(_mm_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)(p))), 16)), (_mm_slli_epi32(_mm_cvtepu16_epi32(_mm_bsrli_si128(_mm_loadu_si128((const __m128i *)(p)), 8)), 16)), 1))
+#endif
+    __m256 c1 = _mm256_setzero_ps();
+    __m256 c2 = _mm256_setzero_ps();
+    __m256 c3 = _mm256_setzero_ps();
+    __m256 c4 = _mm256_setzero_ps();
+    for (; i + 32 <= n; i += 32) {
+        c1 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i), LOAD(y + i)), c1);
+        c2 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i + 8), LOAD(y + i + 8)), c2);
+        c3 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i + 16), LOAD(y + i + 16)), c3);
+        c4 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i + 24), LOAD(y + i + 24)), c4);
+    }
+    __m128 g;
+    c1 = _mm256_add_ps(_mm256_add_ps(c1, c3),
+                       _mm256_add_ps(c2, c4));
+    g = _mm_add_ps(_mm256_extractf128_ps(c1, 1),
+                   _mm256_castps256_ps128(c1));
+    g = _mm_add_ps(g, _mm_movehl_ps(g, g));
+    g = _mm_add_ss(g, _mm_movehdup_ps(g));
+    sumf += (lm_ggml_float)_mm_cvtss_f32(g);
+
+#undef LOAD
+#endif
+
+    for (; i < n; ++i) {
+        sumf += (lm_ggml_float)(LM_GGML_BF16_TO_FP32(x[i]) *
+                             LM_GGML_BF16_TO_FP32(y[i]));
+    }
+    *s = sumf;
+}
+
+void lm_ggml_vec_dot_f16(int n, float * LM_GGML_RESTRICT s, size_t bs, lm_ggml_fp16_t * LM_GGML_RESTRICT x, size_t bx, lm_ggml_fp16_t * LM_GGML_RESTRICT y, size_t by, int nrc) {
+    assert(nrc == 1);
+    LM_GGML_UNUSED(nrc);
+    LM_GGML_UNUSED(bx);
+    LM_GGML_UNUSED(by);
+    LM_GGML_UNUSED(bs);
+
+    lm_ggml_float sumf = 0.0;
+
+#if defined(LM_GGML_SIMD)
+    const int np = (n & ~(LM_GGML_F16_STEP - 1));
+
+    LM_GGML_F16_VEC sum[LM_GGML_F16_ARR] = { LM_GGML_F16_VEC_ZERO };
+
+    LM_GGML_F16_VEC ax[LM_GGML_F16_ARR];
+    LM_GGML_F16_VEC ay[LM_GGML_F16_ARR];
+
+    for (int i = 0; i < np; i += LM_GGML_F16_STEP) {
+        for (int j = 0; j < LM_GGML_F16_ARR; j++) {
+            ax[j] = LM_GGML_F16_VEC_LOAD(x + i + j*LM_GGML_F16_EPR, j);
+            ay[j] = LM_GGML_F16_VEC_LOAD(y + i + j*LM_GGML_F16_EPR, j);
+
+            sum[j] = LM_GGML_F16_VEC_FMA(sum[j], ax[j], ay[j]);
+        }
+    }
+
+    // reduce sum0..sum3 to sum0
+    LM_GGML_F16_VEC_REDUCE(sumf, sum);
+
+    // leftovers
+    for (int i = np; i < n; ++i) {
+        sumf += (lm_ggml_float)(LM_GGML_FP16_TO_FP32(x[i])*LM_GGML_FP16_TO_FP32(y[i]));
+    }
+#else
+    for (int i = 0; i < n; ++i) {
+        sumf += (lm_ggml_float)(LM_GGML_FP16_TO_FP32(x[i])*LM_GGML_FP16_TO_FP32(y[i]));
+    }
+#endif
+
+    *s = sumf;
+}
+
+void lm_ggml_vec_silu_f32(const int n, float * y, const float * x) {
+    int i = 0;
+#if defined(__AVX512F__) && defined(__AVX512DQ__)
+    for (; i + 15 < n; i += 16) {
+        _mm512_storeu_ps(y + i, lm_ggml_v_silu(_mm512_loadu_ps(x + i)));
+    }
+#elif defined(__AVX2__) && defined(__FMA__)
+    for (; i + 7 < n; i += 8) {
+        _mm256_storeu_ps(y + i, lm_ggml_v_silu(_mm256_loadu_ps(x + i)));
+    }
+#elif defined(__SSE2__)
+    for (; i + 3 < n; i += 4) {
+        _mm_storeu_ps(y + i, lm_ggml_v_silu(_mm_loadu_ps(x + i)));
+    }
+#elif defined(__ARM_NEON) && defined(__aarch64__)
+    for (; i + 3 < n; i += 4) {
+        vst1q_f32(y + i, lm_ggml_v_silu(vld1q_f32(x + i)));
+    }
+#endif
+    for (; i < n; ++i) {
+        y[i] = lm_ggml_silu_f32(x[i]);
+    }
+}
+
+lm_ggml_float lm_ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max) {
+    int i = 0;
+    lm_ggml_float sum = 0;
+#if defined(__AVX512F__) && defined(__AVX512DQ__)
+    for (; i + 15 < n; i += 16) {
+        __m512 val = lm_ggml_v_expf(_mm512_sub_ps(_mm512_loadu_ps(x + i),
+                                               _mm512_set1_ps(max)));
+        _mm512_storeu_ps(y + i, val);
+        sum += (lm_ggml_float)_mm512_reduce_add_ps(val);
+    }
+#elif defined(__AVX2__) && defined(__FMA__)
+    for (; i + 7 < n; i += 8) {
+        __m256 val = lm_ggml_v_expf(_mm256_sub_ps(_mm256_loadu_ps(x + i),
+                                               _mm256_set1_ps(max)));
+        _mm256_storeu_ps(y + i, val);
+        __m128 val2 = _mm_add_ps(_mm256_extractf128_ps(val, 1),
+                                 _mm256_castps256_ps128(val));
+        val2 = _mm_add_ps(val2, _mm_movehl_ps(val2, val2));
+        val2 = _mm_add_ss(val2, _mm_movehdup_ps(val2));
+        sum += (lm_ggml_float)_mm_cvtss_f32(val2);
+    }
+#elif defined(__SSE2__)
+    for (; i + 3 < n; i += 4) {
+        __m128 val = lm_ggml_v_expf(_mm_sub_ps(_mm_loadu_ps(x + i),
+                                            _mm_set1_ps(max)));
+        _mm_storeu_ps(y + i, val);
+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
+        val = _mm_add_ps(val, _mm_movehl_ps(val, val));
+        val = _mm_add_ss(val, _mm_movehdup_ps(val));
+#else
+        __m128 tmp = _mm_shuffle_ps(val, val, _MM_SHUFFLE(2, 3, 0, 1));
+        val = _mm_add_ps(val, tmp);
+        tmp = _mm_movehl_ps(tmp, val);
+        val = _mm_add_ss(val, tmp);
+#endif
+        sum += (lm_ggml_float)_mm_cvtss_f32(val);
+    }
+#elif defined(__ARM_NEON) && defined(__aarch64__)
+    for (; i + 3 < n; i += 4) {
+        float32x4_t val = lm_ggml_v_expf(vsubq_f32(vld1q_f32(x + i),
+                                                vdupq_n_f32(max)));
+        vst1q_f32(y + i, val);
+        sum += (lm_ggml_float)vaddvq_f32(val);
+    }
+#endif
+    for (; i < n; ++i) {
+        float val = expf(x[i] - max);
+        sum += (lm_ggml_float)val;
+        y[i] = val;
+    }
+    return sum;
+}
+
+lm_ggml_float lm_ggml_vec_log_soft_max_f32(const int n, float * y, const float * x, float max) {
+    // log(soft_max) = log(soft_max_i / soft_max_sum) = log(soft_max_i) - log(soft_max_sum) = (logit_i - max) - log(soft_max_i)
+
+    int i = 0;
+    lm_ggml_float sum = 0;
+    for (; i < n; ++i) {
+        float val = x[i] - max;
+        y[i] = val;
+        sum += (lm_ggml_float)expf(val);
+    }
+    return sum = (lm_ggml_float)logf(sum);
+}