llama-cpp-pro 0.2.2

package/cpp/ggml-cpu/arch-fallback.h

@@ -0,0 +1,215 @@
+#pragma once
+
+// Rename `_generic` functions if no native implementation is available.
+// This effectively selects the generic implementation.
+
+#if defined(LM_GGML_CPU_GENERIC)
+// quants.c
+#define quantize_row_q8_0_generic quantize_row_q8_0
+#define quantize_row_q8_1_generic quantize_row_q8_1
+#define quantize_row_q8_K_generic quantize_row_q8_K
+#define lm_ggml_vec_dot_q4_0_q8_0_generic lm_ggml_vec_dot_q4_0_q8_0
+#define lm_ggml_vec_dot_q4_1_q8_1_generic lm_ggml_vec_dot_q4_1_q8_1
+#define lm_ggml_vec_dot_q5_0_q8_0_generic lm_ggml_vec_dot_q5_0_q8_0
+#define lm_ggml_vec_dot_q5_1_q8_1_generic lm_ggml_vec_dot_q5_1_q8_1
+#define lm_ggml_vec_dot_q8_0_q8_0_generic lm_ggml_vec_dot_q8_0_q8_0
+#define lm_ggml_vec_dot_mxfp4_q8_0_generic lm_ggml_vec_dot_mxfp4_q8_0
+#define lm_ggml_vec_dot_tq1_0_q8_K_generic lm_ggml_vec_dot_tq1_0_q8_K
+#define lm_ggml_vec_dot_tq2_0_q8_K_generic lm_ggml_vec_dot_tq2_0_q8_K
+#define lm_ggml_vec_dot_q2_K_q8_K_generic lm_ggml_vec_dot_q2_K_q8_K
+#define lm_ggml_vec_dot_q3_K_q8_K_generic lm_ggml_vec_dot_q3_K_q8_K
+#define lm_ggml_vec_dot_q4_K_q8_K_generic lm_ggml_vec_dot_q4_K_q8_K
+#define lm_ggml_vec_dot_q5_K_q8_K_generic lm_ggml_vec_dot_q5_K_q8_K
+#define lm_ggml_vec_dot_q6_K_q8_K_generic lm_ggml_vec_dot_q6_K_q8_K
+#define lm_ggml_vec_dot_iq2_xxs_q8_K_generic lm_ggml_vec_dot_iq2_xxs_q8_K
+#define lm_ggml_vec_dot_iq2_xs_q8_K_generic lm_ggml_vec_dot_iq2_xs_q8_K
+#define lm_ggml_vec_dot_iq2_s_q8_K_generic lm_ggml_vec_dot_iq2_s_q8_K
+#define lm_ggml_vec_dot_iq3_xxs_q8_K_generic lm_ggml_vec_dot_iq3_xxs_q8_K
+#define lm_ggml_vec_dot_iq3_s_q8_K_generic lm_ggml_vec_dot_iq3_s_q8_K
+#define lm_ggml_vec_dot_iq1_s_q8_K_generic lm_ggml_vec_dot_iq1_s_q8_K
+#define lm_ggml_vec_dot_iq1_m_q8_K_generic lm_ggml_vec_dot_iq1_m_q8_K
+#define lm_ggml_vec_dot_iq4_nl_q8_0_generic lm_ggml_vec_dot_iq4_nl_q8_0
+#define lm_ggml_vec_dot_iq4_xs_q8_K_generic lm_ggml_vec_dot_iq4_xs_q8_K
+// repack.cpp
+#define lm_ggml_quantize_mat_q8_0_4x4_generic lm_ggml_quantize_mat_q8_0_4x4
+#define lm_ggml_quantize_mat_q8_0_4x8_generic lm_ggml_quantize_mat_q8_0_4x8
+#define lm_ggml_quantize_mat_q8_K_4x8_generic lm_ggml_quantize_mat_q8_K_4x8
+#define lm_ggml_gemv_q4_0_4x4_q8_0_generic lm_ggml_gemv_q4_0_4x4_q8_0
+#define lm_ggml_gemv_q4_0_4x8_q8_0_generic lm_ggml_gemv_q4_0_4x8_q8_0
+#define lm_ggml_gemv_q4_0_8x8_q8_0_generic lm_ggml_gemv_q4_0_8x8_q8_0
+#define lm_ggml_gemv_q4_K_8x8_q8_K_generic lm_ggml_gemv_q4_K_8x8_q8_K
+#define lm_ggml_gemv_q2_K_8x8_q8_K_generic lm_ggml_gemv_q2_K_8x8_q8_K
+#define lm_ggml_gemv_iq4_nl_4x4_q8_0_generic lm_ggml_gemv_iq4_nl_4x4_q8_0
+#define lm_ggml_gemv_iq4_nl_8x8_q8_0_generic lm_ggml_gemv_iq4_nl_8x8_q8_0
+#define lm_ggml_gemm_q4_0_4x4_q8_0_generic lm_ggml_gemm_q4_0_4x4_q8_0
+#define lm_ggml_gemm_q4_0_4x8_q8_0_generic lm_ggml_gemm_q4_0_4x8_q8_0
+#define lm_ggml_gemm_q4_0_8x8_q8_0_generic lm_ggml_gemm_q4_0_8x8_q8_0
+#define lm_ggml_gemm_q4_K_8x8_q8_K_generic lm_ggml_gemm_q4_K_8x8_q8_K
+#define lm_ggml_gemm_q2_K_8x8_q8_K_generic lm_ggml_gemm_q2_K_8x8_q8_K
+#define lm_ggml_gemm_iq4_nl_4x4_q8_0_generic lm_ggml_gemm_iq4_nl_4x4_q8_0
+#define lm_ggml_gemm_iq4_nl_8x8_q8_0_generic lm_ggml_gemm_iq4_nl_8x8_q8_0
+#elif defined(__aarch64__) || defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64)
+// repack.cpp
+#define lm_ggml_quantize_mat_q8_K_4x8_generic lm_ggml_quantize_mat_q8_K_4x8
+#define lm_ggml_gemv_q4_K_8x8_q8_K_generic lm_ggml_gemv_q4_K_8x8_q8_K
+#define lm_ggml_gemv_iq4_nl_8x8_q8_0_generic lm_ggml_gemv_iq4_nl_8x8_q8_0
+#define lm_ggml_gemv_q2_K_8x8_q8_K_generic lm_ggml_gemv_q2_K_8x8_q8_K
+#define lm_ggml_gemm_q4_K_8x8_q8_K_generic lm_ggml_gemm_q4_K_8x8_q8_K
+#define lm_ggml_gemm_iq4_nl_8x8_q8_0_generic lm_ggml_gemm_iq4_nl_8x8_q8_0
+#define lm_ggml_gemm_q2_K_8x8_q8_K_generic lm_ggml_gemm_q2_K_8x8_q8_K
+#elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64)
+// repack.cpp
+#define lm_ggml_quantize_mat_q8_0_4x4_generic lm_ggml_quantize_mat_q8_0_4x4
+#define lm_ggml_gemv_q4_0_4x4_q8_0_generic lm_ggml_gemv_q4_0_4x4_q8_0
+#define lm_ggml_gemv_q4_0_4x8_q8_0_generic lm_ggml_gemv_q4_0_4x8_q8_0
+#define lm_ggml_gemv_iq4_nl_4x4_q8_0_generic lm_ggml_gemv_iq4_nl_4x4_q8_0
+#define lm_ggml_gemm_q4_0_4x4_q8_0_generic lm_ggml_gemm_q4_0_4x4_q8_0
+#define lm_ggml_gemm_q4_0_4x8_q8_0_generic lm_ggml_gemm_q4_0_4x8_q8_0
+#define lm_ggml_gemm_iq4_nl_4x4_q8_0_generic lm_ggml_gemm_iq4_nl_4x4_q8_0
+#elif defined(__POWERPC__) || defined(__powerpc__)
+// ref: https://github.com/ggml-org/llama.cpp/pull/14146#issuecomment-2972561679
+// quants.c
+#define quantize_row_q8_K_generic quantize_row_q8_K
+#define lm_ggml_vec_dot_tq1_0_q8_K_generic lm_ggml_vec_dot_tq1_0_q8_K
+#define lm_ggml_vec_dot_tq2_0_q8_K_generic lm_ggml_vec_dot_tq2_0_q8_K
+#define lm_ggml_vec_dot_iq1_m_q8_K_generic lm_ggml_vec_dot_iq1_m_q8_K
+// repack.cpp
+#define lm_ggml_quantize_mat_q8_0_4x4_generic lm_ggml_quantize_mat_q8_0_4x4
+#define lm_ggml_quantize_mat_q8_0_4x8_generic lm_ggml_quantize_mat_q8_0_4x8
+#define lm_ggml_quantize_mat_q8_K_4x8_generic lm_ggml_quantize_mat_q8_K_4x8
+#define lm_ggml_gemv_q4_0_4x4_q8_0_generic lm_ggml_gemv_q4_0_4x4_q8_0
+#define lm_ggml_gemv_q4_0_4x8_q8_0_generic lm_ggml_gemv_q4_0_4x8_q8_0
+#define lm_ggml_gemv_q4_0_8x8_q8_0_generic lm_ggml_gemv_q4_0_8x8_q8_0
+#define lm_ggml_gemv_q4_K_8x8_q8_K_generic lm_ggml_gemv_q4_K_8x8_q8_K
+#define lm_ggml_gemv_q2_K_8x8_q8_K_generic lm_ggml_gemv_q2_K_8x8_q8_K
+#define lm_ggml_gemv_iq4_nl_4x4_q8_0_generic lm_ggml_gemv_iq4_nl_4x4_q8_0
+#define lm_ggml_gemv_iq4_nl_8x8_q8_0_generic lm_ggml_gemv_iq4_nl_8x8_q8_0
+#define lm_ggml_gemm_q4_0_4x4_q8_0_generic lm_ggml_gemm_q4_0_4x4_q8_0
+#define lm_ggml_gemm_q4_0_4x8_q8_0_generic lm_ggml_gemm_q4_0_4x8_q8_0
+#define lm_ggml_gemm_q4_0_8x8_q8_0_generic lm_ggml_gemm_q4_0_8x8_q8_0
+#define lm_ggml_gemm_q4_K_8x8_q8_K_generic lm_ggml_gemm_q4_K_8x8_q8_K
+#define lm_ggml_gemm_q2_K_8x8_q8_K_generic lm_ggml_gemm_q2_K_8x8_q8_K
+#define lm_ggml_gemm_iq4_nl_4x4_q8_0_generic lm_ggml_gemm_iq4_nl_4x4_q8_0
+#define lm_ggml_gemm_iq4_nl_8x8_q8_0_generic lm_ggml_gemm_iq4_nl_8x8_q8_0
+#elif defined(__loongarch64)
+// quants.c
+#define quantize_row_q8_K_generic quantize_row_q8_K
+#define lm_ggml_vec_dot_tq1_0_q8_K_generic lm_ggml_vec_dot_tq1_0_q8_K
+#define lm_ggml_vec_dot_tq2_0_q8_K_generic lm_ggml_vec_dot_tq2_0_q8_K
+#define lm_ggml_vec_dot_iq1_m_q8_K_generic lm_ggml_vec_dot_iq1_m_q8_K
+#define lm_ggml_vec_dot_mxfp4_q8_0_generic lm_ggml_vec_dot_mxfp4_q8_0
+// repack.cpp
+#define lm_ggml_quantize_mat_q8_0_4x4_generic lm_ggml_quantize_mat_q8_0_4x4
+#define lm_ggml_quantize_mat_q8_0_4x8_generic lm_ggml_quantize_mat_q8_0_4x8
+#define lm_ggml_quantize_mat_q8_K_4x8_generic lm_ggml_quantize_mat_q8_K_4x8
+#define lm_ggml_gemv_q4_0_4x4_q8_0_generic lm_ggml_gemv_q4_0_4x4_q8_0
+#define lm_ggml_gemv_q4_0_4x8_q8_0_generic lm_ggml_gemv_q4_0_4x8_q8_0
+#define lm_ggml_gemv_q4_0_8x8_q8_0_generic lm_ggml_gemv_q4_0_8x8_q8_0
+#define lm_ggml_gemv_q4_K_8x8_q8_K_generic lm_ggml_gemv_q4_K_8x8_q8_K
+#define lm_ggml_gemv_q2_K_8x8_q8_K_generic lm_ggml_gemv_q2_K_8x8_q8_K
+#define lm_ggml_gemv_iq4_nl_4x4_q8_0_generic lm_ggml_gemv_iq4_nl_4x4_q8_0
+#define lm_ggml_gemv_iq4_nl_8x8_q8_0_generic lm_ggml_gemv_iq4_nl_8x8_q8_0
+#define lm_ggml_gemm_q4_0_4x4_q8_0_generic lm_ggml_gemm_q4_0_4x4_q8_0
+#define lm_ggml_gemm_q4_0_4x8_q8_0_generic lm_ggml_gemm_q4_0_4x8_q8_0
+#define lm_ggml_gemm_q4_0_8x8_q8_0_generic lm_ggml_gemm_q4_0_8x8_q8_0
+#define lm_ggml_gemm_q4_K_8x8_q8_K_generic lm_ggml_gemm_q4_K_8x8_q8_K
+#define lm_ggml_gemm_q2_K_8x8_q8_K_generic lm_ggml_gemm_q2_K_8x8_q8_K
+#define lm_ggml_gemm_iq4_nl_4x4_q8_0_generic lm_ggml_gemm_iq4_nl_4x4_q8_0
+#define lm_ggml_gemm_iq4_nl_8x8_q8_0_generic lm_ggml_gemm_iq4_nl_8x8_q8_0
+#elif defined(__riscv)
+// quants.c
+#define quantize_row_q8_K_generic quantize_row_q8_K
+#define lm_ggml_vec_dot_tq1_0_q8_K_generic lm_ggml_vec_dot_tq1_0_q8_K
+#define lm_ggml_vec_dot_tq2_0_q8_K_generic lm_ggml_vec_dot_tq2_0_q8_K
+#define lm_ggml_vec_dot_iq2_xxs_q8_K_generic lm_ggml_vec_dot_iq2_xxs_q8_K
+#define lm_ggml_vec_dot_iq2_xs_q8_K_generic lm_ggml_vec_dot_iq2_xs_q8_K
+#define lm_ggml_vec_dot_iq2_s_q8_K_generic lm_ggml_vec_dot_iq2_s_q8_K
+#define lm_ggml_vec_dot_iq3_xxs_q8_K_generic lm_ggml_vec_dot_iq3_xxs_q8_K
+#define lm_ggml_vec_dot_iq3_s_q8_K_generic lm_ggml_vec_dot_iq3_s_q8_K
+#define lm_ggml_vec_dot_iq1_s_q8_K_generic lm_ggml_vec_dot_iq1_s_q8_K
+#define lm_ggml_vec_dot_iq1_m_q8_K_generic lm_ggml_vec_dot_iq1_m_q8_K
+#define lm_ggml_vec_dot_iq4_nl_q8_0_generic lm_ggml_vec_dot_iq4_nl_q8_0
+#define lm_ggml_vec_dot_iq4_xs_q8_K_generic lm_ggml_vec_dot_iq4_xs_q8_K
+#define lm_ggml_vec_dot_mxfp4_q8_0_generic lm_ggml_vec_dot_mxfp4_q8_0
+// repack.cpp
+#define lm_ggml_quantize_mat_q8_0_4x4_generic lm_ggml_quantize_mat_q8_0_4x4
+#define lm_ggml_quantize_mat_q8_0_4x8_generic lm_ggml_quantize_mat_q8_0_4x8
+#define lm_ggml_quantize_mat_q8_K_4x8_generic lm_ggml_quantize_mat_q8_K_4x8
+#define lm_ggml_gemv_q4_0_4x4_q8_0_generic lm_ggml_gemv_q4_0_4x4_q8_0
+#define lm_ggml_gemv_q4_0_4x8_q8_0_generic lm_ggml_gemv_q4_0_4x8_q8_0
+#define lm_ggml_gemv_q4_K_8x8_q8_K_generic lm_ggml_gemv_q4_K_8x8_q8_K
+#define lm_ggml_gemv_q2_K_8x8_q8_K_generic lm_ggml_gemv_q2_K_8x8_q8_K
+#define lm_ggml_gemv_iq4_nl_4x4_q8_0_generic lm_ggml_gemv_iq4_nl_4x4_q8_0
+#define lm_ggml_gemv_iq4_nl_8x8_q8_0_generic lm_ggml_gemv_iq4_nl_8x8_q8_0
+#define lm_ggml_gemm_q4_0_4x4_q8_0_generic lm_ggml_gemm_q4_0_4x4_q8_0
+#define lm_ggml_gemm_q4_0_4x8_q8_0_generic lm_ggml_gemm_q4_0_4x8_q8_0
+#define lm_ggml_gemm_q4_K_8x8_q8_K_generic lm_ggml_gemm_q4_K_8x8_q8_K
+#define lm_ggml_gemm_q2_K_8x8_q8_K_generic lm_ggml_gemm_q2_K_8x8_q8_K
+#define lm_ggml_gemm_iq4_nl_4x4_q8_0_generic lm_ggml_gemm_iq4_nl_4x4_q8_0
+#define lm_ggml_gemm_iq4_nl_8x8_q8_0_generic lm_ggml_gemm_iq4_nl_8x8_q8_0
+#elif defined(__s390x__)
+// quants.c
+#define quantize_row_q8_K_generic quantize_row_q8_K
+#define lm_ggml_vec_dot_tq1_0_q8_K_generic lm_ggml_vec_dot_tq1_0_q8_K
+#define lm_ggml_vec_dot_tq2_0_q8_K_generic lm_ggml_vec_dot_tq2_0_q8_K
+#define lm_ggml_vec_dot_q2_K_q8_K_generic lm_ggml_vec_dot_q2_K_q8_K
+#define lm_ggml_vec_dot_iq2_xxs_q8_K_generic lm_ggml_vec_dot_iq2_xxs_q8_K
+#define lm_ggml_vec_dot_iq2_xs_q8_K_generic lm_ggml_vec_dot_iq2_xs_q8_K
+#define lm_ggml_vec_dot_iq2_s_q8_K_generic lm_ggml_vec_dot_iq2_s_q8_K
+#define lm_ggml_vec_dot_iq3_xxs_q8_K_generic lm_ggml_vec_dot_iq3_xxs_q8_K
+#define lm_ggml_vec_dot_iq3_s_q8_K_generic lm_ggml_vec_dot_iq3_s_q8_K
+#define lm_ggml_vec_dot_iq1_s_q8_K_generic lm_ggml_vec_dot_iq1_s_q8_K
+#define lm_ggml_vec_dot_iq1_m_q8_K_generic lm_ggml_vec_dot_iq1_m_q8_K
+#define lm_ggml_vec_dot_mxfp4_q8_0_generic lm_ggml_vec_dot_mxfp4_q8_0
+// repack.cpp
+#define lm_ggml_quantize_mat_q8_0_4x4_generic lm_ggml_quantize_mat_q8_0_4x4
+#define lm_ggml_quantize_mat_q8_0_4x8_generic lm_ggml_quantize_mat_q8_0_4x8
+#define lm_ggml_quantize_mat_q8_K_4x8_generic lm_ggml_quantize_mat_q8_K_4x8
+#define lm_ggml_gemv_q4_0_4x4_q8_0_generic lm_ggml_gemv_q4_0_4x4_q8_0
+#define lm_ggml_gemv_q4_0_4x8_q8_0_generic lm_ggml_gemv_q4_0_4x8_q8_0
+#define lm_ggml_gemv_q4_0_8x8_q8_0_generic lm_ggml_gemv_q4_0_8x8_q8_0
+#define lm_ggml_gemv_q4_K_8x8_q8_K_generic lm_ggml_gemv_q4_K_8x8_q8_K
+#define lm_ggml_gemv_q2_K_8x8_q8_K_generic lm_ggml_gemv_q2_K_8x8_q8_K
+#define lm_ggml_gemv_iq4_nl_4x4_q8_0_generic lm_ggml_gemv_iq4_nl_4x4_q8_0
+#define lm_ggml_gemv_iq4_nl_8x8_q8_0_generic lm_ggml_gemv_iq4_nl_8x8_q8_0
+#define lm_ggml_gemm_q4_0_4x4_q8_0_generic lm_ggml_gemm_q4_0_4x4_q8_0
+#define lm_ggml_gemm_q4_0_4x8_q8_0_generic lm_ggml_gemm_q4_0_4x8_q8_0
+#define lm_ggml_gemm_q4_0_8x8_q8_0_generic lm_ggml_gemm_q4_0_8x8_q8_0
+#define lm_ggml_gemm_q4_K_8x8_q8_K_generic lm_ggml_gemm_q4_K_8x8_q8_K
+#define lm_ggml_gemm_q2_K_8x8_q8_K_generic lm_ggml_gemm_q2_K_8x8_q8_K
+#define lm_ggml_gemm_iq4_nl_4x4_q8_0_generic lm_ggml_gemm_iq4_nl_4x4_q8_0
+#define lm_ggml_gemm_iq4_nl_8x8_q8_0_generic lm_ggml_gemm_iq4_nl_8x8_q8_0
+#elif defined(__wasm__)
+// quants.c
+#define lm_ggml_vec_dot_q4_1_q8_1_generic lm_ggml_vec_dot_q4_1_q8_1
+#define lm_ggml_vec_dot_tq1_0_q8_K_generic lm_ggml_vec_dot_tq1_0_q8_K
+#define lm_ggml_vec_dot_tq2_0_q8_K_generic lm_ggml_vec_dot_tq2_0_q8_K
+#define lm_ggml_vec_dot_iq2_xxs_q8_K_generic lm_ggml_vec_dot_iq2_xxs_q8_K
+#define lm_ggml_vec_dot_iq2_xs_q8_K_generic lm_ggml_vec_dot_iq2_xs_q8_K
+#define lm_ggml_vec_dot_iq2_s_q8_K_generic lm_ggml_vec_dot_iq2_s_q8_K
+#define lm_ggml_vec_dot_iq3_xxs_q8_K_generic lm_ggml_vec_dot_iq3_xxs_q8_K
+#define lm_ggml_vec_dot_iq3_s_q8_K_generic lm_ggml_vec_dot_iq3_s_q8_K
+#define lm_ggml_vec_dot_iq1_s_q8_K_generic lm_ggml_vec_dot_iq1_s_q8_K
+#define lm_ggml_vec_dot_iq1_m_q8_K_generic lm_ggml_vec_dot_iq1_m_q8_K
+#define lm_ggml_vec_dot_iq4_nl_q8_0_generic lm_ggml_vec_dot_iq4_nl_q8_0
+#define lm_ggml_vec_dot_iq4_xs_q8_K_generic lm_ggml_vec_dot_iq4_xs_q8_K
+#define lm_ggml_vec_dot_mxfp4_q8_0_generic lm_ggml_vec_dot_mxfp4_q8_0
+// repack.cpp
+#define lm_ggml_quantize_mat_q8_0_4x4_generic lm_ggml_quantize_mat_q8_0_4x4
+#define lm_ggml_quantize_mat_q8_0_4x8_generic lm_ggml_quantize_mat_q8_0_4x8
+#define lm_ggml_quantize_mat_q8_K_4x8_generic lm_ggml_quantize_mat_q8_K_4x8
+#define lm_ggml_gemv_q4_0_4x4_q8_0_generic lm_ggml_gemv_q4_0_4x4_q8_0
+#define lm_ggml_gemv_q4_0_4x8_q8_0_generic lm_ggml_gemv_q4_0_4x8_q8_0
+#define lm_ggml_gemv_q4_0_8x8_q8_0_generic lm_ggml_gemv_q4_0_8x8_q8_0
+#define lm_ggml_gemv_q4_K_8x8_q8_K_generic lm_ggml_gemv_q4_K_8x8_q8_K
+#define lm_ggml_gemv_q2_K_8x8_q8_K_generic lm_ggml_gemv_q2_K_8x8_q8_K
+#define lm_ggml_gemv_iq4_nl_4x4_q8_0_generic lm_ggml_gemv_iq4_nl_4x4_q8_0
+#define lm_ggml_gemv_iq4_nl_8x8_q8_0_generic lm_ggml_gemv_iq4_nl_8x8_q8_0
+#define lm_ggml_gemm_q4_0_4x4_q8_0_generic lm_ggml_gemm_q4_0_4x4_q8_0
+#define lm_ggml_gemm_q4_0_4x8_q8_0_generic lm_ggml_gemm_q4_0_4x8_q8_0
+#define lm_ggml_gemm_q4_0_8x8_q8_0_generic lm_ggml_gemm_q4_0_8x8_q8_0
+#define lm_ggml_gemm_q4_K_8x8_q8_K_generic lm_ggml_gemm_q4_K_8x8_q8_K
+#define lm_ggml_gemm_q2_K_8x8_q8_K_generic lm_ggml_gemm_q2_K_8x8_q8_K
+#define lm_ggml_gemm_iq4_nl_4x4_q8_0_generic lm_ggml_gemm_iq4_nl_4x4_q8_0
+#define lm_ggml_gemm_iq4_nl_8x8_q8_0_generic lm_ggml_gemm_iq4_nl_8x8_q8_0
+#endif

package/cpp/ggml-cpu/binary-ops.cpp

@@ -0,0 +1,158 @@
+#include "binary-ops.h"
+
+#if defined(LM_GGML_USE_ACCELERATE)
+#include <Accelerate/Accelerate.h>
+
+using vDSP_fn_t = void (*)(const float *, vDSP_Stride, const float *, vDSP_Stride, float *, vDSP_Stride, vDSP_Length);
+#endif
+
+static inline float op_add(float a, float b) {
+    return a + b;
+}
+
+static inline float op_sub(float a, float b) {
+    return a - b;
+}
+
+static inline float op_mul(float a, float b) {
+    return a * b;
+}
+
+static inline float op_div(float a, float b) {
+    return a / b;
+}
+
+template <float (*op)(float, float), typename src0_t, typename src1_t, typename dst_t>
+static inline void vec_binary_op_contiguous(const int64_t n, dst_t * z, const src0_t * x, const src1_t * y) {
+    constexpr auto src0_to_f32 = type_conversion_table<src0_t>::to_f32;
+    constexpr auto src1_to_f32 = type_conversion_table<src1_t>::to_f32;
+    constexpr auto f32_to_dst  = type_conversion_table<dst_t >::from_f32;
+
+    for (int i = 0; i < n; i++) {
+        z[i] = f32_to_dst(op(src0_to_f32(x[i]), src1_to_f32(y[i])));
+    }
+}
+
+template <float (*op)(float, float), typename src0_t, typename src1_t, typename dst_t>
+static inline void vec_binary_op_non_contiguous(const int64_t n, const int64_t ne10, const int64_t nb10, dst_t * z, const src0_t * x, const src1_t * y) {
+    constexpr auto src0_to_f32 = type_conversion_table<src0_t>::to_f32;
+    constexpr auto src1_to_f32 = type_conversion_table<src1_t>::to_f32;
+    constexpr auto f32_to_dst  = type_conversion_table<dst_t >::from_f32;
+
+    for (int i = 0; i < n; i++) {
+        int i10 = i % ne10;
+        const src1_t * y_ptr = (const src1_t *)((const char *)y + i10*nb10);
+        z[i] = f32_to_dst(op(src0_to_f32(x[i]), src1_to_f32(*y_ptr)));
+    }
+}
+
+template <float (*op)(float, float), typename src0_t, typename src1_t, typename dst_t>
+static void apply_binary_op(const lm_ggml_compute_params * params, lm_ggml_tensor * dst) {
+    const lm_ggml_tensor * src0 = dst->src[0];
+    const lm_ggml_tensor * src1 = dst->src[1];
+
+    LM_GGML_ASSERT(lm_ggml_can_repeat(src1, src0) && lm_ggml_are_same_shape(src0, dst));
+
+    LM_GGML_TENSOR_BINARY_OP_LOCALS
+
+    LM_GGML_ASSERT( nb0 == sizeof(dst_t));
+    LM_GGML_ASSERT(nb00 == sizeof(src0_t));
+
+    const auto [ir0, ir1] = get_thread_range(params, src0);
+    const bool is_src1_contiguous = (nb10 == sizeof(src1_t));
+
+    if (!is_src1_contiguous) { // broadcast not implemented yet for non-contiguous
+        LM_GGML_ASSERT(lm_ggml_are_same_shape(src0, src1));
+    }
+
+#ifdef LM_GGML_USE_ACCELERATE
+    vDSP_fn_t vDSP_op = nullptr;
+    // TODO - avoid the f32-only check using type 'trait' lookup tables and row-based src-to-float conversion functions
+    if (src0->type == LM_GGML_TYPE_F32 && src1->type == LM_GGML_TYPE_F32 && dst->type == LM_GGML_TYPE_F32) {
+        if (op == op_add) {
+            vDSP_op = vDSP_vadd;
+        } else if (op == op_sub) {
+            vDSP_op = vDSP_vsub;
+        } else if (op == op_mul) {
+            vDSP_op = vDSP_vmul;
+        } else if (op == op_div) {
+            vDSP_op = vDSP_vdiv;
+        }
+    }
+#endif
+
+    for (int64_t ir = ir0; ir < ir1; ++ir) {
+        const int64_t i03 = ir/(ne02*ne01);
+        const int64_t i02 = (ir - i03*ne02*ne01)/ne01;
+        const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);
+
+        const int64_t i13 = i03 % ne13;
+        const int64_t i12 = i02 % ne12;
+        const int64_t i11 = i01 % ne11;
+
+        dst_t        * dst_ptr  = (dst_t  *)       ((char *)       dst->data  + i03*nb3  + i02*nb2  + i01*nb1 );
+        const src0_t * src0_ptr = (const src0_t *) ((const char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);
+        const src1_t * src1_ptr = (const src1_t *) ((const char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11);
+
+        if (is_src1_contiguous) {
+            // src1 is broadcastable across src0 and dst in i1, i2, i3
+            const int64_t nr0 = ne00 / ne10;
+
+            for (int64_t r = 0; r < nr0; ++r) {
+#ifdef LM_GGML_USE_ACCELERATE
+                if constexpr (std::is_same_v<src0_t, float> && std::is_same_v<src1_t, float> && std::is_same_v<dst_t, float>) {
+                    if (vDSP_op != nullptr) {
+                        vDSP_op(src1_ptr, 1, src0_ptr + r*ne10, 1, dst_ptr + r*ne10, 1, ne10);
+                        continue;
+                    }
+                }
+#endif
+                vec_binary_op_contiguous<op>(ne10, dst_ptr + r*ne10, src0_ptr + r*ne10, src1_ptr);
+            }
+        } else {
+            vec_binary_op_non_contiguous<op>(ne0, ne10, nb10, dst_ptr, src0_ptr, src1_ptr);
+        }
+    }
+}
+
+// TODO: Use the 'traits' lookup table (for type conversion fns), instead of a mass of 'if' conditions with long templates
+template <float (*op)(float, float)>
+static void binary_op(const lm_ggml_compute_params * params, lm_ggml_tensor * dst) {
+    const lm_ggml_tensor * src0 = dst->src[0];
+    const lm_ggml_tensor * src1 = dst->src[1];
+
+    /*  */ if (src0->type == LM_GGML_TYPE_F32  && src1->type == LM_GGML_TYPE_F32  && dst->type == LM_GGML_TYPE_F32) { // all f32
+        apply_binary_op<op, float, float, float>(params, dst);
+    } else if (src0->type == LM_GGML_TYPE_F16  && src1->type == LM_GGML_TYPE_F16  && dst->type == LM_GGML_TYPE_F16) { // all f16
+        apply_binary_op<op, lm_ggml_fp16_t, lm_ggml_fp16_t, lm_ggml_fp16_t>(params, dst);
+    } else if (src0->type == LM_GGML_TYPE_BF16 && src1->type == LM_GGML_TYPE_BF16 && dst->type == LM_GGML_TYPE_BF16) { // all bf16
+        apply_binary_op<op, lm_ggml_bf16_t, lm_ggml_bf16_t, lm_ggml_bf16_t>(params, dst);
+    } else if (src0->type == LM_GGML_TYPE_BF16 && src1->type == LM_GGML_TYPE_F32  && dst->type == LM_GGML_TYPE_BF16) {
+        apply_binary_op<op, lm_ggml_bf16_t, float, lm_ggml_bf16_t>(params, dst);
+    } else if (src0->type == LM_GGML_TYPE_BF16 && src1->type == LM_GGML_TYPE_F32  && dst->type == LM_GGML_TYPE_F32) {
+        apply_binary_op<op, lm_ggml_bf16_t, float, float>(params, dst);
+    } else if (src0->type == LM_GGML_TYPE_F16  && src1->type == LM_GGML_TYPE_F32  && dst->type == LM_GGML_TYPE_F16) {
+        apply_binary_op<op, lm_ggml_fp16_t, float, lm_ggml_fp16_t>(params, dst);
+    } else if (src0->type == LM_GGML_TYPE_F16  && src1->type == LM_GGML_TYPE_F32  && dst->type == LM_GGML_TYPE_F32) {
+        apply_binary_op<op, lm_ggml_fp16_t, float, float>(params, dst);
+    } else {
+        LM_GGML_ABORT("%s: unsupported types: dst: %s, src0: %s, src1: %s\n", __func__,
+            lm_ggml_type_name(dst->type), lm_ggml_type_name(src0->type), lm_ggml_type_name(src1->type));
+    }
+}
+
+void lm_ggml_compute_forward_add_non_quantized(const lm_ggml_compute_params * params, lm_ggml_tensor * dst) {
+    binary_op<op_add>(params, dst);
+}
+
+void lm_ggml_compute_forward_sub(const lm_ggml_compute_params * params, lm_ggml_tensor * dst) {
+    binary_op<op_sub>(params, dst);
+}
+
+void lm_ggml_compute_forward_mul(const lm_ggml_compute_params * params, lm_ggml_tensor * dst) {
+    binary_op<op_mul>(params, dst);
+}
+
+void lm_ggml_compute_forward_div(const lm_ggml_compute_params * params, lm_ggml_tensor * dst) {
+    binary_op<op_div>(params, dst);
+}

package/cpp/ggml-cpu/binary-ops.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include "common.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void lm_ggml_compute_forward_add_non_quantized(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
+void lm_ggml_compute_forward_sub(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
+void lm_ggml_compute_forward_mul(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
+void lm_ggml_compute_forward_div(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
+
+#ifdef __cplusplus
+}
+#endif

package/cpp/ggml-cpu/common.h

@@ -0,0 +1,73 @@
+#pragma once
+
+#include "ggml.h"
+#include "traits.h"
+#include "ggml-cpu-impl.h"
+#include "ggml-impl.h"
+#include "simd-mappings.h"
+
+#ifdef __cplusplus
+
+#include <utility>
+
+// convenience functions/macros for use in template calls
+// note: these won't be required after the 'traits' lookup table is used.
+static inline lm_ggml_fp16_t f32_to_f16(float x) {
+    return LM_GGML_CPU_FP32_TO_FP16(x);
+}
+
+static inline float f16_to_f32(lm_ggml_fp16_t x) {
+    return LM_GGML_CPU_FP16_TO_FP32(x);
+}
+
+static inline lm_ggml_bf16_t f32_to_bf16(float x) {
+    return LM_GGML_FP32_TO_BF16(x);
+}
+
+static inline float bf16_to_f32(lm_ggml_bf16_t x) {
+    return LM_GGML_BF16_TO_FP32(x);
+}
+
+static inline float f32_to_f32(float x) {
+    return x;
+}
+
+// TODO - merge this into the traits table, after using row-based conversions
+template <class T>
+struct type_conversion_table;
+
+template <>
+struct type_conversion_table<lm_ggml_fp16_t> {
+    static constexpr float (*to_f32)(lm_ggml_fp16_t) = f16_to_f32;
+    static constexpr lm_ggml_fp16_t (*from_f32)(float) = f32_to_f16;
+};
+
+template <>
+struct type_conversion_table<float> {
+    static constexpr float (*to_f32)(float) = f32_to_f32;
+    static constexpr float (*from_f32)(float) = f32_to_f32;
+};
+
+template <>
+struct type_conversion_table<lm_ggml_bf16_t> {
+    static constexpr float (*to_f32)(lm_ggml_bf16_t) = bf16_to_f32;
+    static constexpr lm_ggml_bf16_t (*from_f32)(float) = f32_to_bf16;
+};
+
+static std::pair<int64_t, int64_t> get_thread_range(const struct lm_ggml_compute_params * params, const struct lm_ggml_tensor * src0) {
+    const int64_t ith = params->ith;
+    const int64_t nth = params->nth;
+
+    const int64_t nr  = lm_ggml_nrows(src0);
+
+    // rows per thread
+    const int64_t dr = (nr + nth - 1)/nth;
+
+    // row range for this thread
+    const int64_t ir0 = dr*ith;
+    const int64_t ir1 = MIN(ir0 + dr, nr);
+
+    return {ir0, ir1};
+}
+
+#endif