whispercpp 1.3.6 → 1.3.8

data/ext/sources/ggml/src/ggml-webgpu/wgsl-shaders/upscale.wgsl

@@ -0,0 +1,240 @@
+#if defined(SRC_F16) || defined(DST_F16)
+enable f16;
+#endif
+
+#ifdef SRC_F16
+#define SRC_TYPE f16
+#else
+#define SRC_TYPE f32
+#endif
+
+#ifdef DST_F16
+#define DST_TYPE f16
+#else
+#define DST_TYPE f32
+#endif
+
+@group(0) @binding(0)
+var<storage, read_write> input: array<SRC_TYPE>;
+
+@group(0) @binding(1)
+var<storage, read_write> output: array<DST_TYPE>;
+
+struct Params {
+    offset_i: u32,
+    offset_o: u32,
+
+    // element strides
+    si0: u32, si1: u32, si2: u32, si3: u32,
+    so0: u32, so1: u32, so2: u32, so3: u32,
+
+    src_w: u32,
+    src_h: u32,
+    src_z: u32,
+    src_n: u32,
+
+    dst_w: u32,
+    dst_h: u32,
+    dst_z: u32,
+    dst_n: u32,
+
+    mode_flags: u32,
+};
+
+@group(0) @binding(2)
+var<uniform> params: Params;
+
+const GGML_SCALE_FLAG_ALIGN_CORNERS: u32 = 1u << 8u;
+
+fn get_clamped_input(x: i32, y: i32, z: u32, n: u32) -> f32 {
+    let cx = u32(clamp(x, 0, i32(params.src_w) - 1));
+    let cy = u32(clamp(y, 0, i32(params.src_h) - 1));
+    let i = params.offset_i + cx * params.si0 + cy * params.si1 + z * params.si2 + n * params.si3;
+    return f32(input[i]);
+}
+
+fn cubic_weight(t: f32, a: f32) -> f32 {
+    let at = abs(t);
+    if (at <= 1.0) {
+        return (a + 2.0) * at * at * at - (a + 3.0) * at * at + 1.0;
+    } else if (at <= 2.0) {
+        return a * at * at * at - 5.0 * a * at * at + 8.0 * a * at - 4.0 * a;
+    } else {
+        return 0.0;
+    }
+}
+
+@compute @workgroup_size(WG_SIZE)
+fn main(
+    @builtin(global_invocation_id) gid: vec3<u32>,
+    @builtin(num_workgroups) num_wg: vec3<u32>
+) {
+
+    let i_out = gid.x + (num_wg.x * u32(WG_SIZE)) * gid.y;
+    let total = params.dst_w * params.dst_h * params.dst_z * params.dst_n;
+
+    if (i_out >= total) {
+        return;
+    }
+
+    // decode (x, y, z, n)
+    var i = i_out;
+    let x_dst = i % params.dst_w;
+    i = i / params.dst_w;
+    let y_dst = i % params.dst_h;
+    i = i / params.dst_h;
+    let z_dst = i % params.dst_z;
+    let n_dst = i / params.dst_z;
+
+    // scale factors
+    var sf0 = f32(params.dst_w) / f32(params.src_w);
+    var sf1 = f32(params.dst_h) / f32(params.src_h);
+    var sf2 = f32(params.dst_z) / f32(params.src_z);
+    var sf3 = f32(params.dst_n) / f32(params.src_n);
+
+    let align_corners = (params.mode_flags & GGML_SCALE_FLAG_ALIGN_CORNERS) != 0;
+
+    // pixel_offset: 0.5 for half-pixel-center (default), 0.0 for align_corners
+    var pixel_offset = 0.5;
+    if (align_corners) {
+        pixel_offset = 0.0;
+        if (params.dst_w > 1 && params.src_w > 1) {
+            sf0 = f32(params.dst_w - 1) / f32(params.src_w - 1);
+        }
+        if (params.dst_h > 1 && params.src_h > 1) {
+            sf1 = f32(params.dst_h - 1) / f32(params.src_h - 1);
+        }
+    }
+
+    let z_src = min(params.src_z - 1, u32(floor(f32(z_dst) / sf2)));
+    let n_src = min(params.src_n - 1, u32(floor(f32(n_dst) / sf3)));
+
+    var result = 0.0;
+
+#if defined(NEAREST)
+
+    let x_src = min(params.src_w - 1, u32(floor(f32(x_dst) / sf0)));
+    let y_src = min(params.src_h - 1, u32(floor(f32(y_dst) / sf1)));
+
+    result = get_clamped_input(i32(x_src), i32(y_src), z_src, n_src);
+
+#elif defined(BILINEAR)
+
+#if defined(ANTIALIAS)
+
+    // Antialiased bilinear: triangle filter over a variable support region.
+    let support0 = max(1.0f / sf0, 1.0f);
+    let support1 = max(1.0f / sf1, 1.0f);
+    let invscale0 = 1.0 / support0;
+    let invscale1 = 1.0 / support1;
+
+    let fx = (f32(x_dst) + pixel_offset) / sf0;
+    let fy = (f32(y_dst) + pixel_offset) / sf1;
+
+    let x_min = max(i32(fx - support0 + pixel_offset), 0);
+    let y_min = max(i32(fy - support1 + pixel_offset), 0);
+    let x_max = min(i32(fx + support0 + pixel_offset), i32(params.src_w));
+    let y_max = min(i32(fy + support1 + pixel_offset), i32(params.src_h));
+
+    var weighted_sum = 0.0;
+    var total_weight = 0.0;
+
+    for (var x = x_min; x < x_max; x += 1) {
+        let wx = max(1.0 - abs(f32(x) - fx + pixel_offset) * invscale0, 0.0);
+        for (var y = y_min; y < y_max; y += 1) {
+            let wy = max(1.0 - abs(f32(y) - fy + pixel_offset) * invscale1, 0.0);
+            let w = wx * wy;
+            if (w > 0.0) {
+                weighted_sum += get_clamped_input(x, y, z_src, n_src) * w;
+                total_weight += w;
+            }
+        }
+    }
+
+    if (total_weight > 0.0) {
+        result = weighted_sum / total_weight;
+    }
+
+#else
+
+    let fx = (f32(x_dst) + pixel_offset) / sf0 - pixel_offset;
+    let fy = (f32(y_dst) + pixel_offset) / sf1 - pixel_offset;
+    let x0 = i32(floor(fx));
+    let y0 = i32(floor(fy));
+    let dx = clamp(fx - f32(x0), 0.0, 1.0);
+    let dy = clamp(fy - f32(y0), 0.0, 1.0);
+    let a = get_clamped_input(x0, y0, z_src, n_src);
+    let b = get_clamped_input(x0 + 1, y0, z_src, n_src);
+    let c = get_clamped_input(x0, y0 + 1, z_src, n_src);
+    let d = get_clamped_input(x0 + 1, y0 + 1, z_src, n_src);
+
+    let wa = (1.0 - dx) * (1.0 - dy);
+    let wb = dx * (1.0 - dy);
+    let wc = (1.0 - dx) * dy;
+    let wd = dx * dy;
+
+    result = a * wa + b * wb + c * wc + d * wd;
+
+#endif
+
+#elif defined(BICUBIC)
+
+    // bicubic convolution with alpha = -0.75 (PyTorch default)
+    let alpha = -0.75;
+    let fx = (f32(x_dst) + pixel_offset) / sf0 - pixel_offset;
+    let fy = (f32(y_dst) + pixel_offset) / sf1 - pixel_offset;
+
+    let x0 = i32(floor(fx));
+    let y0 = i32(floor(fy));
+    let dx = fx - f32(x0);
+    let dy = fy - f32(y0);
+
+    // horizontal weights for offsets -1, 0, 1, 2
+    let wx0 = cubic_weight(dx + 1.0, alpha);
+    let wx1 = cubic_weight(dx, alpha);
+    let wx2 = cubic_weight(1.0 - dx, alpha);
+    let wx3 = cubic_weight(2.0 - dx, alpha);
+
+    // vertical weights for offsets -1, 0, 1, 2
+    let wy0 = cubic_weight(dy + 1.0, alpha);
+    let wy1 = cubic_weight(dy, alpha);
+    let wy2 = cubic_weight(1.0 - dy, alpha);
+    let wy3 = cubic_weight(2.0 - dy, alpha);
+
+    // intermediate horizontal interpolation for 4x4 grid of pixels
+    // x0-1, x0, x0+1, x0+2, y0-1
+    let p0 = get_clamped_input(x0 - 1, y0 - 1, z_src, n_src);
+    let p1 = get_clamped_input(x0, y0 - 1, z_src, n_src);
+    let p2 = get_clamped_input(x0 + 1, y0 - 1, z_src, n_src);
+    let p3 = get_clamped_input(x0 + 2, y0 - 1, z_src, n_src);
+    let row0 = p0 * wx0 + p1 * wx1 + p2 * wx2 + p3 * wx3;
+
+    // x0-1, x0, x0+1, x0+2, y0
+    let q0 = get_clamped_input(x0 - 1, y0, z_src, n_src);
+    let q1 = get_clamped_input(x0, y0, z_src, n_src);
+    let q2 = get_clamped_input(x0 + 1, y0, z_src, n_src);
+    let q3 = get_clamped_input(x0 + 2, y0, z_src, n_src);
+    let row1 = q0 * wx0 + q1 * wx1 + q2 * wx2 + q3 * wx3;
+
+    // x0-1, x0, x0+1, x0+2, y0+1
+    let r0 = get_clamped_input(x0 - 1, y0 + 1, z_src, n_src);
+    let r1 = get_clamped_input(x0, y0 + 1, z_src, n_src);
+    let r2 = get_clamped_input(x0 + 1, y0 + 1, z_src, n_src);
+    let r3 = get_clamped_input(x0 + 2, y0 + 1, z_src, n_src);
+    let row2 = r0 * wx0 + r1 * wx1 + r2 * wx2 + r3 * wx3;
+
+    // x0-1, x0, x0+1, x0+2, y0+2
+    let s0 = get_clamped_input(x0 - 1, y0 + 2, z_src, n_src);
+    let s1 = get_clamped_input(x0, y0 + 2, z_src, n_src);
+    let s2 = get_clamped_input(x0 + 1, y0 + 2, z_src, n_src);
+    let s3 = get_clamped_input(x0 + 2, y0 + 2, z_src, n_src);
+    let row3 = s0 * wx0 + s1 * wx1 + s2 * wx2 + s3 * wx3;
+
+    // final vertical interpolation
+    result = row0 * wy0 + row1 * wy1 + row2 * wy2 + row3 * wy3;
+
+#endif
+
+    let dst_idx = params.offset_o + x_dst * params.so0 + y_dst * params.so1 + z_dst * params.so2 + n_dst * params.so3;
+    output[dst_idx] = DST_TYPE(result);
+}

data/ext/sources/ggml/src/ggml-zdnn/ggml-zdnn.cpp

@@ -313,6 +313,8 @@ static ggml_backend_buffer_i ggml_backend_zdnn_buffer_i = {
     /* .memset_tensor = */ ggml_backend_zdnn_buffer_memset_tensor,
     /* .set_tensor    = */ ggml_backend_zdnn_buffer_set_tensor,
     /* .get_tensor    = */ ggml_backend_zdnn_buffer_get_tensor,
+    /* .set_tensor_2d = */ NULL,
+    /* .get_tensor_2d = */ NULL,
     /* .cpy_tensor    = */ NULL,
     /* .clear         = */ ggml_backend_zdnn_buffer_clear,
     /* .reset         = */ NULL,
@@ -417,20 +419,22 @@ static enum ggml_status ggml_backend_zdnn_graph_compute(ggml_backend_t backend,
 }
 
 static ggml_backend_i ggml_backend_zdnn_i = {
-    /* .get_name           = */ ggml_backend_zdnn_name,
-    /* .free               = */ ggml_backend_zdnn_free,
-    /* .set_tensor_async   = */ NULL,
-    /* .get_tensor_async   = */ NULL,
-    /* .cpy_tensor_async   = */ NULL,
-    /* .synchronize        = */ NULL,
-    /* .graph_plan_create  = */ NULL,
-    /* .graph_plan_free    = */ NULL,
-    /* .graph_plan_update  = */ NULL,
-    /* .graph_plan_compute = */ NULL,
-    /* .graph_compute      = */ ggml_backend_zdnn_graph_compute,
-    /* .event_record       = */ NULL,
-    /* .event_wait         = */ NULL,
-    /* .graph_optimize     = */ NULL,
+    /* .get_name               = */ ggml_backend_zdnn_name,
+    /* .free                   = */ ggml_backend_zdnn_free,
+    /* .set_tensor_async       = */ NULL,
+    /* .get_tensor_async       = */ NULL,
+    /* .set_tensor_2d_async    = */ NULL,
+    /* .get_tensor_2d_async    = */ NULL,
+    /* .cpy_tensor_async       = */ NULL,
+    /* .synchronize            = */ NULL,
+    /* .graph_plan_create      = */ NULL,
+    /* .graph_plan_free        = */ NULL,
+    /* .graph_plan_update      = */ NULL,
+    /* .graph_plan_compute     = */ NULL,
+    /* .graph_compute          = */ ggml_backend_zdnn_graph_compute,
+    /* .event_record           = */ NULL,
+    /* .event_wait             = */ NULL,
+    /* .graph_optimize         = */ NULL,
 };
 
 static ggml_guid_t ggml_backend_zdnn_guid(void) {

data/ext/sources/ggml/src/ggml-zendnn/CMakeLists.txt

@@ -28,7 +28,7 @@ if (NOT ZENDNN_ROOT OR ZENDNN_ROOT STREQUAL "" OR ZENDNN_ROOT STREQUAL "OFF")
     ExternalProject_Add(
         zendnn
         GIT_REPOSITORY https://github.com/amd/ZenDNN.git
-        GIT_TAG a18adf8c605fb5f5e52cefd7eda08a7b18febbaf    # ZenDNN-2026-WW08
+        GIT_TAG 253b94ce0d7e9284c265fefb485714944caff9d3    # ZenDNN-2026-WW19
         PREFIX      ${ZENDNN_PREFIX}
         SOURCE_DIR  ${ZENDNN_SOURCE_DIR}
         BINARY_DIR  ${ZENDNN_BUILD_DIR}

data/ext/sources/ggml/src/ggml-zendnn/ggml-zendnn.cpp

@@ -2,6 +2,10 @@
 
 #include "ggml-backend-impl.h"
 #include "ggml-impl.h"
+
+#define GGML_COMMON_DECL_CPP
+#include "ggml-common.h"
+
 #include "zendnnl.hpp"
 
 #include <cstring>
@@ -19,6 +23,8 @@ zendnnl::common::data_type_t ggml_to_zendnn_type() {
         return zendnnl::common::data_type_t::f32;
     } else if constexpr (std::is_same_v<T, ggml_bf16_t>) {
         return zendnnl::common::data_type_t::bf16;
+    } else if constexpr (std::is_same_v<T, block_q8_0>) {
+        return zendnnl::common::data_type_t::s8;
     } else {
         return zendnnl::common::data_type_t::none;
     }
@@ -47,6 +53,18 @@ static bool ggml_zendnn_matmul(ggml_backend_zendnn_context * ctx, int64_t m, int
     params.dtypes.dst = ggml_to_zendnn_type<TC>();
     params.num_threads = ctx->n_threads;
 
+    zendnnl::lowoha::matmul::matmul_batch_params_t batch_params;
+
+    if constexpr (std::is_same_v<TA, block_q8_0>) {
+        params.dtypes.compute = zendnnl::common::data_type_t::s8;
+        const int64_t num_groups = k / QK8_0;
+        params.dynamic_quant = true;
+        params.quant_params.src_scale.buff = nullptr;
+        params.quant_params.src_scale.dt   = zendnnl::common::data_type_t::bf16;
+        params.quant_params.src_scale.dims = {n, num_groups};
+        params.packing.pack_format_b = 1;
+    }
+
     zendnnl::error_handling::status_t status = zendnnl::lowoha::matmul::matmul_direct(
         'r', false, true,   // row-major, don't transpose B, transpose A (because it's column-major)
         n,                  // M: rows of B and C
@@ -59,7 +77,7 @@ static bool ggml_zendnn_matmul(ggml_backend_zendnn_context * ctx, int64_t m, int
         0.0f,               // beta
         C, ldc,             // output C[n,m]
         true,               // is_weights_const
-        {},                 // batch_params
+        batch_params,       // batch_params
         params              // params
     );
 
@@ -70,7 +88,7 @@ static bool ggml_zendnn_matmul(ggml_backend_zendnn_context * ctx, int64_t m, int
     return true;
 }
 
-static bool ggml_zendnn_sgemm(ggml_backend_zendnn_context * ctx, int64_t m, int64_t n, int64_t k,
+static bool ggml_zendnn_gemm(ggml_backend_zendnn_context * ctx, int64_t m, int64_t n, int64_t k,
                               const void * A, int64_t lda, const void * B, int64_t ldb, void * C,
                               int64_t ldc, int Atype, int Btype, int Ctype) {
 
@@ -107,6 +125,14 @@ static bool ggml_zendnn_sgemm(ggml_backend_zendnn_context * ctx, int64_t m, int6
                     (const ggml_bf16_t *)B, ldb,
                     (float *)C, ldc);
             return false;
+        case GGML_TYPE_Q8_0:
+            if (Btype != GGML_TYPE_F32 || Ctype != GGML_TYPE_F32)
+                return false;
+            return ggml_zendnn_matmul<block_q8_0, float, float>(
+                ctx, m, n, k,
+                (const block_q8_0 *)A, lda,
+                (const float *)B, ldb,
+                (float *)C, ldc);
         default:
             return false; // unsupported type
     }
@@ -144,7 +170,9 @@ static void ggml_zendnn_compute_forward_mul_mat(
     const int64_t r3 = ne13/ne03;
 
     void * work_data = ctx->work_data.get();
-    if (src1->type != vec_dot_type) {
+
+    // ZenDNN requires FP32 for dynamic quantization, so conversion is skipped
+    if (src1->type != vec_dot_type && src0->type != GGML_TYPE_Q8_0) {
         const size_t nbw1 = ggml_row_size(vec_dot_type, ne10);
         const size_t nbw2 = nbw1 * ne11;
         const size_t nbw3 = nbw2 * ne12;
@@ -170,9 +198,9 @@ static void ggml_zendnn_compute_forward_mul_mat(
 
     for (int64_t i13 = 0; i13 < ne13; i13++) {
         for (int64_t i12 = 0; i12 < ne12; i12++) {
-            const void* wdata = src1->type == vec_dot_type ? src1->data : work_data;
+            const void* wdata = (src1->type == vec_dot_type || src0->type == GGML_TYPE_Q8_0) ? src1->data : work_data;
             const size_t row_size = ggml_row_size(vec_dot_type, ne10);
-            if (!ggml_zendnn_sgemm(ctx,
+            if (!ggml_zendnn_gemm(ctx,
                                   ne01,     // m
                                   ne11,     // n
                                   ne10,     // k
@@ -183,9 +211,179 @@ static void ggml_zendnn_compute_forward_mul_mat(
                                   static_cast<char *>(dst->data) + i12*nb2 + i13*nb3,
                                   ne01,     // ldc
                                   src0->type,
-                                  vec_dot_type,
+                                  src0->type == GGML_TYPE_Q8_0 ? GGML_TYPE_F32 : vec_dot_type,
                                   dst->type))
-                GGML_ABORT("%s: ZenDNN sgemm failed\n", __func__);
+                GGML_ABORT("%s: ZenDNN gemm failed\n", __func__);
+        }
+    }
+}
+
+struct mmid_row_mapping {
+    int32_t i1;
+    int32_t i2;
+};
+
+static void ggml_zendnn_compute_forward_mul_mat_id(
+    ggml_backend_zendnn_context * ctx,
+    ggml_tensor * dst) {
+
+    const ggml_tensor * src0 = dst->src[0];  // expert weights
+    const ggml_tensor * src1 = dst->src[1];  // inputs
+    const ggml_tensor * ids  = dst->src[2];  // expert ids
+
+    GGML_TENSOR_BINARY_OP_LOCALS
+
+    // exit for no tokens to process
+    if (ne2 == 0 || ne11 == 0) {
+        return;
+    }
+
+    ggml_type         const vec_dot_type = src0->type;
+    ggml_from_float_t const from_float = ggml_get_type_traits(vec_dot_type)->from_float_ref;
+
+    // we don't support permuted src0 or src1
+    GGML_ASSERT(nb00 == ggml_type_size(src0->type));
+    GGML_ASSERT(nb10 == ggml_type_size(src1->type));
+
+    // dst cannot be transposed or permuted
+    GGML_ASSERT(nb0 == sizeof(float));
+    GGML_ASSERT(nb0 <= nb1);
+    GGML_ASSERT(nb1 <= nb2);
+    GGML_ASSERT(nb2 <= nb3);
+
+    GGML_ASSERT(ne03 == 1);
+    GGML_ASSERT(ne13 == 1);
+    GGML_ASSERT(ne3  == 1);
+
+    // row groups
+    const int n_ids = ids->ne[0]; // n_expert_used
+    const int n_as  = ne02;       // n_experts
+
+    std::vector<int64_t> matrix_row_counts(n_as, 0);
+    std::vector<std::vector<mmid_row_mapping>> matrix_rows(n_as);
+
+    int64_t max_rows = 0;
+    // group rows by expert (preprocessing step)
+    for (int64_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) {
+        for (int id = 0; id < n_ids; ++id) {
+            const int32_t i02 = *(const int32_t *)((const char *)ids->data + iid1*ids->nb[1] + id*ids->nb[0]);
+
+            GGML_ASSERT(i02 >= 0 && i02 < n_as);
+
+            matrix_rows[i02].push_back({id, iid1});
+            matrix_row_counts[i02]++;
+            if (matrix_row_counts[i02] > max_rows) {
+                max_rows = matrix_row_counts[i02];
+            }
+        }
+    }
+
+    if (max_rows == 0) {
+        return; // no rows to process
+    }
+
+    const size_t row_size = ggml_row_size(vec_dot_type, ne10);
+
+    // size for converting src1 rows to vec_dot_type if needed
+    const size_t nbw1 = row_size;
+    const size_t nbw2 = nbw1 * ne11;
+    const size_t nbw3 = nbw2 * ne12;
+    const size_t src1_conv_size = (src1->type != vec_dot_type && src0->type != GGML_TYPE_Q8_0) ? ne13 * nbw3 : 0;
+
+    // For Q8_0, src1 is always F32; the gather buffer must hold F32 rows (ne10*4 bytes),
+    // not Q8_0-encoded rows (row_size ≈ ne10/32*34 bytes) — they differ by ~4x.
+    const size_t f32_row_size = (size_t)ne10 * sizeof(float);
+    const size_t gather_row_size = (src0->type == GGML_TYPE_Q8_0) ? f32_row_size : row_size;
+
+    // size for MoE gather/scatter buffers
+    const size_t wdata_cur_size = max_rows * gather_row_size;
+    const size_t dst_cur_size = max_rows * ggml_row_size(dst->type, ne01);
+
+    // allocate single buffer for all needs
+    const size_t total_size = src1_conv_size + wdata_cur_size + dst_cur_size;
+    if (ctx->work_size < total_size) {
+        ctx->work_data.reset(new char[total_size]);
+        ctx->work_size = total_size;
+    }
+
+    // partition the buffer
+    char * work_data = ctx->work_data.get();
+    char * wdata_cur = work_data + src1_conv_size;
+    char * dst_cur = wdata_cur + wdata_cur_size;
+
+    // ZenDNN requires FP32 for dynamic quantization, so conversion is skipped
+    if (src1->type != vec_dot_type && src0->type != GGML_TYPE_Q8_0) {
+        GGML_ASSERT(src1->type == GGML_TYPE_F32);
+
+        #pragma omp parallel for collapse(3) num_threads(ctx->n_threads) schedule(static)
+        for (int64_t i13 = 0; i13 < ne13; ++i13) {
+            for (int64_t i12 = 0; i12 < ne12; ++i12) {
+                for (int64_t i11 = 0; i11 < ne11; ++i11) {
+                    const float * src1_f32 = (float *)((char *)src1->data + i11*nb11 + i12*nb12 + i13*nb13);
+                    void * src1_conv = (char *)work_data + i11*nbw1 + i12*nbw2 + i13*nbw3;
+                    from_float(src1_f32, src1_conv, ne10);
+                }
+            }
+        }
+    }
+
+    const void * wdata = (src1->type == vec_dot_type || src0->type == GGML_TYPE_Q8_0) ? src1->data : work_data;
+
+    // process each expert with gather -> gemm -> scatter pattern
+    for (int64_t cur_a = 0; cur_a < n_as; ++cur_a) {
+        const int64_t cne1 = matrix_row_counts[cur_a];
+
+        if (cne1 == 0) {
+            continue;
+        }
+
+        const char * src0_cur = (const char *) src0->data + cur_a*nb02;
+
+        // gather input rows for this expert
+        #pragma omp parallel for num_threads(ctx->n_threads) schedule(static)
+        for (int64_t ir1 = 0; ir1 < cne1; ++ir1) {
+            const mmid_row_mapping & row_mapping = matrix_rows[cur_a][ir1];
+            const int64_t id = row_mapping.i1;
+            const int64_t i11 = id % ne11;
+            const int64_t i12 = row_mapping.i2;
+
+            std::memcpy(
+                wdata_cur + ir1 * gather_row_size,
+                (const char *) wdata + (i11 + i12*ne11) * gather_row_size,
+                gather_row_size
+            );
+        }
+
+        // batched gemm for all tokens in this expert
+        if (!ggml_zendnn_gemm(ctx,
+                              ne01,       // m
+                              cne1,       // n
+                              ne10,       // k
+                              src0_cur,
+                              ne00,       // lda
+                              wdata_cur,
+                              ne10,       // ldb
+                              dst_cur,
+                              ne01,       // ldc
+                              src0->type,
+                              src0->type == GGML_TYPE_Q8_0 ? GGML_TYPE_F32 : vec_dot_type,
+                              dst->type)) {
+            GGML_ABORT("%s: ZenDNN gemm failed\n", __func__);
+        }
+
+        // scatter output rows to destination
+        #pragma omp parallel for num_threads(ctx->n_threads) schedule(static)
+        for (int64_t ir1 = 0; ir1 < cne1; ++ir1) {
+            const mmid_row_mapping & row_mapping = matrix_rows[cur_a][ir1];
+            const int64_t id = row_mapping.i1;
+            const int64_t i1 = id;
+            const int64_t i2 = row_mapping.i2;
+
+            std::memcpy(
+                (char *) dst->data + i1*nb1 + i2*nb2,
+                dst_cur + ir1 * ggml_row_size(dst->type, ne01),
+                ggml_row_size(dst->type, ne01)
+            );
         }
     }
 }
@@ -218,6 +416,9 @@ static ggml_status ggml_backend_zendnn_graph_compute(ggml_backend_t backend, ggm
             case GGML_OP_MUL_MAT:
                 ggml_zendnn_compute_forward_mul_mat(ctx, node);
                 break;
+            case GGML_OP_MUL_MAT_ID:
+                ggml_zendnn_compute_forward_mul_mat_id(ctx, node);
+                break;
             case GGML_OP_NONE:
             case GGML_OP_RESHAPE:
             case GGML_OP_VIEW:
@@ -240,6 +441,8 @@ static struct ggml_backend_i ggml_backend_zendnn_i = {
     /* .free                    = */ ggml_backend_zendnn_free,
     /* .set_tensor_async        = */ NULL,
     /* .get_tensor_async        = */ NULL,
+    /* .set_tensor_2d_async     = */ NULL,
+    /* .get_tensor_2d_async     = */ NULL,
     /* .cpy_tensor_async        = */ NULL,
     /* .synchronize             = */ NULL,
     /* .graph_plan_create       = */ NULL,
@@ -351,6 +554,12 @@ static ggml_backend_buffer_t ggml_backend_zendnn_device_buffer_from_host_ptr(ggm
     GGML_UNUSED(max_tensor_size);
 }
 
+static bool ggml_zendnn_adaptive_fallback_enabled() {
+    static const bool enabled = std::getenv("GGML_ZENDNN_ADAPTIVE_FALLBACK") == nullptr ||
+                                std::atoi(std::getenv("GGML_ZENDNN_ADAPTIVE_FALLBACK")) != 0;
+    return enabled;
+}
+
 static bool ggml_backend_zendnn_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) {
     switch (op->op) {
         case GGML_OP_NONE:
@@ -361,6 +570,7 @@ static bool ggml_backend_zendnn_device_supports_op(ggml_backend_dev_t dev, const
             return true;
 
         case GGML_OP_MUL_MAT:
+        case GGML_OP_MUL_MAT_ID:
         {
             const ggml_tensor * weights = op->src[0];
             const ggml_tensor * inputs = op->src[1];
@@ -368,15 +578,39 @@ static bool ggml_backend_zendnn_device_supports_op(ggml_backend_dev_t dev, const
             const int64_t ne10 = inputs->ne[0];
             const int64_t ne0 = op->ne[0];
             const int64_t ne1 = op->ne[1];
-
             const int64_t min_batch = 1;
-            if (!ggml_is_contiguous(weights) || !ggml_is_contiguous(inputs) ||
-                ne0 < min_batch || ne1 < min_batch || ne10 < min_batch) {
+
+            if(!ggml_is_contiguous(weights) || !ggml_is_contiguous(inputs)) {
+                return false;
+            }
+
+            if (ggml_zendnn_adaptive_fallback_enabled()) {
+                const int64_t K = inputs->ne[0];
+                const int64_t N = (inputs->ne[1]*inputs->ne[2]*inputs->ne[3]);
+                const int64_t M = weights->ne[1];
+                if(K <= 256 || N <= 128 || M <= 96) {
                     return false;
+                }
+            }
+            else if (ne0 < min_batch || ne1 < min_batch || ne10 < min_batch) {
+                return false;
+            }
+
+            // MUL_MAT_ID performs best with a moderate number of experts due to its
+            // gather + batched matmul + scatter approach. Future versions will leverage
+            // ZenDNN's grouped_gemm for better scalability with larger expert counts:
+            // https://github.com/amd/ZenDNN/blob/main/docs/operator/lowoha_group_gemm_operator.md
+            if (op->op == GGML_OP_MUL_MAT_ID) {
+                const int64_t n_experts = weights->ne[2];
+                const int64_t max_experts = 32;
+                if (n_experts > max_experts) {
+                    return false;
+                }
             }
             switch (weights->type) {
                 case GGML_TYPE_F32:
                 case GGML_TYPE_BF16:
+                case GGML_TYPE_Q8_0:
                     return true;
                 default:
                     return false;