toy 0.8.0

data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl

@@ -0,0 +1,193 @@
+#ifdef USE_SUBGROUP_REDUCTION
+enable subgroups;
+#endif
+
+struct Params {
+    offset_s: u32,
+    offset_x: u32,
+    offset_dt: u32,
+    offset_A: u32,
+    offset_B: u32,
+    offset_C: u32,
+    offset_ids: u32,
+    offset_dst: u32,
+
+    stride_s1: u32,
+    stride_s2: u32,
+    stride_s3: u32,
+
+    stride_x1: u32,
+    stride_x2: u32,
+    stride_x3: u32,
+
+    stride_dt1: u32,
+    stride_dt2: u32,
+
+    a_ne0: u32,
+    stride_A1: u32,
+
+    stride_B1: u32,
+    stride_B2: u32,
+    stride_B3: u32,
+
+    stride_C1: u32,
+    stride_C2: u32,
+    stride_C3: u32,
+
+    d_state: u32,
+    d_inner: u32,
+    n_head: u32,
+    n_group: u32,
+    n_seq_tokens: u32,
+    n_seqs: u32,
+
+    y_elems: u32,
+};
+
+@group(0) @binding(0) var<storage, read_write> s_in: array<f32>;
+#ifdef XBC_OVERLAP
+@group(0) @binding(1) var<storage, read_write> x_B_C_merged: array<f32>;
+@group(0) @binding(2) var<storage, read_write> dt: array<f32>;
+@group(0) @binding(3) var<storage, read_write> A: array<f32>;
+@group(0) @binding(4) var<storage, read_write> ids: array<i32>;
+@group(0) @binding(5) var<storage, read_write> dst: array<f32>;
+@group(0) @binding(6) var<uniform> params: Params;
+#else
+@group(0) @binding(1) var<storage, read_write> x: array<f32>;
+@group(0) @binding(2) var<storage, read_write> dt: array<f32>;
+@group(0) @binding(3) var<storage, read_write> A: array<f32>;
+@group(0) @binding(4) var<storage, read_write> B: array<f32>;
+@group(0) @binding(5) var<storage, read_write> C: array<f32>;
+@group(0) @binding(6) var<storage, read_write> ids: array<i32>;
+@group(0) @binding(7) var<storage, read_write> dst: array<f32>;
+@group(0) @binding(8) var<uniform> params: Params;
+#endif
+
+var<workgroup> shared_x_dt: array<f32, TOKENS_PER_TILE>;
+var<workgroup> shared_dtsp: array<f32, TOKENS_PER_TILE>;
+var<workgroup> shared_reduce: array<f32, TOKENS_PER_TILE * WG_SIZE>;
+
+fn reduce_base(token_in_tile: u32) -> u32 {
+    return token_in_tile * WG_SIZE;
+}
+
+@compute @workgroup_size(WG_SIZE)
+fn main(
+    @builtin(local_invocation_id) local_id: vec3<u32>,
+    @builtin(workgroup_id) wg_id: vec3<u32>,
+    @builtin(num_workgroups) num_wg: vec3<u32>
+#ifdef USE_SUBGROUP_REDUCTION
+  , @builtin(subgroup_id) subgroup_id: u32,
+    @builtin(subgroup_invocation_id) subgroup_invocation_id: u32,
+    @builtin(num_subgroups) num_subgroups: u32
+#endif
+) {
+    let tid = local_id.x;
+    let wg_linear = wg_id.y * num_wg.x + wg_id.x;
+
+    let i1 = wg_linear % params.d_inner;
+    let head_seq = wg_linear / params.d_inner;
+    let ir = head_seq % params.n_head;
+    let i3 = head_seq / params.n_head;
+
+    let state_slot = u32(ids[params.offset_ids + i3]);
+    let g = ir / (params.n_head / params.n_group);
+
+    let s_idx = params.offset_s + tid + i1 * params.stride_s1 + ir * params.stride_s2 + state_slot * params.stride_s3;
+    var s_prev = s_in[s_idx];
+
+    let A0 = A[params.offset_A + (tid % params.a_ne0) + ir * params.stride_A1];
+
+    for (var token_base = 0u; token_base < params.n_seq_tokens; token_base += TOKENS_PER_TILE) {
+        if (tid < TOKENS_PER_TILE) {
+            let token = token_base + tid;
+            if (token < params.n_seq_tokens) {
+                let x_idx = params.offset_x + i1 + ir * params.stride_x1 + token * params.stride_x2 + i3 * params.stride_x3;
+                let dt_idx = params.offset_dt + ir + token * params.stride_dt1 + i3 * params.stride_dt2;
+                let dt0 = dt[dt_idx];
+                let dtsp = select(log(1.0 + exp(dt0)), dt0, dt0 > 20.0);
+                shared_dtsp[tid] = dtsp;
+#ifdef XBC_OVERLAP
+                shared_x_dt[tid] = x_B_C_merged[x_idx] * dtsp;
+#else
+                shared_x_dt[tid] = x[x_idx] * dtsp;
+#endif
+            }
+        }
+
+        workgroupBarrier();
+
+        for (var token_in_tile = 0u; token_in_tile < TOKENS_PER_TILE; token_in_tile++) {
+            let token = token_base + token_in_tile;
+            if (token >= params.n_seq_tokens) {
+                break;
+            }
+
+            let x_dt = shared_x_dt[token_in_tile];
+            let dA = exp(shared_dtsp[token_in_tile] * A0);
+            let reduce_idx = reduce_base(token_in_tile) + tid;
+
+            let b_idx = params.offset_B + tid + g * params.stride_B1 + token * params.stride_B2 + i3 * params.stride_B3;
+            let c_idx = params.offset_C + tid + g * params.stride_C1 + token * params.stride_C2 + i3 * params.stride_C3;
+#ifdef XBC_OVERLAP
+            let s = s_prev * dA + x_B_C_merged[b_idx] * x_dt;
+#else
+            let s = s_prev * dA + B[b_idx] * x_dt;
+#endif
+            s_prev = s;
+
+#ifdef USE_SUBGROUP_REDUCTION
+#ifdef XBC_OVERLAP
+            let subgroup_partial = subgroupAdd(s * x_B_C_merged[c_idx]);
+#else
+            let subgroup_partial = subgroupAdd(s * C[c_idx]);
+#endif
+            if (subgroup_invocation_id == 0u) {
+                shared_reduce[reduce_idx - tid + subgroup_id] = subgroup_partial;
+            }
+#else
+#ifdef XBC_OVERLAP
+            shared_reduce[reduce_idx] = s * x_B_C_merged[c_idx];
+#else
+            shared_reduce[reduce_idx] = s * C[c_idx];
+#endif
+#endif
+
+            workgroupBarrier();
+
+#ifdef USE_SUBGROUP_REDUCTION
+            if (tid == 0u) {
+                var sum = 0.0;
+                for (var sg = 0u; sg < num_subgroups; sg++) {
+                    sum += shared_reduce[reduce_base(token_in_tile) + sg];
+                }
+                let y_idx =
+                    params.offset_dst + i1 + ir * params.d_inner + token * (params.n_head * params.d_inner) +
+                    i3 * (params.n_seq_tokens * params.n_head * params.d_inner);
+                dst[y_idx] = sum;
+            }
+#else
+            for (var stride = WG_SIZE / 2u; stride > 0u; stride >>= 1u) {
+                if (tid < stride) {
+                    shared_reduce[reduce_idx] += shared_reduce[reduce_idx + stride];
+                }
+                workgroupBarrier();
+            }
+
+            if (tid == 0u) {
+                let y_idx =
+                    params.offset_dst + i1 + ir * params.d_inner + token * (params.n_head * params.d_inner) +
+                    i3 * (params.n_seq_tokens * params.n_head * params.d_inner);
+                dst[y_idx] = shared_reduce[reduce_base(token_in_tile)];
+            }
+#endif
+
+            workgroupBarrier();
+        }
+    }
+
+    let state_idx =
+        params.offset_dst + params.y_elems + tid + i1 * params.d_state + ir * (params.d_state * params.d_inner) +
+        i3 * (params.d_state * params.d_inner * params.n_head);
+    dst[state_idx] = s_prev;
+}

data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl

@@ -0,0 +1,55 @@
+@group(0) @binding(0)
+var<storage, read_write> src: array<f32>;
+
+@group(0) @binding(1)
+var<storage, read_write> dst: array<f32>;
+
+struct Params {
+    offset_src: u32, // in elements
+    offset_dst: u32, // in elements
+
+    // Strides (in elements)
+    stride_src1: u32,
+    stride_src2: u32,
+    stride_src3: u32,
+
+    ne0: u32,
+    ne1: u32,
+    ne2: u32
+};
+
+@group(0) @binding(2)
+var<uniform> params: Params;
+
+var<workgroup> shared_sum: array<f32, WG_SIZE>;
+
+@compute @workgroup_size(WG_SIZE)
+fn main(@builtin(workgroup_id) wid: vec3<u32>,
+        @builtin(local_invocation_id) lid: vec3<u32>) {
+
+    var i = wid.x;
+    let i3 = i / (params.ne2 * params.ne1);
+    i = i % (params.ne2 * params.ne1);
+    let i2 = i / params.ne1;
+    let i1 = i % params.ne1;
+    let i_src_row = params.offset_src + i3 * params.stride_src3 + i2 * params.stride_src2 + i1 * params.stride_src1;
+    var local_sum: f32 = 0.0;
+    for (var col = lid.x; col < params.ne0; col += WG_SIZE) {
+        local_sum += src[i_src_row + col];
+    }
+    shared_sum[lid.x] = local_sum;
+    workgroupBarrier();
+    // reduce within workgroup
+    var offset: u32 = WG_SIZE >> 1;
+    while (offset > 0) {
+        if (lid.x < offset) {
+            shared_sum[lid.x] = shared_sum[lid.x] + shared_sum[lid.x + offset];
+        }
+        workgroupBarrier();
+        offset >>= 1;
+    }
+
+    if (lid.x == 0) {
+        dst[params.offset_dst + wid.x] = shared_sum[0];
+    }
+}

data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl

@@ -0,0 +1,210 @@
+#ifdef TYPE_F16
+enable f16;
+#define TYPE f16
+#else
+#define TYPE f32
+#endif
+
+@group(0) @binding(0)
+var<storage, read_write> src: array<TYPE>;
+
+#ifndef INPLACE
+@group(0) @binding(1)
+var<storage, read_write> dst: array<TYPE>;
+#define PARAMS_BINDING 2
+#else
+#define PARAMS_BINDING 1
+#endif
+
+struct Params {
+    ne: u32,            // total number of elements
+    offset_src: u32,    // in elements
+    offset_dst: u32,    // in elements
+
+    // Strides (in elements)
+    stride_src0: u32,
+    stride_src1: u32,
+    stride_src2: u32,
+    stride_src3: u32,
+
+    // Logical shapes
+    ne0: u32,
+    ne1: u32,
+    ne2: u32,
+#ifdef CLAMP
+    clamp_min: f32,
+    clamp_max: f32,
+#endif
+#ifdef FILL
+    fill_val: f32,
+#endif
+#ifdef XIELU
+    alpha_n: f32,
+    alpha_p: f32,
+    beta: f32,
+    eps: f32,
+#endif
+
+};
+
+@group(0) @binding(PARAMS_BINDING)
+var<uniform> params: Params;
+
+fn erf_approx(x: TYPE) -> TYPE {
+    let x_f32 = f32(x);
+    let s = select(-1.0, 1.0, x_f32 >= 0.0);
+    let ax = abs(x_f32);
+
+    let t = 1.0 / (1.0 + 0.3275911 * ax);
+
+    let y = 1.0 -
+        (((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t
+            - 0.284496736) * t + 0.254829592) * t) *
+        exp(-ax * ax);
+
+    return TYPE(s * y);
+}
+
+@compute @workgroup_size(WG_SIZE)
+fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
+    if (gid.x >= params.ne) {
+        return;
+    }
+    var i = gid.x;
+    let ne2 = params.ne2;
+#ifdef DIAG
+    let ne1 = params.ne0;
+#else
+    let ne1 = params.ne1;
+#endif
+    let ne0 = params.ne0;
+
+    let i3 = i / (ne2 * ne1 * ne0);
+    i = i % (ne2 * ne1 * ne0);
+    let i2 = i / (ne1 * ne0);
+    i = i % (ne1 * ne0);
+    let i1 = i / ne0;
+    let i0 = i % ne0;
+
+    let src_idx = i0 * params.stride_src0 + i1 * params.stride_src1 + i2 * params.stride_src2 + i3 * params.stride_src3;
+
+#ifdef ABS
+    let res = abs(src[params.offset_src + src_idx]);
+#endif
+#ifdef SGN
+    let res = select(TYPE(select(0.0, -1.0, src[params.offset_src + src_idx] < 0.0)), TYPE(1.0), src[params.offset_src + src_idx] > 0.0);
+#endif
+#ifdef NEG
+    let res = -src[params.offset_src + src_idx];
+#endif
+#ifdef STEP
+    let res = TYPE(select(0.0, 1.0, src[params.offset_src + src_idx] > 0.0));
+#endif
+#ifdef TANH
+    let res = tanh(clamp(src[params.offset_src + src_idx], -9.010913, 9.010913));
+#endif
+#ifdef RELU
+    let res = select(0.0, src[params.offset_src + src_idx], src[params.offset_src + src_idx] > 0.0);
+#endif
+#ifdef ELU
+    let res = select(exp(src[params.offset_src + src_idx]) - 1.0, src[params.offset_src + src_idx], src[params.offset_src + src_idx] > 0.0);
+#endif
+#ifdef HARDSIGMOID
+    let res = min(1.0, max(0.0, (src[params.offset_src + src_idx] + 3.0) / 6.0));
+#endif
+#ifdef SIGMOID
+    let res = 1.0 / (1.0 + exp(-src[params.offset_src + src_idx]));
+#endif
+#ifdef SILU
+    let res = src[params.offset_src + src_idx] / (1.0 + exp(-src[params.offset_src + src_idx]));
+#endif
+#ifdef EXP
+    let src_f32 = f32(src[params.offset_src + src_idx]);
+    let res = TYPE(exp(src_f32));
+#endif
+#ifdef LOG
+    let res = TYPE(log(f32(src[params.offset_src + src_idx])));
+#endif
+#ifdef CLAMP
+    let res = clamp(src[params.offset_src + src_idx], TYPE(params.clamp_min), TYPE(params.clamp_max));
+#endif
+#ifdef FILL
+    let res = TYPE(params.fill_val);
+#endif
+#ifdef HARDSWISH
+    let res = src[params.offset_src + src_idx] * min(1.0, max(0.0, (src[params.offset_src + src_idx] + 3.0) / 6.0));
+#endif
+#ifdef GELU
+    let res = 0.5 * src[params.offset_src + src_idx] * (1.0 + tanh(clamp(0.7978845608028654 * (src[params.offset_src + src_idx] + 0.044715 * src[params.offset_src + src_idx] * src[params.offset_src + src_idx] * src[params.offset_src + src_idx]), -9.010913, 9.010913)));
+#endif
+#ifdef GELU_QUICK
+    let res = src[params.offset_src + src_idx] * (1.0 / (1.0 + exp(clamp(-1.702 * src[params.offset_src + src_idx], -80.0, 80.0))));
+#endif
+#ifdef GELU_ERF
+    let res = 0.5 * src[params.offset_src + src_idx] * (1.0 + erf_approx(src[params.offset_src + src_idx] * 0.7071067811865476));
+#endif
+#ifdef XIELU
+    let val = f32(src[params.offset_src + src_idx]);
+    let res =
+        TYPE(select(
+            ((exp(min(val, params.eps)) - 1.0) - val) * params.alpha_n + params.beta * val,
+            params.alpha_p * val * val + params.beta * val,
+            val > 0.0));
+#endif
+#ifdef SOFTPLUS
+    let src_f32 = f32(src[params.offset_src + src_idx]);
+    let res = TYPE(select(log(1.0 + exp(src_f32)), src_f32, src_f32 > 20.0));
+#endif
+#ifdef EXPM1
+    let src_f32 = f32(src[params.offset_src + src_idx]);
+    let res = TYPE(exp(src_f32) - 1.0);
+#endif
+#ifdef FLOOR
+    let res = floor(src[params.offset_src + src_idx]);
+#endif
+#ifdef CEIL
+    let res = ceil(src[params.offset_src + src_idx]);
+#endif
+#ifdef ROUND
+    let src_f32 = f32(src[params.offset_src + src_idx]);
+    let result = select(ceil(src_f32 - 0.5), floor(src_f32 + 0.5), src_f32 >= 0.0);
+    let res = TYPE(result);
+#endif
+#ifdef TRUNC
+    let res = trunc(src[params.offset_src + src_idx]);
+#endif
+#ifdef SQR
+    let res = src[params.offset_src + src_idx] * src[params.offset_src + src_idx];
+#endif
+#ifdef SQRT
+    let res = TYPE(sqrt(f32(src[params.offset_src + src_idx])));
+#endif
+#ifdef SIN
+    let res_f32 = sin(f32(src[params.offset_src + src_idx]));
+    let res = TYPE(res_f32);
+#endif
+#ifdef COS
+    let res_f32 = cos(f32(src[params.offset_src + src_idx]));
+    let res = TYPE(res_f32);
+#endif
+#ifdef DIAG
+    let res = select(0.0, src[params.offset_src + i0 + i2 * params.stride_src2 + i3 * params.stride_src3], i0 == i1);
+#endif
+#ifdef TRI
+#ifdef TRI_TYPE_LOWER
+    let res = select(0.0, src[params.offset_src + src_idx], i0 < i1);
+#elif TRI_TYPE_LOWER_DIAG
+    let res = select(0.0, src[params.offset_src + src_idx], i0 <= i1);
+#elif TRI_TYPE_UPPER
+    let res = select(0.0, src[params.offset_src + src_idx], i0 > i1);
+#elif TRI_TYPE_UPPER_DIAG
+    let res = select(0.0, src[params.offset_src + src_idx], i0 >= i1);
+#endif
+#endif
+
+#ifdef INPLACE
+    src[params.offset_src + src_idx] = res;
+#else
+    dst[params.offset_dst + gid.x] = res;
+#endif
+}

data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/upscale.wgsl

@@ -0,0 +1,240 @@
+#if defined(SRC_F16) || defined(DST_F16)
+enable f16;
+#endif
+
+#ifdef SRC_F16
+#define SRC_TYPE f16
+#else
+#define SRC_TYPE f32
+#endif
+
+#ifdef DST_F16
+#define DST_TYPE f16
+#else
+#define DST_TYPE f32
+#endif
+
+@group(0) @binding(0)
+var<storage, read_write> input: array<SRC_TYPE>;
+
+@group(0) @binding(1)
+var<storage, read_write> output: array<DST_TYPE>;
+
+struct Params {
+    offset_i: u32,
+    offset_o: u32,
+
+    // element strides
+    si0: u32, si1: u32, si2: u32, si3: u32,
+    so0: u32, so1: u32, so2: u32, so3: u32,
+
+    src_w: u32,
+    src_h: u32,
+    src_z: u32,
+    src_n: u32,
+
+    dst_w: u32,
+    dst_h: u32,
+    dst_z: u32,
+    dst_n: u32,
+
+    mode_flags: u32,
+};
+
+@group(0) @binding(2)
+var<uniform> params: Params;
+
+const GGML_SCALE_FLAG_ALIGN_CORNERS: u32 = 1u << 8u;
+
+fn get_clamped_input(x: i32, y: i32, z: u32, n: u32) -> f32 {
+    let cx = u32(clamp(x, 0, i32(params.src_w) - 1));
+    let cy = u32(clamp(y, 0, i32(params.src_h) - 1));
+    let i = params.offset_i + cx * params.si0 + cy * params.si1 + z * params.si2 + n * params.si3;
+    return f32(input[i]);
+}
+
+fn cubic_weight(t: f32, a: f32) -> f32 {
+    let at = abs(t);
+    if (at <= 1.0) {
+        return (a + 2.0) * at * at * at - (a + 3.0) * at * at + 1.0;
+    } else if (at <= 2.0) {
+        return a * at * at * at - 5.0 * a * at * at + 8.0 * a * at - 4.0 * a;
+    } else {
+        return 0.0;
+    }
+}
+
+@compute @workgroup_size(WG_SIZE)
+fn main(
+    @builtin(global_invocation_id) gid: vec3<u32>,
+    @builtin(num_workgroups) num_wg: vec3<u32>
+) {
+
+    let i_out = gid.x + (num_wg.x * u32(WG_SIZE)) * gid.y;
+    let total = params.dst_w * params.dst_h * params.dst_z * params.dst_n;
+
+    if (i_out >= total) {
+        return;
+    }
+
+    // decode (x, y, z, n)
+    var i = i_out;
+    let x_dst = i % params.dst_w;
+    i = i / params.dst_w;
+    let y_dst = i % params.dst_h;
+    i = i / params.dst_h;
+    let z_dst = i % params.dst_z;
+    let n_dst = i / params.dst_z;
+
+    // scale factors
+    var sf0 = f32(params.dst_w) / f32(params.src_w);
+    var sf1 = f32(params.dst_h) / f32(params.src_h);
+    var sf2 = f32(params.dst_z) / f32(params.src_z);
+    var sf3 = f32(params.dst_n) / f32(params.src_n);
+
+    let align_corners = (params.mode_flags & GGML_SCALE_FLAG_ALIGN_CORNERS) != 0;
+
+    // pixel_offset: 0.5 for half-pixel-center (default), 0.0 for align_corners
+    var pixel_offset = 0.5;
+    if (align_corners) {
+        pixel_offset = 0.0;
+        if (params.dst_w > 1 && params.src_w > 1) {
+            sf0 = f32(params.dst_w - 1) / f32(params.src_w - 1);
+        }
+        if (params.dst_h > 1 && params.src_h > 1) {
+            sf1 = f32(params.dst_h - 1) / f32(params.src_h - 1);
+        }
+    }
+
+    let z_src = min(params.src_z - 1, u32(floor(f32(z_dst) / sf2)));
+    let n_src = min(params.src_n - 1, u32(floor(f32(n_dst) / sf3)));
+
+    var result = 0.0;
+
+#if defined(NEAREST)
+
+    let x_src = min(params.src_w - 1, u32(floor(f32(x_dst) / sf0)));
+    let y_src = min(params.src_h - 1, u32(floor(f32(y_dst) / sf1)));
+
+    result = get_clamped_input(i32(x_src), i32(y_src), z_src, n_src);
+
+#elif defined(BILINEAR)
+
+#if defined(ANTIALIAS)
+
+    // Antialiased bilinear: triangle filter over a variable support region.
+    let support0 = max(1.0f / sf0, 1.0f);
+    let support1 = max(1.0f / sf1, 1.0f);
+    let invscale0 = 1.0 / support0;
+    let invscale1 = 1.0 / support1;
+
+    let fx = (f32(x_dst) + pixel_offset) / sf0;
+    let fy = (f32(y_dst) + pixel_offset) / sf1;
+
+    let x_min = max(i32(fx - support0 + pixel_offset), 0);
+    let y_min = max(i32(fy - support1 + pixel_offset), 0);
+    let x_max = min(i32(fx + support0 + pixel_offset), i32(params.src_w));
+    let y_max = min(i32(fy + support1 + pixel_offset), i32(params.src_h));
+
+    var weighted_sum = 0.0;
+    var total_weight = 0.0;
+
+    for (var x = x_min; x < x_max; x += 1) {
+        let wx = max(1.0 - abs(f32(x) - fx + pixel_offset) * invscale0, 0.0);
+        for (var y = y_min; y < y_max; y += 1) {
+            let wy = max(1.0 - abs(f32(y) - fy + pixel_offset) * invscale1, 0.0);
+            let w = wx * wy;
+            if (w > 0.0) {
+                weighted_sum += get_clamped_input(x, y, z_src, n_src) * w;
+                total_weight += w;
+            }
+        }
+    }
+
+    if (total_weight > 0.0) {
+        result = weighted_sum / total_weight;
+    }
+
+#else
+
+    let fx = (f32(x_dst) + pixel_offset) / sf0 - pixel_offset;
+    let fy = (f32(y_dst) + pixel_offset) / sf1 - pixel_offset;
+    let x0 = i32(floor(fx));
+    let y0 = i32(floor(fy));
+    let dx = clamp(fx - f32(x0), 0.0, 1.0);
+    let dy = clamp(fy - f32(y0), 0.0, 1.0);
+    let a = get_clamped_input(x0, y0, z_src, n_src);
+    let b = get_clamped_input(x0 + 1, y0, z_src, n_src);
+    let c = get_clamped_input(x0, y0 + 1, z_src, n_src);
+    let d = get_clamped_input(x0 + 1, y0 + 1, z_src, n_src);
+
+    let wa = (1.0 - dx) * (1.0 - dy);
+    let wb = dx * (1.0 - dy);
+    let wc = (1.0 - dx) * dy;
+    let wd = dx * dy;
+
+    result = a * wa + b * wb + c * wc + d * wd;
+
+#endif
+
+#elif defined(BICUBIC)
+
+    // bicubic convolution with alpha = -0.75 (PyTorch default)
+    let alpha = -0.75;
+    let fx = (f32(x_dst) + pixel_offset) / sf0 - pixel_offset;
+    let fy = (f32(y_dst) + pixel_offset) / sf1 - pixel_offset;
+
+    let x0 = i32(floor(fx));
+    let y0 = i32(floor(fy));
+    let dx = fx - f32(x0);
+    let dy = fy - f32(y0);
+
+    // horizontal weights for offsets -1, 0, 1, 2
+    let wx0 = cubic_weight(dx + 1.0, alpha);
+    let wx1 = cubic_weight(dx, alpha);
+    let wx2 = cubic_weight(1.0 - dx, alpha);
+    let wx3 = cubic_weight(2.0 - dx, alpha);
+
+    // vertical weights for offsets -1, 0, 1, 2
+    let wy0 = cubic_weight(dy + 1.0, alpha);
+    let wy1 = cubic_weight(dy, alpha);
+    let wy2 = cubic_weight(1.0 - dy, alpha);
+    let wy3 = cubic_weight(2.0 - dy, alpha);
+
+    // intermediate horizontal interpolation for 4x4 grid of pixels
+    // x0-1, x0, x0+1, x0+2, y0-1
+    let p0 = get_clamped_input(x0 - 1, y0 - 1, z_src, n_src);
+    let p1 = get_clamped_input(x0, y0 - 1, z_src, n_src);
+    let p2 = get_clamped_input(x0 + 1, y0 - 1, z_src, n_src);
+    let p3 = get_clamped_input(x0 + 2, y0 - 1, z_src, n_src);
+    let row0 = p0 * wx0 + p1 * wx1 + p2 * wx2 + p3 * wx3;
+
+    // x0-1, x0, x0+1, x0+2, y0
+    let q0 = get_clamped_input(x0 - 1, y0, z_src, n_src);
+    let q1 = get_clamped_input(x0, y0, z_src, n_src);
+    let q2 = get_clamped_input(x0 + 1, y0, z_src, n_src);
+    let q3 = get_clamped_input(x0 + 2, y0, z_src, n_src);
+    let row1 = q0 * wx0 + q1 * wx1 + q2 * wx2 + q3 * wx3;
+
+    // x0-1, x0, x0+1, x0+2, y0+1
+    let r0 = get_clamped_input(x0 - 1, y0 + 1, z_src, n_src);
+    let r1 = get_clamped_input(x0, y0 + 1, z_src, n_src);
+    let r2 = get_clamped_input(x0 + 1, y0 + 1, z_src, n_src);
+    let r3 = get_clamped_input(x0 + 2, y0 + 1, z_src, n_src);
+    let row2 = r0 * wx0 + r1 * wx1 + r2 * wx2 + r3 * wx3;
+
+    // x0-1, x0, x0+1, x0+2, y0+2
+    let s0 = get_clamped_input(x0 - 1, y0 + 2, z_src, n_src);
+    let s1 = get_clamped_input(x0, y0 + 2, z_src, n_src);
+    let s2 = get_clamped_input(x0 + 1, y0 + 2, z_src, n_src);
+    let s3 = get_clamped_input(x0 + 2, y0 + 2, z_src, n_src);
+    let row3 = s0 * wx0 + s1 * wx1 + s2 * wx2 + s3 * wx3;
+
+    // final vertical interpolation
+    result = row0 * wy0 + row1 * wy1 + row2 * wy2 + row3 * wy3;
+
+#endif
+
+    let dst_idx = params.offset_o + x_dst * params.so0 + y_dst * params.so1 + z_dst * params.so2 + n_dst * params.so3;
+    output[dst_idx] = DST_TYPE(result);
+}