@fugood/llama.node 0.6.2 → 1.0.0-beta.1

package/src/llama.cpp/ggml/src/ggml-sycl/gla.cpp

@@ -1,105 +0,0 @@
-#include <sycl/sycl.hpp>
-
-#include "common.hpp"
-
-template <u_int HEAD_SIZE>
-static void gated_linear_attn_f32_kernel(const dpct::queue_ptr stream, u_int B, u_int T, u_int C, u_int H, float scale,
-                                         const float * k, const float * v, const float * r, const float * td,
-                                         const float * s, float * dst) {
-    const u_int head_size    = HEAD_SIZE;
-    const u_int state_size   = C * head_size;
-    const u_int n_seq_tokens = T / B;
-    sycl::range<1> block_dims((C / H));
-    sycl::range<1> grid_dims((B * H));
-    stream->submit([&](sycl::handler & cgh) {
-        /* local memory accessors*/
-        auto _k  = sycl::local_accessor<float, 1>(sycl::range<1>(head_size), cgh);
-        auto _r  = sycl::local_accessor<float, 1>(sycl::range<1>(head_size), cgh);
-        auto _td = sycl::local_accessor<float, 1>(sycl::range<1>(head_size), cgh);
-
-        cgh.parallel_for(sycl::nd_range<1>(grid_dims * block_dims, block_dims), [=](sycl::nd_item<1> item) {
-            u_int tid = item.get_local_id(0);
-            u_int bid = item.get_group(0);
-
-            u_int batch_i = bid / H;
-            u_int head_i  = bid % H;
-
-            float state[head_size];
-
-#pragma unroll
-            for (u_int i = 0; i < head_size; i++) {
-                state[i] = s[batch_i * state_size + head_i * head_size * head_size + i * head_size + tid];
-            }
-
-            for (u_int t = batch_i * n_seq_tokens * C + head_i * head_size + tid;
-                 t < (batch_i + 1) * n_seq_tokens * C + head_i * head_size + tid; t += C) {
-
-                item.barrier(sycl::access::fence_space::local_space);  //sync threads
-                _k[tid]  = k[t];
-                _r[tid]  = r[t];
-                _td[tid] = td[t];
-                item.barrier(sycl::access::fence_space::local_space);  //sync threads
-
-                const float _v = v[t];
-                float       y  = 0;
-
-                for (u_int j = 0; j < head_size; j += 4) {
-                    const sycl::float4 & k  = (sycl::float4 &) (_k[j]);
-                    const sycl::float4 & r  = (sycl::float4 &) (_r[j]);
-                    const sycl::float4 & td = (sycl::float4 &) (_td[j]);
-                    sycl::float4 &       s  = (sycl::float4 &) (state[j]);
-                    sycl::float4         kv;
-
-                    kv.x() = k.x() * _v;
-                    kv.y() = k.y() * _v;
-                    kv.z() = k.z() * _v;
-                    kv.w() = k.w() * _v;
-
-                    s.x() = s.x() * td.x() + kv.x();
-                    s.y() = s.y() * td.y() + kv.y();
-                    s.z() = s.z() * td.z() + kv.z();
-                    s.w() = s.w() * td.w() + kv.w();
-
-                    y += r.x() * s.x();
-                    y += r.y() * s.y();
-                    y += r.z() * s.z();
-                    y += r.w() * s.w();
-                }
-                dst[t] = y * scale;
-            }
-#pragma unroll
-            for (u_int i = 0; i < head_size; i++) {
-                dst[T * C + batch_i * state_size + head_i * head_size * head_size + i * head_size + tid] = state[i];
-            }
-        });
-    });
-}
-
-void ggml_sycl_op_gated_linear_attn(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
-    const float * k_d  = static_cast<const float *>(dst->src[0]->data);
-    const float * v_d  = static_cast<const float *>(dst->src[1]->data);
-    const float * r_d  = static_cast<const float *>(dst->src[2]->data);
-    const float * td_d = static_cast<const float *>(dst->src[3]->data);
-    const float * s_d  = static_cast<const float *>(dst->src[4]->data);
-
-    const int64_t B = dst->src[4]->ne[1];
-    const int64_t T = dst->src[0]->ne[2];
-    const int64_t C = dst->ne[0];
-    const int64_t H = dst->src[0]->ne[1];
-
-    dpct::queue_ptr stream = ctx.stream();
-    GGML_ASSERT(dst->src[4]->type == GGML_TYPE_F32);
-    GGML_ASSERT(C % H == 0);
-    GGML_ASSERT(C / H == 64 || C / H == 128);
-
-    float scale;
-    memcpy(&scale, dst->op_params, sizeof(float));
-
-    float * dst_d = (float *) dst->data;
-
-    if (C / H == 64) {
-        gated_linear_attn_f32_kernel<64>(stream, B, T, C, H, scale, k_d, v_d, r_d, td_d, s_d, dst_d);
-    } else {
-        gated_linear_attn_f32_kernel<128>(stream, B, T, C, H, scale, k_d, v_d, r_d, td_d, s_d, dst_d);
-    }
-}

package/src/llama.cpp/ggml/src/ggml-sycl/gla.hpp

@@ -1,8 +0,0 @@
-#ifndef GGML_SYCL_GLA_HPP
-#define GGML_SYCL_GLA_HPP
-
-#include "common.hpp"
-
-void ggml_sycl_op_gated_linear_attn(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
-
-#endif  // GGML_SYCL_GLA_HPP

package/src/llama.cpp/ggml/src/ggml-sycl/im2col.cpp

@@ -1,136 +0,0 @@
-//
-// MIT license
-// Copyright (C) 2024 Intel Corporation
-// SPDX-License-Identifier: MIT
-//
-
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-
-#include "im2col.hpp"
-
-#include <sycl/sycl.hpp>
-#include <type_traits>  // For std::is_same_v
-
-#include "ggml.h"
-
-template <typename T>
-static void im2col_kernel(const float * x, T * dst, int64_t batch_offset, int64_t offset_delta, int64_t IC, int64_t IW,
-                          int64_t IH, int64_t OH, int64_t OW, int64_t KW, int64_t KH, int64_t pelements, int64_t CHW,
-                          int s0, int s1, int p0, int p1, int d0, int d1, const sycl::nd_item<3> & item_ct1) {
-    const int64_t work_group_size = item_ct1.get_local_range(2);
-    const int64_t global_id       = item_ct1.get_local_id(2) + (work_group_size * item_ct1.get_group(2));
-
-    // make each work-item deal with more elements since sycl global range can not exceed max int
-    for (int64_t i = global_id; i < pelements; i += (work_group_size * item_ct1.get_group_range(2))) {
-        const int64_t ksize = OW * (KH > 1 ? KW : 1);
-        const int64_t kx    = i / ksize;
-        const int64_t kd    = kx * ksize;
-        const int64_t ky    = (i - kd) / OW;
-        const int64_t ix    = i % OW;
-
-        const int64_t oh    = item_ct1.get_group(1);
-        const int64_t batch = item_ct1.get_group(0) / IC;
-        const int64_t ic    = item_ct1.get_group(0) % IC;
-
-        const int64_t iiw = (ix * s0) + (kx * d0) - p0;
-        const int64_t iih = (oh * s1) + (ky * d1) - p1;
-
-        const int64_t offset_dst = (((batch * OH + oh) * OW + ix) * CHW) + (ic * (KW * KH) + ky * KW + kx);
-
-        const int64_t offset_src_base = (ic * offset_delta) + (batch * batch_offset);
-        const int64_t offset_src      = offset_src_base + (iih * IW) + iiw;
-
-        const bool  out_of_bounds = (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW);
-        const float src_val       = out_of_bounds ? 0.0f : x[offset_src];
-
-        if constexpr (std::is_same_v<T, sycl::half>) {
-            dst[offset_dst] = sycl::half(src_val);
-        } else if constexpr (std::is_same_v<T, float>) {
-            dst[offset_dst] = src_val;
-        }
-    }
-}
-
-template <typename T>
-static void im2col_sycl_internal(const float * x, T * dst, int64_t IW, int64_t IH, int64_t OW, int64_t OH, int64_t KW,
-                                 int64_t KH, int64_t IC, int64_t batch, int64_t batch_offset, int64_t offset_delta,
-                                 int s0, int s1, int p0, int p1, int d0, int d1, queue_ptr stream) {
-    const int64_t parallel_elements = OW * KW * KH;
-    const int64_t num_blocks        = (parallel_elements + SYCL_IM2COL_BLOCK_SIZE - 1) / SYCL_IM2COL_BLOCK_SIZE;
-
-    // decrease global range when it exceeds the max int
-    int64_t local_size = downsample_sycl_global_range(batch * IC * OH * num_blocks, SYCL_IM2COL_BLOCK_SIZE);
-
-    sycl::range<3> block_nums(batch * IC, OH, num_blocks);
-    sycl::range<3> local_range(1, 1, local_size);
-
-    const int64_t CHW = IC * KH * KW;
-
-    stream->parallel_for(sycl::nd_range<3>(block_nums * local_range, local_range), [=](sycl::nd_item<3> item_ct1) {
-        im2col_kernel<T>(x, dst, batch_offset, offset_delta, IC, IW, IH, OH, OW, KW, KH, parallel_elements, CHW, s0, s1,
-                         p0, p1, d0, d1, item_ct1);
-    });
-}
-
-static void im2col_sycl_f16(const float * x, sycl::half * dst, int64_t IW, int64_t IH, int64_t OW, int64_t OH,
-                            int64_t KW, int64_t KH, int64_t IC, int64_t batch, int64_t batch_offset,
-                            int64_t offset_delta, int s0, int s1, int p0, int p1, int d0, int d1, queue_ptr stream) {
-    if (!stream->get_device().has(sycl::aspect::fp16)) {
-        throw sycl::exception(sycl::make_error_code(sycl::errc::kernel_not_supported),
-                              "Device does not support half precision (fp16) operations!");
-    }
-    im2col_sycl_internal<sycl::half>(x, dst, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, offset_delta, s0, s1, p0,
-                                     p1, d0, d1, stream);
-}
-
-static void im2col_sycl_f32(const float * x, float * dst, int64_t IW, int64_t IH, int64_t OW, int64_t OH, int64_t KW,
-                            int64_t KH, int64_t IC, int64_t batch, int64_t batch_offset, int64_t offset_delta, int s0,
-                            int s1, int p0, int p1, int d0, int d1, queue_ptr stream) {
-    im2col_sycl_internal<float>(x, dst, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, offset_delta, s0, s1, p0, p1,
-                                d0, d1, stream);
-}
-
-void ggml_sycl_op_im2col(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
-    const ggml_tensor * src0 = dst->src[0];
-    const ggml_tensor * src1 = dst->src[1];
-
-    GGML_ASSERT(src1->type == GGML_TYPE_F32);
-    GGML_ASSERT(dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_F32);
-
-    const int32_t s0 = ((const int32_t *) (dst->op_params))[0];
-    const int32_t s1 = ((const int32_t *) (dst->op_params))[1];
-    const int32_t p0 = ((const int32_t *) (dst->op_params))[2];
-    const int32_t p1 = ((const int32_t *) (dst->op_params))[3];
-    const int32_t d0 = ((const int32_t *) (dst->op_params))[4];
-    const int32_t d1 = ((const int32_t *) (dst->op_params))[5];
-
-    const bool is_2D = ((const int32_t *) (dst->op_params))[6] == 1;
-
-    const int64_t IC = src1->ne[is_2D ? 2 : 1];
-    const int64_t IH = is_2D ? src1->ne[1] : 1;
-    const int64_t IW = src1->ne[0];
-
-    const int64_t KH = is_2D ? src0->ne[1] : 1;
-    const int64_t KW = src0->ne[0];
-
-    const int64_t OH = is_2D ? dst->ne[2] : 1;
-    const int64_t OW = dst->ne[1];
-
-    const size_t  delta_offset = src1->nb[is_2D ? 2 : 1] / sizeof(float);
-    const int64_t batch        = src1->ne[is_2D ? 3 : 2];
-    const size_t  batch_offset = src1->nb[is_2D ? 3 : 2] / sizeof(float);
-
-    queue_ptr stream = ctx.stream();
-
-    if (dst->type == GGML_TYPE_F16) {
-        im2col_sycl_f16((const float *) src1->data, (sycl::half *) dst->data, IW, IH, OW, OH, KW, KH, IC, batch,
-                        batch_offset, delta_offset, s0, s1, p0, p1, d0, d1, stream);
-    } else {
-        im2col_sycl_f32((const float *) src1->data, (float *) dst->data, IW, IH, OW, OH, KW, KH, IC, batch,
-                        batch_offset, delta_offset, s0, s1, p0, p1, d0, d1, stream);
-    }
-}

package/src/llama.cpp/ggml/src/ggml-sycl/im2col.hpp

@@ -1,21 +0,0 @@
-//
-// MIT license
-// Copyright (C) 2024 Intel Corporation
-// SPDX-License-Identifier: MIT
-//
-
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-
-#ifndef GGML_SYCL_IM2COL_HPP
-#define GGML_SYCL_IM2COL_HPP
-
-#include "common.hpp"
-
-void ggml_sycl_op_im2col(
-        ggml_backend_sycl_context & ctx, ggml_tensor *dst);
-
-#endif // GGML_SYCL_IM2COL_HPP