@novastera-oss/llamarn 0.2.9 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (314) hide show
  1. package/android/build.gradle +2 -1
  2. package/android/proguard-rules.pro +12 -0
  3. package/android/src/main/cpp/include/llama.h +15 -47
  4. package/android/src/main/jniLibs/arm64-v8a/libggml-base.so +0 -0
  5. package/android/src/main/jniLibs/arm64-v8a/libggml-cpu.so +0 -0
  6. package/android/src/main/jniLibs/arm64-v8a/libggml.so +0 -0
  7. package/android/src/main/jniLibs/arm64-v8a/libllama.so +0 -0
  8. package/android/src/main/jniLibs/armeabi-v7a/libggml-base.so +0 -0
  9. package/android/src/main/jniLibs/armeabi-v7a/libggml-cpu.so +0 -0
  10. package/android/src/main/jniLibs/armeabi-v7a/libggml.so +0 -0
  11. package/android/src/main/jniLibs/armeabi-v7a/libllama.so +0 -0
  12. package/android/src/main/jniLibs/x86/libggml-base.so +0 -0
  13. package/android/src/main/jniLibs/x86/libggml-cpu.so +0 -0
  14. package/android/src/main/jniLibs/x86/libggml.so +0 -0
  15. package/android/src/main/jniLibs/x86/libllama.so +0 -0
  16. package/android/src/main/jniLibs/x86_64/libggml-base.so +0 -0
  17. package/android/src/main/jniLibs/x86_64/libggml-cpu.so +0 -0
  18. package/android/src/main/jniLibs/x86_64/libggml.so +0 -0
  19. package/android/src/main/jniLibs/x86_64/libllama.so +0 -0
  20. package/cpp/build-info.cpp +2 -2
  21. package/cpp/llama.cpp/CMakeLists.txt +0 -1
  22. package/cpp/llama.cpp/CMakePresets.json +11 -0
  23. package/cpp/llama.cpp/CODEOWNERS +1 -0
  24. package/cpp/llama.cpp/README.md +8 -8
  25. package/cpp/llama.cpp/build-xcframework.sh +1 -1
  26. package/cpp/llama.cpp/common/CMakeLists.txt +4 -5
  27. package/cpp/llama.cpp/common/arg.cpp +62 -1
  28. package/cpp/llama.cpp/common/chat.cpp +37 -20
  29. package/cpp/llama.cpp/common/chat.h +2 -0
  30. package/cpp/llama.cpp/common/common.cpp +22 -6
  31. package/cpp/llama.cpp/common/common.h +22 -4
  32. package/cpp/llama.cpp/convert_hf_to_gguf.py +1250 -43
  33. package/cpp/llama.cpp/convert_hf_to_gguf_update.py +21 -13
  34. package/cpp/llama.cpp/ggml/CMakeLists.txt +13 -3
  35. package/cpp/llama.cpp/ggml/cmake/ggml-config.cmake.in +85 -47
  36. package/cpp/llama.cpp/ggml/include/ggml-backend.h +1 -1
  37. package/cpp/llama.cpp/ggml/include/ggml-webgpu.h +19 -0
  38. package/cpp/llama.cpp/ggml/include/ggml.h +173 -10
  39. package/cpp/llama.cpp/ggml/src/CMakeLists.txt +1 -1
  40. package/cpp/llama.cpp/ggml/src/ggml-alloc.c +0 -15
  41. package/cpp/llama.cpp/ggml/src/ggml-backend-reg.cpp +7 -8
  42. package/cpp/llama.cpp/ggml/src/ggml-backend.cpp +44 -38
  43. package/cpp/llama.cpp/ggml/src/ggml-cann/acl_tensor.cpp +3 -1
  44. package/cpp/llama.cpp/ggml/src/ggml-cann/aclnn_ops.cpp +126 -8
  45. package/cpp/llama.cpp/ggml/src/ggml-cann/aclnn_ops.h +130 -22
  46. package/cpp/llama.cpp/ggml/src/ggml-cann/ggml-cann.cpp +138 -18
  47. package/cpp/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt +11 -3
  48. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/loongarch/quants.c +1 -1
  49. package/cpp/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c +28 -1
  50. package/cpp/llama.cpp/ggml/src/ggml-cpu/kleidiai/kernels.cpp +109 -12
  51. package/cpp/llama.cpp/ggml/src/ggml-cpu/kleidiai/kernels.h +3 -0
  52. package/cpp/llama.cpp/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +88 -10
  53. package/cpp/llama.cpp/ggml/src/ggml-cpu/llamafile/sgemm.cpp +343 -1094
  54. package/cpp/llama.cpp/ggml/src/ggml-cpu/ops.cpp +1206 -163
  55. package/cpp/llama.cpp/ggml/src/ggml-cpu/ops.h +6 -0
  56. package/cpp/llama.cpp/ggml/src/ggml-cpu/repack.cpp +0 -1
  57. package/cpp/llama.cpp/ggml/src/ggml-cpu/simd-mappings.h +1 -1
  58. package/cpp/llama.cpp/ggml/src/ggml-cpu/vec.cpp +36 -9
  59. package/cpp/llama.cpp/ggml/src/ggml-cpu/vec.h +142 -9
  60. package/cpp/llama.cpp/ggml/src/ggml-cuda/CMakeLists.txt +3 -3
  61. package/cpp/llama.cpp/ggml/src/ggml-cuda/common.cuh +31 -4
  62. package/cpp/llama.cpp/ggml/src/ggml-cuda/convert.cu +86 -17
  63. package/cpp/llama.cpp/ggml/src/ggml-cuda/convert.cuh +5 -0
  64. package/cpp/llama.cpp/ggml/src/ggml-cuda/cpy-utils.cuh +225 -0
  65. package/cpp/llama.cpp/ggml/src/ggml-cuda/cpy.cu +41 -301
  66. package/cpp/llama.cpp/ggml/src/ggml-cuda/cross-entropy-loss.cu +2 -14
  67. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh +85 -64
  68. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh +47 -60
  69. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-tile-f16.cu +29 -42
  70. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-tile-f32.cu +46 -59
  71. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-vec-f16.cuh +36 -45
  72. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-vec-f32.cuh +38 -45
  73. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-wmma-f16.cu +23 -36
  74. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn.cu +3 -13
  75. package/cpp/llama.cpp/ggml/src/ggml-cuda/getrows.cu +8 -0
  76. package/cpp/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +255 -99
  77. package/cpp/llama.cpp/ggml/src/ggml-cuda/im2col.cu +1 -1
  78. package/cpp/llama.cpp/ggml/src/ggml-cuda/mma.cuh +111 -3
  79. package/cpp/llama.cpp/ggml/src/ggml-cuda/mmq.cu +6 -4
  80. package/cpp/llama.cpp/ggml/src/ggml-cuda/mmq.cuh +1152 -695
  81. package/cpp/llama.cpp/ggml/src/ggml-cuda/norm.cu +92 -5
  82. package/cpp/llama.cpp/ggml/src/ggml-cuda/norm.cuh +2 -0
  83. package/cpp/llama.cpp/ggml/src/ggml-cuda/rope.cu +21 -27
  84. package/cpp/llama.cpp/ggml/src/ggml-cuda/scale.cu +8 -6
  85. package/cpp/llama.cpp/ggml/src/ggml-cuda/set-rows.cu +275 -0
  86. package/cpp/llama.cpp/ggml/src/ggml-cuda/set-rows.cuh +7 -0
  87. package/cpp/llama.cpp/ggml/src/ggml-cuda/softmax.cu +119 -58
  88. package/cpp/llama.cpp/ggml/src/ggml-cuda/ssm-conv.cu +10 -2
  89. package/cpp/llama.cpp/ggml/src/ggml-cuda/ssm-scan.cu +192 -52
  90. package/cpp/llama.cpp/ggml/src/ggml-cuda/unary.cu +104 -0
  91. package/cpp/llama.cpp/ggml/src/ggml-cuda/unary.cuh +13 -0
  92. package/cpp/llama.cpp/ggml/src/ggml-cuda/upscale.cu +92 -6
  93. package/cpp/llama.cpp/ggml/src/ggml-cuda/vendors/hip.h +27 -6
  94. package/cpp/llama.cpp/ggml/src/ggml-cuda/vendors/musa.h +2 -2
  95. package/cpp/llama.cpp/ggml/src/ggml-impl.h +80 -0
  96. package/cpp/llama.cpp/ggml/src/ggml-metal/CMakeLists.txt +4 -2
  97. package/cpp/llama.cpp/ggml/src/ggml-metal/ggml-metal-impl.h +48 -12
  98. package/cpp/llama.cpp/ggml/src/ggml-metal/ggml-metal.m +572 -106
  99. package/cpp/llama.cpp/ggml/src/ggml-metal/ggml-metal.metal +599 -105
  100. package/cpp/llama.cpp/ggml/src/ggml-musa/CMakeLists.txt +18 -4
  101. package/cpp/llama.cpp/ggml/src/ggml-opencl/CMakeLists.txt +5 -0
  102. package/cpp/llama.cpp/ggml/src/ggml-opencl/ggml-opencl.cpp +800 -42
  103. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/conv2d.cl +185 -0
  104. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl +176 -0
  105. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/gelu.cl +27 -0
  106. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/glu.cl +337 -0
  107. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/im2col_f16.cl +1 -1
  108. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/im2col_f32.cl +1 -1
  109. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul_mat_f16_f32.cl +130 -0
  110. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/rms_norm.cl +79 -0
  111. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/scale.cl +3 -2
  112. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/set_rows.cl +95 -0
  113. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +24 -11
  114. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +24 -11
  115. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/softmax_f16.cl +24 -11
  116. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/softmax_f32.cl +24 -11
  117. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/upscale.cl +2 -3
  118. package/cpp/llama.cpp/ggml/src/ggml-quants.c +6 -6
  119. package/cpp/llama.cpp/ggml/src/ggml-rpc/ggml-rpc.cpp +4 -4
  120. package/cpp/llama.cpp/ggml/src/ggml-sycl/backend.hpp +1 -0
  121. package/cpp/llama.cpp/ggml/src/ggml-sycl/element_wise.cpp +693 -1034
  122. package/cpp/llama.cpp/ggml/src/ggml-sycl/element_wise.hpp +18 -9
  123. package/cpp/llama.cpp/ggml/src/ggml-sycl/gemm.hpp +14 -26
  124. package/cpp/llama.cpp/ggml/src/ggml-sycl/ggml-sycl.cpp +191 -55
  125. package/cpp/llama.cpp/ggml/src/ggml-sycl/im2col.cpp +1 -1
  126. package/cpp/llama.cpp/ggml/src/ggml-sycl/quants.hpp +8 -9
  127. package/cpp/llama.cpp/ggml/src/ggml-sycl/rope.cpp +15 -18
  128. package/cpp/llama.cpp/ggml/src/ggml-sycl/set_rows.cpp +131 -0
  129. package/cpp/llama.cpp/ggml/src/ggml-sycl/set_rows.hpp +8 -0
  130. package/cpp/llama.cpp/ggml/src/ggml-sycl/vecdotq.hpp +2 -6
  131. package/cpp/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan.cpp +991 -307
  132. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp +265 -0
  133. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +59 -12
  134. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +1 -1
  135. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +1 -1
  136. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +1 -1
  137. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +1 -1
  138. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +1 -1
  139. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +28 -23
  140. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.comp +14 -9
  141. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +38 -32
  142. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +32 -27
  143. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +44 -12
  144. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/geglu.comp +13 -0
  145. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/geglu_erf.comp +27 -0
  146. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/geglu_quick.comp +11 -0
  147. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/gelu_erf.comp +39 -0
  148. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.comp +2 -0
  149. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.comp +17 -0
  150. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.comp +29 -0
  151. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +3 -8
  152. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +128 -72
  153. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +38 -9
  154. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp +9 -0
  155. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +18 -3
  156. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp +46 -0
  157. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.comp +1 -4
  158. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +7 -9
  159. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +7 -9
  160. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +7 -9
  161. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rte.comp +5 -0
  162. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +1 -1
  163. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +20 -4
  164. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/swiglu.comp +9 -0
  165. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +69 -5
  166. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +84 -9
  167. package/cpp/llama.cpp/ggml/src/ggml-webgpu/CMakeLists.txt +54 -0
  168. package/cpp/llama.cpp/ggml/src/ggml-webgpu/ggml-webgpu.cpp +907 -0
  169. package/cpp/llama.cpp/ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl +60 -0
  170. package/cpp/llama.cpp/ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py +35 -0
  171. package/cpp/llama.cpp/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +40 -0
  172. package/cpp/llama.cpp/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat.wgsl +56 -0
  173. package/cpp/llama.cpp/ggml/src/ggml.c +386 -67
  174. package/cpp/llama.cpp/ggml/src/gguf.cpp +8 -1
  175. package/cpp/llama.cpp/gguf-py/gguf/constants.py +307 -0
  176. package/cpp/llama.cpp/gguf-py/gguf/gguf_writer.py +8 -2
  177. package/cpp/llama.cpp/gguf-py/gguf/metadata.py +4 -0
  178. package/cpp/llama.cpp/gguf-py/gguf/scripts/gguf_dump.py +24 -1
  179. package/cpp/llama.cpp/gguf-py/gguf/tensor_mapping.py +122 -47
  180. package/cpp/llama.cpp/gguf-py/gguf/vocab.py +12 -3
  181. package/cpp/llama.cpp/include/llama.h +15 -47
  182. package/cpp/llama.cpp/models/templates/llama-cpp-rwkv-world.jinja +34 -0
  183. package/cpp/llama.cpp/models/templates/moonshotai-Kimi-K2.jinja +43 -0
  184. package/cpp/llama.cpp/requirements/requirements-all.txt +1 -0
  185. package/cpp/llama.cpp/requirements/requirements-server-bench.txt +5 -0
  186. package/cpp/llama.cpp/src/llama-arch.cpp +316 -3
  187. package/cpp/llama.cpp/src/llama-arch.h +23 -1
  188. package/cpp/llama.cpp/src/llama-batch.cpp +103 -71
  189. package/cpp/llama.cpp/src/llama-batch.h +31 -18
  190. package/cpp/llama.cpp/src/llama-chat.cpp +58 -1
  191. package/cpp/llama.cpp/src/llama-chat.h +3 -0
  192. package/cpp/llama.cpp/src/llama-context.cpp +180 -106
  193. package/cpp/llama.cpp/src/llama-context.h +26 -16
  194. package/cpp/llama.cpp/src/llama-cparams.h +3 -2
  195. package/cpp/llama.cpp/src/llama-graph.cpp +310 -211
  196. package/cpp/llama.cpp/src/llama-graph.h +184 -122
  197. package/cpp/llama.cpp/src/llama-hparams.cpp +47 -1
  198. package/cpp/llama.cpp/src/llama-hparams.h +13 -2
  199. package/cpp/llama.cpp/src/llama-kv-cache-unified-iswa.cpp +38 -22
  200. package/cpp/llama.cpp/src/llama-kv-cache-unified-iswa.h +7 -2
  201. package/cpp/llama.cpp/src/llama-kv-cache-unified.cpp +849 -304
  202. package/cpp/llama.cpp/src/llama-kv-cache-unified.h +143 -47
  203. package/cpp/llama.cpp/src/llama-kv-cells.h +62 -10
  204. package/cpp/llama.cpp/src/llama-memory-hybrid.cpp +10 -4
  205. package/cpp/llama.cpp/src/llama-memory-hybrid.h +3 -1
  206. package/cpp/llama.cpp/src/llama-memory-recurrent.cpp +36 -11
  207. package/cpp/llama.cpp/src/llama-memory.cpp +17 -0
  208. package/cpp/llama.cpp/src/llama-memory.h +3 -0
  209. package/cpp/llama.cpp/src/llama-model.cpp +3545 -719
  210. package/cpp/llama.cpp/src/llama-model.h +21 -4
  211. package/cpp/llama.cpp/src/llama-quant.cpp +2 -2
  212. package/cpp/llama.cpp/src/llama-vocab.cpp +376 -10
  213. package/cpp/llama.cpp/src/llama-vocab.h +43 -0
  214. package/cpp/llama.cpp/src/unicode.cpp +207 -0
  215. package/cpp/llama.cpp/src/unicode.h +2 -0
  216. package/ios/include/chat.h +2 -0
  217. package/ios/include/common.h +22 -4
  218. package/ios/include/llama.h +15 -47
  219. package/ios/libs/llama.xcframework/Info.plist +13 -13
  220. package/ios/libs/llama.xcframework/ios-arm64/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  221. package/ios/libs/llama.xcframework/ios-arm64/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +5267 -4890
  222. package/ios/libs/llama.xcframework/ios-arm64/llama.framework/Headers/ggml-backend.h +1 -1
  223. package/ios/libs/llama.xcframework/ios-arm64/llama.framework/Headers/ggml.h +173 -10
  224. package/ios/libs/llama.xcframework/ios-arm64/llama.framework/Headers/llama.h +15 -47
  225. package/ios/libs/llama.xcframework/ios-arm64/llama.framework/llama +0 -0
  226. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  227. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +5238 -4861
  228. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/x86_64/llama.yml +4014 -3764
  229. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/ggml-backend.h +1 -1
  230. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/ggml.h +173 -10
  231. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/llama.h +15 -47
  232. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/llama +0 -0
  233. package/ios/libs/llama.xcframework/macos-arm64_x86_64/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  234. package/ios/libs/llama.xcframework/macos-arm64_x86_64/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +5238 -4861
  235. package/ios/libs/llama.xcframework/macos-arm64_x86_64/dSYMs/llama.dSYM/Contents/Resources/Relocations/x86_64/llama.yml +4016 -3766
  236. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Headers/ggml-backend.h +1 -1
  237. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Headers/ggml.h +173 -10
  238. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Headers/llama.h +15 -47
  239. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/ggml-backend.h +1 -1
  240. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/ggml.h +173 -10
  241. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/llama.h +15 -47
  242. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/llama +0 -0
  243. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Headers/ggml-backend.h +1 -1
  244. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Headers/ggml.h +173 -10
  245. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Headers/llama.h +15 -47
  246. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/llama +0 -0
  247. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/llama +0 -0
  248. package/ios/libs/llama.xcframework/tvos-arm64/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  249. package/ios/libs/llama.xcframework/tvos-arm64/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +5267 -4890
  250. package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Headers/ggml-backend.h +1 -1
  251. package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Headers/ggml.h +173 -10
  252. package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Headers/llama.h +15 -47
  253. package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/llama +0 -0
  254. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  255. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +5238 -4861
  256. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/x86_64/llama.yml +4014 -3764
  257. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Headers/ggml-backend.h +1 -1
  258. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Headers/ggml.h +173 -10
  259. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Headers/llama.h +15 -47
  260. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/llama +0 -0
  261. package/ios/libs/llama.xcframework/xros-arm64/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  262. package/ios/libs/llama.xcframework/xros-arm64/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +5303 -4926
  263. package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Headers/ggml-backend.h +1 -1
  264. package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Headers/ggml.h +173 -10
  265. package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Headers/llama.h +15 -47
  266. package/ios/libs/llama.xcframework/xros-arm64/llama.framework/llama +0 -0
  267. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  268. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +5274 -4897
  269. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/x86_64/llama.yml +4044 -3794
  270. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Headers/ggml-backend.h +1 -1
  271. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Headers/ggml.h +173 -10
  272. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Headers/llama.h +15 -47
  273. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/llama +0 -0
  274. package/package.json +4 -4
  275. package/cpp/llama.cpp/ggml/include/ggml-kompute.h +0 -50
  276. package/cpp/llama.cpp/ggml/src/ggml-kompute/CMakeLists.txt +0 -166
  277. package/cpp/llama.cpp/ggml/src/ggml-kompute/ggml-kompute.cpp +0 -2251
  278. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/common.comp +0 -112
  279. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_add.comp +0 -58
  280. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_addrow.comp +0 -25
  281. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f16.comp +0 -52
  282. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f32.comp +0 -52
  283. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f16.comp +0 -52
  284. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f32.comp +0 -52
  285. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_diagmask.comp +0 -30
  286. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_gelu.comp +0 -22
  287. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_getrows.comp +0 -17
  288. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f16.comp +0 -31
  289. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f32.comp +0 -31
  290. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_0.comp +0 -38
  291. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_1.comp +0 -39
  292. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q6_k.comp +0 -44
  293. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_mul.comp +0 -52
  294. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_f16.comp +0 -69
  295. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_mat_f32.comp +0 -51
  296. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_0.comp +0 -33
  297. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_1.comp +0 -35
  298. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_k.comp +0 -140
  299. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q6_k.comp +0 -106
  300. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q8_0.comp +0 -73
  301. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n.comp +0 -52
  302. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n_pre.comp +0 -28
  303. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_norm.comp +0 -84
  304. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_relu.comp +0 -21
  305. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_rmsnorm.comp +0 -53
  306. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f16.comp +0 -52
  307. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f32.comp +0 -52
  308. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f16.comp +0 -52
  309. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f32.comp +0 -52
  310. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_scale.comp +0 -19
  311. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_scale_8.comp +0 -23
  312. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_silu.comp +0 -22
  313. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_softmax.comp +0 -72
  314. package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/rope_common.comp +0 -71
package/cpp/llama.cpp/ggml/src/ggml-sycl/element_wise.cpp CHANGED
@@ -1,12 +1,19 @@
1
1
  #include "common.hpp"
2
+ #include "ggml-sycl/presets.hpp"
2
3
  #include "ggml.h"
3
4
  #include "element_wise.hpp"
4
5
 
6
+ #define SYCL_GLOBAL_ID_LOOP(K, ITEM) \
7
+ for (auto i = ITEM.get_global_id(0); i < (size_t)K; i += ITEM.get_global_range(0))
8
+
9
+ #define SYCL_LOCAL_ID_CALC(ITEM, IDX) \
10
+ (ITEM.get_local_range(IDX) * ITEM.get_group(IDX) + ITEM.get_local_id(IDX))
11
+
12
+
5
13
  static void acc_f32(const float * x, const float * y, float * dst, const int ne,
6
14
  const int ne10, const int ne11, const int ne12,
7
- const int nb1, const int nb2, int offset, const sycl::nd_item<3> &item_ct1) {
8
- const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
9
- item_ct1.get_local_id(2);
15
+ const int nb1, const int nb2, int offset, const sycl::nd_item<1> &item_ct1) {
16
+ const int i = SYCL_LOCAL_ID_CALC(item_ct1, 0);
10
17
  if (i >= ne) {
11
18
  return;
12
19
  }
@@ -21,248 +28,280 @@ static void acc_f32(const float * x, const float * y, float * dst, const int ne,
21
28
  }
22
29
  }
23
30
 
31
+ /* Unary OP funcs */
24
32
  template<typename T>
25
- static void sgn(const T * x, T * dst, const int k, const sycl::nd_item<3> &item_ct1) {
26
- for(auto i = item_ct1.get_global_id(2); i < (const size_t)k; i += item_ct1.get_global_range(2)) {
27
- dst[i] = x[i] > static_cast<T>(0.f) ? static_cast<T>(1.f) : ((x[i] < static_cast<T>(0.f) ? static_cast<T>(-1.f) : static_cast<T>(0.f)));
28
- }
33
+ static __dpct_inline__ T op_sgn(T x) {
34
+ return x > static_cast<T>(0.f) ? static_cast<T>(1.f) : ((x < static_cast<T>(0.f) ? static_cast<T>(-1.f) : static_cast<T>(0.f)));
29
35
  }
30
36
 
31
37
  template<typename T>
32
- static void abs_op(const T * x, T * dst, const int k, const sycl::nd_item<3> &item_ct1) {
33
- for(auto i = item_ct1.get_global_id(2); i < (const size_t)k; i += item_ct1.get_global_range(2)) {
34
- dst[i] = sycl::fabs(x[i]);
35
- }
38
+ static __dpct_inline__ T op_abs(T x) {
39
+ return sycl::fabs(x);
36
40
  }
37
41
 
38
42
  template<typename T>
39
- static void elu_op(const T * x, T * dst, const int k, const sycl::nd_item<3> &item_ct1) {
40
- for(auto i = item_ct1.get_global_id(2); i < (const size_t)k; i += item_ct1.get_global_range(2)) {
41
- dst[i] = (x[i] > static_cast<T>(0.f)) ? x[i] : sycl::expm1(x[i]);
42
- }
43
+ static __dpct_inline__ T op_elu(T x) {
44
+ return (x > static_cast<T>(0.f)) ? x : sycl::expm1(x);
43
45
  }
44
46
 
45
47
  template<typename T>
46
- static void gelu(const T * x, T * dst, const int k,
47
- const sycl::nd_item<3> &item_ct1) {
48
+ static __dpct_inline__ T op_gelu(T x) {
48
49
  const T GELU_COEF_A = static_cast<T>(0.044715f);
49
50
  const T SQRT_2_OVER_PI = static_cast<T>(0.79788456080286535587989211986876f);
50
- const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
51
- item_ct1.get_local_id(2);
51
+ return static_cast<T>(0.5f) * x *
52
+ (static_cast<T>(1.0f) +
53
+ sycl::tanh(SQRT_2_OVER_PI * x * (static_cast<T>(1.0f) + GELU_COEF_A * x * x)));
54
+ }
52
55
 
53
- if (i >= k) {
54
- return;
55
- }
56
+ template<typename T>
57
+ static __dpct_inline__ T op_silu(T x) {
58
+ return x / (static_cast<T>(1.0f) + sycl::native::exp(-x));
59
+ }
56
60
 
57
- float xi = x[i];
58
- dst[i] = static_cast<T>(0.5f) * xi *
59
- (static_cast<T>(1.0f) +
60
- sycl::tanh(SQRT_2_OVER_PI * xi * (static_cast<T>(1.0f) + GELU_COEF_A * xi * xi)));
61
+ template<typename T>
62
+ static __dpct_inline__ T op_gelu_quick(T x) {
63
+ const T GELU_QUICK_COEF_LOCAL = static_cast<T>(-1.702f);
64
+ return x * (static_cast<T>(1.0f) / (static_cast<T>(1.0f) + sycl::native::exp(GELU_QUICK_COEF_LOCAL * x)));
61
65
  }
62
66
 
63
67
  template<typename T>
64
- static void silu(const T * x, T * dst, const int k,
65
- const sycl::nd_item<3> &item_ct1) {
66
- const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
67
- item_ct1.get_local_id(2);
68
+ static __dpct_inline__ T op_gelu_erf(T x) {
69
+ const T SQRT_2_INV = static_cast<T>(0.70710678118654752440084436210484f);
70
+ return static_cast<T>(0.5f) * x * (static_cast<T>(1.0f) + sycl::erf(x * SQRT_2_INV));
71
+ }
68
72
 
69
- if (i >= k) {
70
- return;
71
- }
72
- dst[i] = x[i] / (static_cast<T>(1.0f) + sycl::native::exp(-x[i]));
73
+ template<typename T>
74
+ static __dpct_inline__ T op_tanh(T x) {
75
+ return sycl::tanh(x);
73
76
  }
74
77
 
75
78
  template<typename T>
76
- static void gelu_quick(const T *x, T *dst, int k,
77
- const sycl::nd_item<3> &item_ct1) {
78
- const float GELU_QUICK_COEF = -1.702f;
79
- const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
80
- item_ct1.get_local_id(2);
81
- if (i >= k) {
82
- return;
83
- }
84
- dst[i] = x[i] * (static_cast<T>(1.0f) / (static_cast<T>(1.0f) + sycl::native::exp(GELU_QUICK_COEF * x[i])));
79
+ static __dpct_inline__ T op_relu(T x) {
80
+ return sycl::fmax(x, static_cast<T>(0));
85
81
  }
86
82
 
87
83
  template<typename T>
88
- static void gelu_erf(const T * x, T * dst, const int k, const sycl::nd_item<3> &item_ct1) {
89
- const T SQRT_2_INV = static_cast<T>(0.70710678118654752440084436210484f);
90
- for(auto i = item_ct1.get_global_id(2); i < (const size_t)k; i += item_ct1.get_global_range(2)) {
91
- auto x_i = x[i];
92
- dst[i] = static_cast<T>(0.5f) * x_i * (static_cast<T>(1.0f) + sycl::erf(x_i * SQRT_2_INV));
93
- }
84
+ static __dpct_inline__ T op_sigmoid(T x) {
85
+ return static_cast<T>(1.0f) / (static_cast<T>(1.0f) + sycl::native::exp(-x));
94
86
  }
95
87
 
96
88
  template<typename T>
97
- static void tanh(const T *x, T *dst, int k,
98
- const sycl::nd_item<3> &item_ct1) {
99
- const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
100
- item_ct1.get_local_id(2);
101
- if (i >= k) {
102
- return;
103
- }
104
- dst[i] = sycl::tanh((x[i]));
89
+ static __dpct_inline__ T op_sqrt(T x) {
90
+ return sycl::sqrt(x);
91
+ }
92
+
93
+ template<typename T>
94
+ static __dpct_inline__ T op_sin(T x) {
95
+ return sycl::sin(x);
105
96
  }
106
97
 
107
98
  template<typename T>
108
- static void relu(const T * x, T * dst, const int k,
109
- const sycl::nd_item<3> &item_ct1) {
110
- const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
111
- item_ct1.get_local_id(2);
99
+ static __dpct_inline__ T op_cos(T x) {
100
+ return sycl::cos(x);
101
+ }
112
102
 
113
- if (i >= k) {
114
- return;
115
- }
116
- dst[i] = sycl::fmax((x[i]), static_cast<T>(0));
103
+ template<typename T>
104
+ static __dpct_inline__ T op_hardsigmoid(T x) {
105
+ return sycl::fmin(static_cast<T>(1.0f), sycl::fmax(static_cast<T>(0.0f), (x + static_cast<T>(3.0f)) / static_cast<T>(6.0f)));
117
106
  }
118
107
 
119
108
  template<typename T>
120
- static void sigmoid(const T * x, T * dst, const int k,
121
- const sycl::nd_item<3> &item_ct1) {
122
- const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
123
- item_ct1.get_local_id(2);
109
+ static __dpct_inline__ T op_hardswish(T x) {
110
+ return x * sycl::fmin(static_cast<T>(1.0f), sycl::fmax(static_cast<T>(0.0f), (x + static_cast<T>(3.0f)) / static_cast<T>(6.0f)));
111
+ }
124
112
 
125
- if (i >= k) {
126
- return;
113
+ template<typename T>
114
+ static __dpct_inline__ T op_exp(T x) {
115
+ return sycl::exp(x);
116
+ }
117
+
118
+ template<typename T>
119
+ static __dpct_inline__ T op_log(T x) {
120
+ if (x <= static_cast<T>(0)) {
121
+ return neg_infinity<T>();
127
122
  }
128
- dst[i] = 1.0f / (static_cast<T>(1.0f) + sycl::native::exp(-x[i]));
123
+ return sycl::log(x);
129
124
  }
130
125
 
131
126
  template<typename T>
132
- static void sqrt(const T * x, T * dst, const int k,
133
- const sycl::nd_item<3> &item_ct1) {
134
- const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
135
- item_ct1.get_local_id(2);
127
+ static __dpct_inline__ T op_neg(T x) {
128
+ return -x;
129
+ }
136
130
 
137
- if (i >= k) {
138
- return;
131
+ template<typename T>
132
+ static __dpct_inline__ T op_step(T x) {
133
+ return (x > static_cast<T>(0.0f)) ? static_cast<T>(1.0f) : static_cast<T>(0.0f);
134
+ }
135
+
136
+ template<typename T>
137
+ static __dpct_inline__ T op_leaky_relu(T x, float negative_slope) {
138
+ T neg_slope_T = static_cast<T>(negative_slope);
139
+ return sycl::fmax(x, static_cast<T>(0)) +
140
+ sycl::fmin(x, static_cast<T>(0.0f)) * neg_slope_T;
141
+ }
142
+
143
+ template<typename T>
144
+ static __dpct_inline__ T op_sqr(T x) {
145
+ return x * x;
146
+ }
147
+
148
+ template<typename T>
149
+ static __dpct_inline__ T op_clamp(T x, float min_val, float max_val) {
150
+ return x < static_cast<T>(min_val) ? static_cast<T>(min_val) : (x > static_cast<T>(max_val) ? static_cast<T>(max_val) : x);
151
+ }
152
+
153
+ template<typename T>
154
+ static void unary_op_sgn_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {
155
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
156
+ dst[i] = op_sgn(x[i]);
139
157
  }
140
- dst[i] = sycl::sqrt(x[i]);
141
158
  }
142
159
 
143
160
  template<typename T>
144
- static void sin(const T * x, T * dst, const int k,
145
- const sycl::nd_item<3> &item_ct1) {
146
- const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
147
- item_ct1.get_local_id(2);
161
+ static void unary_op_abs_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {
162
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
163
+ dst[i] = op_abs(x[i]);
164
+ }
165
+ }
148
166
 
149
- if (i >= k) {
150
- return;
167
+ template<typename T>
168
+ static void unary_op_elu_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {
169
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
170
+ dst[i] = op_elu(x[i]);
151
171
  }
152
- dst[i] = sycl::sin(x[i]);
153
172
  }
154
173
 
155
174
  template<typename T>
156
- static void cos(const T * x, T * dst, const int k,
157
- const sycl::nd_item<3> &item_ct1) {
158
- const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
159
- item_ct1.get_local_id(2);
175
+ static void unary_op_gelu_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {
176
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
177
+ dst[i] = op_gelu(x[i]);
178
+ }
179
+ }
160
180
 
161
- if (i >= k) {
162
- return;
181
+ template<typename T>
182
+ static void unary_op_silu_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {
183
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
184
+ dst[i] = op_silu(x[i]);
163
185
  }
164
- dst[i] = sycl::cos(x[i]);
165
186
  }
166
187
 
167
188
  template<typename T>
168
- static void hardsigmoid(const T * x, T * dst, const int k,
169
- const sycl::nd_item<3> &item_ct1) {
170
- const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
171
- item_ct1.get_local_id(2);
189
+ static void unary_op_gelu_quick_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {
190
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
191
+ dst[i] = op_gelu_quick(x[i]);
192
+ }
193
+ }
172
194
 
173
- if (i >= k) {
174
- return;
195
+ template<typename T>
196
+ static void unary_op_gelu_erf_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {
197
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
198
+ dst[i] = op_gelu_erf(x[i]);
199
+ }
200
+ }
201
+
202
+ template<typename T>
203
+ static void unary_op_tanh_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {
204
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
205
+ dst[i] = op_tanh(x[i]);
175
206
  }
176
- dst[i] = sycl::fmin(static_cast<T>(1.0f), sycl::fmax(static_cast<T>(0.0f), (x[i] + static_cast<T>(3.0f)) / static_cast<T>(6.0f)));
177
207
  }
178
208
 
179
209
  template<typename T>
180
- static void hardswish(const T * x, T * dst, const int k,
181
- const sycl::nd_item<3> &item_ct1) {
182
- const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
183
- item_ct1.get_local_id(2);
210
+ static void unary_op_relu_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {
211
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
212
+ dst[i] = op_relu(x[i]);
213
+ }
214
+ }
184
215
 
185
- if (i >= k) {
186
- return;
216
+ template<typename T>
217
+ static void unary_op_sigmoid_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {
218
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
219
+ dst[i] = op_sigmoid(x[i]);
187
220
  }
188
- dst[i] = x[i] * sycl::fmin(static_cast<T>(1.0f), sycl::fmax(static_cast<T>(0.0f), (x[i] + static_cast<T>(3.0f)) / static_cast<T>(6.0f)));
189
221
  }
190
222
 
191
223
  template<typename T>
192
- static void exp(const T * x, T * dst, const int k,
193
- const sycl::nd_item<3> &item_ct1) {
194
- const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
195
- item_ct1.get_local_id(2);
224
+ static void unary_op_sqrt_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {
225
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
226
+ dst[i] = op_sqrt(x[i]);
227
+ }
228
+ }
196
229
 
197
- if (i >= k) {
198
- return;
230
+ template<typename T>
231
+ static void unary_op_sin_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {
232
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
233
+ dst[i] = op_sin(x[i]);
199
234
  }
200
- dst[i] = sycl::exp(x[i]);
201
235
  }
202
236
 
203
237
  template<typename T>
204
- static void log(const T * x, T * dst, const int k,
205
- const sycl::nd_item<3> &item_ct1) {
206
- const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
207
- item_ct1.get_local_id(2);
238
+ static void unary_op_cos_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {
239
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
240
+ dst[i] = op_cos(x[i]);
241
+ }
242
+ }
208
243
 
209
- if (i >= k) {
210
- return;
244
+ template<typename T>
245
+ static void unary_op_hardsigmoid_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {
246
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
247
+ dst[i] = op_hardsigmoid(x[i]);
211
248
  }
212
- T xi = x[i];
213
- if (xi <= 0) {
214
- dst[i] = neg_infinity<T>();
215
- } else {
216
- dst[i] = sycl::log(xi);
249
+ }
250
+
251
+ template<typename T>
252
+ static void unary_op_hardswish_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {
253
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
254
+ dst[i] = op_hardswish(x[i]);
217
255
  }
218
256
  }
219
257
 
220
258
  template<typename T>
221
- static void neg(const T * x, T * dst, const int k,
222
- const sycl::nd_item<3> &item_ct1) {
223
- const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
224
- item_ct1.get_local_id(2);
259
+ static void unary_op_exp_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {
260
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
261
+ dst[i] = op_exp(x[i]);
262
+ }
263
+ }
225
264
 
226
- if (i >= k) {
227
- return;
265
+ template<typename T>
266
+ static void unary_op_log_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {
267
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
268
+ dst[i] = op_log(x[i]);
228
269
  }
229
- dst[i] = -x[i];
230
270
  }
231
271
 
232
272
  template<typename T>
233
- static void step(const T * x, T * dst, const int k,
234
- const sycl::nd_item<3> &item_ct1) {
235
- const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
236
- item_ct1.get_local_id(2);
273
+ static void unary_op_neg_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {
274
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
275
+ dst[i] = op_neg(x[i]);
276
+ }
277
+ }
237
278
 
238
- if (i >= k) {
239
- return;
279
+ template<typename T>
280
+ static void unary_op_step_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {
281
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
282
+ dst[i] = op_step(x[i]);
240
283
  }
241
- dst[i] = x[i] > static_cast<T>(0.0f);
242
284
  }
243
285
 
244
286
  template<typename T>
245
- static void leaky_relu(const T *x, T *dst, const int k, const float negative_slope,
246
- const sycl::nd_item<3> &item_ct1) {
247
- const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
248
- item_ct1.get_local_id(2);
249
- if (i >= k) {
250
- return;
287
+ static void unary_op_leaky_relu_kernel(const T * x, T * dst, const int k, float negative_slope, const sycl::nd_item<1> &item_ct1) {
288
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
289
+ dst[i] = op_leaky_relu(x[i], negative_slope);
251
290
  }
252
- dst[i] = sycl::fmax((x[i]), static_cast<T>(0)) +
253
- sycl::fmin((x[i]), static_cast<T>(0.0f)) * negative_slope;
254
291
  }
255
292
 
256
293
  template<typename T>
257
- static void sqr(const T * x, T * dst, const int k,
258
- const sycl::nd_item<3> &item_ct1) {
259
- const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
260
- item_ct1.get_local_id(2);
294
+ static void unary_op_sqr_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) {
295
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
296
+ dst[i] = op_sqr(x[i]);
297
+ }
298
+ }
261
299
 
262
- if (i >= k) {
263
- return;
300
+ template<typename T>
301
+ static void unary_op_clamp_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1, float min_val, float max_val) {
302
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
303
+ dst[i] = op_clamp(x[i], min_val, max_val);
264
304
  }
265
- dst[i] = x[i] * x[i];
266
305
  }
267
306
 
268
307
  template<typename T>
@@ -281,10 +320,10 @@ static void upscale(const T *x, T *dst, const int nb00, const int nb01,
281
320
  int i12 = (index / (ne10 * ne11)) % ne12;
282
321
  int i13 = (index / (ne10 * ne11 * ne12)) % ne13;
283
322
 
284
- int i00 = i10 / sf0;
285
- int i01 = i11 / sf1;
286
- int i02 = i12 / sf2;
287
- int i03 = i13 / sf3;
323
+ int i00 = static_cast<int>(i10 / sf0);
324
+ int i01 = static_cast<int>(i11 / sf1);
325
+ int i02 = static_cast<int>(i12 / sf2);
326
+ int i03 = static_cast<int>(i13 / sf3);
288
327
 
289
328
  dst[index] = *(const T *)((const char *)x + i03 * nb03 + i02 * nb02 + i01 * nb01 + i00 * nb00);
290
329
  }
@@ -292,8 +331,7 @@ static void upscale(const T *x, T *dst, const int nb00, const int nb01,
292
331
  template <typename T>
293
332
  static void pad(const T *x, T *dst, const int ne0, const int ne00, const int ne01, const int ne02,
294
333
  const sycl::nd_item<3> &item_ct1) {
295
- int nidx = item_ct1.get_local_id(2) +
296
- item_ct1.get_group(2) * item_ct1.get_local_range(2);
334
+ int nidx = SYCL_LOCAL_ID_CALC(item_ct1, 2);
297
335
  if (nidx >= ne0) {
298
336
  return;
299
337
  }
@@ -310,246 +348,73 @@ static void pad(const T *x, T *dst, const int ne0, const int ne00, const int ne
310
348
  }
311
349
  }
312
350
 
313
-
314
351
  template<typename T>
315
352
  static void clamp(const T * x, T * dst, const float min, const float max, const int k,
316
- const sycl::nd_item<3> &item_ct1) {
317
- const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
318
- item_ct1.get_local_id(2);
319
-
320
- if (i >= k) {
321
- return;
353
+ const sycl::nd_item<1> &item_ct1) {
354
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
355
+ dst[i] = x[i] < static_cast<T>(min) ? static_cast<T>(min) : (x[i] > static_cast<T>(max) ? static_cast<T>(max) : x[i]);
322
356
  }
323
-
324
- dst[i] = x[i] < static_cast<T>(min) ? static_cast<T>(min) : (x[i] > static_cast<T>(max) ? static_cast<T>(max) : x[i]);
325
- }
326
-
327
- static void acc_f32_sycl(const float *x, const float *y, float *dst,
328
- const int n_elements, const int ne10, const int ne11,
329
- const int ne12, const int nb1, const int nb2,
330
- const int offset, queue_ptr stream) {
331
- int num_blocks = (n_elements + SYCL_ACC_BLOCK_SIZE - 1) / SYCL_ACC_BLOCK_SIZE;
332
- sycl_parallel_for(stream,
333
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_ACC_BLOCK_SIZE),
334
- sycl::range<3>(1, 1, SYCL_ACC_BLOCK_SIZE)),
335
- [=](sycl::nd_item<3> item_ct1) {
336
- acc_f32(x, y, dst, n_elements, ne10, ne11, ne12, nb1, nb2, offset, item_ct1);
337
- });
338
- }
339
-
340
- template<typename T>
341
- static void gelu_sycl(const T *x, T *dst, const int k,
342
- queue_ptr stream) {
343
- const int num_blocks = (k + SYCL_GELU_BLOCK_SIZE - 1) / SYCL_GELU_BLOCK_SIZE;
344
- sycl_parallel_for(stream,
345
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_GELU_BLOCK_SIZE),
346
- sycl::range<3>(1, 1, SYCL_GELU_BLOCK_SIZE)),
347
- [=](sycl::nd_item<3> item_ct1) { gelu(x, dst, k, item_ct1); });
348
- }
349
-
350
- template<typename T>
351
- static void silu_sycl(const T *x, T *dst, const int k,
352
- queue_ptr stream) {
353
- const int num_blocks = (k + SYCL_SILU_BLOCK_SIZE - 1) / SYCL_SILU_BLOCK_SIZE;
354
- sycl_parallel_for(stream,
355
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_SILU_BLOCK_SIZE),
356
- sycl::range<3>(1, 1, SYCL_SILU_BLOCK_SIZE)),
357
- [=](sycl::nd_item<3> item_ct1) { silu(x, dst, k, item_ct1); });
358
- }
359
-
360
- template<typename T>
361
- static void sgn_sycl(const T * x, T * dst, const int k, queue_ptr stream) {
362
- // hard code for now
363
- const int num_blocks = ceil_div(k, 256);
364
- sycl_parallel_for(
365
- stream, sycl::nd_range<3>((sycl::range<3>(1, 1, num_blocks) * sycl::range(1, 1, 256)), sycl::range(1, 1, 256)),
366
- [=](sycl::nd_item<3> item_ct1) { sgn(x, dst, k, item_ct1); });
367
- }
368
-
369
- template<typename T>
370
- static void abs_sycl(const T * x, T * dst, const int k, queue_ptr stream) {
371
- // hard code for now
372
- const int num_blocks = ceil_div(k, 256);
373
- sycl_parallel_for(
374
- stream,
375
- sycl::nd_range<3>((sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, 256)), sycl::range<3>(1, 1, 256)),
376
- [=](sycl::nd_item<3> item_ct1) { abs_op(x, dst, k, item_ct1); });
377
- }
378
-
379
-
380
- template<typename T>
381
- static void elu_sycl(const T * x, T * dst, const int k, queue_ptr stream) {
382
- // hard code for now
383
- const int num_blocks = ceil_div(k, 256);
384
- sycl_parallel_for(
385
- stream,
386
- sycl::nd_range<3>((sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, 256)), sycl::range<3>(1, 1, 256)),
387
- [=](sycl::nd_item<3> item_ct1) { elu_op(x, dst, k, item_ct1); });
388
- }
389
-
390
- template<typename T>
391
- static void gelu_quick_sycl(const T *x, T *dst, const int k,
392
- queue_ptr stream) {
393
- const int num_blocks = (k + SYCL_GELU_BLOCK_SIZE - 1) / SYCL_GELU_BLOCK_SIZE;
394
- sycl_parallel_for(stream,
395
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_GELU_BLOCK_SIZE),
396
- sycl::range<3>(1, 1, SYCL_GELU_BLOCK_SIZE)),
397
- [=](sycl::nd_item<3> item_ct1) { gelu_quick(x, dst, k, item_ct1); });
398
- }
399
-
400
-
401
- template<typename T>
402
- static void gelu_erf_sycl(const T *x, T *dst, const int k,
403
- queue_ptr stream) {
404
- const int num_blocks = ceil_div(k, SYCL_GELU_BLOCK_SIZE);
405
- sycl_parallel_for(stream,
406
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_GELU_BLOCK_SIZE),
407
- sycl::range<3>(1, 1, SYCL_GELU_BLOCK_SIZE)),
408
- [=](sycl::nd_item<3> item_ct1) { gelu_erf(x, dst, k, item_ct1); });
409
- }
410
-
411
- template<typename T>
412
- static void tanh_sycl(const T *x, T *dst, const int k,
413
- queue_ptr stream) {
414
- const int num_blocks = (k + SYCL_TANH_BLOCK_SIZE - 1) / SYCL_TANH_BLOCK_SIZE;
415
- sycl_parallel_for(stream,
416
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_TANH_BLOCK_SIZE),
417
- sycl::range<3>(1, 1, SYCL_TANH_BLOCK_SIZE)),
418
- [=](sycl::nd_item<3> item_ct1) { tanh(x, dst, k, item_ct1); });
419
- }
420
-
421
- template<typename T>
422
- static void relu_sycl(const T *x, T *dst, const int k,
423
- queue_ptr stream) {
424
- const int num_blocks = (k + SYCL_RELU_BLOCK_SIZE - 1) / SYCL_RELU_BLOCK_SIZE;
425
- sycl_parallel_for(stream,
426
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_RELU_BLOCK_SIZE),
427
- sycl::range<3>(1, 1, SYCL_RELU_BLOCK_SIZE)),
428
- [=](sycl::nd_item<3> item_ct1) { relu(x, dst, k, item_ct1); });
429
- }
430
-
431
- template<typename T>
432
- static void hardsigmoid_sycl(const T *x, T *dst, const int k,
433
- queue_ptr stream) {
434
- const int num_blocks = (k + SYCL_HARDSIGMOID_BLOCK_SIZE - 1) / SYCL_HARDSIGMOID_BLOCK_SIZE;
435
- sycl_parallel_for(
436
- stream,
437
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_HARDSIGMOID_BLOCK_SIZE),
438
- sycl::range<3>(1, 1, SYCL_HARDSIGMOID_BLOCK_SIZE)),
439
- [=](sycl::nd_item<3> item_ct1) { hardsigmoid(x, dst, k, item_ct1); });
440
- }
441
-
442
- template<typename T>
443
- static void hardswish_sycl(const T *x, T *dst, const int k,
444
- queue_ptr stream) {
445
- const int num_blocks = (k + SYCL_HARDSWISH_BLOCK_SIZE - 1) / SYCL_HARDSWISH_BLOCK_SIZE;
446
- sycl_parallel_for(
447
- stream,
448
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_HARDSWISH_BLOCK_SIZE),
449
- sycl::range<3>(1, 1, SYCL_HARDSWISH_BLOCK_SIZE)),
450
- [=](sycl::nd_item<3> item_ct1) { hardswish(x, dst, k, item_ct1); });
451
- }
452
-
453
- template<typename T>
454
- static void exp_sycl(const T *x, T *dst, const int k,
455
- queue_ptr stream) {
456
- const int num_blocks = (k + SYCL_EXP_BLOCK_SIZE - 1) / SYCL_EXP_BLOCK_SIZE;
457
- sycl_parallel_for(stream,
458
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_EXP_BLOCK_SIZE),
459
- sycl::range<3>(1, 1, SYCL_EXP_BLOCK_SIZE)),
460
- [=](sycl::nd_item<3> item_ct1) { exp(x, dst, k, item_ct1); });
461
- }
462
-
463
- template<typename T>
464
- static void log_sycl(const T *x, T *dst, const int k,
465
- queue_ptr stream) {
466
- const int num_blocks = (k + SYCL_EXP_BLOCK_SIZE - 1) / SYCL_EXP_BLOCK_SIZE;
467
- sycl_parallel_for(stream,
468
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_EXP_BLOCK_SIZE),
469
- sycl::range<3>(1, 1, SYCL_EXP_BLOCK_SIZE)),
470
- [=](sycl::nd_item<3> item_ct1) { log(x, dst, k, item_ct1); });
471
- }
472
-
473
- template<typename T>
474
- static void neg_sycl(const T *x, T *dst, const int k,
475
- queue_ptr stream) {
476
- const int num_blocks = (k + SYCL_NEG_BLOCK_SIZE - 1) / SYCL_NEG_BLOCK_SIZE;
477
- sycl_parallel_for(stream,
478
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_NEG_BLOCK_SIZE),
479
- sycl::range<3>(1, 1, SYCL_NEG_BLOCK_SIZE)),
480
- [=](sycl::nd_item<3> item_ct1) { neg(x, dst, k, item_ct1); });
481
- }
482
-
483
- template<typename T>
484
- static void step_sycl(const T *x, T *dst, const int k,
485
- queue_ptr stream) {
486
- const int num_blocks = (k + SYCL_NEG_BLOCK_SIZE - 1) / SYCL_NEG_BLOCK_SIZE;
487
- sycl_parallel_for(stream,
488
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_NEG_BLOCK_SIZE),
489
- sycl::range<3>(1, 1, SYCL_NEG_BLOCK_SIZE)),
490
- [=](sycl::nd_item<3> item_ct1) { step(x, dst, k, item_ct1); });
491
357
  }
492
358
 
493
359
  template<typename T>
494
- static void sigmoid_sycl(const T *x, T *dst, const int k,
495
- queue_ptr stream) {
496
- const int num_blocks = (k + SYCL_SIGMOID_BLOCK_SIZE - 1) / SYCL_SIGMOID_BLOCK_SIZE;
497
- sycl_parallel_for(
498
- stream,
499
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_SIGMOID_BLOCK_SIZE),
500
- sycl::range<3>(1, 1, SYCL_SIGMOID_BLOCK_SIZE)),
501
- [=](sycl::nd_item<3> item_ct1) { sigmoid(x, dst, k, item_ct1); });
360
+ static void gated_op_fused_geglu(const T * x, const T * g, T * dst, const uint64_t k, const uint64_t n, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {
361
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
362
+ const int64_t j0 = (i / n) * o0 + (i % n);
363
+ const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n);
364
+ dst[i] = op_gelu(x[j0]) * g[j1];
365
+ }
502
366
  }
503
367
 
504
368
  template<typename T>
505
- static void sqrt_sycl(const T *x, T *dst, const int k,
506
- queue_ptr stream) {
507
- const int num_blocks = (k + SYCL_SQRT_BLOCK_SIZE - 1) / SYCL_SQRT_BLOCK_SIZE;
508
- sycl_parallel_for(stream,
509
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_SQRT_BLOCK_SIZE),
510
- sycl::range<3>(1, 1, SYCL_SQRT_BLOCK_SIZE)),
511
- [=](sycl::nd_item<3> item_ct1) { sqrt(x, dst, k, item_ct1); });
369
+ static void gated_op_fused_reglu(const T * x, const T * g, T * dst, const uint64_t k, const uint64_t n, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {
370
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
371
+ const int64_t j0 = (i / n) * o0 + (i % n);
372
+ const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n);
373
+ dst[i] = op_relu(x[j0]) * g[j1];
374
+ }
512
375
  }
513
376
 
514
377
  template<typename T>
515
- static void sin_sycl(const T *x, T *dst, const int k,
516
- queue_ptr stream) {
517
- const int num_blocks = (k + SYCL_SIN_BLOCK_SIZE - 1) / SYCL_SIN_BLOCK_SIZE;
518
- sycl_parallel_for(stream,
519
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_SIN_BLOCK_SIZE),
520
- sycl::range<3>(1, 1, SYCL_SIN_BLOCK_SIZE)),
521
- [=](sycl::nd_item<3> item_ct1) { sin(x, dst, k, item_ct1); });
378
+ static void gated_op_fused_swiglu(const T * x, const T * g, T * dst, const uint64_t k, const uint64_t n, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {
379
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
380
+ const int64_t j0 = (i / n) * o0 + (i % n);
381
+ const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n);
382
+ dst[i] = op_silu(x[j0]) * g[j1];
383
+ }
522
384
  }
523
385
 
524
386
  template<typename T>
525
- static void cos_sycl(const T *x, T *dst, const int k,
526
- queue_ptr stream) {
527
- const int num_blocks = (k + SYCL_SIN_BLOCK_SIZE - 1) / SYCL_SIN_BLOCK_SIZE;
528
- sycl_parallel_for(stream,
529
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_SIN_BLOCK_SIZE),
530
- sycl::range<3>(1, 1, SYCL_SIN_BLOCK_SIZE)),
531
- [=](sycl::nd_item<3> item_ct1) { cos(x, dst, k, item_ct1); });
387
+ static void gated_op_fused_geglu_erf(const T * x, const T * g, T * dst, const uint64_t k, const uint64_t n, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {
388
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
389
+ const int64_t j0 = (i / n) * o0 + (i % n);
390
+ const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n);
391
+ dst[i] = op_gelu_erf(x[j0]) * g[j1];
392
+ }
532
393
  }
533
394
 
534
395
  template<typename T>
535
- static void leaky_relu_sycl(const T *x, T *dst, const int k,
536
- const float negative_slope,
537
- queue_ptr stream) {
538
- const int num_blocks = (k + SYCL_RELU_BLOCK_SIZE - 1) / SYCL_RELU_BLOCK_SIZE;
539
- sycl_parallel_for(stream,
540
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_RELU_BLOCK_SIZE),
541
- sycl::range<3>(1, 1, SYCL_RELU_BLOCK_SIZE)),
542
- [=](sycl::nd_item<3> item_ct1) { leaky_relu(x, dst, k, negative_slope, item_ct1); });
396
+ static void gated_op_fused_geglu_quick(const T * x, const T * g, T * dst, const uint64_t k, const uint64_t n, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) {
397
+ SYCL_GLOBAL_ID_LOOP(k, item_ct1) {
398
+ const int64_t j0 = (i / n) * o0 + (i % n);
399
+ const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n);
400
+ dst[i] = op_gelu_quick(x[j0]) * g[j1];
401
+ }
543
402
  }
544
403
 
545
- template<typename T>
546
- static void sqr_sycl(const T *x, T *dst, const int k,
547
- queue_ptr stream) {
548
- const int num_blocks = (k + SYCL_SQR_BLOCK_SIZE - 1) / SYCL_SQR_BLOCK_SIZE;
404
+ namespace ggml_sycl_detail {
405
+ static void acc_f32_sycl(const float *x, const float *y, float *dst,
406
+ const int n_elements, const int ne10, const int ne11,
407
+ const int ne12, const int nb1, const int nb2,
408
+ const int offset, queue_ptr stream) {
409
+ int num_blocks = ceil_div(n_elements, SYCL_ACC_BLOCK_SIZE);
549
410
  sycl_parallel_for(stream,
550
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_SQR_BLOCK_SIZE),
551
- sycl::range<3>(1, 1, SYCL_SQR_BLOCK_SIZE)),
552
- [=](sycl::nd_item<3> item_ct1) { sqr(x, dst, k, item_ct1); });
411
+ sycl::nd_range<1>(sycl::range<1>(num_blocks) *
412
+ sycl::range<1>(SYCL_ACC_BLOCK_SIZE),
413
+ sycl::range<1>(SYCL_ACC_BLOCK_SIZE)),
414
+ [=](sycl::nd_item<1> item_ct1) {
415
+ acc_f32(x, y, dst, n_elements, ne10, ne11, ne12, nb1, nb2, offset,
416
+ item_ct1);
417
+ });
553
418
  }
554
419
 
555
420
  template<typename T>
@@ -558,7 +423,7 @@ static void upscale_sycl(const T *x, T *dst, const int nb00, const int nb01,
558
423
  const int ne12, const int ne13, const float sf0, const float sf1,
559
424
  const float sf2, const float sf3, queue_ptr stream) {
560
425
  int dst_size = ne10 * ne11 * ne12 * ne13;
561
- int num_blocks = (dst_size + SYCL_UPSCALE_BLOCK_SIZE - 1) / SYCL_UPSCALE_BLOCK_SIZE;
426
+ int num_blocks = ceil_div(dst_size, SYCL_UPSCALE_BLOCK_SIZE);
562
427
  sycl::range<1> gridDim(num_blocks * SYCL_UPSCALE_BLOCK_SIZE);
563
428
  sycl_parallel_for<1>(
564
429
  stream, sycl::nd_range<1>(gridDim, sycl::range<1>(SYCL_UPSCALE_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) {
@@ -570,7 +435,7 @@ template<typename T>
570
435
  static void pad_sycl(const T *x, T *dst, const int ne00,
571
436
  const int ne01, const int ne02, const int ne0,
572
437
  const int ne1, const int ne2, queue_ptr stream) {
573
- int num_blocks = (ne0 + SYCL_PAD_BLOCK_SIZE - 1) / SYCL_PAD_BLOCK_SIZE;
438
+ int num_blocks = ceil_div(ne0, SYCL_PAD_BLOCK_SIZE);
574
439
  sycl::range<3> gridDim(ne2, ne1, num_blocks);
575
440
  sycl_parallel_for(stream,
576
441
  sycl::nd_range<3>(gridDim * sycl::range<3>(1, 1, SYCL_PAD_BLOCK_SIZE),
@@ -578,22 +443,11 @@ static void pad_sycl(const T *x, T *dst, const int ne00,
578
443
  [=](sycl::nd_item<3> item_ct1) { pad(x, dst, ne0, ne00, ne01, ne02, item_ct1); });
579
444
  }
580
445
 
581
- template<typename T>
582
- static void clamp_sycl(const T *x, T *dst, const float min,
583
- const float max, const int k,
584
- queue_ptr stream) {
585
- const int num_blocks = (k + SYCL_CLAMP_BLOCK_SIZE - 1) / SYCL_CLAMP_BLOCK_SIZE;
586
- sycl_parallel_for(stream,
587
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CLAMP_BLOCK_SIZE),
588
- sycl::range<3>(1, 1, SYCL_CLAMP_BLOCK_SIZE)),
589
- [=](sycl::nd_item<3> item_ct1) { clamp(x, dst, min, max, k, item_ct1); });
590
- }
591
-
592
- inline void ggml_sycl_op_sgn(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
446
+ template<typename KernelInvoker, typename... Args>
447
+ static inline void dispatch_ggml_sycl_op_unary(ggml_backend_sycl_context & ctx, ggml_tensor * dst, KernelInvoker kernel_invoker, Args&&... args) {
593
448
  #if defined (GGML_SYCL_F16)
594
449
  GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);
595
450
  GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
596
-
597
451
  #else
598
452
  GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);
599
453
  GGML_ASSERT(dst->type == GGML_TYPE_F32);
@@ -606,14 +460,14 @@ inline void ggml_sycl_op_sgn(ggml_backend_sycl_context & ctx, ggml_tensor * dst)
606
460
  case GGML_TYPE_F16:
607
461
  {
608
462
  auto data_pts = cast_data<sycl::half>(dst);
609
- sgn_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
463
+ kernel_invoker(data_pts.src, data_pts.dst, (int)ggml_nelements(dst->src[0]), main_stream, std::forward<Args>(args)...);
610
464
  break;
611
465
  }
612
466
  #endif
613
467
  case GGML_TYPE_F32:
614
468
  {
615
469
  auto data_pts = cast_data<float>(dst);
616
- sgn_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
470
+ kernel_invoker(data_pts.src, data_pts.dst, (int)ggml_nelements(dst->src[0]), main_stream, std::forward<Args>(args)...);
617
471
  break;
618
472
  }
619
473
  default:
@@ -621,11 +475,11 @@ inline void ggml_sycl_op_sgn(ggml_backend_sycl_context & ctx, ggml_tensor * dst)
621
475
  }
622
476
  }
623
477
 
624
- inline void ggml_sycl_op_abs(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
478
+ template<typename KernelInvoker, typename... Args>
479
+ static inline void dispatch_ggml_sycl_op_fused_glu(ggml_backend_sycl_context & ctx, ggml_tensor * dst, KernelInvoker kernel_invoker, Args&&... args) {
625
480
  #if defined (GGML_SYCL_F16)
626
481
  GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);
627
482
  GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
628
-
629
483
  #else
630
484
  GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);
631
485
  GGML_ASSERT(dst->type == GGML_TYPE_F32);
@@ -633,19 +487,66 @@ inline void ggml_sycl_op_abs(ggml_backend_sycl_context & ctx, ggml_tensor * dst)
633
487
  GGML_ASSERT(dst->src[0]->type == dst->type);
634
488
  dpct::queue_ptr main_stream = ctx.stream();
635
489
  SYCL_CHECK(ggml_sycl_set_device(ctx.device));
490
+ const ggml_tensor * src0 = dst->src[0];
491
+ const ggml_tensor * src1 = dst->src[1];
492
+ const int64_t nc = src1 ? src0->ne[0] : src0->ne[0] / 2;;
493
+ GGML_ASSERT(dst->ne[0] == nc);
494
+ GGML_ASSERT(ggml_is_contiguous_1(dst->src[0]));
495
+ GGML_ASSERT(ggml_is_contiguous(dst));
496
+ const int32_t swapped = ((const int32_t *) dst->op_params)[1];
497
+ void * src0_d = src0->data;
498
+ void * src1_d = src1 ? src1->data : src0->data;
499
+ const int64_t src0_o = src0->nb[1];
500
+ const int64_t src1_o = src1 ? src1->nb[1] : src0->nb[1];
501
+ void * dst_d = dst->data;
502
+ if (src1) {
503
+ GGML_ASSERT(ggml_is_contiguous_1(src1));
504
+ GGML_ASSERT(src1->nb[0] == ggml_element_size(src1));
505
+ GGML_ASSERT(src1->ne[0] == nc);
506
+ GGML_ASSERT(src0->type == src1->type);
507
+ }
636
508
  switch (dst->type) {
637
509
  #if defined (GGML_SYCL_F16)
638
510
  case GGML_TYPE_F16:
639
511
  {
640
- auto data_pts = cast_data<sycl::half>(dst);
641
- abs_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
512
+ sycl::half * src0_p = (sycl::half *) src0_d;
513
+ sycl::half * src1_p = (sycl::half *) src1_d;
514
+
515
+ if (!src1) {
516
+ src0_p += swapped ? nc : 0;
517
+ src1_p += swapped ? 0 : nc;
518
+ }
519
+ kernel_invoker(src0_p,
520
+ src1_p,
521
+ (sycl::half *) dst_d,
522
+ ggml_nelements(dst),
523
+ nc,
524
+ src0_o / sizeof(sycl::half),
525
+ src1_o / sizeof(sycl::half),
526
+ main_stream,
527
+ std::forward<Args>(args)...);
642
528
  break;
643
529
  }
644
530
  #endif
645
531
  case GGML_TYPE_F32:
646
532
  {
647
- auto data_pts = cast_data<float>(dst);
648
- abs_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
533
+ float * src0_p = (float *) src0_d;
534
+ float * src1_p = (float *) src1_d;
535
+
536
+ if (!src1) {
537
+ src0_p += swapped ? nc : 0;
538
+ src1_p += swapped ? 0 : nc;
539
+ }
540
+
541
+ kernel_invoker(src0_p,
542
+ src1_p,
543
+ (float *) dst_d,
544
+ ggml_nelements(dst),
545
+ nc,
546
+ src0_o / sizeof(float),
547
+ src1_o / sizeof(float),
548
+ main_stream,
549
+ std::forward<Args>(args)...);
649
550
  break;
650
551
  }
651
552
  default:
@@ -653,32 +554,41 @@ inline void ggml_sycl_op_abs(ggml_backend_sycl_context & ctx, ggml_tensor * dst)
653
554
  }
654
555
  }
655
556
 
656
-
657
- inline void ggml_sycl_op_elu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
557
+ template<typename KernelInvoker, typename... Args>
558
+ static inline void dispatch_ggml_sycl_op_upscale(ggml_backend_sycl_context & ctx, ggml_tensor * dst, KernelInvoker kernel_invoker, Args&&... args) {
658
559
  #if defined (GGML_SYCL_F16)
659
560
  GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);
660
561
  GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
661
-
662
562
  #else
663
563
  GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);
664
564
  GGML_ASSERT(dst->type == GGML_TYPE_F32);
665
565
  #endif
666
566
  GGML_ASSERT(dst->src[0]->type == dst->type);
567
+
667
568
  dpct::queue_ptr main_stream = ctx.stream();
668
569
  SYCL_CHECK(ggml_sycl_set_device(ctx.device));
570
+
571
+ const float sf0 = (float) dst->ne[0] / dst->src[0]->ne[0];
572
+ const float sf1 = (float) dst->ne[1] / dst->src[0]->ne[1];
573
+ const float sf2 = (float) dst->ne[2] / dst->src[0]->ne[2];
574
+ const float sf3 = (float) dst->ne[3] / dst->src[0]->ne[3];
669
575
  switch (dst->type) {
670
576
  #if defined (GGML_SYCL_F16)
671
577
  case GGML_TYPE_F16:
672
578
  {
673
579
  auto data_pts = cast_data<sycl::half>(dst);
674
- elu_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
580
+ kernel_invoker(data_pts.src, data_pts.dst, (int)dst->src[0]->nb[0], (int)dst->src[0]->nb[1], (int)dst->src[0]->nb[2],
581
+ (int)dst->src[0]->nb[3], (int)dst->ne[0], (int)dst->ne[1], (int)dst->ne[2], (int)dst->ne[3], sf0, sf1, sf2, sf3,
582
+ main_stream, std::forward<Args>(args)...);
675
583
  break;
676
584
  }
677
585
  #endif
678
586
  case GGML_TYPE_F32:
679
587
  {
680
588
  auto data_pts = cast_data<float>(dst);
681
- elu_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
589
+ kernel_invoker(data_pts.src, data_pts.dst, (int)dst->src[0]->nb[0], (int)dst->src[0]->nb[1], (int)dst->src[0]->nb[2],
590
+ (int)dst->src[0]->nb[3], (int)dst->ne[0], (int)dst->ne[1], (int)dst->ne[2], (int)dst->ne[3], sf0, sf1, sf2, sf3,
591
+ main_stream, std::forward<Args>(args)...);
682
592
  break;
683
593
  }
684
594
  default:
@@ -686,7 +596,8 @@ inline void ggml_sycl_op_elu(ggml_backend_sycl_context & ctx, ggml_tensor * dst)
686
596
  }
687
597
  }
688
598
 
689
- inline void ggml_sycl_op_silu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
599
+ template<typename KernelInvoker, typename... Args>
600
+ static inline void dispatch_ggml_sycl_op_pad(ggml_backend_sycl_context & ctx, ggml_tensor * dst, KernelInvoker kernel_invoker, Args&&... args) {
690
601
  #if defined (GGML_SYCL_F16)
691
602
  GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);
692
603
  GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
@@ -695,6 +606,7 @@ inline void ggml_sycl_op_silu(ggml_backend_sycl_context & ctx, ggml_tensor * dst
695
606
  GGML_ASSERT(dst->type == GGML_TYPE_F32);
696
607
  #endif
697
608
  GGML_ASSERT(dst->src[0]->type == dst->type);
609
+ GGML_ASSERT(dst->src[0]->ne[3] == 1 && dst->ne[3] == 1); // just 3D tensors
698
610
  dpct::queue_ptr main_stream = ctx.stream();
699
611
  SYCL_CHECK(ggml_sycl_set_device(ctx.device));
700
612
  switch (dst->type) {
@@ -702,14 +614,16 @@ inline void ggml_sycl_op_silu(ggml_backend_sycl_context & ctx, ggml_tensor * dst
702
614
  case GGML_TYPE_F16:
703
615
  {
704
616
  auto data_pts = cast_data<sycl::half>(dst);
705
- silu_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
617
+ kernel_invoker(data_pts.src, data_pts.dst, (int)dst->src[0]->ne[0], (int)dst->src[0]->ne[1], (int)dst->src[0]->ne[2], (int)dst->ne[0],
618
+ (int)dst->ne[1], (int)dst->ne[2], main_stream, std::forward<Args>(args)...);
706
619
  break;
707
620
  }
708
621
  #endif
709
622
  case GGML_TYPE_F32:
710
623
  {
711
624
  auto data_pts = cast_data<float>(dst);
712
- silu_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
625
+ kernel_invoker(data_pts.src, data_pts.dst, (int)dst->src[0]->ne[0], (int)dst->src[0]->ne[1], (int)dst->src[0]->ne[2], (int)dst->ne[0],
626
+ (int)dst->ne[1], (int)dst->ne[2], main_stream, std::forward<Args>(args)...);
713
627
  break;
714
628
  }
715
629
  default:
@@ -717,655 +631,320 @@ inline void ggml_sycl_op_silu(ggml_backend_sycl_context & ctx, ggml_tensor * dst
717
631
  }
718
632
  }
719
633
 
720
- inline void ggml_sycl_op_gelu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
721
- #if defined (GGML_SYCL_F16)
722
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);
723
- GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
724
- #else
725
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);
726
- GGML_ASSERT(dst->type == GGML_TYPE_F32);
727
- #endif
728
- GGML_ASSERT(dst->src[0]->type == dst->type);
729
- dpct::queue_ptr main_stream = ctx.stream();
730
- SYCL_CHECK(ggml_sycl_set_device(ctx.device));
731
- switch (dst->type) {
732
- #if defined (GGML_SYCL_F16)
733
- case GGML_TYPE_F16:
734
- {
735
- auto data_pts = cast_data<sycl::half>(dst);
736
- gelu_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
737
- break;
738
- }
739
- #endif
740
- case GGML_TYPE_F32:
741
- {
742
- auto data_pts = cast_data<float>(dst);
743
- gelu_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
744
- break;
745
- }
746
- default:
747
- GGML_ABORT("GGML tensor type not supported!\n");
748
- }
634
+ } // namespace ggml_sycl_detail
635
+
636
+
637
+
638
+ static inline void ggml_sycl_op_sgn(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
639
+ ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,
640
+ [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {
641
+ const int num_blocks = ceil_div(k_elements, 256);
642
+ sycl_parallel_for(stream,
643
+ sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(256),
644
+ sycl::range<1>(256)),
645
+ [=](sycl::nd_item<1> item_ct1) {
646
+ unary_op_sgn_kernel(src, dst_ptr, k_elements, item_ct1);
647
+ });
648
+ });
749
649
  }
750
650
 
751
- inline void ggml_sycl_op_gelu_quick(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {
752
- #if defined (GGML_SYCL_F16)
753
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);
754
- GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
755
- #else
756
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);
757
- GGML_ASSERT(dst->type == GGML_TYPE_F32);
758
- #endif
759
- GGML_ASSERT(dst->src[0]->type == dst->type);
760
- dpct::queue_ptr main_stream = ctx.stream();
761
- SYCL_CHECK(ggml_sycl_set_device(ctx.device));
762
- switch (dst->type) {
763
- #if defined (GGML_SYCL_F16)
764
- case GGML_TYPE_F16:
765
- {
766
- auto data_pts = cast_data<sycl::half>(dst);
767
- gelu_quick_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
768
- break;
769
- }
770
- #endif
771
- case GGML_TYPE_F32:
772
- {
773
- auto data_pts = cast_data<float>(dst);
774
- gelu_quick_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
775
- break;
776
- }
777
- default:
778
- GGML_ABORT("GGML tensor type not supported!\n");
779
- }
651
+ static inline void ggml_sycl_op_abs(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
652
+ ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,
653
+ [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {
654
+ const int num_blocks = ceil_div(k_elements, 256);
655
+ sycl_parallel_for(stream,
656
+ sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(256),
657
+ sycl::range<1>(256)),
658
+ [=](sycl::nd_item<1> item_ct1) {
659
+ unary_op_abs_kernel(src, dst_ptr, k_elements, item_ct1);
660
+ });
661
+ });
780
662
  }
781
663
 
782
- inline void ggml_sycl_op_gelu_erf(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {
783
- #if defined (GGML_SYCL_F16)
784
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);
785
- GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
786
- #else
787
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);
788
- GGML_ASSERT(dst->type == GGML_TYPE_F32);
789
- #endif
790
- GGML_ASSERT(dst->src[0]->type == dst->type);
791
- dpct::queue_ptr main_stream = ctx.stream();
792
- SYCL_CHECK(ggml_sycl_set_device(ctx.device));
793
- switch (dst->type) {
794
- #if defined (GGML_SYCL_F16)
795
- case GGML_TYPE_F16:
796
- {
797
- auto data_pts = cast_data<sycl::half>(dst);
798
- gelu_erf_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
799
- break;
800
- }
801
- #endif
802
- case GGML_TYPE_F32:
803
- {
804
- auto data_pts = cast_data<float>(dst);
805
- gelu_erf_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
806
- break;
807
- }
808
- default:
809
- GGML_ABORT("GGML tensor type not supported!\n");
810
- }
811
- }
812
-
813
-
814
- inline void ggml_sycl_op_tanh(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
815
- #if defined (GGML_SYCL_F16)
816
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);
817
- GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
818
- #else
819
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);
820
- GGML_ASSERT(dst->type == GGML_TYPE_F32);
821
- #endif
822
- GGML_ASSERT(dst->src[0]->type == dst->type);
823
- dpct::queue_ptr main_stream = ctx.stream();
824
- SYCL_CHECK(ggml_sycl_set_device(ctx.device));
825
- switch (dst->type) {
826
- #if defined (GGML_SYCL_F16)
827
- case GGML_TYPE_F16:
828
- {
829
- auto data_pts = cast_data<sycl::half>(dst);
830
- tanh_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
831
- break;
832
- }
833
- #endif
834
- case GGML_TYPE_F32:
835
- {
836
- auto data_pts = cast_data<float>(dst);
837
- tanh_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
838
- break;
839
- }
840
- default:
841
- GGML_ABORT("GGML tensor type not supported!\n");
842
- }
843
- }
844
-
845
- inline void ggml_sycl_op_relu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
846
- #if defined (GGML_SYCL_F16)
847
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);
848
- GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
849
- #else
850
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);
851
- GGML_ASSERT(dst->type == GGML_TYPE_F32);
852
- #endif
853
- GGML_ASSERT(dst->src[0]->type == dst->type);
854
- dpct::queue_ptr main_stream = ctx.stream();
855
- SYCL_CHECK(ggml_sycl_set_device(ctx.device));
856
-
857
- switch (dst->type) {
858
- #if defined (GGML_SYCL_F16)
859
- case GGML_TYPE_F16:
860
- {
861
- auto data_pts = cast_data<sycl::half>(dst);
862
- relu_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
863
- break;
864
- }
865
- #endif
866
- case GGML_TYPE_F32:
867
- {
868
- auto data_pts = cast_data<float>(dst);
869
- relu_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
870
- break;
871
- }
872
- default:
873
- GGML_ABORT("GGML tensor type not supported!\n");
874
- }
875
- }
876
-
877
- inline void ggml_sycl_op_hardsigmoid(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
878
- #if defined (GGML_SYCL_F16)
879
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);
880
- GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
881
- #else
882
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);
883
- GGML_ASSERT(dst->type == GGML_TYPE_F32);
884
- #endif
885
- GGML_ASSERT(dst->src[0]->type == dst->type);
886
-
887
- dpct::queue_ptr main_stream = ctx.stream();
888
- SYCL_CHECK(ggml_sycl_set_device(ctx.device));
889
-
890
- switch (dst->type) {
891
- #if defined (GGML_SYCL_F16)
892
- case GGML_TYPE_F16:
893
- {
894
- auto data_pts = cast_data<sycl::half>(dst);
895
- hardsigmoid_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
896
- break;
897
- }
898
- #endif
899
- case GGML_TYPE_F32:
900
- {
901
- auto data_pts = cast_data<float>(dst);
902
- hardsigmoid_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
903
- break;
904
- }
905
- default:
906
- GGML_ABORT("GGML tensor type not supported!\n");
907
- }
908
- }
909
-
910
- inline void ggml_sycl_op_hardswish(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
911
- #if defined (GGML_SYCL_F16)
912
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);
913
- GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
914
- #else
915
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);
916
- GGML_ASSERT(dst->type == GGML_TYPE_F32);
917
- #endif
918
- GGML_ASSERT(dst->src[0]->type == dst->type);
919
- dpct::queue_ptr main_stream = ctx.stream();
920
- SYCL_CHECK(ggml_sycl_set_device(ctx.device));
921
- switch (dst->type) {
922
- #if defined (GGML_SYCL_F16)
923
- case GGML_TYPE_F16:
924
- {
925
- auto data_pts = cast_data<sycl::half>(dst);
926
- hardswish_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
927
- break;
928
- }
929
- #endif
930
- case GGML_TYPE_F32:
931
- {
932
- auto data_pts = cast_data<float>(dst);
933
- hardswish_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
934
- break;
935
- }
936
- default:
937
- GGML_ABORT("GGML tensor type not supported!\n");
938
- }
939
- }
940
-
941
- inline void ggml_sycl_op_exp(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
942
- #if defined (GGML_SYCL_F16)
943
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);
944
- GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
945
- #else
946
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);
947
- GGML_ASSERT(dst->type == GGML_TYPE_F32);
948
- #endif
949
- GGML_ASSERT(dst->src[0]->type == dst->type);
950
- dpct::queue_ptr main_stream = ctx.stream();
951
- SYCL_CHECK(ggml_sycl_set_device(ctx.device));
952
- switch (dst->type) {
953
- #if defined (GGML_SYCL_F16)
954
- case GGML_TYPE_F16:
955
- {
956
- auto data_pts = cast_data<sycl::half>(dst);
957
- exp_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
958
- break;
959
- }
960
- #endif
961
- case GGML_TYPE_F32:
962
- {
963
- auto data_pts = cast_data<float>(dst);
964
- exp_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
965
- break;
966
- }
967
- default:
968
- GGML_ABORT("GGML tensor type not supported!\n");
969
- }
970
- }
971
-
972
- inline void ggml_sycl_op_log(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
973
- #if defined (GGML_SYCL_F16)
974
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);
975
- GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
976
- #else
977
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);
978
- GGML_ASSERT(dst->type == GGML_TYPE_F32);
979
- #endif
980
- GGML_ASSERT(dst->src[0]->type == dst->type);
981
- dpct::queue_ptr main_stream = ctx.stream();
982
- SYCL_CHECK(ggml_sycl_set_device(ctx.device));
983
- switch (dst->type) {
984
- #if defined (GGML_SYCL_F16)
985
- case GGML_TYPE_F16:
986
- {
987
- auto data_pts = cast_data<sycl::half>(dst);
988
- log_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
989
- break;
990
- }
991
- #endif
992
- case GGML_TYPE_F32:
993
- {
994
- auto data_pts = cast_data<float>(dst);
995
- log_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
996
- break;
997
- }
998
- default:
999
- GGML_ABORT("GGML tensor type not supported!\n");
1000
- }
1001
- }
1002
-
1003
- inline void ggml_sycl_op_sigmoid(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
1004
- #if defined (GGML_SYCL_F16)
1005
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);
1006
- GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
1007
- #else
1008
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);
1009
- GGML_ASSERT(dst->type == GGML_TYPE_F32);
1010
- #endif
1011
- GGML_ASSERT(dst->src[0]->type == dst->type);
1012
- dpct::queue_ptr main_stream = ctx.stream();
1013
- SYCL_CHECK(ggml_sycl_set_device(ctx.device));
1014
- switch (dst->type) {
1015
- #if defined (GGML_SYCL_F16)
1016
- case GGML_TYPE_F16:
1017
- {
1018
- auto data_pts = cast_data<sycl::half>(dst);
1019
- sigmoid_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
1020
- break;
1021
- }
1022
- #endif
1023
- case GGML_TYPE_F32:
1024
- {
1025
- auto data_pts = cast_data<float>(dst);
1026
- sigmoid_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
1027
- break;
1028
- }
1029
- default:
1030
- GGML_ABORT("GGML tensor type not supported!\n");
1031
- }
1032
- }
1033
-
1034
- inline void ggml_sycl_op_sqrt(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
1035
- #if defined (GGML_SYCL_F16)
1036
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);
1037
- GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
1038
- #else
1039
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);
1040
- GGML_ASSERT(dst->type == GGML_TYPE_F32);
1041
- #endif
1042
- GGML_ASSERT(dst->src[0]->type == dst->type);
1043
-
1044
- dpct::queue_ptr main_stream = ctx.stream();
1045
- SYCL_CHECK(ggml_sycl_set_device(ctx.device));
1046
- switch (dst->type) {
1047
- #if defined (GGML_SYCL_F16)
1048
- case GGML_TYPE_F16:
1049
- {
1050
- auto data_pts = cast_data<sycl::half>(dst);
1051
- sqrt_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
1052
- break;
1053
- }
1054
- #endif
1055
- case GGML_TYPE_F32:
1056
- {
1057
- auto data_pts = cast_data<float>(dst);
1058
- sqrt_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
1059
- break;
1060
- }
1061
- default:
1062
- GGML_ABORT("GGML tensor type not supported!\n");
1063
- }
1064
- }
1065
-
1066
- inline void ggml_sycl_op_sin(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
1067
- #if defined (GGML_SYCL_F16)
1068
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);
1069
- GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
1070
- #else
1071
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);
1072
- GGML_ASSERT(dst->type == GGML_TYPE_F32);
1073
- #endif
1074
- GGML_ASSERT(dst->src[0]->type == dst->type);
1075
- dpct::queue_ptr main_stream = ctx.stream();
1076
- SYCL_CHECK(ggml_sycl_set_device(ctx.device));
1077
- switch (dst->type) {
1078
- #if defined (GGML_SYCL_F16)
1079
- case GGML_TYPE_F16:
1080
- {
1081
- auto data_pts = cast_data<sycl::half>(dst);
1082
- sin_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
1083
- break;
1084
- }
1085
- #endif
1086
- case GGML_TYPE_F32:
1087
- {
1088
- auto data_pts = cast_data<float>(dst);
1089
- sin_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
1090
- break;
1091
- }
1092
- default:
1093
- GGML_ABORT("GGML tensor type not supported!\n");
1094
- }
1095
- }
1096
-
1097
- inline void ggml_sycl_op_cos(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
1098
- #if defined (GGML_SYCL_F16)
1099
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);
1100
- GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
1101
- #else
1102
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);
1103
- GGML_ASSERT(dst->type == GGML_TYPE_F32);
1104
- #endif
1105
- GGML_ASSERT(dst->src[0]->type == dst->type);
1106
- dpct::queue_ptr main_stream = ctx.stream();
1107
- SYCL_CHECK(ggml_sycl_set_device(ctx.device));
1108
- switch (dst->type) {
1109
- #if defined (GGML_SYCL_F16)
1110
- case GGML_TYPE_F16:
1111
- {
1112
- auto data_pts = cast_data<sycl::half>(dst);
1113
- cos_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
1114
- break;
1115
- }
1116
- #endif
1117
- case GGML_TYPE_F32:
1118
- {
1119
- auto data_pts = cast_data<float>(dst);
1120
- cos_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
1121
- break;
1122
- }
1123
- default:
1124
- GGML_ABORT("GGML tensor type not supported!\n");
1125
- }
664
+ static inline void ggml_sycl_op_elu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
665
+ ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,
666
+ [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {
667
+ const int num_blocks = ceil_div(k_elements, 256);
668
+ sycl_parallel_for(stream,
669
+ sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(256),
670
+ sycl::range<1>(256)),
671
+ [=](sycl::nd_item<1> item_ct1) {
672
+ unary_op_elu_kernel(src, dst_ptr, k_elements, item_ct1);
673
+ });
674
+ });
1126
675
  }
1127
676
 
1128
- inline void ggml_sycl_op_step(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
1129
- #if defined (GGML_SYCL_F16)
1130
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);
1131
- GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
1132
- #else
1133
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);
1134
- GGML_ASSERT(dst->type == GGML_TYPE_F32);
1135
- #endif
1136
- GGML_ASSERT(dst->src[0]->type == dst->type);
1137
- dpct::queue_ptr main_stream = ctx.stream();
1138
- SYCL_CHECK(ggml_sycl_set_device(ctx.device));
1139
- switch (dst->type) {
1140
- #if defined (GGML_SYCL_F16)
1141
- case GGML_TYPE_F16:
1142
- {
1143
- auto data_pts = cast_data<sycl::half>(dst);
1144
- step_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
1145
- break;
1146
- }
1147
- #endif
1148
- case GGML_TYPE_F32:
1149
- {
1150
- auto data_pts = cast_data<float>(dst);
1151
- step_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
1152
- break;
1153
- }
1154
- default:
1155
- GGML_ABORT("GGML tensor type not supported!\n");
1156
- }
677
+ static inline void ggml_sycl_op_silu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
678
+ ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,
679
+ [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {
680
+ const int num_blocks = ceil_div(k_elements, SYCL_SILU_BLOCK_SIZE);
681
+ sycl_parallel_for(stream,
682
+ sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_SILU_BLOCK_SIZE),
683
+ sycl::range<1>(SYCL_SILU_BLOCK_SIZE)),
684
+ [=](sycl::nd_item<1> item_ct1) {
685
+ unary_op_silu_kernel(src, dst_ptr, k_elements, item_ct1);
686
+ });
687
+ });
1157
688
  }
1158
689
 
1159
- inline void ggml_sycl_op_neg(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
1160
- #if defined (GGML_SYCL_F16)
1161
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);
1162
- GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
1163
- #else
1164
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);
1165
- GGML_ASSERT(dst->type == GGML_TYPE_F32);
1166
- #endif
1167
- GGML_ASSERT(dst->src[0]->type == dst->type);
1168
- dpct::queue_ptr main_stream = ctx.stream();
1169
- SYCL_CHECK(ggml_sycl_set_device(ctx.device));
1170
- switch (dst->type) {
1171
- #if defined (GGML_SYCL_F16)
1172
- case GGML_TYPE_F16:
1173
- {
1174
- auto data_pts = cast_data<sycl::half>(dst);
1175
- neg_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
1176
- break;
1177
- }
1178
- #endif
1179
- case GGML_TYPE_F32:
1180
- {
1181
- auto data_pts = cast_data<float>(dst);
1182
- neg_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
1183
- break;
1184
- }
1185
- default:
1186
- GGML_ABORT("GGML tensor type not supported!\n");
1187
- }
690
+ static inline void ggml_sycl_op_gelu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
691
+ ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,
692
+ [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {
693
+ const int num_blocks = ceil_div(k_elements, SYCL_GELU_BLOCK_SIZE);
694
+ sycl_parallel_for(stream,
695
+ sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_GELU_BLOCK_SIZE),
696
+ sycl::range<1>(SYCL_GELU_BLOCK_SIZE)),
697
+ [=](sycl::nd_item<1> item_ct1) {
698
+ unary_op_gelu_kernel(src, dst_ptr, k_elements, item_ct1);
699
+ });
700
+ });
1188
701
  }
1189
702
 
1190
- inline void ggml_sycl_op_leaky_relu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
1191
- #if defined (GGML_SYCL_F16)
1192
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);
1193
- GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
1194
- #else
1195
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);
1196
- GGML_ASSERT(dst->type == GGML_TYPE_F32);
1197
- #endif
703
+ static inline void ggml_sycl_op_gelu_quick(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {
704
+ ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,
705
+ [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {
706
+ const int num_blocks = ceil_div(k_elements, SYCL_GELU_BLOCK_SIZE);
707
+ sycl_parallel_for(stream,
708
+ sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_GELU_BLOCK_SIZE),
709
+ sycl::range<1>(SYCL_GELU_BLOCK_SIZE)),
710
+ [=](sycl::nd_item<1> item_ct1) {
711
+ unary_op_gelu_quick_kernel(src, dst_ptr, k_elements, item_ct1);
712
+ });
713
+ });
714
+ }
1198
715
 
1199
- GGML_ASSERT(dst->src[0]->type == dst->type);
1200
- float negative_slope;
1201
- memcpy(&negative_slope, dst->op_params, sizeof(float));
1202
- dpct::queue_ptr main_stream = ctx.stream();
1203
- SYCL_CHECK(ggml_sycl_set_device(ctx.device));
1204
- switch (dst->type) {
1205
- #if defined (GGML_SYCL_F16)
1206
- case GGML_TYPE_F16:
1207
- {
1208
- auto data_pts = cast_data<sycl::half>(dst);
1209
- leaky_relu_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), negative_slope, main_stream);
1210
- break;
1211
- }
1212
- #endif
1213
- case GGML_TYPE_F32:
1214
- {
1215
- auto data_pts = cast_data<float>(dst);
1216
- leaky_relu_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), negative_slope, main_stream);
1217
- break;
1218
- }
1219
- default:
1220
- GGML_ABORT("GGML tensor type not supported!\n");
1221
- }
716
+ static inline void ggml_sycl_op_gelu_erf(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {
717
+ ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,
718
+ [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {
719
+ const int num_blocks = ceil_div(k_elements, SYCL_GELU_BLOCK_SIZE);
720
+ sycl_parallel_for(stream,
721
+ sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_GELU_BLOCK_SIZE),
722
+ sycl::range<1>(SYCL_GELU_BLOCK_SIZE)),
723
+ [=](sycl::nd_item<1> item_ct1) {
724
+ unary_op_gelu_erf_kernel(src, dst_ptr, k_elements, item_ct1);
725
+ });
726
+ });
1222
727
  }
1223
728
 
1224
- inline void ggml_sycl_op_sqr(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
1225
- #if defined (GGML_SYCL_F16)
1226
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);
1227
- GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
1228
- #else
1229
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);
1230
- GGML_ASSERT(dst->type == GGML_TYPE_F32);
1231
- #endif
1232
- GGML_ASSERT(dst->src[0]->type == dst->type);
1233
- dpct::queue_ptr main_stream = ctx.stream();
1234
- SYCL_CHECK(ggml_sycl_set_device(ctx.device));
1235
- switch (dst->type) {
1236
- #if defined (GGML_SYCL_F16)
1237
- case GGML_TYPE_F16:
1238
- {
1239
- auto data_pts = cast_data<sycl::half>(dst);
1240
- sqr_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
1241
- break;
1242
- }
1243
- #endif
1244
- case GGML_TYPE_F32:
1245
- {
1246
- auto data_pts = cast_data<float>(dst);
1247
- sqr_sycl(data_pts.src, data_pts.dst, ggml_nelements(dst->src[0]), main_stream);
1248
- break;
1249
- }
1250
- default:
1251
- GGML_ABORT("GGML tensor type not supported!\n");
1252
- }
729
+ static inline void ggml_sycl_op_tanh(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
730
+ ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,
731
+ [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {
732
+ const int num_blocks = ceil_div(k_elements, SYCL_TANH_BLOCK_SIZE);
733
+ sycl_parallel_for(stream,
734
+ sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_TANH_BLOCK_SIZE),
735
+ sycl::range<1>(SYCL_TANH_BLOCK_SIZE)),
736
+ [=](sycl::nd_item<1> item_ct1) {
737
+ unary_op_tanh_kernel(src, dst_ptr, k_elements, item_ct1);
738
+ });
739
+ });
1253
740
  }
1254
741
 
1255
- inline void ggml_sycl_op_upscale(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
1256
- #if defined (GGML_SYCL_F16)
1257
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);
1258
- GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
1259
- #else
1260
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);
1261
- GGML_ASSERT(dst->type == GGML_TYPE_F32);
1262
- #endif
1263
- GGML_ASSERT(dst->src[0]->type == dst->type);
742
+ static inline void ggml_sycl_op_relu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
743
+ ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,
744
+ [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {
745
+ const int num_blocks = ceil_div(k_elements, SYCL_RELU_BLOCK_SIZE);
746
+ sycl_parallel_for(stream,
747
+ sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_RELU_BLOCK_SIZE),
748
+ sycl::range<1>(SYCL_RELU_BLOCK_SIZE)),
749
+ [=](sycl::nd_item<1> item_ct1) {
750
+ unary_op_relu_kernel(src, dst_ptr, k_elements, item_ct1);
751
+ });
752
+ });
753
+ }
1264
754
 
1265
- dpct::queue_ptr main_stream = ctx.stream();
1266
- SYCL_CHECK(ggml_sycl_set_device(ctx.device));
755
+ static inline void ggml_sycl_op_hardsigmoid(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
756
+ ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,
757
+ [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {
758
+ const int num_blocks = ceil_div(k_elements, SYCL_HARDSIGMOID_BLOCK_SIZE);
759
+ sycl_parallel_for(stream,
760
+ sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_HARDSIGMOID_BLOCK_SIZE),
761
+ sycl::range<1>(SYCL_HARDSIGMOID_BLOCK_SIZE)),
762
+ [=](sycl::nd_item<1> item_ct1) {
763
+ unary_op_hardsigmoid_kernel(src, dst_ptr, k_elements, item_ct1);
764
+ });
765
+ });
766
+ }
1267
767
 
1268
- const float sf0 = (float) dst->ne[0] / dst->src[0]->ne[0];
1269
- const float sf1 = (float) dst->ne[1] / dst->src[0]->ne[1];
1270
- const float sf2 = (float) dst->ne[2] / dst->src[0]->ne[2];
1271
- const float sf3 = (float) dst->ne[3] / dst->src[0]->ne[3];
1272
- switch (dst->type) {
1273
- #if defined (GGML_SYCL_F16)
1274
- case GGML_TYPE_F16:
1275
- {
1276
- auto data_pts = cast_data<sycl::half>(dst);
1277
- upscale_sycl(data_pts.src, data_pts.dst, dst->src[0]->nb[0], dst->src[0]->nb[1], dst->src[0]->nb[2],
1278
- dst->src[0]->nb[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], sf0, sf1, sf2, sf3,
1279
- main_stream);
1280
- break;
1281
- }
1282
- #endif
1283
- case GGML_TYPE_F32:
1284
- {
1285
- auto data_pts = cast_data<float>(dst);
1286
- upscale_sycl(data_pts.src, data_pts.dst, dst->src[0]->nb[0], dst->src[0]->nb[1], dst->src[0]->nb[2],
1287
- dst->src[0]->nb[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], sf0, sf1, sf2, sf3,
1288
- main_stream);
1289
- break;
1290
- }
1291
- default:
1292
- GGML_ABORT("GGML tensor type not supported!\n");
1293
- }
768
+ static inline void ggml_sycl_op_hardswish(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
769
+ ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,
770
+ [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {
771
+ const int num_blocks = ceil_div(k_elements, SYCL_HARDSWISH_BLOCK_SIZE);
772
+ sycl_parallel_for(stream,
773
+ sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_HARDSWISH_BLOCK_SIZE),
774
+ sycl::range<1>(SYCL_HARDSWISH_BLOCK_SIZE)),
775
+ [=](sycl::nd_item<1> item_ct1) {
776
+ unary_op_hardswish_kernel(src, dst_ptr, k_elements, item_ct1);
777
+ });
778
+ });
1294
779
  }
1295
780
 
1296
- inline void ggml_sycl_op_pad(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
1297
- #if defined (GGML_SYCL_F16)
1298
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);
1299
- GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
1300
- #else
1301
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);
1302
- GGML_ASSERT(dst->type == GGML_TYPE_F32);
1303
- #endif
1304
- GGML_ASSERT(dst->src[0]->type == dst->type);
1305
- GGML_ASSERT(dst->src[0]->ne[3] == 1 && dst->ne[3] == 1); // just 3D tensors
1306
- dpct::queue_ptr main_stream = ctx.stream();
1307
- SYCL_CHECK(ggml_sycl_set_device(ctx.device));
1308
- switch (dst->type) {
1309
- #if defined (GGML_SYCL_F16)
1310
- case GGML_TYPE_F16:
1311
- {
1312
- auto data_pts = cast_data<sycl::half>(dst);
1313
- pad_sycl(data_pts.src, data_pts.dst, dst->src[0]->ne[0], dst->src[0]->ne[1], dst->src[0]->ne[2], dst->ne[0],
1314
- dst->ne[1], dst->ne[2], main_stream);
1315
- break;
1316
- }
1317
- #endif
1318
- case GGML_TYPE_F32:
1319
- {
1320
- auto data_pts = cast_data<float>(dst);
1321
- pad_sycl(data_pts.src, data_pts.dst, dst->src[0]->ne[0], dst->src[0]->ne[1], dst->src[0]->ne[2], dst->ne[0],
1322
- dst->ne[1], dst->ne[2], main_stream);
1323
- break;
1324
- }
1325
- default:
1326
- GGML_ABORT("GGML tensor type not supported!\n");
1327
- }
781
+ static inline void ggml_sycl_op_exp(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
782
+ ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,
783
+ [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {
784
+ const int num_blocks = ceil_div(k_elements, SYCL_EXP_BLOCK_SIZE);
785
+ sycl_parallel_for(stream,
786
+ sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_EXP_BLOCK_SIZE),
787
+ sycl::range<1>(SYCL_EXP_BLOCK_SIZE)),
788
+ [=](sycl::nd_item<1> item_ct1) {
789
+ unary_op_exp_kernel(src, dst_ptr, k_elements, item_ct1);
790
+ });
791
+ });
1328
792
  }
1329
793
 
1330
- inline void ggml_sycl_op_clamp(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
1331
- #if defined(GGML_SYCL_F16)
1332
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);
1333
- GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
1334
- #else
794
+ static inline void ggml_sycl_op_log(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
795
+ ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,
796
+ [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {
797
+ const int num_blocks = ceil_div(k_elements, SYCL_EXP_BLOCK_SIZE); // Using EXP block size
798
+ sycl_parallel_for(stream,
799
+ sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_EXP_BLOCK_SIZE),
800
+ sycl::range<1>(SYCL_EXP_BLOCK_SIZE)),
801
+ [=](sycl::nd_item<1> item_ct1) {
802
+ unary_op_log_kernel(src, dst_ptr, k_elements, item_ct1);
803
+ });
804
+ });
805
+ }
1335
806
 
1336
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);
1337
- GGML_ASSERT(dst->type == GGML_TYPE_F32);
1338
- #endif
1339
- GGML_ASSERT(dst->src[0]->type == dst->type);
1340
- dpct::queue_ptr main_stream = ctx.stream();
1341
- SYCL_CHECK(ggml_sycl_set_device(ctx.device));
1342
- float min;
1343
- float max;
1344
- memcpy(&min, dst->op_params, sizeof(float));
1345
- memcpy(&max, (float *) dst->op_params + 1, sizeof(float));
807
+ static inline void ggml_sycl_op_neg(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
808
+ ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,
809
+ [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {
810
+ const int num_blocks = ceil_div(k_elements, SYCL_NEG_BLOCK_SIZE);
811
+ sycl_parallel_for(stream,
812
+ sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_NEG_BLOCK_SIZE),
813
+ sycl::range<1>(SYCL_NEG_BLOCK_SIZE)),
814
+ [=](sycl::nd_item<1> item_ct1) {
815
+ unary_op_neg_kernel(src, dst_ptr, k_elements, item_ct1);
816
+ });
817
+ });
818
+ }
1346
819
 
1347
- switch (dst->type) {
1348
- #if defined(GGML_SYCL_F16)
1349
- case GGML_TYPE_F16:
1350
- {
1351
- auto data_pts = cast_data<sycl::half>(dst);
1352
- clamp_sycl(data_pts.src, data_pts.dst, min, max, ggml_nelements(dst->src[0]), main_stream);
1353
- break;
1354
- }
1355
- #endif
1356
- case GGML_TYPE_F32:
1357
- {
1358
- auto data_pts = cast_data<float>(dst);
1359
- clamp_sycl(data_pts.src, data_pts.dst, min, max, ggml_nelements(dst->src[0]), main_stream);
1360
- break;
1361
- }
1362
- default:
1363
- GGML_ABORT("GGML tensor type not supported!\n");
1364
- }
820
+ static inline void ggml_sycl_op_step(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
821
+ ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,
822
+ [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {
823
+ const int num_blocks = ceil_div(k_elements, SYCL_NEG_BLOCK_SIZE); // Using NEG block size
824
+ sycl_parallel_for(stream,
825
+ sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_NEG_BLOCK_SIZE),
826
+ sycl::range<1>(SYCL_NEG_BLOCK_SIZE)),
827
+ [=](sycl::nd_item<1> item_ct1) {
828
+ unary_op_step_kernel(src, dst_ptr, k_elements, item_ct1);
829
+ });
830
+ });
831
+ }
832
+
833
+ static inline void ggml_sycl_op_sigmoid(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
834
+ ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,
835
+ [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {
836
+ const int num_blocks = ceil_div(k_elements, SYCL_SIGMOID_BLOCK_SIZE);
837
+ sycl_parallel_for(stream,
838
+ sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_SIGMOID_BLOCK_SIZE),
839
+ sycl::range<1>(SYCL_SIGMOID_BLOCK_SIZE)),
840
+ [=](sycl::nd_item<1> item_ct1) {
841
+ unary_op_sigmoid_kernel(src, dst_ptr, k_elements, item_ct1);
842
+ });
843
+ });
844
+ }
845
+
846
+ static inline void ggml_sycl_op_sqrt(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
847
+ ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,
848
+ [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {
849
+ const int num_blocks = ceil_div(k_elements, SYCL_SQRT_BLOCK_SIZE);
850
+ sycl_parallel_for(stream,
851
+ sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_SQRT_BLOCK_SIZE),
852
+ sycl::range<1>(SYCL_SQRT_BLOCK_SIZE)),
853
+ [=](sycl::nd_item<1> item_ct1) {
854
+ unary_op_sqrt_kernel(src, dst_ptr, k_elements, item_ct1);
855
+ });
856
+ });
857
+ }
858
+
859
+ static inline void ggml_sycl_op_sin(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
860
+ ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,
861
+ [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {
862
+ const int num_blocks = ceil_div(k_elements, SYCL_SIN_BLOCK_SIZE);
863
+ sycl_parallel_for(stream,
864
+ sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_SIN_BLOCK_SIZE),
865
+ sycl::range<1>(SYCL_SIN_BLOCK_SIZE)),
866
+ [=](sycl::nd_item<1> item_ct1) {
867
+ unary_op_sin_kernel(src, dst_ptr, k_elements, item_ct1);
868
+ });
869
+ });
870
+ }
871
+
872
+ static inline void ggml_sycl_op_cos(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
873
+ ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,
874
+ [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {
875
+ const int num_blocks = ceil_div(k_elements, SYCL_SIN_BLOCK_SIZE); // Using SIN block size
876
+ sycl_parallel_for(stream,
877
+ sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_SIN_BLOCK_SIZE),
878
+ sycl::range<1>(SYCL_SIN_BLOCK_SIZE)),
879
+ [=](sycl::nd_item<1> item_ct1) {
880
+ unary_op_cos_kernel(src, dst_ptr, k_elements, item_ct1);
881
+ });
882
+ });
883
+ }
884
+
885
+ static inline void ggml_sycl_op_leaky_relu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
886
+ float negative_slope;
887
+ memcpy(&negative_slope, dst->op_params, sizeof(float));
888
+ ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,
889
+ [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream, float slope) {
890
+ const int num_blocks = ceil_div(k_elements, SYCL_RELU_BLOCK_SIZE);
891
+ sycl_parallel_for(stream,
892
+ sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_RELU_BLOCK_SIZE),
893
+ sycl::range<1>(SYCL_RELU_BLOCK_SIZE)),
894
+ [=](sycl::nd_item<1> item_ct1) {
895
+ unary_op_leaky_relu_kernel(src, dst_ptr, k_elements, slope, item_ct1);
896
+ });
897
+ }, negative_slope);
898
+ }
899
+
900
+ static inline void ggml_sycl_op_sqr(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
901
+ ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,
902
+ [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) {
903
+ const int num_blocks = ceil_div(k_elements, SYCL_SQR_BLOCK_SIZE);
904
+ sycl_parallel_for(stream,
905
+ sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_SQR_BLOCK_SIZE),
906
+ sycl::range<1>(SYCL_SQR_BLOCK_SIZE)),
907
+ [=](sycl::nd_item<1> item_ct1) {
908
+ unary_op_sqr_kernel(src, dst_ptr, k_elements, item_ct1);
909
+ });
910
+ });
911
+ }
912
+
913
+ static inline void ggml_sycl_op_upscale(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
914
+ ggml_sycl_detail::dispatch_ggml_sycl_op_upscale(ctx, dst,
915
+ [](const auto* src, auto* dst_ptr, int nb00, int nb01, int nb02, int nb03,
916
+ int ne10, int ne11, int ne12, int ne13, float sf0, float sf1, float sf2, float sf3,
917
+ queue_ptr stream) {
918
+ ggml_sycl_detail::upscale_sycl(src, dst_ptr, nb00, nb01, nb02, nb03, ne10, ne11, ne12, ne13, sf0, sf1, sf2, sf3, stream);
919
+ });
1365
920
  }
1366
921
 
1367
- inline void ggml_sycl_op_acc(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {
922
+ static inline void ggml_sycl_op_pad(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
923
+ ggml_sycl_detail::dispatch_ggml_sycl_op_pad(ctx, dst,
924
+ [](const auto* src, auto* dst_ptr, int ne00, int ne01, int ne02, int ne0, int ne1, int ne2,
925
+ queue_ptr stream) {
926
+ ggml_sycl_detail::pad_sycl(src, dst_ptr, ne00, ne01, ne02, ne0, ne1, ne2, stream);
927
+ });
928
+ }
1368
929
 
930
+ static inline void ggml_sycl_op_clamp(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
931
+ float min_val;
932
+ float max_val;
933
+ memcpy(&min_val, dst->op_params, sizeof(float));
934
+ memcpy(&max_val, (float *) dst->op_params + 1, sizeof(float));
935
+ ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst,
936
+ [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream, float min_arg, float max_arg) {
937
+ const int num_blocks = ceil_div(k_elements, SYCL_CLAMP_BLOCK_SIZE);
938
+ sycl_parallel_for(stream,
939
+ sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_CLAMP_BLOCK_SIZE),
940
+ sycl::range<1>(SYCL_CLAMP_BLOCK_SIZE)),
941
+ [=](sycl::nd_item<1> item_ct1) {
942
+ clamp(src, dst_ptr, min_arg, max_arg, k_elements, item_ct1);
943
+ });
944
+ }, min_val, max_val);
945
+ }
946
+
947
+ static inline void ggml_sycl_op_acc(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {
1369
948
  GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);
1370
949
  GGML_ASSERT(dst->src[1]->type == GGML_TYPE_F32);
1371
950
  GGML_ASSERT( dst->type == GGML_TYPE_F32);
@@ -1381,7 +960,62 @@ inline void ggml_sycl_op_acc(ggml_backend_sycl_context & ctx, ggml_tensor *dst)
1381
960
  // int nb3 = dst->op_params[2] / 4; // 4 bytes of float32 - unused
1382
961
  int offset = dst->op_params[3] / 4; // offset in bytes
1383
962
 
1384
- acc_f32_sycl(src0_dd, src1_dd, dst_dd, ggml_nelements(dst), dst->src[1]->ne[0], dst->src[1]->ne[1], dst->src[1]->ne[2], nb1, nb2, offset, main_stream);
963
+ ggml_sycl_detail::acc_f32_sycl(src0_dd, src1_dd, dst_dd, (int)ggml_nelements(dst), (int)dst->src[1]->ne[0], (int)dst->src[1]->ne[1], (int)dst->src[1]->ne[2], nb1, nb2, offset, main_stream);
964
+ }
965
+
966
+ static inline void ggml_sycl_op_geglu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
967
+ ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst,
968
+ [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) {
969
+ const uint32_t num_blocks = ceil_div(k, SYCL_GELU_BLOCK_SIZE);
970
+ sycl_parallel_for(main_stream,
971
+ sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) {
972
+ gated_op_fused_geglu(x_ptr, g_ptr, dst_ptr, k, n, o0, o1, item_ct1);
973
+ });
974
+ });
975
+ }
976
+
977
+ static inline void ggml_sycl_op_reglu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
978
+ ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst,
979
+ [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) {
980
+ const uint32_t num_blocks = ceil_div((uint32_t)k, SYCL_RELU_BLOCK_SIZE); // Using RELU block size for reglu
981
+ sycl_parallel_for(main_stream,
982
+ sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_RELU_BLOCK_SIZE)), sycl::range<1>(SYCL_RELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) {
983
+ gated_op_fused_reglu(x_ptr, g_ptr, dst_ptr, k, n, o0, o1, item_ct1);
984
+ });
985
+ });
986
+ }
987
+
988
+ static inline void ggml_sycl_op_swiglu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
989
+ ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst,
990
+ [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) {
991
+ const uint32_t num_blocks = ceil_div((uint32_t)k, SYCL_SILU_BLOCK_SIZE); // Using SILU block size for swiglu
992
+ sycl_parallel_for(main_stream,
993
+ sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_SILU_BLOCK_SIZE)), sycl::range<1>(SYCL_SILU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) {
994
+ gated_op_fused_swiglu(x_ptr, g_ptr, dst_ptr, k, n, o0, o1, item_ct1);
995
+ });
996
+ });
997
+ }
998
+
999
+ static inline void ggml_sycl_op_geglu_erf(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
1000
+ ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst,
1001
+ [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) {
1002
+ const uint32_t num_blocks = ceil_div(k, SYCL_GELU_BLOCK_SIZE);
1003
+ sycl_parallel_for(main_stream,
1004
+ sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) {
1005
+ gated_op_fused_geglu_erf(x_ptr, g_ptr, dst_ptr, k, n, o0, o1, item_ct1);
1006
+ });
1007
+ });
1008
+ }
1009
+
1010
+ static inline void ggml_sycl_op_geglu_quick(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
1011
+ ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst,
1012
+ [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) {
1013
+ const uint32_t num_blocks = ceil_div(k, SYCL_GELU_BLOCK_SIZE);
1014
+ sycl_parallel_for(main_stream,
1015
+ sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) {
1016
+ gated_op_fused_geglu_quick(x_ptr, g_ptr, dst_ptr, k, n, o0, o1, item_ct1);
1017
+ });
1018
+ });
1385
1019
  }
1386
1020
 
1387
1021
 
@@ -1509,3 +1143,28 @@ void ggml_sycl_elu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
1509
1143
  scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);
1510
1144
  ggml_sycl_op_elu(ctx, dst);
1511
1145
  }
1146
+
1147
+ void ggml_sycl_geglu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
1148
+ scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);
1149
+ ggml_sycl_op_geglu(ctx, dst);
1150
+ }
1151
+
1152
+ void ggml_sycl_reglu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
1153
+ scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);
1154
+ ggml_sycl_op_reglu(ctx, dst);
1155
+ }
1156
+
1157
+ void ggml_sycl_swiglu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
1158
+ scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);
1159
+ ggml_sycl_op_swiglu(ctx, dst);
1160
+ }
1161
+
1162
+ void ggml_sycl_geglu_erf(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
1163
+ scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);
1164
+ ggml_sycl_op_geglu_erf(ctx, dst);
1165
+ }
1166
+
1167
+ void ggml_sycl_geglu_quick(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
1168
+ scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1);
1169
+ ggml_sycl_op_geglu_quick(ctx, dst);
1170
+ }