@novastera-oss/llamarn 0.2.7 → 0.2.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/android/src/main/cpp/include/llama.h +8 -3
  2. package/android/src/main/jniLibs/arm64-v8a/libggml-base.so +0 -0
  3. package/android/src/main/jniLibs/arm64-v8a/libggml-cpu.so +0 -0
  4. package/android/src/main/jniLibs/arm64-v8a/libggml.so +0 -0
  5. package/android/src/main/jniLibs/arm64-v8a/libllama.so +0 -0
  6. package/android/src/main/jniLibs/x86_64/libggml-base.so +0 -0
  7. package/android/src/main/jniLibs/x86_64/libggml-cpu.so +0 -0
  8. package/android/src/main/jniLibs/x86_64/libggml.so +0 -0
  9. package/android/src/main/jniLibs/x86_64/libllama.so +0 -0
  10. package/cpp/LlamaCppModel.cpp +56 -22
  11. package/cpp/build-info.cpp +2 -2
  12. package/cpp/llama.cpp/CMakeLists.txt +1 -1
  13. package/cpp/llama.cpp/common/arg.cpp +7 -0
  14. package/cpp/llama.cpp/common/common.cpp +3 -0
  15. package/cpp/llama.cpp/common/common.h +1 -0
  16. package/cpp/llama.cpp/common/json-schema-to-grammar.cpp +3 -46
  17. package/cpp/llama.cpp/convert_hf_to_gguf.py +118 -20
  18. package/cpp/llama.cpp/ggml/CMakeLists.txt +1 -0
  19. package/cpp/llama.cpp/ggml/include/ggml-cpu.h +2 -0
  20. package/cpp/llama.cpp/ggml/include/ggml.h +33 -0
  21. package/cpp/llama.cpp/ggml/src/CMakeLists.txt +17 -0
  22. package/cpp/llama.cpp/ggml/src/ggml-cann/common.h +1 -1
  23. package/cpp/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt +31 -2
  24. package/cpp/llama.cpp/ggml/src/ggml-cpu/amx/mmq.cpp +10 -9
  25. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/arm/quants.c +109 -108
  26. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/arm/repack.cpp +1027 -1038
  27. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/loongarch/quants.c +53 -52
  28. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
  29. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/powerpc/quants.c +56 -55
  30. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/riscv/quants.c +42 -41
  31. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/riscv/repack.cpp +24 -23
  32. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/s390/quants.c +29 -28
  33. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/wasm/quants.c +30 -29
  34. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/x86/quants.c +83 -82
  35. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/x86/repack.cpp +20 -19
  36. package/cpp/llama.cpp/ggml/src/ggml-cpu/common.h +3 -2
  37. package/cpp/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-impl.h +9 -3
  38. package/cpp/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c +83 -102
  39. package/cpp/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.cpp +4 -0
  40. package/cpp/llama.cpp/ggml/src/ggml-cpu/llamafile/sgemm.cpp +3 -2
  41. package/cpp/llama.cpp/ggml/src/ggml-cpu/ops.cpp +192 -67
  42. package/cpp/llama.cpp/ggml/src/ggml-cpu/ops.h +2 -0
  43. package/cpp/llama.cpp/ggml/src/ggml-cpu/quants.c +25 -24
  44. package/cpp/llama.cpp/ggml/src/ggml-cpu/repack.cpp +56 -40
  45. package/cpp/llama.cpp/ggml/src/ggml-cpu/simd-mappings.h +211 -33
  46. package/cpp/llama.cpp/ggml/src/ggml-cpu/vec.cpp +2 -2
  47. package/cpp/llama.cpp/ggml/src/ggml-cpu/vec.h +45 -45
  48. package/cpp/llama.cpp/ggml/src/ggml-cuda/common.cuh +54 -29
  49. package/cpp/llama.cpp/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
  50. package/cpp/llama.cpp/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
  51. package/cpp/llama.cpp/ggml/src/ggml-cuda/conv2d-transpose.cu +91 -0
  52. package/cpp/llama.cpp/ggml/src/ggml-cuda/conv2d-transpose.cuh +4 -0
  53. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-wmma-f16.cu +4 -0
  54. package/cpp/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +84 -31
  55. package/cpp/llama.cpp/ggml/src/ggml-cuda/mean.cu +19 -0
  56. package/cpp/llama.cpp/ggml/src/ggml-cuda/mean.cuh +3 -0
  57. package/cpp/llama.cpp/ggml/src/ggml-cuda/mmv.cu +257 -87
  58. package/cpp/llama.cpp/ggml/src/ggml-cuda/mmv.cuh +2 -3
  59. package/cpp/llama.cpp/ggml/src/ggml-cuda/sumrows.cu +5 -18
  60. package/cpp/llama.cpp/ggml/src/ggml-cuda/sumrows.cuh +0 -1
  61. package/cpp/llama.cpp/ggml/src/ggml-impl.h +61 -183
  62. package/cpp/llama.cpp/ggml/src/ggml-metal/ggml-metal-impl.h +16 -0
  63. package/cpp/llama.cpp/ggml/src/ggml-metal/ggml-metal.m +227 -41
  64. package/cpp/llama.cpp/ggml/src/ggml-metal/ggml-metal.metal +362 -182
  65. package/cpp/llama.cpp/ggml/src/ggml-musa/mudnn.cuh +2 -2
  66. package/cpp/llama.cpp/ggml/src/ggml-opencl/ggml-opencl.cpp +240 -535
  67. package/cpp/llama.cpp/ggml/src/ggml-sycl/binbcast.cpp +5 -6
  68. package/cpp/llama.cpp/ggml/src/ggml-sycl/common.hpp +1 -24
  69. package/cpp/llama.cpp/ggml/src/ggml-sycl/concat.cpp +28 -41
  70. package/cpp/llama.cpp/ggml/src/ggml-sycl/conv.cpp +4 -10
  71. package/cpp/llama.cpp/ggml/src/ggml-sycl/convert.cpp +99 -166
  72. package/cpp/llama.cpp/ggml/src/ggml-sycl/cpy.cpp +94 -72
  73. package/cpp/llama.cpp/ggml/src/ggml-sycl/dmmv.cpp +49 -67
  74. package/cpp/llama.cpp/ggml/src/ggml-sycl/dpct/helper.hpp +31 -1
  75. package/cpp/llama.cpp/ggml/src/ggml-sycl/element_wise.cpp +99 -159
  76. package/cpp/llama.cpp/ggml/src/ggml-sycl/getrows.cpp +6 -9
  77. package/cpp/llama.cpp/ggml/src/ggml-sycl/ggml-sycl.cpp +45 -54
  78. package/cpp/llama.cpp/ggml/src/ggml-sycl/gla.cpp +2 -2
  79. package/cpp/llama.cpp/ggml/src/ggml-sycl/im2col.cpp +1 -1
  80. package/cpp/llama.cpp/ggml/src/ggml-sycl/mmq.cpp +60 -80
  81. package/cpp/llama.cpp/ggml/src/ggml-sycl/mmvq.cpp +132 -201
  82. package/cpp/llama.cpp/ggml/src/ggml-sycl/norm.cpp +55 -74
  83. package/cpp/llama.cpp/ggml/src/ggml-sycl/rope.cpp +24 -20
  84. package/cpp/llama.cpp/ggml/src/ggml-sycl/softmax.cpp +3 -3
  85. package/cpp/llama.cpp/ggml/src/ggml-sycl/sycl_hw.cpp +3 -1
  86. package/cpp/llama.cpp/ggml/src/ggml-sycl/sycl_hw.hpp +3 -0
  87. package/cpp/llama.cpp/ggml/src/ggml-sycl/tsembd.cpp +3 -8
  88. package/cpp/llama.cpp/ggml/src/ggml-sycl/wkv.cpp +12 -16
  89. package/cpp/llama.cpp/ggml/src/ggml-vulkan/CMakeLists.txt +12 -1
  90. package/cpp/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan.cpp +57 -1
  91. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +4 -0
  92. package/cpp/llama.cpp/ggml/src/ggml.c +69 -13
  93. package/cpp/llama.cpp/ggml/src/gguf.cpp +5 -1
  94. package/cpp/llama.cpp/gguf-py/gguf/constants.py +76 -0
  95. package/cpp/llama.cpp/gguf-py/gguf/gguf_writer.py +21 -0
  96. package/cpp/llama.cpp/gguf-py/gguf/tensor_mapping.py +64 -0
  97. package/cpp/llama.cpp/gguf-py/gguf/vocab.py +97 -4
  98. package/cpp/llama.cpp/gguf-py/pyproject.toml +2 -2
  99. package/cpp/llama.cpp/include/llama.h +8 -3
  100. package/cpp/llama.cpp/models/templates/Mistral-Small-3.2-24B-Instruct-2506.jinja +124 -0
  101. package/cpp/llama.cpp/src/llama-arch.cpp +55 -0
  102. package/cpp/llama.cpp/src/llama-arch.h +18 -0
  103. package/cpp/llama.cpp/src/llama-batch.cpp +570 -359
  104. package/cpp/llama.cpp/src/llama-batch.h +98 -70
  105. package/cpp/llama.cpp/src/llama-chat.cpp +11 -6
  106. package/cpp/llama.cpp/src/llama-context.cpp +101 -107
  107. package/cpp/llama.cpp/src/llama-context.h +13 -13
  108. package/cpp/llama.cpp/src/llama-graph.cpp +199 -252
  109. package/cpp/llama.cpp/src/llama-graph.h +44 -32
  110. package/cpp/llama.cpp/src/llama-hparams.cpp +4 -0
  111. package/cpp/llama.cpp/src/llama-hparams.h +8 -0
  112. package/cpp/llama.cpp/src/llama-kv-cache-unified-iswa.cpp +51 -53
  113. package/cpp/llama.cpp/src/llama-kv-cache-unified-iswa.h +19 -24
  114. package/cpp/llama.cpp/src/llama-kv-cache-unified.cpp +110 -104
  115. package/cpp/llama.cpp/src/llama-kv-cache-unified.h +17 -22
  116. package/cpp/llama.cpp/src/llama-kv-cells.h +35 -11
  117. package/cpp/llama.cpp/src/llama-memory-hybrid.cpp +66 -67
  118. package/cpp/llama.cpp/src/llama-memory-hybrid.h +16 -21
  119. package/cpp/llama.cpp/src/llama-memory-recurrent.cpp +69 -68
  120. package/cpp/llama.cpp/src/llama-memory-recurrent.h +15 -20
  121. package/cpp/llama.cpp/src/llama-memory.h +18 -22
  122. package/cpp/llama.cpp/src/llama-model-saver.cpp +1 -0
  123. package/cpp/llama.cpp/src/llama-model.cpp +1006 -472
  124. package/cpp/llama.cpp/src/llama-model.h +22 -0
  125. package/cpp/llama.cpp/src/llama-quant.cpp +87 -5
  126. package/cpp/llama.cpp/src/llama-vocab.cpp +26 -3
  127. package/cpp/llama.cpp/src/llama-vocab.h +1 -0
  128. package/cpp/rn-utils.h +3 -0
  129. package/ios/include/common.h +1 -0
  130. package/ios/include/llama.h +8 -3
  131. package/ios/libs/llama.xcframework/Info.plist +19 -19
  132. package/ios/libs/llama.xcframework/ios-arm64/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  133. package/ios/libs/llama.xcframework/ios-arm64/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +4890 -4863
  134. package/ios/libs/llama.xcframework/ios-arm64/llama.framework/Headers/ggml-cpu.h +2 -0
  135. package/ios/libs/llama.xcframework/ios-arm64/llama.framework/Headers/ggml.h +33 -0
  136. package/ios/libs/llama.xcframework/ios-arm64/llama.framework/Headers/llama.h +8 -3
  137. package/ios/libs/llama.xcframework/ios-arm64/llama.framework/llama +0 -0
  138. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  139. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +4861 -4834
  140. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/x86_64/llama.yml +3764 -3742
  141. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/ggml-cpu.h +2 -0
  142. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/ggml.h +33 -0
  143. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/llama.h +8 -3
  144. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/llama +0 -0
  145. package/ios/libs/llama.xcframework/macos-arm64_x86_64/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  146. package/ios/libs/llama.xcframework/macos-arm64_x86_64/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +4861 -4834
  147. package/ios/libs/llama.xcframework/macos-arm64_x86_64/dSYMs/llama.dSYM/Contents/Resources/Relocations/x86_64/llama.yml +3766 -3744
  148. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Headers/ggml-cpu.h +2 -0
  149. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Headers/ggml.h +33 -0
  150. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Headers/llama.h +8 -3
  151. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/ggml-cpu.h +2 -0
  152. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/ggml.h +33 -0
  153. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/llama.h +8 -3
  154. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/llama +0 -0
  155. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Headers/ggml-cpu.h +2 -0
  156. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Headers/ggml.h +33 -0
  157. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Headers/llama.h +8 -3
  158. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/llama +0 -0
  159. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/llama +0 -0
  160. package/ios/libs/llama.xcframework/tvos-arm64/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  161. package/ios/libs/llama.xcframework/tvos-arm64/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +4890 -4863
  162. package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Headers/ggml-cpu.h +2 -0
  163. package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Headers/ggml.h +33 -0
  164. package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Headers/llama.h +8 -3
  165. package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/llama +0 -0
  166. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  167. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +4861 -4834
  168. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/x86_64/llama.yml +3764 -3742
  169. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Headers/ggml-cpu.h +2 -0
  170. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Headers/ggml.h +33 -0
  171. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Headers/llama.h +8 -3
  172. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/llama +0 -0
  173. package/ios/libs/llama.xcframework/xros-arm64/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  174. package/ios/libs/llama.xcframework/xros-arm64/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +4926 -4900
  175. package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Headers/ggml-cpu.h +2 -0
  176. package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Headers/ggml.h +33 -0
  177. package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Headers/llama.h +8 -3
  178. package/ios/libs/llama.xcframework/xros-arm64/llama.framework/llama +0 -0
  179. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  180. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +4897 -4871
  181. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/x86_64/llama.yml +3794 -3773
  182. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Headers/ggml-cpu.h +2 -0
  183. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Headers/ggml.h +33 -0
  184. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Headers/llama.h +8 -3
  185. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/llama +0 -0
  186. package/package.json +1 -1
package/cpp/llama.cpp/ggml/src/ggml-sycl/element_wise.cpp CHANGED
@@ -329,60 +329,51 @@ static void acc_f32_sycl(const float *x, const float *y, float *dst,
329
329
  const int ne12, const int nb1, const int nb2,
330
330
  const int offset, queue_ptr stream) {
331
331
  int num_blocks = (n_elements + SYCL_ACC_BLOCK_SIZE - 1) / SYCL_ACC_BLOCK_SIZE;
332
- stream->parallel_for(
333
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) *
334
- sycl::range<3>(1, 1, SYCL_ACC_BLOCK_SIZE),
335
- sycl::range<3>(1, 1, SYCL_ACC_BLOCK_SIZE)),
336
- [=](sycl::nd_item<3> item_ct1) {
337
- acc_f32(x, y, dst, n_elements, ne10, ne11, ne12, nb1, nb2, offset,
338
- item_ct1);
339
- });
332
+ sycl_parallel_for(stream,
333
+ sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_ACC_BLOCK_SIZE),
334
+ sycl::range<3>(1, 1, SYCL_ACC_BLOCK_SIZE)),
335
+ [=](sycl::nd_item<3> item_ct1) {
336
+ acc_f32(x, y, dst, n_elements, ne10, ne11, ne12, nb1, nb2, offset, item_ct1);
337
+ });
340
338
  }
341
339
 
342
340
  template<typename T>
343
341
  static void gelu_sycl(const T *x, T *dst, const int k,
344
342
  queue_ptr stream) {
345
343
  const int num_blocks = (k + SYCL_GELU_BLOCK_SIZE - 1) / SYCL_GELU_BLOCK_SIZE;
346
- stream->parallel_for(
347
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) *
348
- sycl::range<3>(1, 1, SYCL_GELU_BLOCK_SIZE),
349
- sycl::range<3>(1, 1, SYCL_GELU_BLOCK_SIZE)),
350
- [=](sycl::nd_item<3> item_ct1) {
351
- gelu(x, dst, k, item_ct1);
352
- });
344
+ sycl_parallel_for(stream,
345
+ sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_GELU_BLOCK_SIZE),
346
+ sycl::range<3>(1, 1, SYCL_GELU_BLOCK_SIZE)),
347
+ [=](sycl::nd_item<3> item_ct1) { gelu(x, dst, k, item_ct1); });
353
348
  }
354
349
 
355
350
  template<typename T>
356
351
  static void silu_sycl(const T *x, T *dst, const int k,
357
352
  queue_ptr stream) {
358
353
  const int num_blocks = (k + SYCL_SILU_BLOCK_SIZE - 1) / SYCL_SILU_BLOCK_SIZE;
359
- stream->parallel_for(
360
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) *
361
- sycl::range<3>(1, 1, SYCL_SILU_BLOCK_SIZE),
362
- sycl::range<3>(1, 1, SYCL_SILU_BLOCK_SIZE)),
363
- [=](sycl::nd_item<3> item_ct1) {
364
- silu(x, dst, k, item_ct1);
365
- });
354
+ sycl_parallel_for(stream,
355
+ sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_SILU_BLOCK_SIZE),
356
+ sycl::range<3>(1, 1, SYCL_SILU_BLOCK_SIZE)),
357
+ [=](sycl::nd_item<3> item_ct1) { silu(x, dst, k, item_ct1); });
366
358
  }
367
359
 
368
360
  template<typename T>
369
361
  static void sgn_sycl(const T * x, T * dst, const int k, queue_ptr stream) {
370
362
  // hard code for now
371
363
  const int num_blocks = ceil_div(k, 256);
372
- stream->parallel_for(
373
- sycl::nd_range<3>((sycl::range<3>(1, 1, num_blocks) * sycl::range(1, 1, 256)), sycl::range(1, 1, 256)), [=](sycl::nd_item<3> item_ct1) {
374
- sgn(x, dst, k, item_ct1);
375
- });
364
+ sycl_parallel_for(
365
+ stream, sycl::nd_range<3>((sycl::range<3>(1, 1, num_blocks) * sycl::range(1, 1, 256)), sycl::range(1, 1, 256)),
366
+ [=](sycl::nd_item<3> item_ct1) { sgn(x, dst, k, item_ct1); });
376
367
  }
377
368
 
378
369
  template<typename T>
379
370
  static void abs_sycl(const T * x, T * dst, const int k, queue_ptr stream) {
380
371
  // hard code for now
381
372
  const int num_blocks = ceil_div(k, 256);
382
- stream->parallel_for(
383
- sycl::nd_range<3>((sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, 256)), sycl::range<3>(1, 1, 256)), [=](sycl::nd_item<3> item_ct1) {
384
- abs_op(x, dst, k, item_ct1);
385
- });
373
+ sycl_parallel_for(
374
+ stream,
375
+ sycl::nd_range<3>((sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, 256)), sycl::range<3>(1, 1, 256)),
376
+ [=](sycl::nd_item<3> item_ct1) { abs_op(x, dst, k, item_ct1); });
386
377
  }
387
378
 
388
379
 
@@ -390,23 +381,20 @@ template<typename T>
390
381
  static void elu_sycl(const T * x, T * dst, const int k, queue_ptr stream) {
391
382
  // hard code for now
392
383
  const int num_blocks = ceil_div(k, 256);
393
- stream->parallel_for(
394
- sycl::nd_range<3>((sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, 256)), sycl::range<3>(1, 1, 256)), [=](sycl::nd_item<3> item_ct1) {
395
- elu_op(x, dst, k, item_ct1);
396
- });
384
+ sycl_parallel_for(
385
+ stream,
386
+ sycl::nd_range<3>((sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, 256)), sycl::range<3>(1, 1, 256)),
387
+ [=](sycl::nd_item<3> item_ct1) { elu_op(x, dst, k, item_ct1); });
397
388
  }
398
389
 
399
390
  template<typename T>
400
391
  static void gelu_quick_sycl(const T *x, T *dst, const int k,
401
392
  queue_ptr stream) {
402
393
  const int num_blocks = (k + SYCL_GELU_BLOCK_SIZE - 1) / SYCL_GELU_BLOCK_SIZE;
403
- stream->parallel_for(
404
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) *
405
- sycl::range<3>(1, 1, SYCL_GELU_BLOCK_SIZE),
406
- sycl::range<3>(1, 1, SYCL_GELU_BLOCK_SIZE)),
407
- [=](sycl::nd_item<3> item_ct1) {
408
- gelu_quick(x, dst, k, item_ct1);
409
- });
394
+ sycl_parallel_for(stream,
395
+ sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_GELU_BLOCK_SIZE),
396
+ sycl::range<3>(1, 1, SYCL_GELU_BLOCK_SIZE)),
397
+ [=](sycl::nd_item<3> item_ct1) { gelu_quick(x, dst, k, item_ct1); });
410
398
  }
411
399
 
412
400
 
@@ -414,169 +402,133 @@ template<typename T>
414
402
  static void gelu_erf_sycl(const T *x, T *dst, const int k,
415
403
  queue_ptr stream) {
416
404
  const int num_blocks = ceil_div(k, SYCL_GELU_BLOCK_SIZE);
417
- stream->parallel_for(
418
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) *
419
- sycl::range<3>(1, 1, SYCL_GELU_BLOCK_SIZE),
420
- sycl::range<3>(1, 1, SYCL_GELU_BLOCK_SIZE)),
421
- [=](sycl::nd_item<3> item_ct1) {
422
- gelu_erf(x, dst, k, item_ct1);
423
- });
405
+ sycl_parallel_for(stream,
406
+ sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_GELU_BLOCK_SIZE),
407
+ sycl::range<3>(1, 1, SYCL_GELU_BLOCK_SIZE)),
408
+ [=](sycl::nd_item<3> item_ct1) { gelu_erf(x, dst, k, item_ct1); });
424
409
  }
425
410
 
426
411
  template<typename T>
427
412
  static void tanh_sycl(const T *x, T *dst, const int k,
428
413
  queue_ptr stream) {
429
414
  const int num_blocks = (k + SYCL_TANH_BLOCK_SIZE - 1) / SYCL_TANH_BLOCK_SIZE;
430
- stream->parallel_for(
431
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) *
432
- sycl::range<3>(1, 1, SYCL_TANH_BLOCK_SIZE),
433
- sycl::range<3>(1, 1, SYCL_TANH_BLOCK_SIZE)),
434
- [=](sycl::nd_item<3> item_ct1) {
435
- tanh(x, dst, k, item_ct1);
436
- });
415
+ sycl_parallel_for(stream,
416
+ sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_TANH_BLOCK_SIZE),
417
+ sycl::range<3>(1, 1, SYCL_TANH_BLOCK_SIZE)),
418
+ [=](sycl::nd_item<3> item_ct1) { tanh(x, dst, k, item_ct1); });
437
419
  }
438
420
 
439
421
  template<typename T>
440
422
  static void relu_sycl(const T *x, T *dst, const int k,
441
423
  queue_ptr stream) {
442
424
  const int num_blocks = (k + SYCL_RELU_BLOCK_SIZE - 1) / SYCL_RELU_BLOCK_SIZE;
443
- stream->parallel_for(
444
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) *
445
- sycl::range<3>(1, 1, SYCL_RELU_BLOCK_SIZE),
446
- sycl::range<3>(1, 1, SYCL_RELU_BLOCK_SIZE)),
447
- [=](sycl::nd_item<3> item_ct1) {
448
- relu(x, dst, k, item_ct1);
449
- });
425
+ sycl_parallel_for(stream,
426
+ sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_RELU_BLOCK_SIZE),
427
+ sycl::range<3>(1, 1, SYCL_RELU_BLOCK_SIZE)),
428
+ [=](sycl::nd_item<3> item_ct1) { relu(x, dst, k, item_ct1); });
450
429
  }
451
430
 
452
431
  template<typename T>
453
432
  static void hardsigmoid_sycl(const T *x, T *dst, const int k,
454
433
  queue_ptr stream) {
455
434
  const int num_blocks = (k + SYCL_HARDSIGMOID_BLOCK_SIZE - 1) / SYCL_HARDSIGMOID_BLOCK_SIZE;
456
- stream->parallel_for(
457
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) *
458
- sycl::range<3>(1, 1, SYCL_HARDSIGMOID_BLOCK_SIZE),
435
+ sycl_parallel_for(
436
+ stream,
437
+ sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_HARDSIGMOID_BLOCK_SIZE),
459
438
  sycl::range<3>(1, 1, SYCL_HARDSIGMOID_BLOCK_SIZE)),
460
- [=](sycl::nd_item<3> item_ct1) {
461
- hardsigmoid(x, dst, k, item_ct1);
462
- });
439
+ [=](sycl::nd_item<3> item_ct1) { hardsigmoid(x, dst, k, item_ct1); });
463
440
  }
464
441
 
465
442
  template<typename T>
466
443
  static void hardswish_sycl(const T *x, T *dst, const int k,
467
444
  queue_ptr stream) {
468
445
  const int num_blocks = (k + SYCL_HARDSWISH_BLOCK_SIZE - 1) / SYCL_HARDSWISH_BLOCK_SIZE;
469
- stream->parallel_for(
470
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) *
471
- sycl::range<3>(1, 1, SYCL_HARDSWISH_BLOCK_SIZE),
446
+ sycl_parallel_for(
447
+ stream,
448
+ sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_HARDSWISH_BLOCK_SIZE),
472
449
  sycl::range<3>(1, 1, SYCL_HARDSWISH_BLOCK_SIZE)),
473
- [=](sycl::nd_item<3> item_ct1) {
474
- hardswish(x, dst, k, item_ct1);
475
- });
450
+ [=](sycl::nd_item<3> item_ct1) { hardswish(x, dst, k, item_ct1); });
476
451
  }
477
452
 
478
453
  template<typename T>
479
454
  static void exp_sycl(const T *x, T *dst, const int k,
480
455
  queue_ptr stream) {
481
456
  const int num_blocks = (k + SYCL_EXP_BLOCK_SIZE - 1) / SYCL_EXP_BLOCK_SIZE;
482
- stream->parallel_for(
483
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) *
484
- sycl::range<3>(1, 1, SYCL_EXP_BLOCK_SIZE),
485
- sycl::range<3>(1, 1, SYCL_EXP_BLOCK_SIZE)),
486
- [=](sycl::nd_item<3> item_ct1) {
487
- exp(x, dst, k, item_ct1);
488
- });
457
+ sycl_parallel_for(stream,
458
+ sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_EXP_BLOCK_SIZE),
459
+ sycl::range<3>(1, 1, SYCL_EXP_BLOCK_SIZE)),
460
+ [=](sycl::nd_item<3> item_ct1) { exp(x, dst, k, item_ct1); });
489
461
  }
490
462
 
491
463
  template<typename T>
492
464
  static void log_sycl(const T *x, T *dst, const int k,
493
465
  queue_ptr stream) {
494
466
  const int num_blocks = (k + SYCL_EXP_BLOCK_SIZE - 1) / SYCL_EXP_BLOCK_SIZE;
495
- stream->parallel_for(
496
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) *
497
- sycl::range<3>(1, 1, SYCL_EXP_BLOCK_SIZE),
498
- sycl::range<3>(1, 1, SYCL_EXP_BLOCK_SIZE)),
499
- [=](sycl::nd_item<3> item_ct1) {
500
- log(x, dst, k, item_ct1);
501
- });
467
+ sycl_parallel_for(stream,
468
+ sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_EXP_BLOCK_SIZE),
469
+ sycl::range<3>(1, 1, SYCL_EXP_BLOCK_SIZE)),
470
+ [=](sycl::nd_item<3> item_ct1) { log(x, dst, k, item_ct1); });
502
471
  }
503
472
 
504
473
  template<typename T>
505
474
  static void neg_sycl(const T *x, T *dst, const int k,
506
475
  queue_ptr stream) {
507
476
  const int num_blocks = (k + SYCL_NEG_BLOCK_SIZE - 1) / SYCL_NEG_BLOCK_SIZE;
508
- stream->parallel_for(
509
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) *
510
- sycl::range<3>(1, 1, SYCL_NEG_BLOCK_SIZE),
511
- sycl::range<3>(1, 1, SYCL_NEG_BLOCK_SIZE)),
512
- [=](sycl::nd_item<3> item_ct1) {
513
- neg(x, dst, k, item_ct1);
514
- });
477
+ sycl_parallel_for(stream,
478
+ sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_NEG_BLOCK_SIZE),
479
+ sycl::range<3>(1, 1, SYCL_NEG_BLOCK_SIZE)),
480
+ [=](sycl::nd_item<3> item_ct1) { neg(x, dst, k, item_ct1); });
515
481
  }
516
482
 
517
483
  template<typename T>
518
484
  static void step_sycl(const T *x, T *dst, const int k,
519
485
  queue_ptr stream) {
520
486
  const int num_blocks = (k + SYCL_NEG_BLOCK_SIZE - 1) / SYCL_NEG_BLOCK_SIZE;
521
- stream->parallel_for(
522
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) *
523
- sycl::range<3>(1, 1, SYCL_NEG_BLOCK_SIZE),
524
- sycl::range<3>(1, 1, SYCL_NEG_BLOCK_SIZE)),
525
- [=](sycl::nd_item<3> item_ct1) {
526
- step(x, dst, k, item_ct1);
527
- });
487
+ sycl_parallel_for(stream,
488
+ sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_NEG_BLOCK_SIZE),
489
+ sycl::range<3>(1, 1, SYCL_NEG_BLOCK_SIZE)),
490
+ [=](sycl::nd_item<3> item_ct1) { step(x, dst, k, item_ct1); });
528
491
  }
529
492
 
530
493
  template<typename T>
531
494
  static void sigmoid_sycl(const T *x, T *dst, const int k,
532
495
  queue_ptr stream) {
533
496
  const int num_blocks = (k + SYCL_SIGMOID_BLOCK_SIZE - 1) / SYCL_SIGMOID_BLOCK_SIZE;
534
- stream->parallel_for(
535
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) *
536
- sycl::range<3>(1, 1, SYCL_SIGMOID_BLOCK_SIZE),
497
+ sycl_parallel_for(
498
+ stream,
499
+ sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_SIGMOID_BLOCK_SIZE),
537
500
  sycl::range<3>(1, 1, SYCL_SIGMOID_BLOCK_SIZE)),
538
- [=](sycl::nd_item<3> item_ct1) {
539
- sigmoid(x, dst, k, item_ct1);
540
- });
501
+ [=](sycl::nd_item<3> item_ct1) { sigmoid(x, dst, k, item_ct1); });
541
502
  }
542
503
 
543
504
  template<typename T>
544
505
  static void sqrt_sycl(const T *x, T *dst, const int k,
545
506
  queue_ptr stream) {
546
507
  const int num_blocks = (k + SYCL_SQRT_BLOCK_SIZE - 1) / SYCL_SQRT_BLOCK_SIZE;
547
- stream->parallel_for(
548
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) *
549
- sycl::range<3>(1, 1, SYCL_SQRT_BLOCK_SIZE),
550
- sycl::range<3>(1, 1, SYCL_SQRT_BLOCK_SIZE)),
551
- [=](sycl::nd_item<3> item_ct1) {
552
- sqrt(x, dst, k, item_ct1);
553
- });
508
+ sycl_parallel_for(stream,
509
+ sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_SQRT_BLOCK_SIZE),
510
+ sycl::range<3>(1, 1, SYCL_SQRT_BLOCK_SIZE)),
511
+ [=](sycl::nd_item<3> item_ct1) { sqrt(x, dst, k, item_ct1); });
554
512
  }
555
513
 
556
514
  template<typename T>
557
515
  static void sin_sycl(const T *x, T *dst, const int k,
558
516
  queue_ptr stream) {
559
517
  const int num_blocks = (k + SYCL_SIN_BLOCK_SIZE - 1) / SYCL_SIN_BLOCK_SIZE;
560
- stream->parallel_for(
561
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) *
562
- sycl::range<3>(1, 1, SYCL_SIN_BLOCK_SIZE),
563
- sycl::range<3>(1, 1, SYCL_SIN_BLOCK_SIZE)),
564
- [=](sycl::nd_item<3> item_ct1) {
565
- sin(x, dst, k, item_ct1);
566
- });
518
+ sycl_parallel_for(stream,
519
+ sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_SIN_BLOCK_SIZE),
520
+ sycl::range<3>(1, 1, SYCL_SIN_BLOCK_SIZE)),
521
+ [=](sycl::nd_item<3> item_ct1) { sin(x, dst, k, item_ct1); });
567
522
  }
568
523
 
569
524
  template<typename T>
570
525
  static void cos_sycl(const T *x, T *dst, const int k,
571
526
  queue_ptr stream) {
572
527
  const int num_blocks = (k + SYCL_SIN_BLOCK_SIZE - 1) / SYCL_SIN_BLOCK_SIZE;
573
- stream->parallel_for(
574
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) *
575
- sycl::range<3>(1, 1, SYCL_SIN_BLOCK_SIZE),
576
- sycl::range<3>(1, 1, SYCL_SIN_BLOCK_SIZE)),
577
- [=](sycl::nd_item<3> item_ct1) {
578
- cos(x, dst, k, item_ct1);
579
- });
528
+ sycl_parallel_for(stream,
529
+ sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_SIN_BLOCK_SIZE),
530
+ sycl::range<3>(1, 1, SYCL_SIN_BLOCK_SIZE)),
531
+ [=](sycl::nd_item<3> item_ct1) { cos(x, dst, k, item_ct1); });
580
532
  }
581
533
 
582
534
  template<typename T>
@@ -584,26 +536,20 @@ static void leaky_relu_sycl(const T *x, T *dst, const int k,
584
536
  const float negative_slope,
585
537
  queue_ptr stream) {
586
538
  const int num_blocks = (k + SYCL_RELU_BLOCK_SIZE - 1) / SYCL_RELU_BLOCK_SIZE;
587
- stream->parallel_for(
588
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) *
589
- sycl::range<3>(1, 1, SYCL_RELU_BLOCK_SIZE),
590
- sycl::range<3>(1, 1, SYCL_RELU_BLOCK_SIZE)),
591
- [=](sycl::nd_item<3> item_ct1) {
592
- leaky_relu(x, dst, k, negative_slope, item_ct1);
593
- });
539
+ sycl_parallel_for(stream,
540
+ sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_RELU_BLOCK_SIZE),
541
+ sycl::range<3>(1, 1, SYCL_RELU_BLOCK_SIZE)),
542
+ [=](sycl::nd_item<3> item_ct1) { leaky_relu(x, dst, k, negative_slope, item_ct1); });
594
543
  }
595
544
 
596
545
  template<typename T>
597
546
  static void sqr_sycl(const T *x, T *dst, const int k,
598
547
  queue_ptr stream) {
599
548
  const int num_blocks = (k + SYCL_SQR_BLOCK_SIZE - 1) / SYCL_SQR_BLOCK_SIZE;
600
- stream->parallel_for(
601
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) *
602
- sycl::range<3>(1, 1, SYCL_SQR_BLOCK_SIZE),
603
- sycl::range<3>(1, 1, SYCL_SQR_BLOCK_SIZE)),
604
- [=](sycl::nd_item<3> item_ct1) {
605
- sqr(x, dst, k, item_ct1);
606
- });
549
+ sycl_parallel_for(stream,
550
+ sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_SQR_BLOCK_SIZE),
551
+ sycl::range<3>(1, 1, SYCL_SQR_BLOCK_SIZE)),
552
+ [=](sycl::nd_item<3> item_ct1) { sqr(x, dst, k, item_ct1); });
607
553
  }
608
554
 
609
555
  template<typename T>
@@ -614,9 +560,8 @@ static void upscale_sycl(const T *x, T *dst, const int nb00, const int nb01,
614
560
  int dst_size = ne10 * ne11 * ne12 * ne13;
615
561
  int num_blocks = (dst_size + SYCL_UPSCALE_BLOCK_SIZE - 1) / SYCL_UPSCALE_BLOCK_SIZE;
616
562
  sycl::range<1> gridDim(num_blocks * SYCL_UPSCALE_BLOCK_SIZE);
617
- stream->parallel_for(
618
- sycl::nd_range<1>(gridDim, sycl::range<1>(SYCL_UPSCALE_BLOCK_SIZE)),
619
- [=](sycl::nd_item<1> item_ct1) {
563
+ sycl_parallel_for<1>(
564
+ stream, sycl::nd_range<1>(gridDim, sycl::range<1>(SYCL_UPSCALE_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) {
620
565
  upscale(x, dst, nb00, nb01, nb02, nb03, ne10, ne11, ne12, ne13, sf0, sf1, sf2, sf3, item_ct1);
621
566
  });
622
567
  }
@@ -627,12 +572,10 @@ static void pad_sycl(const T *x, T *dst, const int ne00,
627
572
  const int ne1, const int ne2, queue_ptr stream) {
628
573
  int num_blocks = (ne0 + SYCL_PAD_BLOCK_SIZE - 1) / SYCL_PAD_BLOCK_SIZE;
629
574
  sycl::range<3> gridDim(ne2, ne1, num_blocks);
630
- stream->parallel_for(
631
- sycl::nd_range<3>(gridDim * sycl::range<3>(1, 1, SYCL_PAD_BLOCK_SIZE),
632
- sycl::range<3>(1, 1, SYCL_PAD_BLOCK_SIZE)),
633
- [=](sycl::nd_item<3> item_ct1) {
634
- pad(x, dst, ne0, ne00, ne01, ne02, item_ct1);
635
- });
575
+ sycl_parallel_for(stream,
576
+ sycl::nd_range<3>(gridDim * sycl::range<3>(1, 1, SYCL_PAD_BLOCK_SIZE),
577
+ sycl::range<3>(1, 1, SYCL_PAD_BLOCK_SIZE)),
578
+ [=](sycl::nd_item<3> item_ct1) { pad(x, dst, ne0, ne00, ne01, ne02, item_ct1); });
636
579
  }
637
580
 
638
581
  template<typename T>
@@ -640,13 +583,10 @@ static void clamp_sycl(const T *x, T *dst, const float min,
640
583
  const float max, const int k,
641
584
  queue_ptr stream) {
642
585
  const int num_blocks = (k + SYCL_CLAMP_BLOCK_SIZE - 1) / SYCL_CLAMP_BLOCK_SIZE;
643
- stream->parallel_for(
644
- sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) *
645
- sycl::range<3>(1, 1, SYCL_CLAMP_BLOCK_SIZE),
646
- sycl::range<3>(1, 1, SYCL_CLAMP_BLOCK_SIZE)),
647
- [=](sycl::nd_item<3> item_ct1) {
648
- clamp(x, dst, min, max, k, item_ct1);
649
- });
586
+ sycl_parallel_for(stream,
587
+ sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CLAMP_BLOCK_SIZE),
588
+ sycl::range<3>(1, 1, SYCL_CLAMP_BLOCK_SIZE)),
589
+ [=](sycl::nd_item<3> item_ct1) { clamp(x, dst, min, max, k, item_ct1); });
650
590
  }
651
591
 
652
592
  inline void ggml_sycl_op_sgn(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
package/cpp/llama.cpp/ggml/src/ggml-sycl/getrows.cpp CHANGED
@@ -118,12 +118,10 @@ static void get_rows_sycl(ggml_backend_sycl_context & ctx, const ggml_tensor *sr
118
118
 
119
119
  GGML_ASSERT(ne00 % 2 == 0);
120
120
 
121
- stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
122
- [=](sycl::nd_item<3> item_ct1) {
123
- k_get_rows<qk, qr, dq>(
124
- src0_dd, src1_dd, dst_dd, ne00, ne12, s1, s2,
125
- s3, nb01, nb02, nb03, s10, s11, s12, item_ct1);
126
- });
121
+ sycl_parallel_for(stream, sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) {
122
+ k_get_rows<qk, qr, dq>(src0_dd, src1_dd, dst_dd, ne00, ne12, s1, s2, s3, nb01, nb02, nb03, s10, s11, s12,
123
+ item_ct1);
124
+ });
127
125
 
128
126
  GGML_UNUSED(dst);
129
127
  GGML_UNUSED(ctx);
@@ -156,9 +154,8 @@ static void get_rows_sycl_float(ggml_backend_sycl_context & ctx, const ggml_tens
156
154
  dpct::has_capability_or_fail(stream->get_device(),
157
155
  {sycl::aspect::fp16});
158
156
 
159
- stream->parallel_for(
160
- sycl::nd_range<3>(block_nums * block_dims, block_dims),
161
- [=](sycl::nd_item<3> item_ct1) {
157
+ sycl_parallel_for(
158
+ stream, sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) {
162
159
  k_get_rows_float(src0_dd, src1_dd, dst_dd, ne00, ne12, s1, s2,
163
160
  s3, nb01, nb02, nb03, s10, s11, s12, item_ct1);
164
161
  });
package/cpp/llama.cpp/ggml/src/ggml-sycl/ggml-sycl.cpp CHANGED
@@ -83,9 +83,7 @@ static ggml_sycl_device_info ggml_sycl_init() {
83
83
 
84
84
  info.devices[i].cc =
85
85
  100 * prop.get_major_version() + 10 * prop.get_minor_version();
86
- info.devices[i].hw_info = get_device_hw_info(&device);
87
- info.devices[i].opt_feature = check_gpu_optimize_feature(info.devices[i].hw_info.arch);
88
-
86
+ info.devices[i].opt_feature.reorder = !device.ext_oneapi_architecture_is(syclex::arch_category::intel_gpu);
89
87
  info.max_work_group_sizes[i] = prop.get_max_work_group_size();
90
88
  }
91
89
 
@@ -195,7 +193,7 @@ static void ggml_check_sycl() try {
195
193
 
196
194
  if (!initialized) {
197
195
  g_ggml_sycl_debug = get_sycl_env("GGML_SYCL_DEBUG", 0);
198
- g_ggml_sycl_disable_optimize= get_sycl_env("GGML_SYCL_DISABLE_OPT", 1);
196
+ g_ggml_sycl_disable_optimize = get_sycl_env("GGML_SYCL_DISABLE_OPT", 0);
199
197
  g_ggml_sycl_disable_graph = get_sycl_env("GGML_SYCL_DISABLE_GRAPH", 1);
200
198
  g_ggml_sycl_disable_dnn = get_sycl_env("GGML_SYCL_DISABLE_DNN", 0);
201
199
  g_ggml_sycl_prioritize_dmmv = get_sycl_env("GGML_SYCL_PRIORITIZE_DMMV", 0);
@@ -1887,13 +1885,12 @@ static void argsort_f32_i32_sycl(const float *x, int *dst, const int ncols,
1887
1885
  const size_t shared_mem = ncols_pad * sizeof(int);
1888
1886
 
1889
1887
  if (order == GGML_SORT_ORDER_ASC) {
1890
- stream->submit([&](sycl::handler &cgh) {
1888
+ sycl_launch(stream, [&](sycl::handler & cgh) {
1891
1889
  sycl::local_accessor<uint8_t, 1> dpct_local_acc_ct1(
1892
1890
  sycl::range<1>(shared_mem), cgh);
1893
1891
 
1894
- cgh.parallel_for(
1895
- sycl::nd_range<3>(block_nums * block_dims, block_dims),
1896
- [=](sycl::nd_item<3> item_ct1) {
1892
+ sycl_parallel_for(
1893
+ cgh, sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) {
1897
1894
  k_argsort_f32_i32<GGML_SORT_ORDER_ASC>(
1898
1895
  x, dst, ncols, ncols_pad, item_ct1,
1899
1896
  dpct_local_acc_ct1.get_multi_ptr<sycl::access::decorated::no>()
@@ -1901,13 +1898,12 @@ static void argsort_f32_i32_sycl(const float *x, int *dst, const int ncols,
1901
1898
  });
1902
1899
  });
1903
1900
  } else if (order == GGML_SORT_ORDER_DESC) {
1904
- stream->submit([&](sycl::handler &cgh) {
1901
+ sycl_launch(stream, [&](sycl::handler & cgh) {
1905
1902
  sycl::local_accessor<uint8_t, 1> dpct_local_acc_ct1(
1906
1903
  sycl::range<1>(shared_mem), cgh);
1907
1904
 
1908
- cgh.parallel_for(
1909
- sycl::nd_range<3>(block_nums * block_dims, block_dims),
1910
- [=](sycl::nd_item<3> item_ct1) {
1905
+ sycl_parallel_for(
1906
+ cgh, sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) {
1911
1907
  k_argsort_f32_i32<GGML_SORT_ORDER_DESC>(
1912
1908
  x, dst, ncols, ncols_pad, item_ct1,
1913
1909
  dpct_local_acc_ct1.get_multi_ptr<sycl::access::decorated::no>()
@@ -1925,50 +1921,47 @@ static void argmax_f32_i32_sycl(const float *x, int *dst, const int ncols,
1925
1921
  const sycl::range<3> block_nums(1, nrows, 1);
1926
1922
  const size_t shared_mem = 256 * sizeof(float);
1927
1923
 
1928
- stream->submit([&](sycl::handler &cgh) {
1924
+ sycl_launch(stream, [&](sycl::handler & cgh) {
1929
1925
  sycl::local_accessor<float, 1> shared_data(
1930
1926
  sycl::range<1>(shared_mem/sizeof(float)), cgh);
1931
1927
  sycl::local_accessor<int, 1> shared_indices(
1932
1928
  sycl::range<1>(shared_mem/sizeof(float)), cgh);
1933
1929
 
1934
- cgh.parallel_for(
1935
- sycl::nd_range<3>(block_nums * block_dims, block_dims),
1936
- [=](sycl::nd_item<3> item_ct1) {
1937
- const int tid = item_ct1.get_local_id(2);
1938
- const int row = item_ct1.get_global_id(1);
1939
-
1940
- float max_val = -INFINITY;
1941
- int max_idx = -1;
1942
-
1943
- for (int col = tid; col < ncols; col += 256) {
1944
- float val = x[row * ncols + col];
1945
- if (val > max_val) {
1946
- max_val = val;
1947
- max_idx = col;
1948
- }
1949
- }
1930
+ sycl_parallel_for(cgh, sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) {
1931
+ const int tid = item_ct1.get_local_id(2);
1932
+ const int row = item_ct1.get_global_id(1);
1950
1933
 
1951
- shared_data[tid] = max_val;
1952
- shared_indices[tid] = max_idx;
1953
- item_ct1.barrier(sycl::access::fence_space::local_space);
1934
+ float max_val = -INFINITY;
1935
+ int max_idx = -1;
1954
1936
 
1955
- for (int stride = 256/2; stride > 0; stride >>= 1) {
1956
- if (tid < stride) {
1957
- float val1 = shared_data[tid];
1958
- float val2 = shared_data[tid + stride];
1959
- if (val2 > val1) {
1960
- shared_data[tid] = val2;
1961
- shared_indices[tid] = shared_indices[tid + stride];
1962
- }
1963
- }
1964
- item_ct1.barrier(sycl::access::fence_space::local_space);
1937
+ for (int col = tid; col < ncols; col += 256) {
1938
+ float val = x[row * ncols + col];
1939
+ if (val > max_val) {
1940
+ max_val = val;
1941
+ max_idx = col;
1965
1942
  }
1943
+ }
1966
1944
 
1945
+ shared_data[tid] = max_val;
1946
+ shared_indices[tid] = max_idx;
1947
+ item_ct1.barrier(sycl::access::fence_space::local_space);
1967
1948
 
1968
- if (tid == 0) {
1969
- dst[row] = shared_indices[0];
1949
+ for (int stride = 256 / 2; stride > 0; stride >>= 1) {
1950
+ if (tid < stride) {
1951
+ float val1 = shared_data[tid];
1952
+ float val2 = shared_data[tid + stride];
1953
+ if (val2 > val1) {
1954
+ shared_data[tid] = val2;
1955
+ shared_indices[tid] = shared_indices[tid + stride];
1956
+ }
1970
1957
  }
1971
- });
1958
+ item_ct1.barrier(sycl::access::fence_space::local_space);
1959
+ }
1960
+
1961
+ if (tid == 0) {
1962
+ dst[row] = shared_indices[0];
1963
+ }
1964
+ });
1972
1965
  });
1973
1966
  }
1974
1967
  static void diag_mask_inf_f32_sycl(const float *x, float *dst,
@@ -2952,7 +2945,7 @@ static void ggml_sycl_mul_mat_batched_sycl(ggml_backend_sycl_context & ctx, cons
2952
2945
  void ** ptrs_dst_get = ptrs_dst.get();
2953
2946
  size_t nb12_scaled = src1->type == GGML_TYPE_F16 ? nb12 : s12 * sizeof(sycl::half);
2954
2947
  size_t nb13_scaled = src1->type == GGML_TYPE_F16 ? nb13 : s13 * sizeof(sycl::half);
2955
- cgh.parallel_for(sycl::nd_range<3>(block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) {
2948
+ sycl_parallel_for(cgh, sycl::nd_range<3>(block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) {
2956
2949
  k_compute_batched_ptrs(src0_f16, src1_f16, dst_ddf, ptrs_src_get, ptrs_dst_get, ne12, ne13, ne23, nb02,
2957
2950
  nb03, nb12_scaled, nb13_scaled, nbd2, nbd3, r2, r3, item_ct1);
2958
2951
  });
@@ -3456,7 +3449,7 @@ static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx,
3456
3449
  {
3457
3450
  sycl::range<3> block_dims(1, 1, std::min((unsigned int)ne10, 768u));
3458
3451
  sycl::range<3> grid_dims(1, n_ids, ids->ne[1]);
3459
- stream->submit([&](sycl::handler &cgh) {
3452
+ sycl_launch(stream, [&](sycl::handler & cgh) {
3460
3453
  sycl::local_accessor<int, 0> src1_row_acc(cgh);
3461
3454
 
3462
3455
  char *__restrict src1_contiguous_get =
@@ -3468,9 +3461,8 @@ static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx,
3468
3461
  size_t ids_nb_ct6 = ids->nb[1];
3469
3462
  size_t ids_nb_ct7 = ids->nb[0];
3470
3463
 
3471
- cgh.parallel_for(
3472
- sycl::nd_range<3>(grid_dims * block_dims, block_dims),
3473
- [=](sycl::nd_item<3> item_ct1) {
3464
+ sycl_parallel_for(
3465
+ cgh, sycl::nd_range<3>(grid_dims * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) {
3474
3466
  k_copy_src1_to_contiguous(
3475
3467
  src1_original, src1_contiguous_get,
3476
3468
  dev_cur_src1_row_get,
@@ -3501,15 +3493,14 @@ static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx,
3501
3493
  {
3502
3494
  sycl::range<3> block_dims(1, 1, std::min((unsigned int)ne0, 768u));
3503
3495
  sycl::range<3> grid_dims(1, 1, num_src1_rows);
3504
- stream->submit([&](sycl::handler &cgh) {
3496
+ sycl_launch(stream, [&](sycl::handler & cgh) {
3505
3497
  const char *__restrict dst_contiguous_get =
3506
3498
  dst_contiguous.get();
3507
3499
  const mmid_row_mapping *__restrict dev_row_mapping_get =
3508
3500
  dev_row_mapping.get();
3509
3501
 
3510
- cgh.parallel_for(
3511
- sycl::nd_range<3>(grid_dims * block_dims, block_dims),
3512
- [=](sycl::nd_item<3> item_ct1) {
3502
+ sycl_parallel_for(
3503
+ cgh, sycl::nd_range<3>(grid_dims * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) {
3513
3504
  k_copy_dst_from_contiguous(dst_original,
3514
3505
  dst_contiguous_get,
3515
3506
  dev_row_mapping_get,