@fugood/llama.node 0.3.13 → 0.3.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/bin/darwin/arm64/llama-node.node +0 -0
  2. package/bin/darwin/x64/llama-node.node +0 -0
  3. package/bin/linux/arm64/llama-node.node +0 -0
  4. package/bin/linux/x64/llama-node.node +0 -0
  5. package/bin/linux-cuda/arm64/llama-node.node +0 -0
  6. package/bin/linux-cuda/x64/llama-node.node +0 -0
  7. package/bin/linux-vulkan/arm64/llama-node.node +0 -0
  8. package/bin/linux-vulkan/x64/llama-node.node +0 -0
  9. package/bin/win32/arm64/llama-node.node +0 -0
  10. package/bin/win32/arm64/node.lib +0 -0
  11. package/bin/win32/x64/llama-node.node +0 -0
  12. package/bin/win32/x64/node.lib +0 -0
  13. package/bin/win32-vulkan/arm64/llama-node.node +0 -0
  14. package/bin/win32-vulkan/arm64/node.lib +0 -0
  15. package/bin/win32-vulkan/x64/llama-node.node +0 -0
  16. package/bin/win32-vulkan/x64/node.lib +0 -0
  17. package/lib/binding.ts +1 -1
  18. package/package.json +1 -1
  19. package/src/LlamaContext.cpp +98 -76
  20. package/src/LlamaContext.h +1 -1
  21. package/src/common.hpp +1 -2
  22. package/src/llama.cpp/.github/workflows/build.yml +60 -10
  23. package/src/llama.cpp/.github/workflows/server.yml +2 -0
  24. package/src/llama.cpp/common/CMakeLists.txt +3 -3
  25. package/src/llama.cpp/common/arg.cpp +112 -11
  26. package/src/llama.cpp/common/chat.cpp +960 -266
  27. package/src/llama.cpp/common/chat.h +135 -0
  28. package/src/llama.cpp/common/common.cpp +27 -171
  29. package/src/llama.cpp/common/common.h +27 -67
  30. package/src/llama.cpp/common/json-schema-to-grammar.cpp +4 -5
  31. package/src/llama.cpp/common/json-schema-to-grammar.h +0 -1
  32. package/src/llama.cpp/common/{minja.hpp → minja/minja.hpp} +37 -5
  33. package/src/llama.cpp/common/ngram-cache.cpp +1 -0
  34. package/src/llama.cpp/common/sampling.cpp +45 -7
  35. package/src/llama.cpp/common/speculative.cpp +6 -5
  36. package/src/llama.cpp/common/speculative.h +1 -1
  37. package/src/llama.cpp/docs/build.md +45 -7
  38. package/src/llama.cpp/examples/cvector-generator/cvector-generator.cpp +3 -1
  39. package/src/llama.cpp/examples/embedding/embedding.cpp +1 -0
  40. package/src/llama.cpp/examples/export-lora/export-lora.cpp +4 -2
  41. package/src/llama.cpp/examples/imatrix/imatrix.cpp +2 -3
  42. package/src/llama.cpp/examples/llama.android/llama/src/main/cpp/llama-android.cpp +1 -1
  43. package/src/llama.cpp/examples/llava/CMakeLists.txt +7 -0
  44. package/src/llama.cpp/examples/llava/clip.cpp +373 -107
  45. package/src/llama.cpp/examples/llava/clip.h +19 -3
  46. package/src/llama.cpp/examples/llava/gemma3-cli.cpp +341 -0
  47. package/src/llama.cpp/examples/llava/llava.cpp +4 -2
  48. package/src/llama.cpp/examples/llava/minicpmv-cli.cpp +30 -11
  49. package/src/llama.cpp/examples/lookahead/lookahead.cpp +1 -0
  50. package/src/llama.cpp/examples/main/main.cpp +73 -28
  51. package/src/llama.cpp/examples/parallel/parallel.cpp +1 -0
  52. package/src/llama.cpp/examples/passkey/passkey.cpp +1 -0
  53. package/src/llama.cpp/examples/quantize/quantize.cpp +1 -0
  54. package/src/llama.cpp/examples/run/linenoise.cpp/linenoise.cpp +882 -237
  55. package/src/llama.cpp/examples/run/linenoise.cpp/linenoise.h +35 -26
  56. package/src/llama.cpp/examples/run/run.cpp +110 -67
  57. package/src/llama.cpp/examples/server/server.cpp +82 -87
  58. package/src/llama.cpp/examples/server/utils.hpp +94 -107
  59. package/src/llama.cpp/examples/sycl/run-llama2.sh +2 -2
  60. package/src/llama.cpp/examples/tts/tts.cpp +251 -142
  61. package/src/llama.cpp/ggml/CMakeLists.txt +13 -1
  62. package/src/llama.cpp/ggml/include/ggml-alloc.h +1 -1
  63. package/src/llama.cpp/ggml/include/ggml-backend.h +3 -3
  64. package/src/llama.cpp/ggml/include/ggml-cpu.h +3 -0
  65. package/src/llama.cpp/ggml/include/ggml.h +5 -1
  66. package/src/llama.cpp/ggml/src/CMakeLists.txt +10 -7
  67. package/src/llama.cpp/ggml/src/ggml-alloc.c +24 -15
  68. package/src/llama.cpp/ggml/src/ggml-backend-impl.h +1 -1
  69. package/src/llama.cpp/ggml/src/ggml-backend-reg.cpp +58 -54
  70. package/src/llama.cpp/ggml/src/ggml-backend.cpp +10 -8
  71. package/src/llama.cpp/ggml/src/ggml-cann/ggml-cann.cpp +3 -2
  72. package/src/llama.cpp/ggml/src/ggml-cann/kernels/dup.cpp +3 -5
  73. package/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt +132 -17
  74. package/src/llama.cpp/ggml/src/ggml-cpu/amx/amx.cpp +2 -1
  75. package/src/llama.cpp/ggml/src/ggml-cpu/cpu-feats-x86.cpp +4 -0
  76. package/src/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +2 -1
  77. package/src/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-impl.h +151 -0
  78. package/src/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-quants.c +1396 -386
  79. package/src/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c +1432 -151
  80. package/src/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.cpp +22 -0
  81. package/src/llama.cpp/ggml/src/ggml-cpu/kleidiai/kernels.cpp +259 -0
  82. package/src/llama.cpp/ggml/src/ggml-cpu/kleidiai/kernels.h +61 -0
  83. package/src/llama.cpp/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +288 -0
  84. package/src/llama.cpp/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
  85. package/src/llama.cpp/ggml/src/ggml-cuda/CMakeLists.txt +15 -2
  86. package/src/llama.cpp/ggml/src/ggml-hip/CMakeLists.txt +14 -0
  87. package/src/llama.cpp/ggml/src/ggml-impl.h +1 -1
  88. package/src/llama.cpp/ggml/src/ggml-metal/CMakeLists.txt +4 -5
  89. package/src/llama.cpp/ggml/src/ggml-metal/ggml-metal-impl.h +235 -0
  90. package/src/llama.cpp/ggml/src/ggml-musa/CMakeLists.txt +6 -2
  91. package/src/llama.cpp/ggml/src/ggml-opencl/CMakeLists.txt +1 -0
  92. package/src/llama.cpp/ggml/src/ggml-opencl/ggml-opencl.cpp +220 -116
  93. package/src/llama.cpp/ggml/src/ggml-quants.c +114 -114
  94. package/src/llama.cpp/ggml/src/ggml-rpc/ggml-rpc.cpp +2 -1
  95. package/src/llama.cpp/ggml/src/ggml-sycl/CMakeLists.txt +2 -0
  96. package/src/llama.cpp/ggml/src/ggml-sycl/backend.hpp +1 -0
  97. package/src/llama.cpp/ggml/src/ggml-sycl/common.cpp +17 -0
  98. package/src/llama.cpp/ggml/src/ggml-sycl/common.hpp +51 -10
  99. package/src/llama.cpp/ggml/src/ggml-sycl/convert.cpp +33 -4
  100. package/src/llama.cpp/ggml/src/ggml-sycl/convert.hpp +2 -2
  101. package/src/llama.cpp/ggml/src/ggml-sycl/cpy.cpp +701 -0
  102. package/src/llama.cpp/ggml/src/ggml-sycl/cpy.hpp +11 -0
  103. package/src/llama.cpp/ggml/src/ggml-sycl/dequantize.hpp +55 -0
  104. package/src/llama.cpp/ggml/src/ggml-sycl/dmmv.cpp +136 -4
  105. package/src/llama.cpp/ggml/src/ggml-sycl/getrows.cpp +308 -0
  106. package/src/llama.cpp/ggml/src/ggml-sycl/getrows.hpp +23 -0
  107. package/src/llama.cpp/ggml/src/ggml-sycl/ggml-sycl.cpp +168 -721
  108. package/src/llama.cpp/ggml/src/ggml-sycl/mmvq.cpp +75 -77
  109. package/src/llama.cpp/ggml/src/ggml-sycl/softmax.cpp +3 -0
  110. package/src/llama.cpp/ggml/src/ggml-sycl/sycl_hw.cpp +13 -0
  111. package/src/llama.cpp/ggml/src/ggml-sycl/sycl_hw.hpp +23 -0
  112. package/src/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan.cpp +146 -42
  113. package/src/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +13 -3
  114. package/src/llama.cpp/ggml/src/ggml.c +8 -3
  115. package/src/llama.cpp/include/llama.h +19 -5
  116. package/src/llama.cpp/models/ggml-vocab-gpt-4o.gguf.inp +112 -0
  117. package/src/llama.cpp/models/ggml-vocab-gpt-4o.gguf.out +46 -0
  118. package/src/llama.cpp/requirements/requirements-all.txt +1 -0
  119. package/src/llama.cpp/requirements/requirements-tool_bench.txt +12 -0
  120. package/src/llama.cpp/requirements.txt +1 -0
  121. package/src/llama.cpp/src/llama-arch.cpp +21 -0
  122. package/src/llama.cpp/src/llama-arch.h +1 -0
  123. package/src/llama.cpp/src/llama-chat.cpp +1 -0
  124. package/src/llama.cpp/src/llama-grammar.cpp +182 -182
  125. package/src/llama.cpp/src/llama-grammar.h +12 -3
  126. package/src/llama.cpp/src/llama-kv-cache.h +1 -0
  127. package/src/llama.cpp/src/llama-mmap.cpp +11 -1
  128. package/src/llama.cpp/src/llama-model.cpp +69 -5
  129. package/src/llama.cpp/src/llama-sampling.cpp +43 -10
  130. package/src/llama.cpp/src/llama-vocab.cpp +12 -0
  131. package/src/llama.cpp/src/llama.cpp +147 -0
  132. package/src/llama.cpp/tests/test-backend-ops.cpp +166 -110
  133. package/src/llama.cpp/tests/test-chat-template.cpp +32 -22
  134. package/src/llama.cpp/tests/test-chat.cpp +593 -395
  135. package/src/llama.cpp/tests/test-json-schema-to-grammar.cpp +63 -63
  136. package/src/llama.cpp/tests/test-quantize-fns.cpp +1 -9
  137. package/src/llama.cpp/Sources/llama/llama.h +0 -4
  138. package/src/llama.cpp/common/chat.hpp +0 -55
  139. /package/src/llama.cpp/common/{chat-template.hpp → minja/chat-template.hpp} +0 -0
@@ -91,7 +91,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
91
91
  })""",
92
92
  R"""(
93
93
  root ::= ([0] | [1-9] [0-9]{0,15}) space
94
- space ::= | " " | "\n" [ \t]{0,20}
94
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
95
95
  )"""
96
96
  });
97
97
 
@@ -104,7 +104,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
104
104
  })""",
105
105
  R"""(
106
106
  root ::= ([1-9] [0-9]{0,15}) space
107
- space ::= | " " | "\n" [ \t]{0,20}
107
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
108
108
  )"""
109
109
  });
110
110
 
@@ -117,7 +117,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
117
117
  })""",
118
118
  R"""(
119
119
  root ::= ([1-2] [0-9]{1,15} | [3-9] [0-9]{0,15}) space
120
- space ::= | " " | "\n" [ \t]{0,20}
120
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
121
121
  )"""
122
122
  });
123
123
 
@@ -130,7 +130,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
130
130
  })""",
131
131
  R"""(
132
132
  root ::= ([1-8] [0-9]{1,15} | [9] [0-9]{0,15}) space
133
- space ::= | " " | "\n" [ \t]{0,20}
133
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
134
134
  )"""
135
135
  });
136
136
 
@@ -143,7 +143,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
143
143
  })""",
144
144
  R"""(
145
145
  root ::= ([1] ([0-9]{1,15}) | [2-9] [0-9]{1,15}) space
146
- space ::= | " " | "\n" [ \t]{0,20}
146
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
147
147
  )"""
148
148
  });
149
149
 
@@ -156,7 +156,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
156
156
  })""",
157
157
  R"""(
158
158
  root ::= ([1] [0-9]{2,15} | [2] ([0-4] [0-9]{1,14} | [5-9] [0-9]{0,14}) | [3-9] [0-9]{1,15}) space
159
- space ::= | " " | "\n" [ \t]{0,20}
159
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
160
160
  )"""
161
161
  });
162
162
 
@@ -169,7 +169,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
169
169
  })""",
170
170
  R"""(
171
171
  root ::= ("-" [1-9] [0-9]{0,15} | [0-9] | ([1-2] [0-9] | [3] "0")) space
172
- space ::= | " " | "\n" [ \t]{0,20}
172
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
173
173
  )"""
174
174
  });
175
175
 
@@ -182,7 +182,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
182
182
  })""",
183
183
  R"""(
184
184
  root ::= ("-" ([0-5]) | [0] | [1-9] [0-9]{0,15}) space
185
- space ::= | " " | "\n" [ \t]{0,20}
185
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
186
186
  )"""
187
187
  });
188
188
 
@@ -195,7 +195,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
195
195
  })""",
196
196
  R"""(
197
197
  root ::= ("-" ([0-9] | ([1-8] [0-9] | [9] [0-9]) | "1" ([0-1] [0-9] | [2] [0-3])) | [0] | [1-9] [0-9]{0,15}) space
198
- space ::= | " " | "\n" [ \t]{0,20}
198
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
199
199
  )"""
200
200
  });
201
201
 
@@ -208,7 +208,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
208
208
  })""",
209
209
  R"""(
210
210
  root ::= ("-" ([0-4] [0-9]{1,15} | [5-9] [0-9]{0,15})) space
211
- space ::= | " " | "\n" [ \t]{0,20}
211
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
212
212
  )"""
213
213
  });
214
214
 
@@ -221,7 +221,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
221
221
  })""",
222
222
  R"""(
223
223
  root ::= ("-" [1-9] [0-9]{0,15} | [0-1]) space
224
- space ::= | " " | "\n" [ \t]{0,20}
224
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
225
225
  )"""
226
226
  });
227
227
 
@@ -234,7 +234,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
234
234
  })""",
235
235
  R"""(
236
236
  root ::= ("-" [1-9] [0-9]{0,15} | [0-9] | ([1-8] [0-9] | [9] [0-9]) | "100") space
237
- space ::= | " " | "\n" [ \t]{0,20}
237
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
238
238
  )"""
239
239
  });
240
240
 
@@ -248,7 +248,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
248
248
  })""",
249
249
  R"""(
250
250
  root ::= ([0-9] | ([1] [0-9] | [2] [0-3])) space
251
- space ::= | " " | "\n" [ \t]{0,20}
251
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
252
252
  )"""
253
253
  });
254
254
 
@@ -262,7 +262,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
262
262
  })""",
263
263
  R"""(
264
264
  root ::= (([1] ([5-9]) | [2-9] [0-9]) | ([1-2] [0-9]{2} | [3] "00")) space
265
- space ::= | " " | "\n" [ \t]{0,20}
265
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
266
266
  )"""
267
267
  });
268
268
 
@@ -276,7 +276,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
276
276
  })""",
277
277
  R"""(
278
278
  root ::= ([5-9] | ([1-2] [0-9] | [3] "0")) space
279
- space ::= | " " | "\n" [ \t]{0,20}
279
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
280
280
  )"""
281
281
  });
282
282
 
@@ -290,7 +290,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
290
290
  })""",
291
291
  R"""(
292
292
  root ::= ("-" ([0-9] | ([1-8] [0-9] | [9] [0-9]) | "1" ([0-1] [0-9] | [2] [0-3])) | [0-9] | ([1-3] [0-9] | [4] [0-2])) space
293
- space ::= | " " | "\n" [ \t]{0,20}
293
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
294
294
  )"""
295
295
  });
296
296
 
@@ -304,7 +304,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
304
304
  })""",
305
305
  R"""(
306
306
  root ::= ("-" ([0-9] | "10") | [0-9] | "10") space
307
- space ::= | " " | "\n" [ \t]{0,20}
307
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
308
308
  )"""
309
309
  });
310
310
 
@@ -340,7 +340,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
340
340
  number ::= ("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space
341
341
  object ::= "{" space ( string ":" space value ("," space string ":" space value)* )? "}" space
342
342
  root ::= object
343
- space ::= | " " | "\n" [ \t]{0,20}
343
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
344
344
  string ::= "\"" char* "\"" space
345
345
  value ::= object | array | string | number | boolean | null
346
346
  )"""
@@ -363,7 +363,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
363
363
  date-time ::= date "T" time
364
364
  date-time-string ::= "\"" date-time "\"" space
365
365
  root ::= "[" space tuple-0 "," space uuid "," space tuple-2 "," space tuple-3 "]" space
366
- space ::= | " " | "\n" [ \t]{0,20}
366
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
367
367
  time ::= ([01] [0-9] | "2" [0-3]) ":" [0-5] [0-9] ":" [0-5] [0-9] ( "." [0-9]{3} )? ( "Z" | ( "+" | "-" ) ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] )
368
368
  time-string ::= "\"" time "\"" space
369
369
  tuple-0 ::= date-string
@@ -382,7 +382,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
382
382
  R"""(
383
383
  char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
384
384
  root ::= "\"" char* "\"" space
385
- space ::= | " " | "\n" [ \t]{0,20}
385
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
386
386
  )"""
387
387
  });
388
388
 
@@ -396,7 +396,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
396
396
  R"""(
397
397
  char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
398
398
  root ::= "\"" char+ "\"" space
399
- space ::= | " " | "\n" [ \t]{0,20}
399
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
400
400
  )"""
401
401
  });
402
402
 
@@ -410,7 +410,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
410
410
  R"""(
411
411
  char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
412
412
  root ::= "\"" char{3,} "\"" space
413
- space ::= | " " | "\n" [ \t]{0,20}
413
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
414
414
  )"""
415
415
  });
416
416
 
@@ -424,7 +424,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
424
424
  R"""(
425
425
  char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
426
426
  root ::= "\"" char{0,3} "\"" space
427
- space ::= | " " | "\n" [ \t]{0,20}
427
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
428
428
  )"""
429
429
  });
430
430
 
@@ -439,7 +439,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
439
439
  R"""(
440
440
  char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
441
441
  root ::= "\"" char{1,4} "\"" space
442
- space ::= | " " | "\n" [ \t]{0,20}
442
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
443
443
  )"""
444
444
  });
445
445
 
@@ -451,7 +451,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
451
451
  })""",
452
452
  R"""(
453
453
  root ::= ("true" | "false") space
454
- space ::= | " " | "\n" [ \t]{0,20}
454
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
455
455
  )"""
456
456
  });
457
457
 
@@ -464,7 +464,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
464
464
  R"""(
465
465
  integral-part ::= [0] | [1-9] [0-9]{0,15}
466
466
  root ::= ("-"? integral-part) space
467
- space ::= | " " | "\n" [ \t]{0,20}
467
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
468
468
  )"""
469
469
  });
470
470
 
@@ -476,7 +476,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
476
476
  })""",
477
477
  R"""(
478
478
  root ::= "\"foo\"" space
479
- space ::= | " " | "\n" [ \t]{0,20}
479
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
480
480
  )"""
481
481
  });
482
482
 
@@ -488,7 +488,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
488
488
  })""",
489
489
  R"""(
490
490
  root ::= "123" space
491
- space ::= | " " | "\n" [ \t]{0,20}
491
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
492
492
  )"""
493
493
  });
494
494
 
@@ -500,7 +500,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
500
500
  })""",
501
501
  R"""(
502
502
  root ::= ("\"red\"" | "\"amber\"" | "\"green\"" | "null" | "42" | "[\"foo\"]") space
503
- space ::= | " " | "\n" [ \t]{0,20}
503
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
504
504
  )"""
505
505
  });
506
506
 
@@ -514,7 +514,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
514
514
  R"""(
515
515
  char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
516
516
  root ::= "[" space (string ("," space string)*)? "]" space
517
- space ::= | " " | "\n" [ \t]{0,20}
517
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
518
518
  string ::= "\"" char* "\"" space
519
519
  )"""
520
520
  });
@@ -531,7 +531,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
531
531
  char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
532
532
  null ::= "null" space
533
533
  root ::= alternative-0 | null
534
- space ::= | " " | "\n" [ \t]{0,20}
534
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
535
535
  string ::= "\"" char* "\"" space
536
536
  )"""
537
537
  });
@@ -545,7 +545,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
545
545
  R"""(
546
546
  char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
547
547
  root ::= "[" space string "]" space
548
- space ::= | " " | "\n" [ \t]{0,20}
548
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
549
549
  string ::= "\"" char* "\"" space
550
550
  )"""
551
551
  });
@@ -562,7 +562,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
562
562
  integral-part ::= [0] | [1-9] [0-9]{0,15}
563
563
  number ::= ("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space
564
564
  root ::= "[" space string "," space number "]" space
565
- space ::= | " " | "\n" [ \t]{0,20}
565
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
566
566
  string ::= "\"" char* "\"" space
567
567
  )"""
568
568
  });
@@ -577,7 +577,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
577
577
  decimal-part ::= [0-9]{1,16}
578
578
  integral-part ::= [0] | [1-9] [0-9]{0,15}
579
579
  root ::= ("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space
580
- space ::= | " " | "\n" [ \t]{0,20}
580
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
581
581
  )"""
582
582
  });
583
583
 
@@ -593,7 +593,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
593
593
  R"""(
594
594
  boolean ::= ("true" | "false") space
595
595
  root ::= "[" space boolean ("," space boolean)+ "]" space
596
- space ::= | " " | "\n" [ \t]{0,20}
596
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
597
597
  )"""
598
598
  });
599
599
 
@@ -609,7 +609,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
609
609
  R"""(
610
610
  boolean ::= ("true" | "false") space
611
611
  root ::= "[" space boolean? "]" space
612
- space ::= | " " | "\n" [ \t]{0,20}
612
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
613
613
  )"""
614
614
  });
615
615
 
@@ -625,7 +625,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
625
625
  R"""(
626
626
  boolean ::= ("true" | "false") space
627
627
  root ::= "[" space (boolean ("," space boolean)?)? "]" space
628
- space ::= | " " | "\n" [ \t]{0,20}
628
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
629
629
  )"""
630
630
  });
631
631
 
@@ -646,7 +646,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
646
646
  item ::= number | integer
647
647
  number ::= ("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space
648
648
  root ::= "[" space item ("," space item){2,4} "]" space
649
- space ::= | " " | "\n" [ \t]{0,20}
649
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
650
650
  )"""
651
651
  });
652
652
 
@@ -665,7 +665,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
665
665
  R"""(
666
666
  item ::= ("-" ([0-9] | "1" [0-2]) | [0-9] | ([1-8] [0-9] | [9] [0-9]) | ([1] [0-9]{2} | [2] "0" [0-7])) space
667
667
  root ::= "[" space item ("," space item){2,4} "]" space
668
- space ::= | " " | "\n" [ \t]{0,20}
668
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
669
669
  )"""
670
670
  });
671
671
 
@@ -684,7 +684,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
684
684
  R"""(
685
685
  item ::= (([1] ([2-9]) | [2-9] [0-9]) | ([1] [0-9]{2} | [2] "0" [0-7])) space
686
686
  root ::= "[" space item ("," space item){2,4} "]" space
687
- space ::= | " " | "\n" [ \t]{0,20}
687
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
688
688
  )"""
689
689
  });
690
690
 
@@ -697,7 +697,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
697
697
  })""",
698
698
  R"""(
699
699
  root ::= "\"" ("ab" "c"? "d"* "ef" "g"+ ("hij")? "kl") "\"" space
700
- space ::= | " " | "\n" [ \t]{0,20}
700
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
701
701
  )"""
702
702
  });
703
703
 
@@ -710,7 +710,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
710
710
  })""",
711
711
  R"""(
712
712
  root ::= "\"" ("[]{}()|+*?") "\"" space
713
- space ::= | " " | "\n" [ \t]{0,20}
713
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
714
714
  )"""
715
715
  });
716
716
 
@@ -723,7 +723,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
723
723
  })""",
724
724
  R"""(
725
725
  root ::= "\"" ("\"") "\"" space
726
- space ::= | " " | "\n" [ \t]{0,20}
726
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
727
727
  )"""
728
728
  });
729
729
 
@@ -736,7 +736,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
736
736
  })""",
737
737
  R"""(
738
738
  root ::= "\"" ("A" | "B" | "C" | "D") "\"" space
739
- space ::= | " " | "\n" [ \t]{0,20}
739
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
740
740
  )"""
741
741
  });
742
742
 
@@ -751,7 +751,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
751
751
  dot ::= [^\x0A\x0D]
752
752
  root ::= "\"" (("(" root-1{1,3} ")")? root-1{3,3} "-" root-1{4,4} " " "a"{3,5} "nd" dot dot dot) "\"" space
753
753
  root-1 ::= [0-9]
754
- space ::= | " " | "\n" [ \t]{0,20}
754
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
755
755
  )"""
756
756
  });
757
757
 
@@ -779,7 +779,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
779
779
  c-kv ::= "\"c\"" space ":" space string
780
780
  char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
781
781
  root ::= "{" space b-kv "," space c-kv "," space a-kv "}" space
782
- space ::= | " " | "\n" [ \t]{0,20}
782
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
783
783
  string ::= "\"" char* "\"" space
784
784
  )"""
785
785
  });
@@ -799,7 +799,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
799
799
  a-kv ::= "\"a\"" space ":" space string
800
800
  char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
801
801
  root ::= "{" space (a-kv )? "}" space
802
- space ::= | " " | "\n" [ \t]{0,20}
802
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
803
803
  string ::= "\"" char* "\"" space
804
804
  )"""
805
805
  });
@@ -823,7 +823,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
823
823
  c-kv ::= "\"c\"" space ":" space string
824
824
  char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
825
825
  root ::= "{" space (a-kv a-rest | b-kv b-rest | c-kv )? "}" space
826
- space ::= | " " | "\n" [ \t]{0,20}
826
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
827
827
  string ::= "\"" char* "\"" space
828
828
  )"""
829
829
  });
@@ -849,7 +849,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
849
849
  d-kv ::= "\"d\"" space ":" space string
850
850
  d-rest ::= ( "," space c-kv )?
851
851
  root ::= "{" space b-kv "," space a-kv ( "," space ( d-kv d-rest | c-kv ) )? "}" space
852
- space ::= | " " | "\n" [ \t]{0,20}
852
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
853
853
  string ::= "\"" char* "\"" space
854
854
  )"""
855
855
  });
@@ -869,7 +869,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
869
869
  integral-part ::= [0] | [1-9] [0-9]{0,15}
870
870
  number ::= ("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space
871
871
  root ::= "{" space (additional-kv ( "," space additional-kv )* )? "}" space
872
- space ::= | " " | "\n" [ \t]{0,20}
872
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
873
873
  string ::= "\"" char* "\"" space
874
874
  )"""
875
875
  });
@@ -891,7 +891,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
891
891
  number ::= ("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space
892
892
  object ::= "{" space ( string ":" space value ("," space string ":" space value)* )? "}" space
893
893
  root ::= object
894
- space ::= | " " | "\n" [ \t]{0,20}
894
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
895
895
  string ::= "\"" char* "\"" space
896
896
  value ::= object | array | string | number | boolean | null
897
897
  )"""
@@ -913,7 +913,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
913
913
  number ::= ("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space
914
914
  object ::= "{" space ( string ":" space value ("," space string ":" space value)* )? "}" space
915
915
  root ::= object
916
- space ::= | " " | "\n" [ \t]{0,20}
916
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
917
917
  string ::= "\"" char* "\"" space
918
918
  value ::= object | array | string | number | boolean | null
919
919
  )"""
@@ -928,7 +928,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
928
928
  })""",
929
929
  R"""(
930
930
  root ::= "{" space "}" space
931
- space ::= | " " | "\n" [ \t]{0,20}
931
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
932
932
  )"""
933
933
  });
934
934
 
@@ -952,7 +952,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
952
952
  integral-part ::= [0] | [1-9] [0-9]{0,15}
953
953
  number ::= ("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space
954
954
  root ::= "{" space a-kv ( "," space ( additional-kv ( "," space additional-kv )* ) )? "}" space
955
- space ::= | " " | "\n" [ \t]{0,20}
955
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
956
956
  string ::= "\"" char* "\"" space
957
957
  )"""
958
958
  });
@@ -977,7 +977,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
977
977
  integral-part ::= [0] | [1-9] [0-9]{0,15}
978
978
  number ::= ("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space
979
979
  root ::= "{" space (a-kv a-rest | additional-kv ( "," space additional-kv )* )? "}" space
980
- space ::= | " " | "\n" [ \t]{0,20}
980
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
981
981
  )"""
982
982
  });
983
983
 
@@ -1004,7 +1004,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
1004
1004
  integral-part ::= [0] | [1-9] [0-9]{0,15}
1005
1005
  number ::= ("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space
1006
1006
  root ::= "{" space and-kv ( "," space ( also-kv also-rest | additional-kv ( "," space additional-kv )* ) )? "}" space
1007
- space ::= | " " | "\n" [ \t]{0,20}
1007
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
1008
1008
  )"""
1009
1009
  });
1010
1010
 
@@ -1030,7 +1030,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
1030
1030
  integral-part ::= [0] | [1-9] [0-9]{0,15}
1031
1031
  root ::= ("-"? integral-part) space
1032
1032
  root0 ::= "{" space (-kv -rest | a-kv a-rest | additional-kv ( "," space additional-kv )* )? "}" space
1033
- space ::= | " " | "\n" [ \t]{0,20}
1033
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
1034
1034
  )"""
1035
1035
  });
1036
1036
 
@@ -1055,7 +1055,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
1055
1055
  integer ::= ("-"? integral-part) space
1056
1056
  integral-part ::= [0] | [1-9] [0-9]{0,15}
1057
1057
  root ::= "{" space (a-kv a-rest | aa-kv aa-rest | additional-kv ( "," space additional-kv )* )? "}" space
1058
- space ::= | " " | "\n" [ \t]{0,20}
1058
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
1059
1059
  )"""
1060
1060
  });
1061
1061
 
@@ -1080,7 +1080,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
1080
1080
  integer ::= ("-"? integral-part) space
1081
1081
  integral-part ::= [0] | [1-9] [0-9]{0,15}
1082
1082
  root ::= "{" space (ab-kv ab-rest | ac-kv ac-rest | additional-kv ( "," space additional-kv )* )? "}" space
1083
- space ::= | " " | "\n" [ \t]{0,20}
1083
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
1084
1084
  )"""
1085
1085
  });
1086
1086
 
@@ -1109,7 +1109,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
1109
1109
  foo ::= "{" space foo-a-kv "}" space
1110
1110
  foo-a-kv ::= "\"a\"" space ":" space string
1111
1111
  root ::= foo
1112
- space ::= | " " | "\n" [ \t]{0,20}
1112
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
1113
1113
  string ::= "\"" char* "\"" space
1114
1114
  )"""
1115
1115
  });
@@ -1143,7 +1143,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
1143
1143
  integral-part ::= [0] | [1-9] [0-9]{0,15}
1144
1144
  number ::= ("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space
1145
1145
  root ::= alternative-0 | alternative-1
1146
- space ::= | " " | "\n" [ \t]{0,20}
1146
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
1147
1147
  )"""
1148
1148
  });
1149
1149
 
@@ -1187,7 +1187,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
1187
1187
  integral-part ::= [0] | [1-9] [0-9]{0,15}
1188
1188
  number ::= ("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space
1189
1189
  root ::= "{" space a-kv "," space b-kv ( "," space ( d-kv d-rest | c-kv ) )? "}" space
1190
- space ::= | " " | "\n" [ \t]{0,20}
1190
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
1191
1191
  )"""
1192
1192
  });
1193
1193
 
@@ -1235,7 +1235,7 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
1235
1235
  number-number-kv ::= "\"number\"" space ":" space number-number
1236
1236
  number-number-root-kv ::= "\"root\"" space ":" space number
1237
1237
  root ::= "{" space number-kv "}" space
1238
- space ::= | " " | "\n" [ \t]{0,20}
1238
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
1239
1239
  )"""
1240
1240
  });
1241
1241
  }
@@ -120,13 +120,7 @@ int main(int argc, char * argv[]) {
120
120
  generate_data(0.0, test_data.size(), test_data.data());
121
121
  generate_data(1.0, test_data2.size(), test_data2.data());
122
122
 
123
- // Initialize GGML, ensures float conversion tables are initialized
124
- struct ggml_init_params ggml_params = {
125
- /* .mem_size = */ 1*1024,
126
- /* .mem_buffer = */ NULL,
127
- /* .no_alloc = */ true,
128
- };
129
- struct ggml_context * ctx = ggml_init(ggml_params);
123
+ ggml_cpu_init();
130
124
 
131
125
  int num_failed = 0;
132
126
  bool failed = false;
@@ -188,7 +182,5 @@ int main(int argc, char * argv[]) {
188
182
  printf("%d tests failed\n", num_failed);
189
183
  }
190
184
 
191
- ggml_free(ctx);
192
-
193
185
  return num_failed > 0;
194
186
  }
@@ -1,4 +0,0 @@
1
- #pragma once
2
-
3
- #include <llama.h>
4
-
@@ -1,55 +0,0 @@
1
- // Chat support (incl. tool call grammar constraining & output parsing) w/ generic & custom template handlers.
2
-
3
- #pragma once
4
-
5
- #include "common.h"
6
- #include <json.hpp>
7
- #include <optional>
8
- #include <string>
9
- #include <vector>
10
-
11
- using json = nlohmann::ordered_json;
12
-
13
- struct common_chat_inputs {
14
- json messages;
15
- json tools;
16
- json tool_choice;
17
- json json_schema;
18
- bool parallel_tool_calls;
19
- bool stream;
20
- std::string grammar;
21
- bool add_generation_prompt = true;
22
- bool extract_reasoning = true;
23
- };
24
-
25
- enum common_chat_format {
26
- COMMON_CHAT_FORMAT_CONTENT_ONLY,
27
- COMMON_CHAT_FORMAT_GENERIC,
28
- COMMON_CHAT_FORMAT_MISTRAL_NEMO,
29
- COMMON_CHAT_FORMAT_LLAMA_3_X,
30
- COMMON_CHAT_FORMAT_LLAMA_3_X_WITH_BUILTIN_TOOLS,
31
- COMMON_CHAT_FORMAT_DEEPSEEK_R1,
32
- COMMON_CHAT_FORMAT_DEEPSEEK_R1_EXTRACT_REASONING,
33
- COMMON_CHAT_FORMAT_FIREFUNCTION_V2,
34
- COMMON_CHAT_FORMAT_FUNCTIONARY_V3_2,
35
- COMMON_CHAT_FORMAT_FUNCTIONARY_V3_1_LLAMA_3_1,
36
- COMMON_CHAT_FORMAT_HERMES_2_PRO,
37
- COMMON_CHAT_FORMAT_COMMAND_R7B,
38
- COMMON_CHAT_FORMAT_COMMAND_R7B_EXTRACT_REASONING,
39
-
40
- COMMON_CHAT_FORMAT_COUNT, // Not a format, just the # formats
41
- };
42
-
43
- struct common_chat_params {
44
- common_chat_format format = COMMON_CHAT_FORMAT_CONTENT_ONLY;
45
- json prompt;
46
- std::string grammar;
47
- bool grammar_lazy = false;
48
- std::vector<common_grammar_trigger> grammar_triggers;
49
- std::vector<std::string> preserved_tokens;
50
- std::vector<std::string> additional_stops;
51
- };
52
-
53
- struct common_chat_params common_chat_params_init(const common_chat_template & tmpl, const struct common_chat_inputs & params);
54
- std::string common_chat_format_name(common_chat_format format);
55
- common_chat_msg common_chat_parse( const std::string & input, common_chat_format format);