@delali/narsil-react-native 0.3.1-next.155 → 0.3.1-next.156

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (253) hide show
  1. package/NOTICE +144 -0
  2. package/NarsilReactNative.podspec +2 -2
  3. package/android/CMakeLists.txt +1 -1
  4. package/cpp/HybridNarsilCoreAnalysis.cpp +277 -0
  5. package/cpp/HybridNarsilCoreImpl.hpp +43 -0
  6. package/cpp/HybridNarsilCoreKeyword.cpp +253 -0
  7. package/cpp/HybridNarsilCoreMath.cpp +47 -0
  8. package/cpp/core/include/narsil_analysis.h +86 -0
  9. package/cpp/core/include/narsil_core.h +1 -1
  10. package/cpp/core/include/narsil_keyword.h +110 -0
  11. package/cpp/core/include/narsil_math.h +36 -0
  12. package/cpp/core/src/analysis/analyse.c +281 -0
  13. package/cpp/core/src/analysis/analyser.h +49 -0
  14. package/cpp/core/src/analysis/languages.c +79 -0
  15. package/cpp/core/src/analysis/languages.h +42 -0
  16. package/cpp/core/src/analysis/normalise.c +242 -0
  17. package/cpp/core/src/analysis/normalise.h +28 -0
  18. package/cpp/core/src/analysis/split.c +183 -0
  19. package/cpp/core/src/analysis/split.h +27 -0
  20. package/cpp/core/src/analysis/stop_words.c +82 -0
  21. package/cpp/core/src/analysis/stop_words.h +19 -0
  22. package/cpp/core/src/analysis/term_cache.c +88 -0
  23. package/cpp/core/src/analysis/term_cache.h +36 -0
  24. package/cpp/core/src/analysis/text.c +161 -0
  25. package/cpp/core/src/analysis/text.h +45 -0
  26. package/cpp/core/src/analysis/transform.c +103 -0
  27. package/cpp/core/src/analysis/unicode.c +252 -0
  28. package/cpp/core/src/analysis/unicode.h +54 -0
  29. package/cpp/core/src/internal.h +4 -0
  30. package/cpp/core/src/keyword/pass.h +117 -0
  31. package/cpp/core/src/keyword/prune.c +324 -0
  32. package/cpp/core/src/keyword/request.c +179 -0
  33. package/cpp/core/src/keyword/rows.c +83 -0
  34. package/cpp/core/src/keyword/rows.h +18 -0
  35. package/cpp/core/src/keyword/score.c +257 -0
  36. package/cpp/core/src/keyword/score.h +17 -0
  37. package/cpp/core/src/keyword/scratch.c +226 -0
  38. package/cpp/core/src/keyword/scratch.h +55 -0
  39. package/cpp/core/src/keyword/select.c +155 -0
  40. package/cpp/core/src/keyword/select.h +17 -0
  41. package/cpp/core/src/languages/bulgarian_stemmer.c +114 -0
  42. package/cpp/core/src/languages/data/albanian.c +54 -0
  43. package/cpp/core/src/languages/data/amharic.c +93 -0
  44. package/cpp/core/src/languages/data/arabic.c +146 -0
  45. package/cpp/core/src/languages/data/armenian.c +46 -0
  46. package/cpp/core/src/languages/data/azerbaijani.c +68 -0
  47. package/cpp/core/src/languages/data/bambara.c +48 -0
  48. package/cpp/core/src/languages/data/basque.c +62 -0
  49. package/cpp/core/src/languages/data/belarusian.c +56 -0
  50. package/cpp/core/src/languages/data/bengali.c +63 -0
  51. package/cpp/core/src/languages/data/bosnian.c +73 -0
  52. package/cpp/core/src/languages/data/breton.c +52 -0
  53. package/cpp/core/src/languages/data/bulgarian.c +144 -0
  54. package/cpp/core/src/languages/data/burmese.c +48 -0
  55. package/cpp/core/src/languages/data/catalan.c +84 -0
  56. package/cpp/core/src/languages/data/chinese.c +70 -0
  57. package/cpp/core/src/languages/data/croatian.c +72 -0
  58. package/cpp/core/src/languages/data/czech.c +72 -0
  59. package/cpp/core/src/languages/data/dagbani.c +88 -0
  60. package/cpp/core/src/languages/data/danish.c +56 -0
  61. package/cpp/core/src/languages/data/declarations.h +114 -0
  62. package/cpp/core/src/languages/data/dutch.c +58 -0
  63. package/cpp/core/src/languages/data/english.c +172 -0
  64. package/cpp/core/src/languages/data/esperanto.c +71 -0
  65. package/cpp/core/src/languages/data/estonian.c +417 -0
  66. package/cpp/core/src/languages/data/ewe.c +89 -0
  67. package/cpp/core/src/languages/data/faroese.c +54 -0
  68. package/cpp/core/src/languages/data/fijian.c +44 -0
  69. package/cpp/core/src/languages/data/finnish.c +100 -0
  70. package/cpp/core/src/languages/data/french.c +74 -0
  71. package/cpp/core/src/languages/data/ga.c +83 -0
  72. package/cpp/core/src/languages/data/galician.c +70 -0
  73. package/cpp/core/src/languages/data/georgian.c +57 -0
  74. package/cpp/core/src/languages/data/german.c +96 -0
  75. package/cpp/core/src/languages/data/greek.c +202 -0
  76. package/cpp/core/src/languages/data/guarani.c +110 -0
  77. package/cpp/core/src/languages/data/gujarati.c +90 -0
  78. package/cpp/core/src/languages/data/haitian-creole.c +54 -0
  79. package/cpp/core/src/languages/data/hausa.c +85 -0
  80. package/cpp/core/src/languages/data/hawaiian.c +50 -0
  81. package/cpp/core/src/languages/data/hebrew.c +85 -0
  82. package/cpp/core/src/languages/data/hindi.c +91 -0
  83. package/cpp/core/src/languages/data/hungarian.c +98 -0
  84. package/cpp/core/src/languages/data/icelandic.c +73 -0
  85. package/cpp/core/src/languages/data/igbo.c +90 -0
  86. package/cpp/core/src/languages/data/indonesian.c +296 -0
  87. package/cpp/core/src/languages/data/irish.c +60 -0
  88. package/cpp/core/src/languages/data/italian.c +107 -0
  89. package/cpp/core/src/languages/data/japanese.c +52 -0
  90. package/cpp/core/src/languages/data/kannada.c +60 -0
  91. package/cpp/core/src/languages/data/kazakh.c +57 -0
  92. package/cpp/core/src/languages/data/khmer.c +71 -0
  93. package/cpp/core/src/languages/data/kinyarwanda.c +54 -0
  94. package/cpp/core/src/languages/data/kirundi.c +53 -0
  95. package/cpp/core/src/languages/data/korean.c +74 -0
  96. package/cpp/core/src/languages/data/kurmanji.c +43 -0
  97. package/cpp/core/src/languages/data/kyrgyz.c +65 -0
  98. package/cpp/core/src/languages/data/lao.c +46 -0
  99. package/cpp/core/src/languages/data/latin.c +46 -0
  100. package/cpp/core/src/languages/data/latvian.c +75 -0
  101. package/cpp/core/src/languages/data/lingala.c +88 -0
  102. package/cpp/core/src/languages/data/lithuanian.c +65 -0
  103. package/cpp/core/src/languages/data/luxembourgish.c +80 -0
  104. package/cpp/core/src/languages/data/macedonian.c +286 -0
  105. package/cpp/core/src/languages/data/malagasy.c +51 -0
  106. package/cpp/core/src/languages/data/malay.c +184 -0
  107. package/cpp/core/src/languages/data/malayalam.c +64 -0
  108. package/cpp/core/src/languages/data/maltese.c +102 -0
  109. package/cpp/core/src/languages/data/maori.c +52 -0
  110. package/cpp/core/src/languages/data/marathi.c +62 -0
  111. package/cpp/core/src/languages/data/nepali.c +72 -0
  112. package/cpp/core/src/languages/data/norwegian.c +74 -0
  113. package/cpp/core/src/languages/data/oromo.c +51 -0
  114. package/cpp/core/src/languages/data/persian.c +150 -0
  115. package/cpp/core/src/languages/data/polish.c +111 -0
  116. package/cpp/core/src/languages/data/portuguese.c +91 -0
  117. package/cpp/core/src/languages/data/punjabi.c +80 -0
  118. package/cpp/core/src/languages/data/registry.c +116 -0
  119. package/cpp/core/src/languages/data/romanian.c +93 -0
  120. package/cpp/core/src/languages/data/russian.c +76 -0
  121. package/cpp/core/src/languages/data/samoan.c +47 -0
  122. package/cpp/core/src/languages/data/sanskrit.c +53 -0
  123. package/cpp/core/src/languages/data/scottish-gaelic.c +119 -0
  124. package/cpp/core/src/languages/data/serbian.c +84 -0
  125. package/cpp/core/src/languages/data/shona.c +55 -0
  126. package/cpp/core/src/languages/data/sinhala.c +78 -0
  127. package/cpp/core/src/languages/data/slovak.c +135 -0
  128. package/cpp/core/src/languages/data/slovenian.c +139 -0
  129. package/cpp/core/src/languages/data/sorani.c +55 -0
  130. package/cpp/core/src/languages/data/spanish.c +84 -0
  131. package/cpp/core/src/languages/data/swahili.c +59 -0
  132. package/cpp/core/src/languages/data/swedish.c +60 -0
  133. package/cpp/core/src/languages/data/tagalog.c +75 -0
  134. package/cpp/core/src/languages/data/tamil.c +69 -0
  135. package/cpp/core/src/languages/data/tatar.c +548 -0
  136. package/cpp/core/src/languages/data/telugu.c +54 -0
  137. package/cpp/core/src/languages/data/thai.c +46 -0
  138. package/cpp/core/src/languages/data/tibetan.c +55 -0
  139. package/cpp/core/src/languages/data/tigrinya.c +73 -0
  140. package/cpp/core/src/languages/data/tongan.c +47 -0
  141. package/cpp/core/src/languages/data/turkish.c +100 -0
  142. package/cpp/core/src/languages/data/twi.c +88 -0
  143. package/cpp/core/src/languages/data/ukrainian.c +99 -0
  144. package/cpp/core/src/languages/data/urdu.c +174 -0
  145. package/cpp/core/src/languages/data/vietnamese.c +88 -0
  146. package/cpp/core/src/languages/data/welsh.c +143 -0
  147. package/cpp/core/src/languages/data/wolof.c +51 -0
  148. package/cpp/core/src/languages/data/xhosa.c +56 -0
  149. package/cpp/core/src/languages/data/yoruba.c +93 -0
  150. package/cpp/core/src/languages/data/zulu.c +58 -0
  151. package/cpp/core/src/languages/marks.c +64 -0
  152. package/cpp/core/src/languages/rules.c +63 -0
  153. package/cpp/core/src/languages/rules.h +68 -0
  154. package/cpp/core/src/languages/sanskrit_stemmer.c +50 -0
  155. package/cpp/core/src/languages/scripts.c +143 -0
  156. package/cpp/core/src/languages/slovenian_stemmer.c +46 -0
  157. package/cpp/core/src/languages/snowball/api.c +52 -0
  158. package/cpp/core/src/languages/snowball/api.h +43 -0
  159. package/cpp/core/src/languages/snowball/arabic_stemmer.c +1506 -0
  160. package/cpp/core/src/languages/snowball/arabic_stemmer.h +24 -0
  161. package/cpp/core/src/languages/snowball/armenian_stemmer.c +523 -0
  162. package/cpp/core/src/languages/snowball/armenian_stemmer.h +24 -0
  163. package/cpp/core/src/languages/snowball/basque_stemmer.c +1089 -0
  164. package/cpp/core/src/languages/snowball/basque_stemmer.h +24 -0
  165. package/cpp/core/src/languages/snowball/catalan_stemmer.c +1380 -0
  166. package/cpp/core/src/languages/snowball/catalan_stemmer.h +24 -0
  167. package/cpp/core/src/languages/snowball/czech_stemmer.c +582 -0
  168. package/cpp/core/src/languages/snowball/czech_stemmer.h +24 -0
  169. package/cpp/core/src/languages/snowball/danish_stemmer.c +352 -0
  170. package/cpp/core/src/languages/snowball/danish_stemmer.h +24 -0
  171. package/cpp/core/src/languages/snowball/dutch_stemmer.c +1657 -0
  172. package/cpp/core/src/languages/snowball/dutch_stemmer.h +24 -0
  173. package/cpp/core/src/languages/snowball/english_stemmer.c +1019 -0
  174. package/cpp/core/src/languages/snowball/english_stemmer.h +24 -0
  175. package/cpp/core/src/languages/snowball/esperanto_stemmer.c +716 -0
  176. package/cpp/core/src/languages/snowball/esperanto_stemmer.h +24 -0
  177. package/cpp/core/src/languages/snowball/estonian_stemmer.c +1361 -0
  178. package/cpp/core/src/languages/snowball/estonian_stemmer.h +24 -0
  179. package/cpp/core/src/languages/snowball/finnish_stemmer.c +800 -0
  180. package/cpp/core/src/languages/snowball/finnish_stemmer.h +24 -0
  181. package/cpp/core/src/languages/snowball/french_stemmer.c +1217 -0
  182. package/cpp/core/src/languages/snowball/french_stemmer.h +24 -0
  183. package/cpp/core/src/languages/snowball/german_stemmer.c +567 -0
  184. package/cpp/core/src/languages/snowball/german_stemmer.h +24 -0
  185. package/cpp/core/src/languages/snowball/greek_stemmer.c +3450 -0
  186. package/cpp/core/src/languages/snowball/greek_stemmer.h +24 -0
  187. package/cpp/core/src/languages/snowball/hindi_stemmer.c +325 -0
  188. package/cpp/core/src/languages/snowball/hindi_stemmer.h +24 -0
  189. package/cpp/core/src/languages/snowball/hungarian_stemmer.c +742 -0
  190. package/cpp/core/src/languages/snowball/hungarian_stemmer.h +24 -0
  191. package/cpp/core/src/languages/snowball/indonesian_stemmer.c +443 -0
  192. package/cpp/core/src/languages/snowball/indonesian_stemmer.h +24 -0
  193. package/cpp/core/src/languages/snowball/irish_stemmer.c +428 -0
  194. package/cpp/core/src/languages/snowball/irish_stemmer.h +24 -0
  195. package/cpp/core/src/languages/snowball/italian_stemmer.c +983 -0
  196. package/cpp/core/src/languages/snowball/italian_stemmer.h +24 -0
  197. package/cpp/core/src/languages/snowball/lithuanian_stemmer.c +794 -0
  198. package/cpp/core/src/languages/snowball/lithuanian_stemmer.h +24 -0
  199. package/cpp/core/src/languages/snowball/nepali_stemmer.c +361 -0
  200. package/cpp/core/src/languages/snowball/nepali_stemmer.h +24 -0
  201. package/cpp/core/src/languages/snowball/norwegian_stemmer.c +332 -0
  202. package/cpp/core/src/languages/snowball/norwegian_stemmer.h +24 -0
  203. package/cpp/core/src/languages/snowball/persian_stemmer.c +538 -0
  204. package/cpp/core/src/languages/snowball/persian_stemmer.h +24 -0
  205. package/cpp/core/src/languages/snowball/polish_stemmer.c +501 -0
  206. package/cpp/core/src/languages/snowball/polish_stemmer.h +24 -0
  207. package/cpp/core/src/languages/snowball/portuguese_stemmer.c +866 -0
  208. package/cpp/core/src/languages/snowball/portuguese_stemmer.h +24 -0
  209. package/cpp/core/src/languages/snowball/romanian_stemmer.c +940 -0
  210. package/cpp/core/src/languages/snowball/romanian_stemmer.h +24 -0
  211. package/cpp/core/src/languages/snowball/russian_stemmer.c +598 -0
  212. package/cpp/core/src/languages/snowball/russian_stemmer.h +24 -0
  213. package/cpp/core/src/languages/snowball/serbian_stemmer.c +6765 -0
  214. package/cpp/core/src/languages/snowball/serbian_stemmer.h +24 -0
  215. package/cpp/core/src/languages/snowball/snowball_runtime.h +122 -0
  216. package/cpp/core/src/languages/snowball/spanish_stemmer.c +955 -0
  217. package/cpp/core/src/languages/snowball/spanish_stemmer.h +24 -0
  218. package/cpp/core/src/languages/snowball/swedish_stemmer.c +364 -0
  219. package/cpp/core/src/languages/snowball/swedish_stemmer.h +24 -0
  220. package/cpp/core/src/languages/snowball/tamil_stemmer.c +1305 -0
  221. package/cpp/core/src/languages/snowball/tamil_stemmer.h +24 -0
  222. package/cpp/core/src/languages/snowball/turkish_stemmer.c +1700 -0
  223. package/cpp/core/src/languages/snowball/turkish_stemmer.h +24 -0
  224. package/cpp/core/src/languages/snowball/utilities.c +716 -0
  225. package/cpp/core/src/languages/substitutions.c +131 -0
  226. package/cpp/core/src/languages/swahili_stemmer.c +85 -0
  227. package/cpp/core/src/languages/ukrainian_stemmer.c +168 -0
  228. package/cpp/core/src/locks.c +2 -15
  229. package/cpp/core/src/math/apply.c +44 -0
  230. package/cpp/core/src/math/fdlibm.h +16 -0
  231. package/cpp/core/src/math/inverse.c +242 -0
  232. package/cpp/core/src/math/log.c +90 -0
  233. package/cpp/core/src/math/reduce.c +325 -0
  234. package/cpp/core/src/math/trig.c +188 -0
  235. package/cpp/core/src/math/words.h +45 -0
  236. package/cpp/core/src/thread_yield.h +18 -0
  237. package/cpp/core/src/unicode/decompositions.c +310 -0
  238. package/cpp/core/src/unicode/lowercase.c +172 -0
  239. package/cpp/core/src/unicode/properties.c +214 -0
  240. package/cpp/core/src/workspace.c +5 -1
  241. package/lib/engine/analysis.d.ts +3 -0
  242. package/lib/engine/analysis.js +76 -0
  243. package/lib/engine/core.js +5 -0
  244. package/lib/engine/keyword.d.ts +3 -0
  245. package/lib/engine/keyword.js +25 -0
  246. package/lib/specs/core.nitro.d.ts +1 -0
  247. package/nitrogen/generated/shared/c++/HybridNarsilCoreSpec.cpp +1 -0
  248. package/nitrogen/generated/shared/c++/HybridNarsilCoreSpec.hpp +1 -0
  249. package/package.json +4 -3
  250. package/src/engine/analysis.ts +87 -0
  251. package/src/engine/core.ts +6 -0
  252. package/src/engine/keyword.ts +26 -0
  253. package/src/specs/core.nitro.ts +1 -0
@@ -0,0 +1,85 @@
1
+ #include "../../analysis/languages.h"
2
+ #include "declarations.h"
3
+ #include "../rules.h"
4
+
5
+ #include <stddef.h>
6
+ #include <stdint.h>
7
+
8
+ static const uint16_t stop_word_units[731] = {
9
+ 1488, 1489, 1500, 1488, 1493, 1488, 1493, 1500, 1497, 1488, 1493, 1514, 1492, 1488, 1493, 1514, 1493, 1488, 1493,
10
+ 1514, 1497, 1488, 1493, 1514, 1498, 1488, 1493, 1514, 1501, 1488, 1493, 1514, 1503, 1488, 1493, 1514, 1504, 1493,
11
+ 1488, 1494, 1488, 1495, 1512, 1488, 1495, 1512, 1493, 1514, 1488, 1495, 1512, 1497, 1488, 1495, 1512, 1497, 1499,
12
+ 1503, 1488, 1495, 1512, 1497, 1501, 1488, 1495, 1512, 1514, 1488, 1497, 1488, 1497, 1494, 1492, 1488, 1497, 1498,
13
+ 1488, 1497, 1503, 1488, 1497, 1508, 1492, 1488, 1497, 1514, 1492, 1488, 1497, 1514, 1493, 1488, 1497, 1514, 1497,
14
+ 1488, 1497, 1514, 1498, 1488, 1497, 1514, 1499, 1501, 1488, 1497, 1514, 1499, 1503, 1488, 1497, 1514, 1501, 1488,
15
+ 1497, 1514, 1503, 1488, 1497, 1514, 1504, 1493, 1488, 1498, 1488, 1500, 1488, 1500, 1492, 1488, 1500, 1493, 1488,
16
+ 1501, 1488, 1504, 1495, 1504, 1493, 1488, 1504, 1497, 1488, 1505, 1488, 1507, 1488, 1510, 1500, 1488, 1513, 1512,
17
+ 1488, 1514, 1488, 1514, 1492, 1488, 1514, 1499, 1501, 1488, 1514, 1499, 1503, 1488, 1514, 1501, 1488, 1514, 1503,
18
+ 1489, 1488, 1497, 1494, 1493, 1502, 1497, 1491, 1492, 1489, 1488, 1502, 1510, 1506, 1489, 1488, 1502, 1510, 1506,
19
+ 1493, 1514, 1489, 1490, 1500, 1500, 1489, 1497, 1503, 1489, 1500, 1497, 1489, 1502, 1497, 1491, 1492, 1489, 1502,
20
+ 1511, 1493, 1501, 1513, 1489, 1493, 1489, 1512, 1501, 1489, 1513, 1489, 1497, 1500, 1489, 1513, 1506, 1492, 1513,
21
+ 1489, 1514, 1493, 1498, 1490, 1501, 1491, 1512, 1498, 1492, 1493, 1488, 1492, 1497, 1488, 1492, 1497, 1492, 1492,
22
+ 1497, 1499, 1503, 1492, 1497, 1514, 1492, 1492, 1497, 1514, 1497, 1492, 1501, 1492, 1503, 1492, 1504, 1492, 1492,
23
+ 1505, 1497, 1489, 1492, 1513, 1489, 1490, 1500, 1500, 1492, 1492, 1512, 1497, 1493, 1488, 1497, 1500, 1493, 1493,
24
+ 1488, 1514, 1494, 1488, 1514, 1494, 1492, 1494, 1493, 1514, 1497, 1492, 1497, 1492, 1497, 1493, 1499, 1500, 1497,
25
+ 1493, 1499, 1500, 1493, 1497, 1493, 1514, 1512, 1502, 1491, 1497, 1497, 1499, 1493, 1500, 1497, 1499, 1493, 1500,
26
+ 1492, 1497, 1499, 1493, 1500, 1493, 1514, 1497, 1499, 1493, 1500, 1497, 1501, 1497, 1499, 1500, 1497, 1499, 1500,
27
+ 1492, 1497, 1499, 1500, 1493, 1497, 1513, 1499, 1488, 1503, 1499, 1488, 1513, 1512, 1499, 1493, 1500, 1501, 1499,
28
+ 1493, 1500, 1503, 1499, 1494, 1492, 1499, 1497, 1499, 1497, 1510, 1491, 1499, 1498, 1499, 1499, 1492, 1499, 1500,
29
+ 1499, 1500, 1500, 1499, 1502, 1493, 1499, 1503, 1499, 1508, 1497, 1499, 1513, 1500, 1488, 1500, 1488, 1493, 1500,
30
+ 1488, 1497, 1494, 1493, 1514, 1499, 1500, 1497, 1514, 1500, 1488, 1503, 1500, 1489, 1497, 1503, 1500, 1492, 1500,
31
+ 1492, 1497, 1493, 1514, 1500, 1492, 1501, 1500, 1492, 1503, 1500, 1493, 1500, 1497, 1500, 1499, 1501, 1500, 1499,
32
+ 1503, 1500, 1502, 1492, 1500, 1502, 1496, 1492, 1500, 1502, 1506, 1500, 1492, 1500, 1502, 1511, 1493, 1501, 1513,
33
+ 1489, 1493, 1500, 1502, 1512, 1493, 1514, 1500, 1504, 1493, 1500, 1506, 1489, 1512, 1500, 1506, 1497, 1499, 1503,
34
+ 1500, 1508, 1497, 1499, 1498, 1500, 1508, 1504, 1497, 1502, 1488, 1491, 1502, 1488, 1495, 1493, 1512, 1497, 1502,
35
+ 1488, 1497, 1494, 1493, 1505, 1497, 1489, 1492, 1502, 1488, 1497, 1503, 1502, 1488, 1497, 1508, 1492, 1502, 1489,
36
+ 1500, 1497, 1502, 1489, 1506, 1491, 1502, 1491, 1493, 1506, 1502, 1492, 1502, 1492, 1497, 1499, 1503, 1502, 1493,
37
+ 1500, 1502, 1495, 1493, 1509, 1502, 1497, 1502, 1499, 1488, 1503, 1502, 1499, 1497, 1493, 1493, 1503, 1502, 1500,
38
+ 1489, 1491, 1502, 1503, 1502, 1504, 1497, 1503, 1502, 1505, 1493, 1490, 1500, 1502, 1506, 1496, 1502, 1506, 1496,
39
+ 1497, 1501, 1502, 1506, 1500, 1502, 1510, 1491, 1502, 1511, 1493, 1501, 1489, 1493, 1502, 1514, 1495, 1514, 1502,
40
+ 1514, 1497, 1504, 1490, 1491, 1504, 1490, 1512, 1504, 1493, 1506, 1491, 1506, 1494, 1506, 1500, 1506, 1500, 1497,
41
+ 1506, 1500, 1497, 1492, 1506, 1500, 1497, 1492, 1501, 1506, 1500, 1497, 1492, 1503, 1506, 1500, 1497, 1493, 1506,
42
+ 1500, 1497, 1498, 1506, 1500, 1497, 1499, 1501, 1506, 1500, 1497, 1504, 1493, 1506, 1501, 1506, 1510, 1502, 1492,
43
+ 1506, 1510, 1502, 1492, 1501, 1506, 1510, 1502, 1492, 1503, 1506, 1510, 1502, 1493, 1506, 1510, 1502, 1497, 1506,
44
+ 1510, 1502, 1501, 1506, 1510, 1502, 1503, 1506, 1510, 1502, 1504, 1493, 1508, 1492, 1512, 1511, 1513, 1493, 1489,
45
+ 1513, 1500, 1513, 1500, 1492, 1513, 1500, 1492, 1501, 1513, 1500, 1492, 1503, 1513, 1500, 1493, 1513, 1500, 1497,
46
+ 1513, 1500, 1498, 1513, 1500, 1499, 1492, 1513, 1500, 1499, 1501, 1513, 1500, 1499, 1503, 1513, 1500, 1504, 1493,
47
+ 1513, 1501, 1514, 1492, 1497, 1492, 1514, 1495, 1514,
48
+ };
49
+
50
+ static const uint32_t stop_word_offsets[195] = {
51
+ 0, 3, 5, 9, 13, 17, 21, 25, 29, 33, 38, 40, 43, 48, 52, 58, 63, 67, 69, 73, 76, 79, 83, 87, 91, 95, 99, 104, 109, 113,
52
+ 117, 122, 124, 126, 129, 132, 134, 139, 142, 144, 146, 149, 152, 154, 157, 161, 165, 168, 171, 180, 185, 192, 196,
53
+ 199, 202, 207, 215, 218, 223, 228, 232, 234, 237, 240, 243, 246, 250, 254, 258, 260, 262, 265, 276, 279, 284, 287,
54
+ 290, 292, 295, 299, 303, 308, 315, 319, 324, 330, 336, 339, 343, 347, 349, 352, 356, 360, 364, 367, 369, 373, 375,
55
+ 378, 380, 383, 386, 388, 391, 393, 395, 398, 408, 411, 415, 417, 422, 425, 428, 430, 432, 435, 438, 441, 445, 450,
56
+ 458, 463, 466, 470, 475, 480, 484, 487, 493, 502, 506, 511, 515, 519, 523, 525, 530, 533, 537, 539, 543, 549, 553,
57
+ 555, 559, 564, 567, 572, 575, 578, 584, 588, 591, 594, 597, 599, 601, 603, 605, 608, 612, 617, 622, 626, 630, 635,
58
+ 640, 642, 646, 651, 656, 660, 664, 668, 672, 677, 679, 681, 684, 686, 689, 693, 697, 700, 703, 706, 710, 714, 718,
59
+ 722, 724, 728, 731,
60
+ };
61
+
62
+ static const uint32_t keep_ranges[14] = {
63
+ 48, 57, 65, 90, 97, 122, 1425, 1469, 1473, 1474, 1479, 1514, 1520, 1524,
64
+ };
65
+
66
+ const narsil_language narsil_language_hebrew = {
67
+ .name = "hebrew",
68
+ .revision = "1",
69
+ .stop_word_units = stop_word_units,
70
+ .stop_word_offsets = stop_word_offsets,
71
+ .stop_word_count = 194,
72
+ .ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
73
+ .keep_ranges = keep_ranges,
74
+ .keep_range_count = 7,
75
+ .splits_code_points = 0,
76
+ .strips_possessive = 0,
77
+ .normalises_diacritics = 0,
78
+ .min_token_length = 1,
79
+ .ngram_size = 0,
80
+ .normalise = narsil_hebrew_normalise,
81
+ .stem_by_rules = NULL,
82
+ .create_snowball = NULL,
83
+ .snowball_stem = NULL,
84
+ .close_snowball = NULL,
85
+ };
@@ -0,0 +1,91 @@
1
+ #include "../../analysis/languages.h"
2
+ #include "declarations.h"
3
+ #include "../rules.h"
4
+ #include "../snowball/hindi_stemmer.h"
5
+
6
+ #include <stddef.h>
7
+ #include <stdint.h>
8
+
9
+ static const uint16_t stop_word_units[815] = {
10
+ 2309, 2306, 2342, 2352, 2309, 2340, 2309, 2342, 2367, 2309, 2346, 2309, 2346, 2344, 2366, 2309, 2346, 2344, 2367,
11
+ 2309, 2346, 2344, 2368, 2309, 2346, 2344, 2375, 2309, 2349, 2367, 2309, 2349, 2368, 2310, 2342, 2367, 2310, 2346,
12
+ 2311, 2306, 2361, 2367, 2306, 2311, 2306, 2361, 2375, 2306, 2311, 2306, 2361, 2379, 2306, 2311, 2340, 2351, 2366,
13
+ 2342, 2367, 2311, 2340, 2381, 2351, 2366, 2342, 2367, 2311, 2344, 2311, 2344, 2325, 2366, 2311, 2344, 2381, 2361,
14
+ 2368, 2306, 2311, 2344, 2381, 2361, 2375, 2306, 2311, 2344, 2381, 2361, 2379, 2306, 2311, 2360, 2311, 2360, 2325,
15
+ 2366, 2311, 2360, 2325, 2367, 2311, 2360, 2325, 2368, 2311, 2360, 2325, 2375, 2311, 2360, 2350, 2375, 2306, 2311,
16
+ 2360, 2367, 2311, 2360, 2368, 2311, 2360, 2375, 2313, 2306, 2361, 2367, 2306, 2313, 2306, 2361, 2375, 2306, 2313,
17
+ 2306, 2361, 2379, 2306, 2313, 2344, 2313, 2344, 2325, 2366, 2313, 2344, 2325, 2367, 2313, 2344, 2325, 2368, 2313,
18
+ 2344, 2325, 2375, 2313, 2344, 2325, 2379, 2313, 2344, 2381, 2361, 2368, 2306, 2313, 2344, 2381, 2361, 2375, 2306,
19
+ 2313, 2344, 2381, 2361, 2379, 2306, 2313, 2360, 2313, 2360, 2325, 2375, 2313, 2360, 2367, 2313, 2360, 2368, 2313,
20
+ 2360, 2375, 2319, 2325, 2319, 2357, 2306, 2319, 2360, 2319, 2360, 2375, 2320, 2360, 2375, 2323, 2352, 2324, 2352,
21
+ 2325, 2311, 2325, 2312, 2325, 2352, 2325, 2352, 2340, 2366, 2325, 2352, 2340, 2375, 2325, 2352, 2344, 2366, 2325,
22
+ 2352, 2344, 2375, 2325, 2352, 2375, 2306, 2325, 2361, 2340, 2375, 2325, 2361, 2366, 2325, 2366, 2325, 2366, 2347,
23
+ 2364, 2368, 2325, 2366, 2347, 2367, 2325, 2367, 2325, 2367, 2306, 2361, 2375, 2306, 2325, 2367, 2306, 2361, 2379,
24
+ 2306, 2325, 2367, 2340, 2344, 2366, 2325, 2367, 2344, 2381, 2361, 2375, 2306, 2325, 2367, 2344, 2381, 2361, 2379,
25
+ 2306, 2325, 2367, 2351, 2366, 2325, 2367, 2352, 2325, 2367, 2360, 2325, 2367, 2360, 2367, 2325, 2367, 2360, 2368,
26
+ 2325, 2367, 2360, 2375, 2325, 2368, 2325, 2369, 2331, 2325, 2369, 2354, 2325, 2375, 2325, 2379, 2325, 2379, 2311,
27
+ 2325, 2379, 2312, 2325, 2379, 2344, 2325, 2379, 2344, 2360, 2366, 2325, 2380, 2344, 2325, 2380, 2344, 2360, 2366,
28
+ 2327, 2351, 2366, 2328, 2352, 2332, 2348, 2332, 2361, 2366, 2305, 2332, 2361, 2366, 2306, 2332, 2366, 2332, 2367,
29
+ 2306, 2361, 2375, 2306, 2332, 2367, 2306, 2361, 2379, 2306, 2332, 2367, 2340, 2344, 2366, 2332, 2367, 2343, 2352,
30
+ 2332, 2367, 2344, 2332, 2367, 2344, 2381, 2361, 2375, 2306, 2332, 2367, 2344, 2381, 2361, 2379, 2306, 2332, 2367,
31
+ 2360, 2332, 2367, 2360, 2375, 2332, 2368, 2343, 2352, 2332, 2375, 2360, 2366, 2332, 2375, 2360, 2375, 2332, 2376,
32
+ 2360, 2366, 2332, 2376, 2360, 2375, 2332, 2379, 2340, 2325, 2340, 2348, 2340, 2352, 2361, 2340, 2367, 2306, 2361,
33
+ 2375, 2306, 2340, 2367, 2306, 2361, 2379, 2306, 2340, 2367, 2344, 2340, 2367, 2344, 2381, 2361, 2375, 2306, 2340,
34
+ 2367, 2344, 2381, 2361, 2379, 2306, 2340, 2367, 2360, 2340, 2367, 2360, 2375, 2340, 2379, 2341, 2366, 2341, 2367,
35
+ 2341, 2368, 2341, 2375, 2342, 2348, 2366, 2352, 2366, 2342, 2357, 2366, 2352, 2366, 2342, 2367, 2351, 2366, 2342,
36
+ 2369, 2360, 2352, 2366, 2342, 2369, 2360, 2352, 2375, 2342, 2370, 2360, 2352, 2375, 2342, 2379, 2342, 2381, 2357,
37
+ 2366, 2352, 2366, 2344, 2344, 2361, 2367, 2306, 2344, 2361, 2368, 2306, 2344, 2366, 2344, 2367, 2330, 2375, 2344,
38
+ 2367, 2361, 2366, 2351, 2340, 2344, 2368, 2330, 2375, 2344, 2375, 2346, 2352, 2346, 2361, 2354, 2375, 2346, 2369,
39
+ 2352, 2366, 2346, 2370, 2352, 2366, 2346, 2375, 2347, 2367, 2352, 2348, 2344, 2367, 2348, 2344, 2368, 2348, 2361,
40
+ 2367, 2348, 2361, 2368, 2348, 2361, 2369, 2340, 2348, 2366, 2342, 2348, 2366, 2354, 2366, 2348, 2367, 2354, 2325,
41
+ 2369, 2354, 2349, 2367, 2349, 2367, 2340, 2352, 2349, 2368, 2349, 2368, 2340, 2352, 2350, 2327, 2352, 2350, 2366,
42
+ 2344, 2379, 2350, 2375, 2350, 2375, 2306, 2351, 2342, 2367, 2351, 2361, 2351, 2361, 2366, 2305, 2351, 2361, 2366,
43
+ 2306, 2351, 2361, 2367, 2351, 2361, 2368, 2351, 2366, 2351, 2367, 2361, 2351, 2375, 2352, 2326, 2375, 2306, 2352,
44
+ 2357, 2366, 2360, 2366, 2352, 2361, 2366, 2352, 2361, 2375, 2353, 2381, 2357, 2366, 2360, 2366, 2354, 2367, 2319,
45
+ 2354, 2367, 2351, 2375, 2354, 2375, 2325, 2367, 2344, 2357, 2357, 2327, 2364, 2376, 2352, 2361, 2357, 2327, 2375,
46
+ 2352, 2361, 2357, 2352, 2327, 2357, 2352, 2381, 2327, 2357, 2361, 2357, 2361, 2366, 2305, 2357, 2361, 2366, 2306,
47
+ 2357, 2361, 2367, 2306, 2357, 2361, 2368, 2306, 2357, 2366, 2354, 2375, 2357, 2369, 2361, 2357, 2375, 2360, 2306,
48
+ 2327, 2360, 2325, 2340, 2366, 2360, 2325, 2340, 2375, 2360, 2348, 2360, 2375, 2360, 2349, 2367, 2360, 2349, 2368,
49
+ 2360, 2366, 2341, 2360, 2366, 2348, 2369, 2340, 2360, 2366, 2349, 2360, 2366, 2352, 2366, 2360, 2375, 2360, 2379,
50
+ 2361, 2367, 2361, 2368, 2361, 2369, 2309, 2361, 2369, 2310, 2361, 2369, 2311, 2361, 2369, 2312, 2361, 2369, 2319,
51
+ 2361, 2375, 2361, 2375, 2306, 2361, 2376, 2361, 2376, 2306, 2361, 2379, 2361, 2379, 2340, 2366, 2361, 2379, 2340,
52
+ 2367, 2361, 2379, 2340, 2368, 2361, 2379, 2340, 2375, 2361, 2379, 2344, 2366, 2361, 2379, 2344, 2375,
53
+ };
54
+
55
+ static const uint32_t stop_word_offsets[226] = {
56
+ 0, 4, 6, 9, 11, 15, 19, 23, 27, 30, 33, 36, 38, 43, 48, 53, 59, 66, 68, 72, 78, 84, 90, 92, 96, 100, 104, 108, 113,
57
+ 116, 119, 122, 127, 132, 137, 139, 143, 147, 151, 155, 159, 165, 171, 177, 179, 183, 186, 189, 192, 194, 197, 199,
58
+ 202, 205, 207, 209, 211, 213, 215, 219, 223, 227, 231, 235, 239, 242, 244, 249, 253, 255, 261, 267, 272, 279, 286,
59
+ 290, 293, 296, 300, 304, 308, 310, 313, 316, 318, 320, 323, 326, 329, 334, 337, 342, 345, 347, 349, 353, 357, 359,
60
+ 365, 371, 376, 380, 383, 390, 397, 400, 404, 408, 412, 416, 420, 424, 426, 428, 430, 433, 439, 445, 448, 455, 462,
61
+ 465, 469, 471, 473, 475, 477, 479, 484, 489, 493, 498, 503, 508, 510, 516, 517, 521, 525, 527, 531, 537, 541, 543,
62
+ 545, 549, 553, 557, 559, 562, 565, 568, 571, 574, 578, 581, 585, 591, 593, 597, 599, 603, 606, 610, 612, 615, 618,
63
+ 620, 624, 628, 631, 634, 636, 639, 641, 645, 650, 653, 656, 662, 665, 669, 674, 675, 681, 686, 689, 693, 695, 699,
64
+ 703, 707, 711, 715, 718, 720, 723, 727, 731, 735, 738, 741, 744, 749, 752, 756, 758, 760, 762, 764, 767, 770, 773,
65
+ 776, 779, 781, 784, 786, 789, 791, 795, 799, 803, 807, 811, 815,
66
+ };
67
+
68
+ static const uint32_t keep_ranges[10] = {
69
+ 48, 57, 65, 90, 97, 122, 2304, 2403, 2406, 2431,
70
+ };
71
+
72
+ const narsil_language narsil_language_hindi = {
73
+ .name = "hindi",
74
+ .revision = "b7e1a82abe08c76a",
75
+ .stop_word_units = stop_word_units,
76
+ .stop_word_offsets = stop_word_offsets,
77
+ .stop_word_count = 225,
78
+ .ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
79
+ .keep_ranges = keep_ranges,
80
+ .keep_range_count = 5,
81
+ .splits_code_points = 0,
82
+ .strips_possessive = 0,
83
+ .normalises_diacritics = 0,
84
+ .min_token_length = 1,
85
+ .ngram_size = 0,
86
+ .normalise = narsil_hindi_normalise,
87
+ .stem_by_rules = NULL,
88
+ .create_snowball = hindi_create_env,
89
+ .snowball_stem = hindi_stem,
90
+ .close_snowball = hindi_close_env,
91
+ };
@@ -0,0 +1,98 @@
1
+ #include "../../analysis/languages.h"
2
+ #include "declarations.h"
3
+ #include "../snowball/hungarian_stemmer.h"
4
+
5
+ #include <stddef.h>
6
+ #include <stdint.h>
7
+
8
+ static const uint16_t stop_word_units[1164] = {
9
+ 97, 97, 98, 98, 97, 97, 98, 98, 97, 110, 97, 98, 98, 243, 108, 97, 100, 100, 105, 103, 97, 104, 104, 111, 122, 97,
10
+ 104, 111, 103, 121, 97, 104, 111, 108, 97, 107, 105, 97, 107, 105, 107, 97, 107, 107, 111, 114, 97, 107, 225, 114, 97,
11
+ 108, 97, 112, 106, 225, 110, 97, 108, 97, 116, 116, 97, 108, 97, 116, 116, 97, 97, 108, 225, 97, 109, 101, 108, 121,
12
+ 97, 109, 101, 108, 121, 101, 107, 97, 109, 101, 108, 121, 101, 116, 97, 109, 101, 108, 121, 105, 107, 97, 109, 101,
13
+ 108, 121, 110, 101, 107, 97, 109, 105, 97, 109, 105, 107, 111, 114, 97, 109, 105, 116, 97, 109, 237, 103, 97, 110,
14
+ 110, 97, 107, 97, 110, 110, 225, 108, 97, 114, 114, 97, 97, 114, 114, 243, 108, 97, 116, 116, 243, 108, 97, 122, 97,
15
+ 122, 111, 107, 97, 122, 111, 107, 97, 116, 97, 122, 111, 110, 97, 122, 111, 110, 98, 97, 110, 97, 122, 116, 97, 122,
16
+ 116, 225, 110, 97, 122, 117, 116, 225, 110, 97, 122, 122, 97, 108, 97, 122, 233, 114, 116, 98, 101, 98, 101, 108, 252,
17
+ 108, 98, 101, 110, 110, 101, 98, 225, 114, 99, 115, 97, 107, 100, 101, 101, 101, 100, 100, 105, 103, 101, 103, 121,
18
+ 101, 103, 121, 101, 100, 252, 108, 101, 103, 121, 101, 115, 101, 103, 121, 101, 116, 108, 101, 110, 101, 103, 121,
19
+ 105, 107, 101, 103, 121, 109, 225, 115, 101, 103, 121, 114, 101, 101, 103, 121, 233, 98, 101, 103, 121, 252, 116, 116,
20
+ 101, 103, 233, 115, 122, 101, 104, 104, 101, 122, 101, 107, 107, 111, 114, 101, 108, 101, 108, 108, 101, 110, 101,
21
+ 108, 233, 103, 101, 108, 337, 101, 108, 337, 98, 98, 101, 108, 337, 115, 122, 246, 114, 101, 108, 337, 116, 116, 101,
22
+ 108, 337, 122, 337, 101, 110, 103, 101, 109, 101, 110, 110, 101, 107, 101, 114, 114, 101, 101, 122, 101, 122, 101,
23
+ 107, 101, 122, 101, 110, 101, 122, 116, 101, 122, 117, 116, 225, 110, 101, 122, 122, 101, 108, 101, 122, 233, 114,
24
+ 116, 102, 101, 108, 102, 101, 108, 233, 104, 97, 104, 97, 110, 101, 109, 104, 105, 115, 122, 101, 110, 104, 111, 103,
25
+ 121, 104, 111, 103, 121, 97, 110, 104, 111, 108, 104, 111, 108, 110, 97, 112, 104, 111, 110, 110, 97, 110, 104, 111,
26
+ 118, 97, 104, 111, 122, 122, 225, 105, 100, 101, 105, 103, 101, 110, 105, 108, 108, 105, 108, 108, 101, 116, 118, 101,
27
+ 105, 108, 121, 101, 110, 105, 109, 109, 225, 114, 105, 110, 107, 225, 98, 98, 105, 115, 105, 115, 109, 233, 116, 105,
28
+ 116, 116, 106, 111, 98, 98, 97, 110, 106, 243, 108, 107, 101, 108, 108, 107, 101, 108, 108, 101, 116, 116, 107, 101,
29
+ 114, 101, 115, 122, 116, 252, 108, 107, 101, 116, 116, 337, 107, 101, 118, 233, 115, 107, 105, 107, 105, 116, 107,
30
+ 246, 122, 98, 101, 110, 107, 246, 122, 101, 108, 107, 246, 122, 233, 107, 246, 122, 246, 116, 116, 107, 246, 122, 252,
31
+ 108, 107, 252, 108, 246, 110, 98, 101, 110, 107, 252, 108, 246, 110, 98, 246, 122, 337, 108, 101, 108, 101, 103, 97,
32
+ 108, 225, 98, 98, 108, 101, 103, 121, 101, 110, 108, 101, 104, 101, 116, 108, 101, 110, 110, 101, 108, 101, 110, 110,
33
+ 105, 108, 101, 115, 122, 108, 101, 116, 116, 109, 97, 109, 97, 103, 97, 109, 97, 103, 225, 116, 109, 97, 105, 109, 97,
34
+ 106, 100, 109, 101, 103, 109, 101, 103, 105, 110, 116, 109, 101, 108, 108, 101, 116, 116, 109, 101, 108, 108, 233,
35
+ 109, 101, 108, 121, 109, 101, 108, 121, 101, 107, 109, 101, 108, 121, 105, 107, 109, 101, 110, 110, 121, 105, 109,
36
+ 101, 114, 116, 109, 105, 109, 105, 97, 116, 116, 109, 105, 107, 111, 114, 109, 105, 108, 121, 101, 110, 109, 105, 110,
37
+ 100, 109, 105, 110, 100, 101, 110, 109, 105, 110, 100, 101, 110, 107, 105, 109, 105, 110, 100, 101, 110, 116, 109,
38
+ 105, 110, 100, 105, 103, 109, 105, 110, 101, 107, 109, 105, 110, 107, 101, 116, 109, 105, 110, 116, 109, 105, 110,
39
+ 116, 104, 97, 109, 105, 116, 109, 105, 118, 101, 108, 109, 105, 233, 114, 116, 109, 111, 110, 100, 116, 97, 109, 111,
40
+ 115, 116, 109, 225, 114, 109, 225, 115, 109, 225, 115, 105, 107, 109, 225, 115, 111, 100, 105, 107, 109, 233, 103,
41
+ 109, 233, 103, 105, 115, 109, 237, 103, 109, 250, 108, 118, 97, 110, 97, 103, 121, 110, 97, 103, 121, 111, 110, 110,
42
+ 101, 110, 101, 107, 101, 100, 110, 101, 107, 101, 109, 110, 101, 107, 105, 110, 101, 107, 105, 107, 110, 101, 107,
43
+ 252, 110, 107, 110, 101, 109, 110, 105, 110, 99, 115, 110, 225, 108, 117, 110, 107, 110, 233, 104, 97, 110, 233, 104,
44
+ 225, 110, 121, 110, 233, 108, 107, 252, 108, 111, 100, 97, 111, 108, 121, 97, 110, 111, 110, 110, 97, 110, 111, 116,
45
+ 116, 112, 101, 100, 105, 103, 112, 101, 114, 115, 122, 101, 114, 97, 106, 116, 97, 114, 243, 108, 97, 115, 97, 106,
46
+ 225, 116, 115, 101, 115, 101, 109, 115, 101, 109, 109, 105, 115, 101, 110, 107, 105, 115, 111, 104, 97, 115, 111, 107,
47
+ 115, 111, 107, 97, 116, 115, 111, 107, 107, 97, 108, 115, 111, 114, 225, 110, 115, 122, 101, 109, 98, 101, 110, 115,
48
+ 122, 101, 114, 105, 110, 116, 115, 122, 105, 110, 116, 101, 115, 122, 225, 109, 225, 114, 97, 115, 337, 116, 116, 97,
49
+ 108, 225, 110, 116, 101, 116, 101, 104, 225, 116, 116, 105, 116, 111, 118, 225, 98, 98, 116, 111, 118, 225, 98, 98,
50
+ 105, 116, 111, 118, 225, 98, 98, 225, 116, 233, 103, 101, 100, 116, 337, 108, 101, 117, 103, 121, 97, 110, 105, 115,
51
+ 117, 103, 121, 101, 117, 116, 225, 110, 117, 116, 225, 110, 97, 118, 97, 103, 121, 118, 97, 103, 121, 105, 115, 118,
52
+ 97, 103, 121, 111, 107, 118, 97, 103, 121, 117, 110, 107, 118, 97, 106, 111, 110, 118, 97, 108, 97, 107, 105, 118, 97,
53
+ 108, 97, 109, 105, 118, 97, 108, 97, 109, 105, 110, 116, 118, 97, 108, 243, 118, 97, 110, 118, 97, 110, 110, 97, 107,
54
+ 118, 101, 108, 101, 118, 101, 108, 101, 109, 118, 101, 108, 252, 110, 107, 118, 105, 115, 115, 122, 97, 118, 105, 115,
55
+ 122, 111, 110, 116, 118, 111, 108, 110, 97, 118, 111, 108, 116, 118, 111, 108, 116, 97, 107, 118, 233, 103, 114, 101,
56
+ 118, 233, 103, 252, 108, 225, 108, 116, 97, 108, 225, 108, 116, 97, 108, 225, 98, 97, 110, 225, 116, 233, 110, 233,
57
+ 115, 237, 103, 121, 337, 337, 107, 337, 107, 101, 116, 337, 116,
58
+ };
59
+
60
+ static const uint32_t stop_word_offsets[246] = {
61
+ 0, 1, 5, 10, 15, 20, 25, 30, 34, 37, 41, 46, 50, 57, 62, 68, 71, 76, 83, 90, 97, 105, 108, 114, 118, 122, 127, 132,
62
+ 136, 141, 146, 148, 152, 158, 162, 169, 172, 177, 183, 188, 193, 195, 200, 205, 208, 212, 214, 215, 220, 223, 230,
63
+ 235, 243, 248, 254, 259, 264, 270, 275, 280, 285, 287, 292, 296, 299, 304, 311, 316, 321, 326, 331, 335, 337, 341,
64
+ 345, 348, 354, 359, 364, 367, 371, 373, 378, 384, 388, 394, 397, 403, 409, 413, 418, 421, 425, 428, 435, 440, 445,
65
+ 451, 453, 458, 461, 467, 470, 474, 481, 490, 495, 500, 502, 505, 511, 516, 520, 526, 531, 539, 548, 550, 558, 564,
66
+ 569, 574, 579, 583, 587, 589, 593, 598, 601, 605, 608, 614, 621, 626, 630, 636, 642, 648, 652, 654, 659, 664, 670,
67
+ 674, 680, 688, 695, 701, 706, 712, 716, 722, 725, 730, 735, 741, 745, 748, 751, 756, 763, 766, 771, 774, 779, 783,
68
+ 789, 791, 796, 801, 805, 810, 816, 819, 824, 830, 834, 840, 846, 849, 854, 859, 862, 867, 873, 878, 882, 887, 889,
69
+ 892, 897, 902, 906, 909, 914, 920, 925, 932, 939, 945, 952, 955, 960, 962, 967, 969, 975, 982, 989, 994, 998, 1005,
70
+ 1009, 1013, 1018, 1022, 1028, 1034, 1041, 1046, 1052, 1058, 1066, 1070, 1073, 1079, 1083, 1088, 1094, 1100, 1107,
71
+ 1112, 1116, 1122, 1127, 1132, 1137, 1146, 1148, 1150, 1152, 1155, 1156, 1158, 1162, 1164,
72
+ };
73
+
74
+ static const uint32_t keep_ranges[38] = {
75
+ 48, 57, 65, 90, 97, 122, 193, 193, 201, 201, 205, 205, 211, 211, 214, 214, 218, 218, 220, 220, 225, 225, 233, 233,
76
+ 237, 237, 243, 243, 246, 246, 250, 250, 252, 252, 336, 337, 368, 369,
77
+ };
78
+
79
+ const narsil_language narsil_language_hungarian = {
80
+ .name = "hungarian",
81
+ .revision = "7b9c2fc79181fc11",
82
+ .stop_word_units = stop_word_units,
83
+ .stop_word_offsets = stop_word_offsets,
84
+ .stop_word_count = 245,
85
+ .ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
86
+ .keep_ranges = keep_ranges,
87
+ .keep_range_count = 19,
88
+ .splits_code_points = 0,
89
+ .strips_possessive = 0,
90
+ .normalises_diacritics = 0,
91
+ .min_token_length = 1,
92
+ .ngram_size = 0,
93
+ .normalise = NULL,
94
+ .stem_by_rules = NULL,
95
+ .create_snowball = hungarian_create_env,
96
+ .snowball_stem = hungarian_stem,
97
+ .close_snowball = hungarian_close_env,
98
+ };
@@ -0,0 +1,73 @@
1
+ #include "../../analysis/languages.h"
2
+ #include "declarations.h"
3
+
4
+ #include <stddef.h>
5
+ #include <stdint.h>
6
+
7
+ static const uint16_t stop_word_units[681] = {
8
+ 97, 102, 104, 118, 101, 114, 106, 117, 97, 102, 116, 97, 110, 97, 102, 116, 117, 114, 97, 102, 254, 118, 237, 97, 108,
9
+ 100, 114, 101, 105, 97, 108, 108, 105, 114, 97, 108, 108, 116, 97, 108, 118, 101, 103, 97, 110, 110, 97, 114, 115, 97,
10
+ 110, 110, 97, 240, 98, 97, 114, 97, 100, 97, 103, 101, 102, 116, 105, 114, 101, 105, 103, 97, 101, 105, 110, 104, 118,
11
+ 101, 114, 101, 105, 110, 104, 118, 101, 114, 106, 105, 114, 101, 105, 110, 104, 118, 101, 114, 115, 101, 105, 110,
12
+ 115, 101, 105, 110, 117, 101, 105, 116, 116, 104, 118, 97, 240, 101, 107, 107, 101, 114, 116, 101, 107, 107, 105, 101,
13
+ 110, 110, 254, 225, 101, 114, 117, 101, 240, 97, 102, 97, 114, 97, 102, 101, 114, 102, 105, 110, 110, 97, 102, 106,
14
+ 246, 108, 100, 105, 102, 114, 97, 109, 97, 110, 102, 114, 101, 107, 97, 114, 102, 114, 225, 102, 121, 114, 105, 114,
15
+ 102, 243, 108, 107, 103, 101, 103, 110, 117, 109, 103, 101, 116, 97, 103, 101, 116, 117, 114, 103, 109, 103, 103, 111,
16
+ 116, 116, 104, 97, 102, 97, 104, 97, 110, 110, 104, 101, 102, 104, 101, 102, 117, 114, 104, 101, 121, 114, 97, 104,
17
+ 106, 225, 104, 118, 97, 114, 104, 118, 97, 240, 104, 118, 101, 114, 104, 118, 101, 114, 106, 105, 114, 104, 118, 101,
18
+ 114, 106, 117, 109, 104, 118, 101, 114, 110, 105, 103, 104, 118, 111, 114, 104, 118, 111, 114, 116, 104, 230, 103,
19
+ 116, 104, 233, 114, 104, 233, 114, 110, 97, 104, 250, 110, 105, 109, 103, 105, 110, 110, 107, 97, 110, 110, 115, 107,
20
+ 105, 107, 111, 109, 97, 108, 111, 108, 108, 237, 107, 97, 109, 97, 240, 117, 114, 109, 101, 103, 97, 109, 101, 105,
21
+ 114, 97, 109, 101, 240, 109, 105, 103, 109, 105, 107, 105, 240, 109, 105, 110, 110, 97, 109, 105, 110, 110, 105, 109,
22
+ 105, 115, 115, 97, 109, 106, 246, 103, 109, 225, 116, 116, 109, 233, 114, 110, 101, 105, 110, 105, 240, 117, 114, 110,
23
+ 250, 110, 97, 111, 102, 116, 111, 107, 107, 97, 114, 111, 107, 107, 117, 114, 112, 243, 115, 116, 112, 243, 115, 116,
24
+ 117, 114, 114, 111, 102, 108, 115, 97, 109, 97, 110, 115, 101, 109, 115, 105, 103, 115, 105, 110, 110, 105, 115, 106,
25
+ 225, 115, 109, 225, 115, 109, 225, 116, 116, 115, 112, 117, 114, 106, 97, 115, 112, 121, 114, 106, 97, 115, 116, 97,
26
+ 240, 97, 114, 115, 116, 243, 114, 116, 115, 118, 111, 115, 118, 111, 110, 97, 115, 230, 108, 105, 114, 115, 230, 108,
27
+ 108, 115, 233, 114, 115, 237, 240, 97, 110, 116, 97, 107, 97, 116, 97, 107, 107, 116, 105, 108, 116, 105, 108, 118,
28
+ 105, 116, 110, 117, 110, 116, 105, 116, 108, 97, 114, 117, 112, 112, 118, 97, 114, 118, 101, 108, 118, 101, 108, 107,
29
+ 111, 109, 105, 110, 118, 101, 108, 107, 111, 109, 105, 110, 110, 118, 101, 114, 97, 118, 101, 114, 105, 240, 118, 101,
30
+ 114, 240, 117, 114, 118, 105, 108, 118, 105, 108, 106, 97, 118, 105, 108, 108, 118, 105, 116, 97, 118, 105, 240, 118,
31
+ 230, 114, 105, 121, 102, 105, 114, 121, 107, 107, 97, 114, 230, 116, 116, 105, 254, 97, 107, 107, 97, 254, 97, 107,
32
+ 107, 105, 114, 254, 97, 110, 110, 105, 103, 254, 97, 114, 254, 97, 114, 102, 254, 97, 117, 254, 97, 240, 254, 101,
33
+ 103, 97, 114, 254, 101, 105, 109, 254, 101, 105, 114, 254, 101, 105, 114, 114, 97, 254, 101, 115, 115, 254, 101, 115,
34
+ 115, 97, 254, 101, 115, 115, 105, 254, 101, 115, 115, 117, 254, 101, 115, 115, 117, 109, 254, 101, 116, 116, 97, 254,
35
+ 105, 110, 110, 254, 105, 116, 116, 254, 105, 240, 254, 114, 225, 240, 254, 114, 225, 240, 117, 114, 254, 118, 237,
36
+ 254, 230, 114, 254, 233, 114, 254, 237, 110,
37
+ };
38
+
39
+ static const uint32_t stop_word_offsets[150] = {
40
+ 0, 8, 13, 18, 23, 29, 34, 38, 43, 49, 54, 58, 61, 66, 70, 77, 87, 95, 99, 103, 111, 117, 121, 126, 129, 132, 136, 139,
41
+ 144, 150, 156, 162, 165, 170, 174, 180, 184, 189, 192, 196, 200, 204, 207, 212, 217, 220, 224, 228, 232, 239, 246,
42
+ 253, 257, 262, 266, 269, 274, 277, 280, 283, 290, 294, 297, 301, 306, 310, 315, 318, 321, 326, 331, 336, 341, 345,
43
+ 349, 352, 355, 360, 364, 367, 372, 377, 381, 387, 391, 396, 399, 402, 407, 410, 413, 418, 424, 430, 436, 441, 444,
44
+ 449, 454, 458, 461, 466, 470, 474, 477, 486, 492, 495, 498, 501, 509, 518, 522, 527, 533, 536, 541, 545, 549, 552,
45
+ 556, 560, 565, 569, 574, 580, 586, 589, 593, 596, 599, 604, 608, 612, 618, 622, 627, 632, 637, 643, 648, 652, 656,
46
+ 659, 663, 669, 672, 675, 678, 681,
47
+ };
48
+
49
+ static const uint32_t keep_ranges[42] = {
50
+ 48, 57, 65, 90, 97, 122, 193, 193, 198, 198, 201, 201, 205, 205, 208, 208, 211, 211, 214, 214, 218, 218, 221, 222,
51
+ 225, 225, 230, 230, 233, 233, 237, 237, 240, 240, 243, 243, 246, 246, 250, 250, 253, 254,
52
+ };
53
+
54
+ const narsil_language narsil_language_icelandic = {
55
+ .name = "icelandic",
56
+ .revision = "1",
57
+ .stop_word_units = stop_word_units,
58
+ .stop_word_offsets = stop_word_offsets,
59
+ .stop_word_count = 149,
60
+ .ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
61
+ .keep_ranges = keep_ranges,
62
+ .keep_range_count = 21,
63
+ .splits_code_points = 0,
64
+ .strips_possessive = 0,
65
+ .normalises_diacritics = 0,
66
+ .min_token_length = 1,
67
+ .ngram_size = 0,
68
+ .normalise = NULL,
69
+ .stem_by_rules = NULL,
70
+ .create_snowball = NULL,
71
+ .snowball_stem = NULL,
72
+ .close_snowball = NULL,
73
+ };
@@ -0,0 +1,90 @@
1
+ #include "../../analysis/languages.h"
2
+ #include "declarations.h"
3
+
4
+ #include <stddef.h>
5
+ #include <stdint.h>
6
+
7
+ static const uint16_t stop_word_units[254] = {
8
+ 97, 97, 103, 97, 97, 107, 97, 97, 108, 97, 97, 110, 121, 97, 97, 110, 121, 105, 97, 110, 121, 7883, 98, 97, 110, 121,
9
+ 101, 114, 101, 98, 117, 98, 7909, 100, 105, 100, 7883, 101, 101, 98, 101, 101, 106, 105, 101, 108, 117, 101, 109, 101,
10
+ 101, 116, 105, 116, 105, 103, 97, 103, 97, 114, 97, 103, 104, 105, 103, 104, 7883, 103, 105, 103, 119, 97, 103, 7883,
11
+ 104, 97, 105, 105, 104, 101, 105, 107, 101, 105, 115, 105, 106, 105, 107, 97, 107, 119, 97, 107, 119, 97, 114, 97,
12
+ 107, 119, 117, 107, 119, 117, 114, 117, 109, 109, 97, 109, 97, 107, 97, 109, 97, 110, 97, 109, 101, 114, 101, 109,
13
+ 103, 98, 101, 110, 110, 97, 110, 97, 110, 105, 110, 97, 110, 7883, 110, 99, 104, 97, 110, 100, 105, 110, 100, 7883,
14
+ 110, 105, 108, 101, 110, 105, 109, 101, 110, 107, 101, 110, 111, 110, 119, 97, 110, 119, 101, 114, 101, 110, 121, 101,
15
+ 111, 111, 98, 105, 111, 98, 117, 108, 97, 111, 103, 101, 111, 107, 112, 117, 114, 117, 111, 107, 119, 117, 111, 110,
16
+ 119, 101, 111, 110, 121, 101, 111, 116, 117, 111, 122, 111, 112, 117, 114, 117, 115, 105, 115, 105, 116, 101, 116, 97,
17
+ 97, 116, 117, 112, 117, 117, 102, 111, 100, 117, 117, 103, 98, 117, 117, 110, 117, 119, 101, 121, 97, 7883, 7885,
18
+ 7885, 98, 7909, 108, 97, 7885, 122, 7885, 7909, 102, 7885, 100, 7909,
19
+ };
20
+
21
+ static const uint32_t stop_word_offsets[82] = {
22
+ 0, 1, 4, 7, 10, 14, 18, 22, 29, 31, 33, 35, 37, 38, 41, 44, 47, 50, 55, 57, 61, 64, 67, 69, 72, 74, 76, 77, 80, 83,
23
+ 86, 88, 90, 93, 98, 101, 106, 107, 109, 113, 117, 121, 125, 126, 128, 132, 136, 140, 143, 146, 150, 154, 157, 159,
24
+ 162, 167, 170, 171, 174, 179, 182, 188, 192, 196, 200, 203, 206, 210, 212, 216, 219, 223, 228, 232, 235, 237, 239,
25
+ 240, 241, 246, 249, 254,
26
+ };
27
+
28
+ static const uint32_t keep_ranges[694] = {
29
+ 48, 57, 65, 90, 97, 122, 192, 192, 200, 201, 210, 211, 224, 224, 232, 233, 242, 243, 296, 297, 360, 361, 383, 383,
30
+ 558, 559, 768, 879, 921, 921, 953, 953, 1155, 1161, 1425, 1469, 1471, 1471, 1473, 1474, 1476, 1477, 1479, 1479, 1552,
31
+ 1562, 1611, 1631, 1648, 1648, 1750, 1756, 1759, 1764, 1767, 1768, 1770, 1773, 1809, 1809, 1840, 1866, 1958, 1968,
32
+ 2027, 2035, 2045, 2045, 2070, 2073, 2075, 2083, 2085, 2087, 2089, 2093, 2137, 2139, 2199, 2207, 2250, 2273, 2275,
33
+ 2307, 2362, 2364, 2366, 2383, 2385, 2391, 2402, 2403, 2433, 2435, 2492, 2492, 2494, 2500, 2503, 2504, 2507, 2509,
34
+ 2519, 2519, 2530, 2531, 2558, 2558, 2561, 2563, 2620, 2620, 2622, 2626, 2631, 2632, 2635, 2637, 2641, 2641, 2672,
35
+ 2673, 2677, 2677, 2689, 2691, 2748, 2748, 2750, 2757, 2759, 2761, 2763, 2765, 2786, 2787, 2810, 2815, 2817, 2819,
36
+ 2876, 2876, 2878, 2884, 2887, 2888, 2891, 2893, 2901, 2903, 2914, 2915, 2946, 2946, 3006, 3010, 3014, 3016, 3018,
37
+ 3021, 3031, 3031, 3072, 3076, 3132, 3132, 3134, 3140, 3142, 3144, 3146, 3149, 3157, 3158, 3170, 3171, 3201, 3203,
38
+ 3260, 3260, 3262, 3268, 3270, 3272, 3274, 3277, 3285, 3286, 3298, 3299, 3315, 3315, 3328, 3331, 3387, 3388, 3390,
39
+ 3396, 3398, 3400, 3402, 3405, 3415, 3415, 3426, 3427, 3457, 3459, 3530, 3530, 3535, 3540, 3542, 3542, 3544, 3551,
40
+ 3570, 3571, 3633, 3633, 3636, 3642, 3655, 3662, 3761, 3761, 3764, 3772, 3784, 3790, 3864, 3865, 3893, 3893, 3895,
41
+ 3895, 3897, 3897, 3902, 3903, 3953, 3972, 3974, 3975, 3981, 3991, 3993, 4028, 4038, 4038, 4139, 4158, 4182, 4185,
42
+ 4190, 4192, 4194, 4196, 4199, 4205, 4209, 4212, 4226, 4237, 4239, 4239, 4250, 4253, 4957, 4959, 5906, 5909, 5938,
43
+ 5940, 5970, 5971, 6002, 6003, 6068, 6099, 6109, 6109, 6155, 6157, 6159, 6159, 6277, 6278, 6313, 6313, 6432, 6443,
44
+ 6448, 6459, 6679, 6683, 6741, 6750, 6752, 6780, 6783, 6783, 6832, 6877, 6880, 6891, 6912, 6916, 6964, 6980, 7019,
45
+ 7027, 7040, 7042, 7073, 7085, 7142, 7155, 7204, 7223, 7376, 7378, 7380, 7400, 7405, 7405, 7412, 7412, 7415, 7417,
46
+ 7616, 7679, 7748, 7749, 7882, 7885, 7908, 7909, 8126, 8126, 8400, 8432, 8490, 8490, 11503, 11505, 11647, 11647, 11744,
47
+ 11775, 12330, 12335, 12441, 12442, 42607, 42610, 42612, 42621, 42654, 42655, 42736, 42737, 43010, 43010, 43014, 43014,
48
+ 43019, 43019, 43043, 43047, 43052, 43052, 43136, 43137, 43188, 43205, 43232, 43249, 43263, 43263, 43302, 43309, 43335,
49
+ 43347, 43392, 43395, 43443, 43456, 43493, 43493, 43561, 43574, 43587, 43587, 43596, 43597, 43643, 43645, 43696, 43696,
50
+ 43698, 43700, 43703, 43704, 43710, 43711, 43713, 43713, 43755, 43759, 43765, 43766, 44003, 44010, 44012, 44013, 64286,
51
+ 64286, 65024, 65039, 65056, 65071, 66045, 66045, 66272, 66272, 66422, 66426, 68097, 68099, 68101, 68102, 68108, 68111,
52
+ 68152, 68154, 68159, 68159, 68325, 68326, 68900, 68903, 68969, 68973, 69291, 69292, 69370, 69375, 69446, 69456, 69506,
53
+ 69509, 69632, 69634, 69688, 69702, 69744, 69744, 69747, 69748, 69759, 69762, 69808, 69818, 69826, 69826, 69888, 69890,
54
+ 69927, 69940, 69957, 69958, 70003, 70003, 70016, 70018, 70067, 70080, 70089, 70092, 70094, 70095, 70188, 70199, 70206,
55
+ 70206, 70209, 70209, 70367, 70378, 70400, 70403, 70459, 70460, 70462, 70468, 70471, 70472, 70475, 70477, 70487, 70487,
56
+ 70498, 70499, 70502, 70508, 70512, 70516, 70584, 70592, 70594, 70594, 70597, 70597, 70599, 70602, 70604, 70608, 70610,
57
+ 70610, 70625, 70626, 70709, 70726, 70750, 70750, 70832, 70851, 71087, 71093, 71096, 71104, 71132, 71133, 71216, 71232,
58
+ 71339, 71351, 71453, 71467, 71724, 71738, 71984, 71989, 71991, 71992, 71995, 71998, 72000, 72000, 72002, 72003, 72145,
59
+ 72151, 72154, 72160, 72164, 72164, 72193, 72202, 72243, 72249, 72251, 72254, 72263, 72263, 72273, 72283, 72330, 72345,
60
+ 72544, 72551, 72751, 72758, 72760, 72767, 72850, 72871, 72873, 72886, 73009, 73014, 73018, 73018, 73020, 73021, 73023,
61
+ 73029, 73031, 73031, 73098, 73102, 73104, 73105, 73107, 73111, 73459, 73462, 73472, 73473, 73475, 73475, 73524, 73530,
62
+ 73534, 73538, 73562, 73562, 78912, 78912, 78919, 78933, 90398, 90415, 92912, 92916, 92976, 92982, 94031, 94031, 94033,
63
+ 94087, 94095, 94098, 94180, 94180, 94192, 94193, 113821, 113822, 118528, 118573, 118576, 118598, 119141, 119145,
64
+ 119149, 119154, 119163, 119170, 119173, 119179, 119210, 119213, 119362, 119364, 121344, 121398, 121403, 121452,
65
+ 121461, 121461, 121476, 121476, 121499, 121503, 121505, 121519, 122880, 122886, 122888, 122904, 122907, 122913,
66
+ 122915, 122916, 122918, 122922, 123023, 123023, 123184, 123190, 123566, 123566, 123628, 123631, 124140, 124143,
67
+ 124398, 124399, 124643, 124643, 124646, 124646, 124654, 124655, 124661, 124661, 125136, 125142, 125252, 125258,
68
+ 917760, 917999,
69
+ };
70
+
71
+ const narsil_language narsil_language_igbo = {
72
+ .name = "igbo",
73
+ .revision = "1",
74
+ .stop_word_units = stop_word_units,
75
+ .stop_word_offsets = stop_word_offsets,
76
+ .stop_word_count = 81,
77
+ .ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
78
+ .keep_ranges = keep_ranges,
79
+ .keep_range_count = 347,
80
+ .splits_code_points = 1,
81
+ .strips_possessive = 0,
82
+ .normalises_diacritics = 0,
83
+ .min_token_length = 1,
84
+ .ngram_size = 0,
85
+ .normalise = NULL,
86
+ .stem_by_rules = NULL,
87
+ .create_snowball = NULL,
88
+ .snowball_stem = NULL,
89
+ .close_snowball = NULL,
90
+ };