@delali/narsil-react-native 0.3.1-next.154 → 0.3.1-next.156

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (253) hide show
  1. package/NOTICE +144 -0
  2. package/NarsilReactNative.podspec +2 -2
  3. package/android/CMakeLists.txt +1 -1
  4. package/cpp/HybridNarsilCoreAnalysis.cpp +277 -0
  5. package/cpp/HybridNarsilCoreImpl.hpp +43 -0
  6. package/cpp/HybridNarsilCoreKeyword.cpp +253 -0
  7. package/cpp/HybridNarsilCoreMath.cpp +47 -0
  8. package/cpp/core/include/narsil_analysis.h +86 -0
  9. package/cpp/core/include/narsil_core.h +1 -1
  10. package/cpp/core/include/narsil_keyword.h +110 -0
  11. package/cpp/core/include/narsil_math.h +36 -0
  12. package/cpp/core/src/analysis/analyse.c +281 -0
  13. package/cpp/core/src/analysis/analyser.h +49 -0
  14. package/cpp/core/src/analysis/languages.c +79 -0
  15. package/cpp/core/src/analysis/languages.h +42 -0
  16. package/cpp/core/src/analysis/normalise.c +242 -0
  17. package/cpp/core/src/analysis/normalise.h +28 -0
  18. package/cpp/core/src/analysis/split.c +183 -0
  19. package/cpp/core/src/analysis/split.h +27 -0
  20. package/cpp/core/src/analysis/stop_words.c +82 -0
  21. package/cpp/core/src/analysis/stop_words.h +19 -0
  22. package/cpp/core/src/analysis/term_cache.c +88 -0
  23. package/cpp/core/src/analysis/term_cache.h +36 -0
  24. package/cpp/core/src/analysis/text.c +161 -0
  25. package/cpp/core/src/analysis/text.h +45 -0
  26. package/cpp/core/src/analysis/transform.c +103 -0
  27. package/cpp/core/src/analysis/unicode.c +252 -0
  28. package/cpp/core/src/analysis/unicode.h +54 -0
  29. package/cpp/core/src/internal.h +4 -0
  30. package/cpp/core/src/keyword/pass.h +117 -0
  31. package/cpp/core/src/keyword/prune.c +324 -0
  32. package/cpp/core/src/keyword/request.c +179 -0
  33. package/cpp/core/src/keyword/rows.c +83 -0
  34. package/cpp/core/src/keyword/rows.h +18 -0
  35. package/cpp/core/src/keyword/score.c +257 -0
  36. package/cpp/core/src/keyword/score.h +17 -0
  37. package/cpp/core/src/keyword/scratch.c +226 -0
  38. package/cpp/core/src/keyword/scratch.h +55 -0
  39. package/cpp/core/src/keyword/select.c +155 -0
  40. package/cpp/core/src/keyword/select.h +17 -0
  41. package/cpp/core/src/languages/bulgarian_stemmer.c +114 -0
  42. package/cpp/core/src/languages/data/albanian.c +54 -0
  43. package/cpp/core/src/languages/data/amharic.c +93 -0
  44. package/cpp/core/src/languages/data/arabic.c +146 -0
  45. package/cpp/core/src/languages/data/armenian.c +46 -0
  46. package/cpp/core/src/languages/data/azerbaijani.c +68 -0
  47. package/cpp/core/src/languages/data/bambara.c +48 -0
  48. package/cpp/core/src/languages/data/basque.c +62 -0
  49. package/cpp/core/src/languages/data/belarusian.c +56 -0
  50. package/cpp/core/src/languages/data/bengali.c +63 -0
  51. package/cpp/core/src/languages/data/bosnian.c +73 -0
  52. package/cpp/core/src/languages/data/breton.c +52 -0
  53. package/cpp/core/src/languages/data/bulgarian.c +144 -0
  54. package/cpp/core/src/languages/data/burmese.c +48 -0
  55. package/cpp/core/src/languages/data/catalan.c +84 -0
  56. package/cpp/core/src/languages/data/chinese.c +70 -0
  57. package/cpp/core/src/languages/data/croatian.c +72 -0
  58. package/cpp/core/src/languages/data/czech.c +72 -0
  59. package/cpp/core/src/languages/data/dagbani.c +88 -0
  60. package/cpp/core/src/languages/data/danish.c +56 -0
  61. package/cpp/core/src/languages/data/declarations.h +114 -0
  62. package/cpp/core/src/languages/data/dutch.c +58 -0
  63. package/cpp/core/src/languages/data/english.c +172 -0
  64. package/cpp/core/src/languages/data/esperanto.c +71 -0
  65. package/cpp/core/src/languages/data/estonian.c +417 -0
  66. package/cpp/core/src/languages/data/ewe.c +89 -0
  67. package/cpp/core/src/languages/data/faroese.c +54 -0
  68. package/cpp/core/src/languages/data/fijian.c +44 -0
  69. package/cpp/core/src/languages/data/finnish.c +100 -0
  70. package/cpp/core/src/languages/data/french.c +74 -0
  71. package/cpp/core/src/languages/data/ga.c +83 -0
  72. package/cpp/core/src/languages/data/galician.c +70 -0
  73. package/cpp/core/src/languages/data/georgian.c +57 -0
  74. package/cpp/core/src/languages/data/german.c +96 -0
  75. package/cpp/core/src/languages/data/greek.c +202 -0
  76. package/cpp/core/src/languages/data/guarani.c +110 -0
  77. package/cpp/core/src/languages/data/gujarati.c +90 -0
  78. package/cpp/core/src/languages/data/haitian-creole.c +54 -0
  79. package/cpp/core/src/languages/data/hausa.c +85 -0
  80. package/cpp/core/src/languages/data/hawaiian.c +50 -0
  81. package/cpp/core/src/languages/data/hebrew.c +85 -0
  82. package/cpp/core/src/languages/data/hindi.c +91 -0
  83. package/cpp/core/src/languages/data/hungarian.c +98 -0
  84. package/cpp/core/src/languages/data/icelandic.c +73 -0
  85. package/cpp/core/src/languages/data/igbo.c +90 -0
  86. package/cpp/core/src/languages/data/indonesian.c +296 -0
  87. package/cpp/core/src/languages/data/irish.c +60 -0
  88. package/cpp/core/src/languages/data/italian.c +107 -0
  89. package/cpp/core/src/languages/data/japanese.c +52 -0
  90. package/cpp/core/src/languages/data/kannada.c +60 -0
  91. package/cpp/core/src/languages/data/kazakh.c +57 -0
  92. package/cpp/core/src/languages/data/khmer.c +71 -0
  93. package/cpp/core/src/languages/data/kinyarwanda.c +54 -0
  94. package/cpp/core/src/languages/data/kirundi.c +53 -0
  95. package/cpp/core/src/languages/data/korean.c +74 -0
  96. package/cpp/core/src/languages/data/kurmanji.c +43 -0
  97. package/cpp/core/src/languages/data/kyrgyz.c +65 -0
  98. package/cpp/core/src/languages/data/lao.c +46 -0
  99. package/cpp/core/src/languages/data/latin.c +46 -0
  100. package/cpp/core/src/languages/data/latvian.c +75 -0
  101. package/cpp/core/src/languages/data/lingala.c +88 -0
  102. package/cpp/core/src/languages/data/lithuanian.c +65 -0
  103. package/cpp/core/src/languages/data/luxembourgish.c +80 -0
  104. package/cpp/core/src/languages/data/macedonian.c +286 -0
  105. package/cpp/core/src/languages/data/malagasy.c +51 -0
  106. package/cpp/core/src/languages/data/malay.c +184 -0
  107. package/cpp/core/src/languages/data/malayalam.c +64 -0
  108. package/cpp/core/src/languages/data/maltese.c +102 -0
  109. package/cpp/core/src/languages/data/maori.c +52 -0
  110. package/cpp/core/src/languages/data/marathi.c +62 -0
  111. package/cpp/core/src/languages/data/nepali.c +72 -0
  112. package/cpp/core/src/languages/data/norwegian.c +74 -0
  113. package/cpp/core/src/languages/data/oromo.c +51 -0
  114. package/cpp/core/src/languages/data/persian.c +150 -0
  115. package/cpp/core/src/languages/data/polish.c +111 -0
  116. package/cpp/core/src/languages/data/portuguese.c +91 -0
  117. package/cpp/core/src/languages/data/punjabi.c +80 -0
  118. package/cpp/core/src/languages/data/registry.c +116 -0
  119. package/cpp/core/src/languages/data/romanian.c +93 -0
  120. package/cpp/core/src/languages/data/russian.c +76 -0
  121. package/cpp/core/src/languages/data/samoan.c +47 -0
  122. package/cpp/core/src/languages/data/sanskrit.c +53 -0
  123. package/cpp/core/src/languages/data/scottish-gaelic.c +119 -0
  124. package/cpp/core/src/languages/data/serbian.c +84 -0
  125. package/cpp/core/src/languages/data/shona.c +55 -0
  126. package/cpp/core/src/languages/data/sinhala.c +78 -0
  127. package/cpp/core/src/languages/data/slovak.c +135 -0
  128. package/cpp/core/src/languages/data/slovenian.c +139 -0
  129. package/cpp/core/src/languages/data/sorani.c +55 -0
  130. package/cpp/core/src/languages/data/spanish.c +84 -0
  131. package/cpp/core/src/languages/data/swahili.c +59 -0
  132. package/cpp/core/src/languages/data/swedish.c +60 -0
  133. package/cpp/core/src/languages/data/tagalog.c +75 -0
  134. package/cpp/core/src/languages/data/tamil.c +69 -0
  135. package/cpp/core/src/languages/data/tatar.c +548 -0
  136. package/cpp/core/src/languages/data/telugu.c +54 -0
  137. package/cpp/core/src/languages/data/thai.c +46 -0
  138. package/cpp/core/src/languages/data/tibetan.c +55 -0
  139. package/cpp/core/src/languages/data/tigrinya.c +73 -0
  140. package/cpp/core/src/languages/data/tongan.c +47 -0
  141. package/cpp/core/src/languages/data/turkish.c +100 -0
  142. package/cpp/core/src/languages/data/twi.c +88 -0
  143. package/cpp/core/src/languages/data/ukrainian.c +99 -0
  144. package/cpp/core/src/languages/data/urdu.c +174 -0
  145. package/cpp/core/src/languages/data/vietnamese.c +88 -0
  146. package/cpp/core/src/languages/data/welsh.c +143 -0
  147. package/cpp/core/src/languages/data/wolof.c +51 -0
  148. package/cpp/core/src/languages/data/xhosa.c +56 -0
  149. package/cpp/core/src/languages/data/yoruba.c +93 -0
  150. package/cpp/core/src/languages/data/zulu.c +58 -0
  151. package/cpp/core/src/languages/marks.c +64 -0
  152. package/cpp/core/src/languages/rules.c +63 -0
  153. package/cpp/core/src/languages/rules.h +68 -0
  154. package/cpp/core/src/languages/sanskrit_stemmer.c +50 -0
  155. package/cpp/core/src/languages/scripts.c +143 -0
  156. package/cpp/core/src/languages/slovenian_stemmer.c +46 -0
  157. package/cpp/core/src/languages/snowball/api.c +52 -0
  158. package/cpp/core/src/languages/snowball/api.h +43 -0
  159. package/cpp/core/src/languages/snowball/arabic_stemmer.c +1506 -0
  160. package/cpp/core/src/languages/snowball/arabic_stemmer.h +24 -0
  161. package/cpp/core/src/languages/snowball/armenian_stemmer.c +523 -0
  162. package/cpp/core/src/languages/snowball/armenian_stemmer.h +24 -0
  163. package/cpp/core/src/languages/snowball/basque_stemmer.c +1089 -0
  164. package/cpp/core/src/languages/snowball/basque_stemmer.h +24 -0
  165. package/cpp/core/src/languages/snowball/catalan_stemmer.c +1380 -0
  166. package/cpp/core/src/languages/snowball/catalan_stemmer.h +24 -0
  167. package/cpp/core/src/languages/snowball/czech_stemmer.c +582 -0
  168. package/cpp/core/src/languages/snowball/czech_stemmer.h +24 -0
  169. package/cpp/core/src/languages/snowball/danish_stemmer.c +352 -0
  170. package/cpp/core/src/languages/snowball/danish_stemmer.h +24 -0
  171. package/cpp/core/src/languages/snowball/dutch_stemmer.c +1657 -0
  172. package/cpp/core/src/languages/snowball/dutch_stemmer.h +24 -0
  173. package/cpp/core/src/languages/snowball/english_stemmer.c +1019 -0
  174. package/cpp/core/src/languages/snowball/english_stemmer.h +24 -0
  175. package/cpp/core/src/languages/snowball/esperanto_stemmer.c +716 -0
  176. package/cpp/core/src/languages/snowball/esperanto_stemmer.h +24 -0
  177. package/cpp/core/src/languages/snowball/estonian_stemmer.c +1361 -0
  178. package/cpp/core/src/languages/snowball/estonian_stemmer.h +24 -0
  179. package/cpp/core/src/languages/snowball/finnish_stemmer.c +800 -0
  180. package/cpp/core/src/languages/snowball/finnish_stemmer.h +24 -0
  181. package/cpp/core/src/languages/snowball/french_stemmer.c +1217 -0
  182. package/cpp/core/src/languages/snowball/french_stemmer.h +24 -0
  183. package/cpp/core/src/languages/snowball/german_stemmer.c +567 -0
  184. package/cpp/core/src/languages/snowball/german_stemmer.h +24 -0
  185. package/cpp/core/src/languages/snowball/greek_stemmer.c +3450 -0
  186. package/cpp/core/src/languages/snowball/greek_stemmer.h +24 -0
  187. package/cpp/core/src/languages/snowball/hindi_stemmer.c +325 -0
  188. package/cpp/core/src/languages/snowball/hindi_stemmer.h +24 -0
  189. package/cpp/core/src/languages/snowball/hungarian_stemmer.c +742 -0
  190. package/cpp/core/src/languages/snowball/hungarian_stemmer.h +24 -0
  191. package/cpp/core/src/languages/snowball/indonesian_stemmer.c +443 -0
  192. package/cpp/core/src/languages/snowball/indonesian_stemmer.h +24 -0
  193. package/cpp/core/src/languages/snowball/irish_stemmer.c +428 -0
  194. package/cpp/core/src/languages/snowball/irish_stemmer.h +24 -0
  195. package/cpp/core/src/languages/snowball/italian_stemmer.c +983 -0
  196. package/cpp/core/src/languages/snowball/italian_stemmer.h +24 -0
  197. package/cpp/core/src/languages/snowball/lithuanian_stemmer.c +794 -0
  198. package/cpp/core/src/languages/snowball/lithuanian_stemmer.h +24 -0
  199. package/cpp/core/src/languages/snowball/nepali_stemmer.c +361 -0
  200. package/cpp/core/src/languages/snowball/nepali_stemmer.h +24 -0
  201. package/cpp/core/src/languages/snowball/norwegian_stemmer.c +332 -0
  202. package/cpp/core/src/languages/snowball/norwegian_stemmer.h +24 -0
  203. package/cpp/core/src/languages/snowball/persian_stemmer.c +538 -0
  204. package/cpp/core/src/languages/snowball/persian_stemmer.h +24 -0
  205. package/cpp/core/src/languages/snowball/polish_stemmer.c +501 -0
  206. package/cpp/core/src/languages/snowball/polish_stemmer.h +24 -0
  207. package/cpp/core/src/languages/snowball/portuguese_stemmer.c +866 -0
  208. package/cpp/core/src/languages/snowball/portuguese_stemmer.h +24 -0
  209. package/cpp/core/src/languages/snowball/romanian_stemmer.c +940 -0
  210. package/cpp/core/src/languages/snowball/romanian_stemmer.h +24 -0
  211. package/cpp/core/src/languages/snowball/russian_stemmer.c +598 -0
  212. package/cpp/core/src/languages/snowball/russian_stemmer.h +24 -0
  213. package/cpp/core/src/languages/snowball/serbian_stemmer.c +6765 -0
  214. package/cpp/core/src/languages/snowball/serbian_stemmer.h +24 -0
  215. package/cpp/core/src/languages/snowball/snowball_runtime.h +122 -0
  216. package/cpp/core/src/languages/snowball/spanish_stemmer.c +955 -0
  217. package/cpp/core/src/languages/snowball/spanish_stemmer.h +24 -0
  218. package/cpp/core/src/languages/snowball/swedish_stemmer.c +364 -0
  219. package/cpp/core/src/languages/snowball/swedish_stemmer.h +24 -0
  220. package/cpp/core/src/languages/snowball/tamil_stemmer.c +1305 -0
  221. package/cpp/core/src/languages/snowball/tamil_stemmer.h +24 -0
  222. package/cpp/core/src/languages/snowball/turkish_stemmer.c +1700 -0
  223. package/cpp/core/src/languages/snowball/turkish_stemmer.h +24 -0
  224. package/cpp/core/src/languages/snowball/utilities.c +716 -0
  225. package/cpp/core/src/languages/substitutions.c +131 -0
  226. package/cpp/core/src/languages/swahili_stemmer.c +85 -0
  227. package/cpp/core/src/languages/ukrainian_stemmer.c +168 -0
  228. package/cpp/core/src/locks.c +2 -15
  229. package/cpp/core/src/math/apply.c +44 -0
  230. package/cpp/core/src/math/fdlibm.h +16 -0
  231. package/cpp/core/src/math/inverse.c +242 -0
  232. package/cpp/core/src/math/log.c +90 -0
  233. package/cpp/core/src/math/reduce.c +325 -0
  234. package/cpp/core/src/math/trig.c +188 -0
  235. package/cpp/core/src/math/words.h +45 -0
  236. package/cpp/core/src/thread_yield.h +18 -0
  237. package/cpp/core/src/unicode/decompositions.c +310 -0
  238. package/cpp/core/src/unicode/lowercase.c +172 -0
  239. package/cpp/core/src/unicode/properties.c +214 -0
  240. package/cpp/core/src/workspace.c +5 -1
  241. package/lib/engine/analysis.d.ts +3 -0
  242. package/lib/engine/analysis.js +76 -0
  243. package/lib/engine/core.js +5 -0
  244. package/lib/engine/keyword.d.ts +3 -0
  245. package/lib/engine/keyword.js +25 -0
  246. package/lib/specs/core.nitro.d.ts +1 -0
  247. package/nitrogen/generated/shared/c++/HybridNarsilCoreSpec.cpp +1 -0
  248. package/nitrogen/generated/shared/c++/HybridNarsilCoreSpec.hpp +1 -0
  249. package/package.json +4 -3
  250. package/src/engine/analysis.ts +87 -0
  251. package/src/engine/core.ts +6 -0
  252. package/src/engine/keyword.ts +26 -0
  253. package/src/specs/core.nitro.ts +1 -0
package/cpp/core/src/languages/data/malay.c ADDED
@@ -0,0 +1,184 @@
1
+ #include "../../analysis/languages.h"
2
+ #include "declarations.h"
3
+
4
+ #include <stddef.h>
5
+ #include <stdint.h>
6
+
7
+ static const uint16_t stop_word_units[3044] = {
8
+ 97, 98, 100, 117, 108, 97, 98, 100, 117, 108, 108, 97, 104, 97, 99, 97, 114, 97, 97, 100, 97, 97, 100, 97, 108, 97,
9
+ 104, 97, 104, 109, 97, 100, 97, 105, 114, 97, 107, 97, 110, 97, 107, 104, 98, 97, 114, 97, 107, 104, 105, 114, 97,
10
+ 107, 116, 105, 118, 105, 116, 105, 97, 108, 97, 109, 97, 109, 97, 116, 97, 109, 101, 114, 105, 107, 97, 97, 110, 97,
11
+ 107, 97, 110, 103, 103, 111, 116, 97, 97, 110, 116, 97, 114, 97, 97, 110, 116, 97, 114, 97, 98, 97, 110, 103, 115, 97,
12
+ 97, 112, 97, 97, 112, 97, 98, 105, 108, 97, 97, 112, 114, 105, 108, 97, 115, 97, 115, 97, 115, 97, 115, 101, 97, 110,
13
+ 97, 115, 105, 97, 97, 115, 105, 110, 103, 97, 116, 97, 115, 97, 116, 97, 117, 97, 117, 115, 116, 114, 97, 108, 105,
14
+ 97, 97, 119, 97, 108, 97, 119, 97, 109, 98, 97, 103, 97, 105, 109, 97, 110, 97, 112, 117, 110, 98, 97, 103, 105, 98,
15
+ 97, 104, 97, 103, 105, 97, 110, 98, 97, 104, 97, 110, 98, 97, 104, 97, 114, 117, 98, 97, 104, 97, 119, 97, 98, 97,
16
+ 105, 107, 98, 97, 110, 100, 97, 114, 98, 97, 110, 107, 98, 97, 110, 121, 97, 107, 98, 97, 114, 97, 110, 103, 97, 110,
17
+ 98, 97, 114, 117, 98, 97, 119, 97, 104, 98, 101, 98, 101, 114, 97, 112, 97, 98, 101, 107, 97, 115, 98, 101, 108, 105,
18
+ 97, 117, 98, 101, 108, 117, 109, 98, 101, 114, 97, 100, 97, 98, 101, 114, 97, 107, 104, 105, 114, 98, 101, 114, 98,
19
+ 97, 110, 100, 105, 110, 103, 98, 101, 114, 100, 97, 115, 97, 114, 107, 97, 110, 98, 101, 114, 104, 97, 114, 97, 112,
20
+ 98, 101, 114, 105, 107, 117, 116, 97, 110, 98, 101, 114, 106, 97, 121, 97, 98, 101, 114, 106, 117, 109, 108, 97, 104,
21
+ 98, 101, 114, 107, 97, 105, 116, 97, 110, 98, 101, 114, 107, 97, 116, 97, 98, 101, 114, 107, 101, 110, 97, 97, 110,
22
+ 98, 101, 114, 108, 97, 107, 117, 98, 101, 114, 109, 117, 108, 97, 98, 101, 114, 110, 97, 109, 97, 98, 101, 114, 110,
23
+ 105, 108, 97, 105, 98, 101, 114, 115, 97, 109, 97, 98, 101, 114, 117, 98, 97, 104, 98, 101, 115, 97, 114, 98, 104,
24
+ 100, 98, 105, 100, 97, 110, 103, 98, 105, 108, 105, 111, 110, 98, 110, 98, 111, 108, 101, 104, 98, 117, 107, 97, 110,
25
+ 98, 117, 108, 97, 110, 98, 117, 114, 115, 97, 99, 97, 100, 97, 110, 103, 97, 110, 99, 104, 105, 110, 97, 100, 97, 103,
26
+ 97, 110, 103, 97, 110, 100, 97, 108, 97, 109, 100, 97, 110, 100, 97, 110, 97, 100, 97, 112, 97, 116, 100, 97, 114,
27
+ 105, 100, 97, 114, 105, 112, 97, 100, 97, 100, 97, 115, 97, 114, 100, 97, 116, 97, 110, 103, 100, 97, 116, 117, 107,
28
+ 100, 101, 109, 105, 107, 105, 97, 110, 100, 101, 110, 103, 97, 110, 100, 101, 112, 97, 110, 100, 101, 114, 105, 118,
29
+ 97, 116, 105, 118, 101, 115, 100, 101, 119, 97, 110, 100, 105, 100, 105, 97, 100, 97, 107, 97, 110, 100, 105, 98, 117,
30
+ 107, 97, 100, 105, 99, 97, 116, 97, 116, 107, 97, 110, 100, 105, 106, 97, 110, 103, 107, 97, 100, 105, 110, 105, 97,
31
+ 103, 97, 107, 97, 110, 100, 105, 115, 100, 105, 115, 101, 109, 98, 101, 114, 100, 105, 116, 117, 116, 117, 112, 100,
32
+ 111, 108, 97, 114, 100, 114, 100, 117, 97, 100, 117, 110, 105, 97, 101, 107, 111, 110, 111, 109, 105, 101, 107, 115,
33
+ 101, 107, 117, 116, 105, 102, 101, 107, 115, 112, 111, 114, 116, 101, 109, 112, 97, 116, 101, 110, 97, 109, 102, 97,
34
+ 101, 100, 97, 104, 102, 101, 98, 103, 108, 111, 98, 97, 108, 104, 97, 100, 97, 112, 97, 110, 104, 97, 110, 121, 97,
35
+ 104, 97, 114, 103, 97, 104, 97, 114, 105, 104, 97, 115, 105, 108, 104, 105, 110, 103, 103, 97, 104, 117, 98, 117, 110,
36
+ 103, 97, 110, 105, 97, 105, 97, 105, 116, 117, 105, 97, 108, 97, 104, 105, 110, 100, 101, 107, 115, 105, 110, 100,
37
+ 105, 97, 105, 110, 100, 111, 110, 101, 115, 105, 97, 105, 110, 100, 117, 115, 116, 114, 105, 105, 110, 105, 105, 115,
38
+ 108, 97, 109, 105, 115, 110, 105, 110, 105, 115, 117, 105, 116, 117, 106, 97, 98, 97, 116, 97, 110, 106, 97, 108, 97,
39
+ 110, 106, 97, 110, 106, 97, 119, 97, 116, 97, 110, 106, 97, 119, 97, 116, 97, 110, 107, 117, 97, 115, 97, 106, 101,
40
+ 112, 117, 110, 106, 105, 107, 97, 106, 117, 97, 108, 97, 110, 106, 117, 103, 97, 106, 117, 108, 97, 105, 106, 117,
41
+ 109, 97, 97, 116, 106, 117, 109, 108, 97, 104, 106, 117, 110, 106, 117, 116, 97, 107, 97, 100, 97, 114, 107, 97, 108,
42
+ 97, 110, 103, 97, 110, 107, 97, 108, 105, 107, 97, 109, 105, 107, 97, 116, 97, 107, 97, 116, 97, 110, 121, 97, 107,
43
+ 97, 117, 110, 116, 101, 114, 107, 97, 119, 97, 115, 97, 110, 107, 101, 107, 101, 97, 100, 97, 97, 110, 107, 101, 99,
44
+ 105, 108, 107, 101, 100, 117, 97, 107, 101, 100, 117, 100, 117, 107, 97, 110, 107, 101, 107, 97, 108, 107, 101, 109,
45
+ 101, 110, 116, 101, 114, 105, 97, 110, 107, 101, 109, 117, 100, 97, 104, 97, 110, 107, 101, 110, 97, 105, 107, 97,
46
+ 110, 107, 101, 110, 121, 97, 116, 97, 97, 110, 107, 101, 112, 97, 100, 97, 107, 101, 112, 101, 110, 116, 105, 110,
47
+ 103, 97, 110, 107, 101, 112, 117, 116, 117, 115, 97, 110, 107, 101, 114, 97, 106, 97, 97, 110, 107, 101, 114, 97, 110,
48
+ 97, 107, 101, 114, 101, 116, 97, 107, 101, 114, 106, 97, 107, 101, 114, 106, 97, 115, 97, 109, 97, 107, 101, 115, 107,
49
+ 101, 115, 101, 108, 97, 109, 97, 116, 97, 110, 107, 101, 115, 101, 108, 117, 114, 117, 104, 97, 110, 107, 101, 115,
50
+ 105, 104, 97, 116, 97, 110, 107, 101, 116, 105, 107, 97, 107, 101, 116, 117, 97, 107, 101, 117, 110, 116, 117, 110,
51
+ 103, 97, 110, 107, 101, 119, 97, 110, 103, 97, 110, 107, 104, 97, 109, 105, 115, 107, 105, 110, 105, 107, 105, 116,
52
+ 97, 107, 108, 99, 105, 107, 108, 105, 98, 111, 114, 107, 111, 109, 112, 111, 115, 105, 116, 107, 111, 110, 116, 114,
53
+ 97, 107, 107, 111, 115, 107, 117, 97, 108, 97, 107, 117, 97, 115, 97, 107, 117, 107, 117, 104, 107, 117, 109, 112,
54
+ 117, 108, 97, 110, 108, 97, 103, 105, 108, 97, 105, 110, 108, 97, 110, 103, 107, 97, 104, 108, 97, 112, 111, 114, 97,
55
+ 110, 108, 101, 98, 105, 104, 108, 101, 112, 97, 115, 108, 105, 109, 97, 108, 111, 116, 108, 117, 97, 114, 108, 117,
56
+ 109, 112, 117, 114, 109, 97, 99, 109, 97, 104, 107, 97, 109, 97, 104, 109, 97, 104, 117, 109, 97, 106, 108, 105, 115,
57
+ 109, 97, 107, 97, 110, 97, 110, 109, 97, 107, 108, 117, 109, 97, 116, 109, 97, 108, 97, 109, 109, 97, 108, 97, 121,
58
+ 115, 105, 97, 109, 97, 110, 97, 109, 97, 110, 97, 107, 97, 108, 97, 109, 97, 115, 97, 109, 97, 115, 97, 108, 97, 104,
59
+ 109, 97, 115, 105, 104, 109, 97, 115, 121, 97, 114, 97, 107, 97, 116, 109, 97, 116, 97, 109, 101, 100, 105, 97, 109,
60
+ 101, 105, 109, 101, 108, 97, 108, 117, 105, 109, 101, 108, 105, 104, 97, 116, 109, 101, 109, 97, 110, 100, 97, 110,
61
+ 103, 107, 97, 110, 109, 101, 109, 97, 115, 116, 105, 107, 97, 110, 109, 101, 109, 98, 97, 110, 116, 117, 109, 101,
62
+ 109, 98, 97, 119, 97, 109, 101, 109, 98, 101, 114, 105, 109, 101, 109, 98, 101, 114, 105, 107, 97, 110, 109, 101, 109,
63
+ 98, 111, 108, 101, 104, 107, 97, 110, 109, 101, 109, 98, 117, 97, 116, 109, 101, 109, 112, 117, 110, 121, 97, 105,
64
+ 109, 101, 110, 97, 109, 98, 97, 104, 109, 101, 110, 97, 114, 105, 107, 109, 101, 110, 97, 119, 97, 114, 107, 97, 110,
65
+ 109, 101, 110, 99, 97, 112, 97, 105, 109, 101, 110, 99, 97, 116, 97, 116, 107, 97, 110, 109, 101, 110, 100, 97, 112,
66
+ 97, 116, 109, 101, 110, 100, 97, 112, 97, 116, 107, 97, 110, 109, 101, 110, 101, 114, 105, 109, 97, 109, 101, 110,
67
+ 101, 114, 117, 115, 105, 109, 101, 110, 103, 97, 100, 97, 107, 97, 110, 109, 101, 110, 103, 97, 109, 98, 105, 108,
68
+ 109, 101, 110, 103, 101, 110, 97, 105, 109, 101, 110, 103, 103, 97, 108, 97, 107, 107, 97, 110, 109, 101, 110, 103,
69
+ 103, 117, 110, 97, 107, 97, 110, 109, 101, 110, 103, 105, 107, 117, 116, 109, 101, 110, 103, 117, 109, 117, 109, 107,
70
+ 97, 110, 109, 101, 110, 103, 117, 114, 97, 110, 103, 107, 97, 110, 109, 101, 110, 105, 110, 103, 107, 97, 116, 109,
71
+ 101, 110, 105, 110, 103, 107, 97, 116, 107, 97, 110, 109, 101, 110, 106, 97, 100, 105, 109, 101, 110, 106, 101, 108,
72
+ 97, 110, 103, 109, 101, 110, 111, 107, 111, 107, 109, 101, 110, 116, 101, 114, 105, 109, 101, 110, 117, 110, 106, 117,
73
+ 107, 107, 97, 110, 109, 101, 110, 117, 114, 117, 116, 109, 101, 110, 121, 97, 107, 115, 105, 107, 97, 110, 109, 101,
74
+ 110, 121, 101, 100, 105, 97, 107, 97, 110, 109, 101, 114, 101, 107, 97, 109, 101, 114, 111, 115, 111, 116, 109, 101,
75
+ 114, 117, 112, 97, 107, 97, 110, 109, 101, 115, 121, 117, 97, 114, 97, 116, 109, 105, 110, 97, 116, 109, 105, 110,
76
+ 103, 103, 117, 109, 105, 110, 121, 97, 107, 109, 111, 100, 97, 108, 109, 111, 104, 100, 109, 117, 100, 97, 104, 109,
77
+ 117, 110, 103, 107, 105, 110, 110, 97, 105, 107, 110, 97, 106, 105, 98, 110, 97, 115, 105, 111, 110, 97, 108, 110,
78
+ 101, 103, 97, 114, 97, 110, 101, 103, 101, 114, 105, 110, 105, 97, 103, 97, 110, 105, 108, 97, 105, 110, 111, 118,
79
+ 111, 103, 111, 115, 111, 107, 116, 111, 108, 101, 104, 111, 112, 101, 114, 97, 115, 105, 111, 114, 97, 110, 103, 112,
80
+ 97, 100, 97, 112, 97, 103, 105, 112, 97, 108, 105, 110, 103, 112, 97, 109, 101, 114, 97, 110, 112, 97, 112, 97, 110,
81
+ 112, 97, 114, 97, 112, 97, 114, 97, 115, 112, 97, 114, 108, 105, 109, 101, 110, 112, 97, 114, 116, 105, 112, 97, 115,
82
+ 97, 114, 97, 110, 112, 97, 115, 117, 107, 97, 110, 112, 101, 103, 97, 119, 97, 105, 112, 101, 106, 97, 98, 97, 116,
83
+ 112, 101, 107, 101, 114, 106, 97, 112, 101, 108, 97, 98, 117, 114, 112, 101, 108, 97, 98, 117, 114, 97, 110, 112, 101,
84
+ 108, 97, 110, 99, 111, 110, 103, 97, 110, 112, 101, 108, 97, 110, 103, 103, 97, 110, 112, 101, 108, 98, 97, 103, 97,
85
+ 105, 112, 101, 108, 117, 97, 110, 103, 112, 101, 109, 98, 97, 110, 103, 117, 110, 97, 110, 112, 101, 109, 98, 101,
86
+ 114, 105, 116, 97, 112, 101, 109, 98, 105, 110, 97, 97, 110, 112, 101, 109, 105, 109, 112, 105, 110, 112, 101, 110,
87
+ 100, 97, 112, 97, 116, 97, 110, 112, 101, 110, 100, 105, 100, 105, 107, 97, 110, 112, 101, 110, 100, 117, 100, 117,
88
+ 107, 112, 101, 110, 101, 114, 98, 97, 110, 103, 97, 110, 112, 101, 110, 103, 97, 114, 97, 104, 112, 101, 110, 103,
89
+ 101, 108, 117, 97, 114, 97, 110, 112, 101, 110, 103, 101, 114, 117, 115, 105, 112, 101, 110, 103, 103, 117, 110, 97,
90
+ 112, 101, 110, 103, 117, 114, 117, 115, 97, 110, 112, 101, 110, 105, 97, 103, 97, 112, 101, 110, 105, 110, 103, 107,
91
+ 97, 116, 97, 110, 112, 101, 110, 116, 105, 110, 103, 112, 101, 114, 97, 116, 117, 115, 112, 101, 114, 100, 97, 103,
92
+ 97, 110, 103, 97, 110, 112, 101, 114, 100, 97, 110, 97, 112, 101, 114, 105, 110, 103, 107, 97, 116, 112, 101, 114,
93
+ 106, 97, 110, 106, 105, 97, 110, 112, 101, 114, 107, 97, 114, 97, 112, 101, 114, 107, 104, 105, 100, 109, 97, 116, 97,
94
+ 110, 112, 101, 114, 108, 97, 100, 97, 110, 103, 97, 110, 112, 101, 114, 108, 117, 112, 101, 114, 109, 105, 110, 116,
95
+ 97, 97, 110, 112, 101, 114, 110, 105, 97, 103, 97, 97, 110, 112, 101, 114, 115, 101, 107, 117, 116, 117, 97, 110, 112,
96
+ 101, 114, 115, 105, 100, 97, 110, 103, 97, 110, 112, 101, 114, 116, 97, 109, 97, 112, 101, 114, 116, 117, 98, 117,
97
+ 104, 97, 110, 112, 101, 114, 116, 117, 109, 98, 117, 104, 97, 110, 112, 101, 114, 117, 115, 97, 104, 97, 97, 110, 112,
98
+ 101, 115, 101, 114, 116, 97, 112, 101, 116, 97, 110, 103, 112, 105, 104, 97, 107, 112, 105, 108, 105, 104, 97, 110,
99
+ 112, 105, 110, 106, 97, 109, 97, 110, 112, 111, 108, 105, 115, 112, 111, 108, 105, 116, 105, 107, 112, 114, 101, 115,
100
+ 105, 100, 101, 110, 112, 114, 101, 115, 116, 97, 115, 105, 112, 114, 111, 100, 117, 107, 112, 114, 111, 103, 114, 97,
101
+ 109, 112, 114, 111, 106, 101, 107, 112, 114, 111, 115, 101, 115, 112, 114, 111, 116, 111, 110, 112, 117, 107, 117,
102
+ 108, 112, 117, 108, 97, 112, 117, 115, 97, 116, 114, 97, 98, 117, 114, 97, 107, 97, 110, 114, 97, 107, 121, 97, 116,
103
+ 114, 97, 109, 97, 105, 114, 97, 110, 116, 97, 117, 114, 97, 121, 97, 114, 101, 110, 100, 97, 104, 114, 105, 110, 103,
104
+ 103, 105, 116, 114, 117, 109, 97, 104, 115, 97, 98, 97, 104, 115, 97, 104, 97, 106, 97, 115, 97, 104, 97, 109, 115,
105
+ 97, 109, 97, 115, 97, 114, 97, 119, 97, 107, 115, 97, 116, 117, 115, 97, 119, 105, 116, 115, 97, 121, 97, 115, 100,
106
+ 110, 115, 101, 98, 97, 103, 97, 105, 115, 101, 98, 97, 104, 97, 103, 105, 97, 110, 115, 101, 98, 97, 110, 121, 97,
107
+ 107, 115, 101, 98, 97, 114, 97, 110, 103, 115, 101, 98, 101, 108, 117, 109, 115, 101, 98, 101, 108, 117, 109, 110,
108
+ 121, 97, 115, 101, 98, 117, 97, 104, 115, 101, 99, 97, 114, 97, 115, 101, 100, 97, 110, 103, 115, 101, 103, 105, 115,
109
+ 101, 104, 105, 110, 103, 103, 97, 115, 101, 106, 97, 107, 115, 101, 107, 97, 114, 97, 110, 103, 115, 101, 107, 116,
110
+ 111, 114, 115, 101, 107, 117, 114, 105, 116, 105, 115, 101, 108, 97, 105, 110, 115, 101, 108, 97, 109, 97, 115, 101,
111
+ 108, 97, 115, 97, 115, 101, 108, 97, 116, 97, 110, 115, 101, 108, 101, 112, 97, 115, 115, 101, 108, 117, 114, 117,
112
+ 104, 115, 101, 109, 97, 107, 105, 110, 115, 101, 109, 97, 108, 97, 109, 115, 101, 109, 97, 115, 97, 115, 101, 109,
113
+ 101, 110, 116, 97, 114, 97, 115, 101, 109, 117, 97, 115, 101, 109, 117, 108, 97, 115, 101, 110, 115, 101, 110, 100,
114
+ 105, 114, 105, 115, 101, 111, 114, 97, 110, 103, 115, 101, 112, 97, 110, 106, 97, 110, 103, 115, 101, 112, 101, 114,
115
+ 116, 105, 115, 101, 112, 116, 115, 101, 112, 116, 101, 109, 98, 101, 114, 115, 101, 114, 97, 110, 116, 97, 117, 115,
116
+ 101, 114, 105, 115, 101, 114, 116, 97, 115, 101, 115, 105, 115, 101, 116, 105, 97, 112, 115, 101, 116, 105, 97, 117,
117
+ 115, 97, 104, 97, 115, 105, 100, 97, 110, 103, 115, 105, 110, 103, 97, 112, 117, 114, 97, 115, 105, 110, 105, 115,
118
+ 105, 115, 116, 101, 109, 115, 111, 107, 111, 110, 103, 97, 110, 115, 114, 105, 115, 117, 100, 97, 104, 115, 117, 107,
119
+ 97, 110, 115, 117, 107, 117, 115, 117, 109, 98, 101, 114, 115, 117, 112, 97, 121, 97, 115, 117, 115, 117, 116, 115,
120
+ 121, 97, 114, 105, 107, 97, 116, 115, 121, 101, 100, 116, 97, 104, 97, 112, 116, 97, 104, 117, 110, 116, 97, 110, 116,
121
+ 97, 110, 97, 104, 116, 97, 110, 112, 97, 116, 97, 119, 97, 114, 97, 110, 116, 101, 107, 110, 111, 108, 111, 103, 105,
122
+ 116, 101, 108, 97, 104, 116, 101, 109, 112, 97, 116, 116, 101, 109, 112, 97, 116, 97, 110, 116, 101, 109, 112, 111,
123
+ 104, 116, 101, 110, 97, 103, 97, 116, 101, 110, 103, 97, 104, 116, 101, 110, 116, 97, 110, 103, 116, 101, 114, 98, 97,
124
+ 105, 107, 116, 101, 114, 98, 97, 110, 103, 116, 101, 114, 98, 101, 115, 97, 114, 116, 101, 114, 98, 117, 107, 97, 116,
125
+ 101, 114, 100, 97, 112, 97, 116, 116, 101, 114, 104, 97, 100, 97, 112, 116, 101, 114, 109, 97, 115, 117, 107, 116,
126
+ 101, 114, 115, 101, 98, 117, 116, 116, 101, 114, 117, 115, 116, 101, 116, 97, 112, 105, 116, 104, 97, 105, 108, 97,
127
+ 110, 100, 116, 105, 97, 100, 97, 116, 105, 100, 97, 107, 116, 105, 103, 97, 116, 105, 109, 98, 97, 108, 97, 110, 116,
128
+ 105, 109, 117, 114, 116, 105, 110, 100, 97, 107, 97, 110, 116, 105, 110, 103, 103, 105, 116, 117, 110, 116, 117, 110,
129
+ 97, 105, 116, 117, 114, 117, 110, 116, 117, 114, 117, 116, 117, 109, 110, 111, 117, 110, 105, 116, 117, 110, 116, 117,
130
+ 107, 117, 110, 116, 117, 110, 103, 117, 114, 117, 115, 117, 115, 97, 104, 97, 117, 116, 97, 109, 97, 119, 97, 108, 97,
131
+ 117, 112, 117, 110, 119, 97, 110, 103, 119, 97, 110, 105, 116, 97, 119, 105, 108, 97, 121, 97, 104, 121, 97, 110, 103,
132
+ };
133
+
134
+ static const uint32_t stop_word_offsets[471] = {
135
+ 0, 5, 13, 18, 21, 27, 32, 35, 39, 45, 50, 58, 62, 66, 73, 77, 84, 90, 102, 105, 112, 117, 119, 123, 128, 132, 137,
136
+ 141, 145, 154, 158, 162, 174, 178, 186, 191, 197, 203, 207, 213, 217, 223, 231, 235, 240, 248, 253, 259, 264, 270,
137
+ 278, 288, 299, 307, 316, 323, 332, 341, 348, 357, 364, 371, 378, 386, 393, 400, 405, 408, 414, 420, 422, 427, 432,
138
+ 437, 442, 450, 455, 463, 468, 471, 475, 480, 484, 492, 497, 503, 508, 516, 522, 527, 538, 543, 545, 553, 559, 569,
139
+ 577, 587, 590, 598, 605, 610, 612, 615, 620, 627, 636, 643, 648, 652, 658, 661, 667, 674, 679, 684, 688, 693, 699,
140
+ 707, 709, 714, 719, 725, 730, 739, 747, 750, 755, 760, 763, 766, 773, 778, 781, 788, 800, 805, 809, 815, 819, 824,
141
+ 830, 836, 839, 843, 848, 856, 860, 864, 868, 875, 882, 889, 891, 898, 903, 908, 917, 922, 933, 942, 950, 959, 965,
142
+ 976, 985, 993, 999, 1005, 1010, 1019, 1022, 1033, 1044, 1053, 1059, 1064, 1074, 1082, 1088, 1092, 1096, 1100, 1106,
143
+ 1114, 1121, 1124, 1129, 1134, 1139, 1147, 1151, 1155, 1162, 1169, 1174, 1179, 1183, 1186, 1190, 1196, 1199, 1207,
144
+ 1211, 1217, 1224, 1232, 1237, 1245, 1249, 1257, 1261, 1268, 1273, 1283, 1287, 1292, 1295, 1302, 1309, 1321, 1331,
145
+ 1339, 1346, 1353, 1363, 1374, 1381, 1390, 1398, 1405, 1415, 1423, 1434, 1442, 1453, 1461, 1469, 1479, 1488, 1496,
146
+ 1508, 1519, 1527, 1538, 1550, 1559, 1571, 1578, 1587, 1594, 1601, 1612, 1619, 1630, 1641, 1647, 1654, 1663, 1672,
147
+ 1677, 1683, 1689, 1694, 1698, 1703, 1710, 1714, 1719, 1727, 1733, 1739, 1744, 1749, 1752, 1756, 1759, 1763, 1770,
148
+ 1775, 1779, 1783, 1789, 1796, 1801, 1805, 1810, 1818, 1823, 1830, 1837, 1844, 1851, 1858, 1865, 1874, 1885, 1894,
149
+ 1902, 1909, 1920, 1929, 1938, 1946, 1956, 1966, 1974, 1985, 1993, 2004, 2013, 2021, 2031, 2038, 2049, 2056, 2063,
150
+ 2074, 2081, 2090, 2100, 2107, 2119, 2130, 2135, 2145, 2155, 2166, 2177, 2184, 2194, 2205, 2215, 2222, 2228, 2233,
151
+ 2240, 2248, 2253, 2260, 2268, 2276, 2282, 2289, 2295, 2301, 2307, 2312, 2316, 2321, 2325, 2330, 2336, 2341, 2347,
152
+ 2351, 2357, 2364, 2369, 2374, 2380, 2385, 2389, 2396, 2400, 2405, 2409, 2412, 2419, 2429, 2437, 2445, 2452, 2462,
153
+ 2468, 2474, 2480, 2484, 2492, 2497, 2505, 2511, 2519, 2525, 2531, 2537, 2544, 2551, 2558, 2565, 2572, 2578, 2587,
154
+ 2592, 2598, 2601, 2608, 2615, 2624, 2631, 2635, 2644, 2652, 2656, 2661, 2665, 2671, 2681, 2687, 2696, 2700, 2706,
155
+ 2714, 2717, 2722, 2727, 2731, 2737, 2743, 2748, 2756, 2760, 2765, 2770, 2773, 2778, 2783, 2790, 2799, 2804, 2810,
156
+ 2818, 2824, 2830, 2836, 2843, 2850, 2857, 2865, 2872, 2880, 2888, 2896, 2904, 2909, 2915, 2923, 2928, 2933, 2937,
157
+ 2945, 2950, 2958, 2964, 2967, 2972, 2977, 2982, 2986, 2990, 2995, 3001, 3005, 3010, 3015, 3023, 3027, 3033, 3040,
158
+ 3044,
159
+ };
160
+
161
+ static const uint32_t keep_ranges[6] = {
162
+ 48, 57, 65, 90, 97, 122,
163
+ };
164
+
165
+ const narsil_language narsil_language_malay = {
166
+ .name = "malay",
167
+ .revision = "1",
168
+ .stop_word_units = stop_word_units,
169
+ .stop_word_offsets = stop_word_offsets,
170
+ .stop_word_count = 470,
171
+ .ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
172
+ .keep_ranges = keep_ranges,
173
+ .keep_range_count = 3,
174
+ .splits_code_points = 0,
175
+ .strips_possessive = 0,
176
+ .normalises_diacritics = 0,
177
+ .min_token_length = 1,
178
+ .ngram_size = 0,
179
+ .normalise = NULL,
180
+ .stem_by_rules = NULL,
181
+ .create_snowball = NULL,
182
+ .snowball_stem = NULL,
183
+ .close_snowball = NULL,
184
+ };
package/cpp/core/src/languages/data/malayalam.c ADDED
@@ -0,0 +1,64 @@
1
+ #include "../../analysis/languages.h"
2
+ #include "declarations.h"
3
+ #include "../rules.h"
4
+
5
+ #include <stddef.h>
6
+ #include <stdint.h>
7
+
8
+ static const uint16_t stop_word_units[435] = {
9
+ 3333, 3353, 3405, 3353, 3368, 3398, 3333, 3364, 3391, 3368, 3390, 3453, 3333, 3364, 3391, 3368, 3405, 3333, 3364,
10
+ 3391, 3368, 3405, 3377, 3398, 3333, 3364, 3391, 3453, 3333, 3364, 3393, 3330, 3333, 3364, 3393, 3349, 3402, 3363,
11
+ 3405, 3359, 3405, 3333, 3364, 3405, 3333, 3368, 3405, 3368, 3399, 3376, 3330, 3333, 3368, 3405, 3368, 3405, 3333,
12
+ 3370, 3405, 3370, 3403, 3454, 3333, 3378, 3405, 3378, 3333, 3378, 3405, 3378, 3398, 3353, 3405, 3349, 3391, 3453,
13
+ 3333, 3381, 3368, 3405, 3377, 3398, 3333, 3381, 3376, 3393, 3359, 3398, 3333, 3381, 3379, 3393, 3359, 3398, 3333,
14
+ 3381, 3391, 3359, 3398, 3333, 3381, 3451, 3333, 3381, 3452, 3333, 3381, 3454, 3334, 3334, 3349, 3393, 3330, 3334,
15
+ 3349, 3393, 3368, 3405, 3368, 3393, 3334, 3363, 3405, 3334, 3375, 3334, 3375, 3391, 3334, 3375, 3391, 3376, 3393,
16
+ 3368, 3405, 3368, 3393, 3334, 3376, 3405, 3335, 3353, 3405, 3353, 3368, 3398, 3335, 3364, 3391, 3368, 3405, 3335,
17
+ 3364, 3391, 3368, 3405, 3377, 3398, 3335, 3364, 3391, 3453, 3335, 3364, 3393, 3330, 3335, 3364, 3405, 3335, 3368,
18
+ 3391, 3335, 3368, 3405, 3368, 3405, 3335, 3370, 3405, 3370, 3403, 3454, 3335, 3378, 3405, 3378, 3335, 3381, 3391,
19
+ 3359, 3398, 3335, 3381, 3452, 3336, 3337, 3363, 3405, 3359, 3405, 3337, 3379, 3405, 3379, 3342, 3353, 3405, 3349,
20
+ 3391, 3378, 3393, 3330, 3342, 3353, 3405, 3349, 3391, 3453, 3342, 3353, 3405, 3353, 3368, 3398, 3342, 3368, 3405,
21
+ 3364, 3405, 3342, 3368, 3405, 3368, 3342, 3368, 3405, 3368, 3390, 3453, 3342, 3368, 3405, 3368, 3391, 3381, 3342,
22
+ 3368, 3405, 3368, 3393, 3342, 3368, 3405, 3368, 3393, 3330, 3342, 3368, 3405, 3368, 3405, 3342, 3368, 3405, 3377,
23
+ 3398, 3342, 3370, 3405, 3370, 3403, 3454, 3342, 3378, 3405, 3378, 3390, 3342, 3381, 3391, 3359, 3398, 3343, 3364,
24
+ 3405, 3343, 3377, 3405, 3377, 3381, 3393, 3330, 3346, 3370, 3405, 3370, 3330, 3346, 3376, 3393, 3347, 3376, 3403,
25
+ 3349, 3380, 3391, 3375, 3393, 3330, 3349, 3394, 3359, 3390, 3364, 3398, 3349, 3394, 3359, 3391, 3349, 3394, 3359,
26
+ 3398, 3354, 3391, 3378, 3358, 3353, 3405, 3353, 3454, 3358, 3390, 3451, 3364, 3368, 3405, 3368, 3398, 3364, 3368,
27
+ 3405, 3377, 3398, 3364, 3390, 3451, 3368, 3374, 3405, 3374, 3393, 3359, 3398, 3368, 3390, 3330, 3368, 3391, 3353,
28
+ 3405, 3353, 3454, 3368, 3391, 3368, 3405, 3377, 3398, 3368, 3392, 3370, 3349, 3405, 3383, 3399, 3370, 3378, 3370,
29
+ 3391, 3368, 3405, 3368, 3398, 3370, 3403, 3378, 3393, 3330, 3370, 3403, 3378, 3398, 3374, 3377, 3405, 3377, 3393,
30
+ 3374, 3377, 3405, 3377, 3405, 3374, 3390, 3364, 3405, 3376, 3330, 3374, 3393, 3364, 3453, 3381, 3376, 3398, 3381,
31
+ 3379, 3376, 3398, 3381, 3399, 3363, 3330, 3381, 3399, 3363, 3405, 3359, 3391, 3382, 3399, 3383, 3330,
32
+ };
33
+
34
+ static const uint32_t stop_word_offsets[92] = {
35
+ 0, 6, 12, 17, 24, 28, 32, 41, 44, 51, 56, 62, 66, 76, 82, 88, 94, 99, 102, 105, 108, 109, 113, 120, 123, 125, 128,
36
+ 137, 140, 146, 151, 158, 162, 166, 169, 172, 177, 183, 187, 192, 195, 196, 201, 205, 213, 219, 225, 230, 234, 240,
37
+ 246, 251, 257, 262, 267, 273, 278, 283, 286, 293, 298, 301, 304, 310, 316, 320, 324, 327, 332, 335, 340, 345, 348,
38
+ 355, 358, 364, 370, 372, 377, 379, 385, 390, 394, 399, 404, 410, 414, 417, 421, 425, 431, 435,
39
+ };
40
+
41
+ static const uint32_t keep_ranges[10] = {
42
+ 48, 57, 65, 90, 97, 122, 3328, 3455, 8204, 8205,
43
+ };
44
+
45
+ const narsil_language narsil_language_malayalam = {
46
+ .name = "malayalam",
47
+ .revision = "1",
48
+ .stop_word_units = stop_word_units,
49
+ .stop_word_offsets = stop_word_offsets,
50
+ .stop_word_count = 91,
51
+ .ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
52
+ .keep_ranges = keep_ranges,
53
+ .keep_range_count = 5,
54
+ .splits_code_points = 0,
55
+ .strips_possessive = 0,
56
+ .normalises_diacritics = 0,
57
+ .min_token_length = 1,
58
+ .ngram_size = 0,
59
+ .normalise = narsil_malayalam_normalise,
60
+ .stem_by_rules = NULL,
61
+ .create_snowball = NULL,
62
+ .snowball_stem = NULL,
63
+ .close_snowball = NULL,
64
+ };
package/cpp/core/src/languages/data/maltese.c ADDED
@@ -0,0 +1,102 @@
1
+ #include "../../analysis/languages.h"
2
+ #include "declarations.h"
3
+
4
+ #include <stddef.h>
5
+ #include <stdint.h>
6
+
7
+ static const uint16_t stop_word_units[456] = {
8
+ 97, 107, 116, 97, 114, 97, 110, 107, 101, 97, 110, 107, 105, 97, 295, 110, 97, 98, 101, 106, 110, 98, 105, 98, 105,
9
+ 100, 98, 105, 101, 120, 98, 105, 108, 98, 105, 108, 108, 105, 98, 105, 110, 98, 105, 114, 98, 105, 115, 98, 105, 115,
10
+ 115, 98, 105, 116, 98, 105, 120, 98, 295, 97, 108, 98, 295, 97, 108, 97, 100, 97, 107, 100, 97, 110, 100, 97, 119,
11
+ 107, 100, 97, 119, 110, 100, 105, 107, 100, 105, 110, 100, 105, 289, 224, 100, 119, 97, 114, 102, 101, 106, 110, 102,
12
+ 105, 102, 105, 100, 102, 105, 108, 102, 105, 110, 102, 105, 114, 102, 105, 115, 102, 105, 116, 102, 105, 120, 102,
13
+ 108, 102, 116, 105, 116, 102, 117, 113, 103, 295, 97, 108, 103, 295, 97, 108, 105, 101, 120, 103, 295, 97, 108, 108,
14
+ 103, 295, 97, 115, 103, 295, 97, 116, 103, 295, 97, 120, 104, 97, 119, 110, 104, 101, 109, 109, 104, 105, 104, 105,
15
+ 106, 97, 104, 117, 104, 117, 109, 97, 104, 117, 119, 97, 105, 100, 105, 107, 116, 97, 114, 105, 107, 117, 110, 105,
16
+ 108, 105, 109, 109, 97, 105, 110, 105, 110, 116, 105, 110, 116, 105, 105, 110, 116, 111, 109, 105, 114, 105, 115, 105,
17
+ 115, 115, 97, 105, 116, 105, 120, 105, 267, 105, 380, 105, 380, 100, 97, 106, 101, 107, 107, 106, 101, 119, 106, 105,
18
+ 101, 110, 106, 105, 101, 110, 97, 107, 105, 101, 110, 107, 105, 101, 110, 101, 116, 107, 105, 101, 110, 117, 107, 105,
19
+ 102, 107, 111, 108, 108, 104, 97, 107, 117, 108, 108, 108, 108, 105, 108, 105, 108, 108, 105, 108, 108, 109, 97, 109,
20
+ 97, 39, 109, 97, 100, 109, 97, 108, 109, 97, 110, 109, 97, 114, 109, 97, 115, 109, 97, 116, 109, 101, 116, 97, 109,
21
+ 104, 105, 120, 109, 104, 117, 120, 109, 105, 100, 109, 105, 108, 108, 109, 105, 110, 109, 105, 110, 110, 109, 105,
22
+ 114, 109, 105, 115, 109, 105, 116, 109, 105, 120, 113, 97, 98, 101, 108, 115, 97, 115, 97, 108, 116, 97, 116, 97, 39,
23
+ 116, 97, 100, 116, 97, 103, 295, 104, 97, 116, 97, 103, 295, 104, 111, 109, 116, 97, 103, 295, 107, 111, 109, 116, 97,
24
+ 103, 295, 110, 97, 116, 97, 108, 116, 97, 110, 116, 97, 114, 116, 97, 115, 116, 97, 116, 116, 97, 120, 116, 97, 295,
25
+ 116, 116, 105, 101, 103, 295, 101, 107, 116, 105, 101, 103, 295, 105, 116, 105, 101, 103, 295, 117, 116, 107, 117,
26
+ 110, 117, 117, 107, 111, 108, 108, 119, 97, 114, 97, 120, 101, 106, 110, 295, 97, 102, 110, 97,
27
+ };
28
+
29
+ static const uint32_t stop_word_offsets[128] = {
30
+ 0, 5, 9, 13, 17, 21, 23, 26, 30, 33, 38, 41, 44, 47, 51, 54, 57, 61, 66, 69, 72, 76, 80, 83, 86, 90, 94, 98, 100, 103,
31
+ 106, 109, 112, 115, 118, 121, 123, 127, 130, 134, 141, 146, 150, 154, 158, 162, 166, 168, 172, 174, 178, 182, 184,
32
+ 189, 193, 195, 199, 201, 204, 208, 213, 215, 217, 221, 223, 225, 227, 229, 233, 237, 240, 244, 249, 253, 259, 264,
33
+ 267, 273, 277, 278, 280, 283, 287, 289, 292, 295, 298, 301, 304, 307, 310, 314, 318, 322, 325, 329, 332, 336, 339,
34
+ 342, 345, 348, 353, 355, 358, 360, 363, 366, 372, 379, 386, 392, 395, 398, 401, 404, 407, 410, 414, 421, 427, 433,
35
+ 437, 438, 443, 447, 451, 456,
36
+ };
37
+
38
+ static const uint32_t keep_ranges[734] = {
39
+ 39, 39, 48, 57, 65, 90, 97, 122, 170, 170, 186, 186, 192, 214, 216, 246, 248, 696, 736, 740, 768, 879, 921, 921, 953,
40
+ 953, 1155, 1161, 1425, 1469, 1471, 1471, 1473, 1474, 1476, 1477, 1479, 1479, 1552, 1562, 1611, 1631, 1648, 1648, 1750,
41
+ 1756, 1759, 1764, 1767, 1768, 1770, 1773, 1809, 1809, 1840, 1866, 1958, 1968, 2027, 2035, 2045, 2045, 2070, 2073,
42
+ 2075, 2083, 2085, 2087, 2089, 2093, 2137, 2139, 2199, 2207, 2250, 2273, 2275, 2307, 2362, 2364, 2366, 2383, 2385,
43
+ 2391, 2402, 2403, 2433, 2435, 2492, 2492, 2494, 2500, 2503, 2504, 2507, 2509, 2519, 2519, 2530, 2531, 2558, 2558,
44
+ 2561, 2563, 2620, 2620, 2622, 2626, 2631, 2632, 2635, 2637, 2641, 2641, 2672, 2673, 2677, 2677, 2689, 2691, 2748,
45
+ 2748, 2750, 2757, 2759, 2761, 2763, 2765, 2786, 2787, 2810, 2815, 2817, 2819, 2876, 2876, 2878, 2884, 2887, 2888,
46
+ 2891, 2893, 2901, 2903, 2914, 2915, 2946, 2946, 3006, 3010, 3014, 3016, 3018, 3021, 3031, 3031, 3072, 3076, 3132,
47
+ 3132, 3134, 3140, 3142, 3144, 3146, 3149, 3157, 3158, 3170, 3171, 3201, 3203, 3260, 3260, 3262, 3268, 3270, 3272,
48
+ 3274, 3277, 3285, 3286, 3298, 3299, 3315, 3315, 3328, 3331, 3387, 3388, 3390, 3396, 3398, 3400, 3402, 3405, 3415,
49
+ 3415, 3426, 3427, 3457, 3459, 3530, 3530, 3535, 3540, 3542, 3542, 3544, 3551, 3570, 3571, 3633, 3633, 3636, 3642,
50
+ 3655, 3662, 3761, 3761, 3764, 3772, 3784, 3790, 3864, 3865, 3893, 3893, 3895, 3895, 3897, 3897, 3902, 3903, 3953,
51
+ 3972, 3974, 3975, 3981, 3991, 3993, 4028, 4038, 4038, 4139, 4158, 4182, 4185, 4190, 4192, 4194, 4196, 4199, 4205,
52
+ 4209, 4212, 4226, 4237, 4239, 4239, 4250, 4253, 4957, 4959, 5906, 5909, 5938, 5940, 5970, 5971, 6002, 6003, 6068,
53
+ 6099, 6109, 6109, 6155, 6157, 6159, 6159, 6277, 6278, 6313, 6313, 6432, 6443, 6448, 6459, 6679, 6683, 6741, 6750,
54
+ 6752, 6780, 6783, 6783, 6832, 6877, 6880, 6891, 6912, 6916, 6964, 6980, 7019, 7027, 7040, 7042, 7073, 7085, 7142,
55
+ 7155, 7204, 7223, 7376, 7378, 7380, 7400, 7405, 7405, 7412, 7412, 7415, 7417, 7424, 7461, 7468, 7516, 7522, 7525,
56
+ 7531, 7543, 7545, 7614, 7616, 7935, 8126, 8126, 8305, 8305, 8319, 8319, 8336, 8348, 8400, 8432, 8490, 8491, 8498,
57
+ 8498, 8526, 8526, 8544, 8584, 11360, 11391, 11503, 11505, 11647, 11647, 11744, 11775, 12330, 12335, 12441, 12442,
58
+ 42607, 42610, 42612, 42621, 42654, 42655, 42736, 42737, 42786, 42887, 42891, 42972, 42993, 43007, 43010, 43010, 43014,
59
+ 43014, 43019, 43019, 43043, 43047, 43052, 43052, 43136, 43137, 43188, 43205, 43232, 43249, 43263, 43263, 43302, 43309,
60
+ 43335, 43347, 43392, 43395, 43443, 43456, 43493, 43493, 43561, 43574, 43587, 43587, 43596, 43597, 43643, 43645, 43696,
61
+ 43696, 43698, 43700, 43703, 43704, 43710, 43711, 43713, 43713, 43755, 43759, 43765, 43766, 43824, 43866, 43868, 43876,
62
+ 43878, 43881, 44003, 44010, 44012, 44013, 64256, 64262, 64286, 64286, 65024, 65039, 65056, 65071, 65313, 65338, 65345,
63
+ 65370, 66045, 66045, 66272, 66272, 66422, 66426, 67456, 67461, 67463, 67504, 67506, 67514, 68097, 68099, 68101, 68102,
64
+ 68108, 68111, 68152, 68154, 68159, 68159, 68325, 68326, 68900, 68903, 68969, 68973, 69291, 69292, 69370, 69375, 69446,
65
+ 69456, 69506, 69509, 69632, 69634, 69688, 69702, 69744, 69744, 69747, 69748, 69759, 69762, 69808, 69818, 69826, 69826,
66
+ 69888, 69890, 69927, 69940, 69957, 69958, 70003, 70003, 70016, 70018, 70067, 70080, 70089, 70092, 70094, 70095, 70188,
67
+ 70199, 70206, 70206, 70209, 70209, 70367, 70378, 70400, 70403, 70459, 70460, 70462, 70468, 70471, 70472, 70475, 70477,
68
+ 70487, 70487, 70498, 70499, 70502, 70508, 70512, 70516, 70584, 70592, 70594, 70594, 70597, 70597, 70599, 70602, 70604,
69
+ 70608, 70610, 70610, 70625, 70626, 70709, 70726, 70750, 70750, 70832, 70851, 71087, 71093, 71096, 71104, 71132, 71133,
70
+ 71216, 71232, 71339, 71351, 71453, 71467, 71724, 71738, 71984, 71989, 71991, 71992, 71995, 71998, 72000, 72000, 72002,
71
+ 72003, 72145, 72151, 72154, 72160, 72164, 72164, 72193, 72202, 72243, 72249, 72251, 72254, 72263, 72263, 72273, 72283,
72
+ 72330, 72345, 72544, 72551, 72751, 72758, 72760, 72767, 72850, 72871, 72873, 72886, 73009, 73014, 73018, 73018, 73020,
73
+ 73021, 73023, 73029, 73031, 73031, 73098, 73102, 73104, 73105, 73107, 73111, 73459, 73462, 73472, 73473, 73475, 73475,
74
+ 73524, 73530, 73534, 73538, 73562, 73562, 78912, 78912, 78919, 78933, 90398, 90415, 92912, 92916, 92976, 92982, 94031,
75
+ 94031, 94033, 94087, 94095, 94098, 94180, 94180, 94192, 94193, 113821, 113822, 118528, 118573, 118576, 118598, 119141,
76
+ 119145, 119149, 119154, 119163, 119170, 119173, 119179, 119210, 119213, 119362, 119364, 121344, 121398, 121403,
77
+ 121452, 121461, 121461, 121476, 121476, 121499, 121503, 121505, 121519, 122624, 122654, 122661, 122666, 122880,
78
+ 122886, 122888, 122904, 122907, 122913, 122915, 122916, 122918, 122922, 123023, 123023, 123184, 123190, 123566,
79
+ 123566, 123628, 123631, 124140, 124143, 124398, 124399, 124643, 124643, 124646, 124646, 124654, 124655, 124661,
80
+ 124661, 125136, 125142, 125252, 125258, 917760, 917999,
81
+ };
82
+
83
+ const narsil_language narsil_language_maltese = {
84
+ .name = "maltese",
85
+ .revision = "1",
86
+ .stop_word_units = stop_word_units,
87
+ .stop_word_offsets = stop_word_offsets,
88
+ .stop_word_count = 127,
89
+ .ascii_keep = {0x03FF008000000000ULL, 0x07FFFFFE07FFFFFEULL},
90
+ .keep_ranges = keep_ranges,
91
+ .keep_range_count = 367,
92
+ .splits_code_points = 1,
93
+ .strips_possessive = 0,
94
+ .normalises_diacritics = 0,
95
+ .min_token_length = 1,
96
+ .ngram_size = 0,
97
+ .normalise = NULL,
98
+ .stem_by_rules = NULL,
99
+ .create_snowball = NULL,
100
+ .snowball_stem = NULL,
101
+ .close_snowball = NULL,
102
+ };
package/cpp/core/src/languages/data/maori.c ADDED
@@ -0,0 +1,52 @@
1
+ #include "../../analysis/languages.h"
2
+ #include "declarations.h"
3
+
4
+ #include <stddef.h>
5
+ #include <stdint.h>
6
+
7
+ static const uint16_t stop_word_units[237] = {
8
+ 97, 97, 104, 97, 107, 111, 97, 97, 104, 97, 117, 97, 105, 97, 110, 97, 97, 110, 97, 107, 101, 97, 110, 333, 97, 117,
9
+ 97, 117, 97, 101, 101, 110, 103, 97, 114, 105, 104, 101, 104, 101, 105, 104, 111, 107, 105, 105, 105, 97, 107, 97,
10
+ 107, 97, 116, 111, 97, 107, 97, 117, 97, 107, 101, 105, 107, 105, 107, 105, 97, 107, 111, 107, 111, 101, 107, 111,
11
+ 114, 101, 107, 111, 117, 116, 111, 117, 107, 117, 97, 107, 257, 104, 111, 114, 101, 107, 257, 111, 114, 101, 107, 333,
12
+ 114, 117, 97, 109, 101, 109, 101, 104, 101, 109, 101, 97, 109, 257, 109, 257, 116, 111, 117, 109, 257, 117, 97, 109,
13
+ 333, 110, 101, 105, 110, 103, 257, 110, 257, 110, 333, 111, 111, 116, 105, 114, 257, 112, 101, 97, 114, 257, 114, 257,
14
+ 110, 101, 105, 114, 257, 116, 111, 117, 114, 257, 117, 97, 116, 97, 107, 117, 116, 97, 110, 97, 116, 97, 117, 97, 116,
15
+ 101, 116, 111, 110, 117, 116, 257, 107, 117, 116, 257, 110, 97, 116, 257, 116, 111, 117, 116, 257, 117, 97, 116, 275,
16
+ 110, 101, 105, 116, 275, 110, 257, 116, 275, 114, 257, 116, 275, 116, 97, 104, 105, 116, 333, 116, 333, 107, 117, 116,
17
+ 333, 110, 97, 275, 110, 101, 105, 275, 110, 257, 275, 114, 257, 275, 116, 97, 104, 105,
18
+ };
19
+
20
+ static const uint32_t stop_word_offsets[68] = {
21
+ 0, 1, 7, 11, 13, 16, 21, 24, 26, 29, 30, 36, 38, 41, 45, 46, 48, 50, 55, 59, 62, 64, 67, 69, 72, 76, 82, 85, 91, 96,
22
+ 101, 103, 110, 112, 117, 121, 123, 126, 129, 131, 133, 134, 139, 142, 144, 149, 154, 158, 162, 166, 170, 172, 176,
23
+ 180, 184, 189, 193, 198, 202, 206, 212, 214, 218, 222, 226, 229, 232, 237,
24
+ };
25
+
26
+ static const uint32_t keep_ranges[74] = {
27
+ 48, 57, 65, 90, 97, 122, 170, 170, 186, 186, 192, 214, 216, 246, 248, 696, 736, 740, 7424, 7461, 7468, 7516, 7522,
28
+ 7525, 7531, 7543, 7545, 7614, 7680, 7935, 8305, 8305, 8319, 8319, 8336, 8348, 8490, 8491, 8498, 8498, 8526, 8526,
29
+ 8544, 8584, 11360, 11391, 42786, 42887, 42891, 42972, 42993, 43007, 43824, 43866, 43868, 43876, 43878, 43881, 64256,
30
+ 64262, 65313, 65338, 65345, 65370, 67456, 67461, 67463, 67504, 67506, 67514, 122624, 122654, 122661, 122666,
31
+ };
32
+
33
+ const narsil_language narsil_language_maori = {
34
+ .name = "maori",
35
+ .revision = "1",
36
+ .stop_word_units = stop_word_units,
37
+ .stop_word_offsets = stop_word_offsets,
38
+ .stop_word_count = 67,
39
+ .ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
40
+ .keep_ranges = keep_ranges,
41
+ .keep_range_count = 37,
42
+ .splits_code_points = 1,
43
+ .strips_possessive = 0,
44
+ .normalises_diacritics = 0,
45
+ .min_token_length = 1,
46
+ .ngram_size = 0,
47
+ .normalise = NULL,
48
+ .stem_by_rules = NULL,
49
+ .create_snowball = NULL,
50
+ .snowball_stem = NULL,
51
+ .close_snowball = NULL,
52
+ };
package/cpp/core/src/languages/data/marathi.c ADDED
@@ -0,0 +1,62 @@
1
+ #include "../../analysis/languages.h"
2
+ #include "declarations.h"
3
+
4
+ #include <stddef.h>
5
+ #include <stdint.h>
6
+
7
+ static const uint16_t stop_word_units[389] = {
8
+ 2309, 2343, 2367, 2325, 2309, 2344, 2375, 2325, 2309, 2358, 2368, 2309, 2360, 2354, 2351, 2366, 2330, 2375, 2309,
9
+ 2360, 2354, 2375, 2354, 2381, 2351, 2366, 2309, 2360, 2366, 2309, 2360, 2370, 2344, 2309, 2360, 2375, 2310, 2332,
10
+ 2310, 2339, 2367, 2310, 2340, 2366, 2310, 2346, 2354, 2381, 2351, 2366, 2310, 2354, 2366, 2310, 2354, 2368, 2310,
11
+ 2354, 2375, 2310, 2361, 2375, 2310, 2361, 2375, 2340, 2319, 2325, 2319, 2325, 2366, 2325, 2350, 2368, 2325, 2352,
12
+ 2339, 2351, 2366, 2340, 2325, 2352, 2370, 2344, 2325, 2366, 2325, 2366, 2350, 2325, 2366, 2351, 2325, 2366, 2361,
13
+ 2368, 2325, 2367, 2357, 2366, 2325, 2368, 2325, 2375, 2354, 2366, 2325, 2375, 2354, 2368, 2325, 2375, 2354, 2375,
14
+ 2325, 2379, 2335, 2368, 2327, 2375, 2354, 2381, 2351, 2366, 2328, 2375, 2314, 2344, 2332, 2366, 2340, 2333, 2366,
15
+ 2354, 2366, 2333, 2366, 2354, 2368, 2333, 2366, 2354, 2375, 2333, 2366, 2354, 2375, 2354, 2381, 2351, 2366, 2335,
16
+ 2366, 2337, 2377, 2340, 2352, 2340, 2352, 2368, 2340, 2360, 2375, 2330, 2340, 2366, 2340, 2368, 2340, 2368, 2344,
17
+ 2340, 2375, 2340, 2379, 2340, 2381, 2351, 2366, 2340, 2381, 2351, 2366, 2330, 2366, 2340, 2381, 2351, 2366, 2330,
18
+ 2368, 2340, 2381, 2351, 2366, 2330, 2381, 2351, 2366, 2340, 2381, 2351, 2366, 2344, 2366, 2340, 2381, 2351, 2366,
19
+ 2344, 2368, 2340, 2381, 2351, 2366, 2350, 2369, 2355, 2375, 2340, 2381, 2352, 2368, 2342, 2367, 2354, 2368, 2342,
20
+ 2379, 2344, 2344, 2344, 2366, 2361, 2368, 2344, 2367, 2352, 2381, 2339, 2381, 2351, 2346, 2339, 2346, 2350, 2346,
21
+ 2352, 2351, 2340, 2344, 2346, 2366, 2335, 2368, 2354, 2350, 2350, 2366, 2340, 2381, 2352, 2350, 2366, 2361, 2367,
22
+ 2340, 2368, 2350, 2368, 2350, 2369, 2348, 2368, 2350, 2381, 2361, 2339, 2332, 2375, 2350, 2381, 2361, 2339, 2366,
23
+ 2354, 2375, 2350, 2381, 2361, 2339, 2370, 2344, 2351, 2366, 2351, 2366, 2330, 2366, 2351, 2366, 2330, 2368, 2351,
24
+ 2366, 2330, 2381, 2351, 2366, 2351, 2366, 2344, 2366, 2351, 2366, 2344, 2368, 2351, 2375, 2339, 2366, 2352, 2351,
25
+ 2375, 2340, 2351, 2375, 2341, 2368, 2354, 2351, 2375, 2341, 2375, 2354, 2366, 2326, 2357, 2357, 2381, 2351, 2325,
26
+ 2340, 2360, 2352, 2381, 2357, 2360, 2366, 2327, 2367, 2340, 2381, 2354, 2375, 2360, 2369, 2352, 2370, 2361, 2332,
27
+ 2366, 2352, 2361, 2366, 2361, 2368, 2361, 2375, 2361, 2379, 2339, 2366, 2352, 2361, 2379, 2340, 2361, 2379, 2340,
28
+ 2366, 2361, 2379, 2340, 2368, 2361, 2379, 2340, 2375,
29
+ };
30
+
31
+ static const uint32_t stop_word_offsets[100] = {
32
+ 0, 4, 8, 11, 18, 26, 29, 33, 36, 38, 41, 44, 50, 53, 56, 59, 62, 66, 68, 71, 74, 80, 84, 86, 89, 92, 96, 100, 102,
33
+ 106, 110, 114, 118, 124, 128, 131, 135, 139, 143, 151, 153, 155, 157, 160, 164, 166, 168, 171, 173, 175, 179, 185,
34
+ 191, 199, 205, 211, 219, 223, 227, 230, 231, 235, 242, 244, 246, 251, 256, 257, 262, 268, 270, 274, 280, 287, 293,
35
+ 295, 299, 303, 309, 313, 317, 322, 325, 330, 334, 337, 338, 343, 347, 355, 359, 363, 365, 367, 369, 374, 377, 381,
36
+ 385, 389,
37
+ };
38
+
39
+ static const uint32_t keep_ranges[10] = {
40
+ 48, 57, 65, 90, 97, 122, 2304, 2403, 2406, 2431,
41
+ };
42
+
43
+ const narsil_language narsil_language_marathi = {
44
+ .name = "marathi",
45
+ .revision = "1",
46
+ .stop_word_units = stop_word_units,
47
+ .stop_word_offsets = stop_word_offsets,
48
+ .stop_word_count = 99,
49
+ .ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
50
+ .keep_ranges = keep_ranges,
51
+ .keep_range_count = 5,
52
+ .splits_code_points = 0,
53
+ .strips_possessive = 0,
54
+ .normalises_diacritics = 0,
55
+ .min_token_length = 1,
56
+ .ngram_size = 0,
57
+ .normalise = NULL,
58
+ .stem_by_rules = NULL,
59
+ .create_snowball = NULL,
60
+ .snowball_stem = NULL,
61
+ .close_snowball = NULL,
62
+ };
package/cpp/core/src/languages/data/nepali.c ADDED
@@ -0,0 +1,72 @@
1
+ #include "../../analysis/languages.h"
2
+ #include "declarations.h"
3
+ #include "../snowball/nepali_stemmer.h"
4
+
5
+ #include <stddef.h>
6
+ #include <stdint.h>
7
+
8
+ static const uint16_t stop_word_units[532] = {
9
+ 2309, 2325, 2381, 2360, 2352, 2309, 2327, 2366, 2337, 2367, 2309, 2333, 2376, 2309, 2344, 2369, 2360, 2366, 2352,
10
+ 2309, 2344, 2381, 2340, 2352, 2381, 2327, 2340, 2309, 2344, 2381, 2351, 2309, 2348, 2309, 2352, 2370, 2309, 2352,
11
+ 2381, 2325, 2379, 2309, 2352, 2381, 2341, 2366, 2340, 2309, 2354, 2327, 2310, 2319, 2310, 2336, 2310, 2342, 2367,
12
+ 2310, 2347, 2370, 2310, 2347, 2381, 2344, 2379, 2310, 2351, 2379, 2313, 2342, 2366, 2361, 2352, 2339, 2313, 2344,
13
+ 2313, 2344, 2325, 2379, 2313, 2344, 2354, 2375, 2313, 2346, 2319, 2325, 2319, 2325, 2342, 2350, 2325, 2340, 2376,
14
+ 2325, 2350, 2325, 2360, 2352, 2368, 2325, 2360, 2376, 2325, 2360, 2376, 2354, 2375, 2325, 2361, 2366, 2305, 2348,
15
+ 2366, 2335, 2325, 2366, 2325, 2367, 2325, 2367, 2344, 2325, 2367, 2344, 2349, 2344, 2375, 2325, 2369, 2344, 2376,
16
+ 2325, 2369, 2352, 2366, 2325, 2371, 2346, 2351, 2366, 2325, 2375, 2325, 2375, 2361, 2367, 2325, 2379, 2325, 2379,
17
+ 2361, 2368, 2327, 2319, 2327, 2352, 2368, 2327, 2352, 2375, 2325, 2379, 2327, 2352, 2375, 2352, 2327, 2352, 2381,
18
+ 2331, 2327, 2352, 2381, 2344, 2327, 2352, 2381, 2344, 2369, 2327, 2352, 2381, 2344, 2375, 2330, 2366, 2352, 2331,
19
+ 2331, 2344, 2381, 2331, 2369, 2331, 2376, 2344, 2331, 2380, 2306, 2332, 2348, 2332, 2360, 2325, 2379, 2332, 2360,
20
+ 2354, 2366, 2312, 2332, 2360, 2354, 2375, 2332, 2360, 2381, 2340, 2376, 2332, 2360, 2381, 2340, 2379, 2332, 2361,
21
+ 2366, 2305, 2332, 2366, 2344, 2332, 2369, 2344, 2332, 2375, 2332, 2379, 2336, 2368, 2325, 2340, 2340, 2341, 2366,
22
+ 2340, 2346, 2366, 2312, 2306, 2340, 2352, 2340, 2354, 2340, 2367, 2344, 2368, 2340, 2367, 2344, 2368, 2361, 2352,
23
+ 2370, 2340, 2368, 2344, 2340, 2381, 2351, 2360, 2346, 2331, 2367, 2340, 2381, 2351, 2361, 2366, 2305, 2340, 2381,
24
+ 2351, 2379, 2341, 2367, 2319, 2341, 2367, 2351, 2379, 2342, 2367, 2319, 2342, 2369, 2312, 2342, 2375, 2326, 2367,
25
+ 2343, 2375, 2352, 2376, 2344, 2344, 2367, 2344, 2367, 2350, 2381, 2340, 2367, 2344, 2376, 2346, 2331, 2367, 2346,
26
+ 2344, 2367, 2346, 2352, 2381, 2331, 2346, 2361, 2367, 2354, 2375, 2346, 2361, 2367, 2354, 2379, 2346, 2366, 2305,
27
+ 2330, 2346, 2381, 2352, 2340, 2367, 2346, 2381, 2352, 2340, 2381, 2351, 2375, 2325, 2347, 2375, 2352, 2367, 2348,
28
+ 2344, 2375, 2348, 2366, 2352, 2375, 2348, 2366, 2361, 2375, 2325, 2348, 2368, 2330, 2349, 2319, 2349, 2319, 2325,
29
+ 2379, 2349, 2344, 2381, 2342, 2366, 2349, 2344, 2381, 2344, 2375, 2349, 2352, 2349, 2367, 2340, 2381, 2352, 2350,
30
+ 2350, 2354, 2366, 2312, 2350, 2366, 2350, 2366, 2340, 2381, 2352, 2350, 2366, 2341, 2367, 2350, 2375, 2352, 2379,
31
+ 2351, 2340, 2367, 2351, 2342, 2367, 2351, 2360, 2351, 2360, 2325, 2379, 2351, 2360, 2381, 2340, 2379, 2351, 2361,
32
+ 2366, 2305, 2351, 2366, 2351, 2368, 2351, 2379, 2352, 2352, 2361, 2375, 2325, 2379, 2352, 2366, 2350, 2381, 2352,
33
+ 2379, 2354, 2366, 2312, 2354, 2366, 2327, 2367, 2354, 2375, 2360, 2305, 2327, 2360, 2325, 2381, 2331, 2360, 2348,
34
+ 2376, 2360, 2350, 2351, 2360, 2350, 2381, 2350, 2360, 2361, 2368, 2360, 2366, 2340, 2360, 2366, 2341, 2360, 2379,
35
+ 2361, 2352, 2375, 2361, 2352, 2375, 2325, 2361, 2366, 2350, 2368, 2361, 2366, 2350, 2381, 2352, 2379, 2361, 2369,
36
+ 2305, 2361, 2369, 2344, 2361, 2369, 2344, 2375, 2361, 2369, 2344, 2381, 2331, 2361, 2379, 2361, 2379, 2311, 2344,
37
+ };
38
+
39
+ static const uint32_t stop_word_offsets[148] = {
40
+ 0, 5, 10, 13, 19, 27, 31, 33, 36, 41, 47, 50, 52, 54, 57, 60, 65, 68, 74, 76, 80, 84, 86, 88, 92, 95, 97, 101, 104,
41
+ 109, 116, 118, 120, 123, 129, 133, 137, 142, 144, 148, 150, 154, 156, 159, 164, 168, 172, 176, 181, 186, 189, 190,
42
+ 193, 195, 198, 201, 203, 207, 212, 216, 221, 226, 230, 233, 236, 238, 240, 243, 244, 247, 252, 254, 256, 260, 267,
43
+ 270, 277, 283, 287, 290, 294, 297, 300, 304, 308, 309, 311, 317, 319, 322, 325, 329, 334, 339, 343, 348, 356, 360,
44
+ 363, 367, 372, 375, 377, 381, 386, 391, 393, 398, 399, 403, 405, 410, 414, 418, 421, 424, 426, 430, 435, 439, 441,
45
+ 443, 445, 446, 451, 457, 460, 464, 466, 469, 473, 476, 479, 483, 486, 489, 492, 494, 497, 501, 505, 511, 514, 517,
46
+ 521, 526, 528, 532,
47
+ };
48
+
49
+ static const uint32_t keep_ranges[10] = {
50
+ 48, 57, 65, 90, 97, 122, 2304, 2403, 2406, 2431,
51
+ };
52
+
53
+ const narsil_language narsil_language_nepali = {
54
+ .name = "nepali",
55
+ .revision = "12453388370b3da0",
56
+ .stop_word_units = stop_word_units,
57
+ .stop_word_offsets = stop_word_offsets,
58
+ .stop_word_count = 147,
59
+ .ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
60
+ .keep_ranges = keep_ranges,
61
+ .keep_range_count = 5,
62
+ .splits_code_points = 0,
63
+ .strips_possessive = 0,
64
+ .normalises_diacritics = 0,
65
+ .min_token_length = 1,
66
+ .ngram_size = 0,
67
+ .normalise = NULL,
68
+ .stem_by_rules = NULL,
69
+ .create_snowball = nepali_create_env,
70
+ .snowball_stem = nepali_stem,
71
+ .close_snowball = nepali_close_env,
72
+ };
package/cpp/core/src/languages/data/norwegian.c ADDED
@@ -0,0 +1,74 @@
1
+ #include "../../analysis/languages.h"
2
+ #include "declarations.h"
3
+ #include "../snowball/norwegian_stemmer.h"
4
+
5
+ #include <stddef.h>
6
+ #include <stdint.h>
7
+
8
+ static const uint16_t stop_word_units[644] = {
9
+ 97, 108, 108, 101, 97, 116, 97, 118, 98, 97, 114, 101, 98, 101, 103, 103, 101, 98, 108, 101, 98, 108, 101, 105, 98,
10
+ 108, 105, 98, 108, 105, 114, 98, 108, 105, 116, 116, 98, 229, 100, 101, 98, 229, 101, 100, 97, 100, 101, 100, 101,
11
+ 103, 100, 101, 105, 100, 101, 105, 109, 100, 101, 105, 114, 97, 100, 101, 105, 114, 101, 115, 100, 101, 109, 100, 101,
12
+ 110, 100, 101, 110, 110, 101, 100, 101, 114, 100, 101, 114, 101, 100, 101, 114, 101, 115, 100, 101, 116, 100, 101,
13
+ 116, 116, 101, 100, 105, 100, 105, 110, 100, 105, 115, 115, 101, 100, 105, 116, 116, 100, 117, 100, 121, 107, 107,
14
+ 100, 121, 107, 107, 97, 114, 100, 229, 101, 103, 101, 105, 110, 101, 105, 116, 101, 105, 116, 116, 101, 108, 108, 101,
15
+ 114, 101, 108, 108, 101, 115, 101, 110, 101, 110, 110, 101, 114, 101, 116, 101, 116, 116, 101, 116, 116, 101, 114,
16
+ 102, 111, 114, 102, 111, 114, 100, 105, 102, 114, 97, 102, 248, 114, 104, 97, 104, 97, 100, 100, 101, 104, 97, 110,
17
+ 104, 97, 110, 115, 104, 97, 114, 104, 101, 110, 110, 97, 114, 104, 101, 110, 110, 101, 104, 101, 110, 110, 101, 115,
18
+ 104, 101, 114, 104, 106, 229, 104, 111, 104, 111, 101, 104, 111, 110, 111, 109, 104, 111, 115, 115, 104, 111, 115,
19
+ 115, 101, 110, 104, 117, 110, 104, 118, 97, 104, 118, 101, 109, 104, 118, 101, 114, 104, 118, 105, 108, 107, 101, 104,
20
+ 118, 105, 108, 107, 101, 110, 104, 118, 105, 115, 104, 118, 111, 114, 104, 118, 111, 114, 100, 97, 110, 104, 118, 111,
21
+ 114, 102, 111, 114, 105, 105, 107, 107, 101, 105, 107, 107, 106, 101, 105, 110, 103, 101, 110, 105, 110, 103, 105,
22
+ 105, 110, 107, 106, 101, 105, 110, 110, 105, 110, 110, 105, 106, 97, 106, 101, 103, 107, 97, 110, 107, 111, 109, 107,
23
+ 111, 114, 108, 101, 105, 115, 107, 111, 114, 115, 111, 107, 117, 110, 107, 117, 110, 110, 101, 107, 118, 97, 107, 118,
24
+ 97, 114, 107, 118, 97, 114, 104, 101, 108, 115, 116, 107, 118, 101, 110, 107, 118, 105, 107, 118, 105, 102, 111, 114,
25
+ 109, 97, 110, 109, 97, 110, 103, 101, 109, 101, 109, 101, 100, 109, 101, 100, 97, 110, 109, 101, 103, 109, 101, 103,
26
+ 101, 116, 109, 101, 108, 108, 111, 109, 109, 101, 110, 109, 105, 109, 105, 110, 109, 105, 110, 101, 109, 105, 116,
27
+ 116, 109, 111, 116, 109, 121, 107, 106, 101, 110, 101, 100, 110, 111, 110, 111, 101, 110, 111, 101, 110, 110, 111,
28
+ 107, 97, 110, 111, 107, 111, 110, 111, 107, 111, 110, 110, 111, 107, 111, 114, 110, 111, 107, 114, 101, 110, 229, 110,
29
+ 229, 114, 111, 103, 111, 103, 115, 229, 111, 109, 111, 112, 112, 111, 115, 115, 111, 118, 101, 114, 112, 229, 115, 97,
30
+ 109, 109, 101, 115, 101, 103, 115, 101, 108, 118, 115, 105, 115, 105, 97, 115, 105, 100, 97, 110, 115, 105, 100, 101,
31
+ 110, 115, 105, 110, 115, 105, 110, 101, 115, 105, 116, 116, 115, 106, 248, 108, 115, 107, 97, 108, 115, 107, 117, 108,
32
+ 108, 101, 115, 108, 105, 107, 115, 111, 115, 111, 109, 115, 111, 109, 109, 101, 115, 111, 109, 116, 115, 229, 115,
33
+ 229, 110, 110, 116, 105, 108, 117, 109, 117, 112, 112, 117, 116, 117, 116, 101, 110, 118, 97, 114, 118, 97, 114, 116,
34
+ 118, 97, 114, 116, 101, 118, 101, 100, 118, 101, 114, 101, 118, 101, 114, 116, 101, 118, 105, 118, 105, 108, 118, 105,
35
+ 108, 108, 101, 118, 111, 114, 101, 118, 111, 114, 115, 118, 111, 114, 116, 118, 229, 114, 118, 230, 114, 101, 118,
36
+ 230, 114, 116, 229,
37
+ };
38
+
39
+ static const uint32_t stop_word_offsets[173] = {
40
+ 0, 4, 6, 8, 12, 17, 20, 24, 27, 31, 36, 40, 43, 45, 47, 50, 53, 57, 62, 68, 71, 74, 79, 82, 86, 91, 94, 99, 101, 104,
41
+ 109, 113, 115, 119, 125, 127, 129, 132, 135, 139, 144, 149, 151, 154, 156, 158, 161, 166, 169, 174, 177, 180, 182,
42
+ 187, 190, 194, 197, 203, 208, 214, 217, 220, 222, 225, 230, 234, 240, 243, 246, 250, 254, 260, 267, 271, 275, 282,
43
+ 289, 290, 294, 299, 304, 308, 313, 316, 320, 322, 325, 328, 331, 338, 343, 346, 351, 354, 358, 367, 371, 374, 380,
44
+ 383, 388, 390, 393, 398, 401, 406, 412, 415, 417, 420, 424, 428, 431, 436, 439, 441, 444, 448, 452, 456, 461, 466,
45
+ 471, 473, 476, 478, 482, 484, 487, 490, 494, 496, 501, 504, 508, 510, 513, 518, 523, 526, 530, 534, 538, 542, 548,
46
+ 552, 554, 557, 562, 566, 568, 572, 575, 577, 580, 582, 586, 589, 593, 598, 601, 605, 610, 612, 615, 620, 624, 628,
47
+ 632, 635, 639, 643, 644,
48
+ };
49
+
50
+ static const uint32_t keep_ranges[30] = {
51
+ 48, 57, 65, 90, 97, 122, 196, 198, 201, 201, 211, 211, 214, 214, 216, 216, 220, 220, 228, 230, 233, 233, 243, 243,
52
+ 246, 246, 248, 248, 252, 252,
53
+ };
54
+
55
+ const narsil_language narsil_language_norwegian = {
56
+ .name = "norwegian",
57
+ .revision = "8e862aa3bcec10dd",
58
+ .stop_word_units = stop_word_units,
59
+ .stop_word_offsets = stop_word_offsets,
60
+ .stop_word_count = 172,
61
+ .ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
62
+ .keep_ranges = keep_ranges,
63
+ .keep_range_count = 15,
64
+ .splits_code_points = 0,
65
+ .strips_possessive = 0,
66
+ .normalises_diacritics = 0,
67
+ .min_token_length = 1,
68
+ .ngram_size = 0,
69
+ .normalise = NULL,
70
+ .stem_by_rules = NULL,
71
+ .create_snowball = norwegian_create_env,
72
+ .snowball_stem = norwegian_stem,
73
+ .close_snowball = norwegian_close_env,
74
+ };