@delali/narsil-react-native 0.3.1-next.155 → 0.3.1-next.156
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +144 -0
- package/NarsilReactNative.podspec +2 -2
- package/android/CMakeLists.txt +1 -1
- package/cpp/HybridNarsilCoreAnalysis.cpp +277 -0
- package/cpp/HybridNarsilCoreImpl.hpp +43 -0
- package/cpp/HybridNarsilCoreKeyword.cpp +253 -0
- package/cpp/HybridNarsilCoreMath.cpp +47 -0
- package/cpp/core/include/narsil_analysis.h +86 -0
- package/cpp/core/include/narsil_core.h +1 -1
- package/cpp/core/include/narsil_keyword.h +110 -0
- package/cpp/core/include/narsil_math.h +36 -0
- package/cpp/core/src/analysis/analyse.c +281 -0
- package/cpp/core/src/analysis/analyser.h +49 -0
- package/cpp/core/src/analysis/languages.c +79 -0
- package/cpp/core/src/analysis/languages.h +42 -0
- package/cpp/core/src/analysis/normalise.c +242 -0
- package/cpp/core/src/analysis/normalise.h +28 -0
- package/cpp/core/src/analysis/split.c +183 -0
- package/cpp/core/src/analysis/split.h +27 -0
- package/cpp/core/src/analysis/stop_words.c +82 -0
- package/cpp/core/src/analysis/stop_words.h +19 -0
- package/cpp/core/src/analysis/term_cache.c +88 -0
- package/cpp/core/src/analysis/term_cache.h +36 -0
- package/cpp/core/src/analysis/text.c +161 -0
- package/cpp/core/src/analysis/text.h +45 -0
- package/cpp/core/src/analysis/transform.c +103 -0
- package/cpp/core/src/analysis/unicode.c +252 -0
- package/cpp/core/src/analysis/unicode.h +54 -0
- package/cpp/core/src/internal.h +4 -0
- package/cpp/core/src/keyword/pass.h +117 -0
- package/cpp/core/src/keyword/prune.c +324 -0
- package/cpp/core/src/keyword/request.c +179 -0
- package/cpp/core/src/keyword/rows.c +83 -0
- package/cpp/core/src/keyword/rows.h +18 -0
- package/cpp/core/src/keyword/score.c +257 -0
- package/cpp/core/src/keyword/score.h +17 -0
- package/cpp/core/src/keyword/scratch.c +226 -0
- package/cpp/core/src/keyword/scratch.h +55 -0
- package/cpp/core/src/keyword/select.c +155 -0
- package/cpp/core/src/keyword/select.h +17 -0
- package/cpp/core/src/languages/bulgarian_stemmer.c +114 -0
- package/cpp/core/src/languages/data/albanian.c +54 -0
- package/cpp/core/src/languages/data/amharic.c +93 -0
- package/cpp/core/src/languages/data/arabic.c +146 -0
- package/cpp/core/src/languages/data/armenian.c +46 -0
- package/cpp/core/src/languages/data/azerbaijani.c +68 -0
- package/cpp/core/src/languages/data/bambara.c +48 -0
- package/cpp/core/src/languages/data/basque.c +62 -0
- package/cpp/core/src/languages/data/belarusian.c +56 -0
- package/cpp/core/src/languages/data/bengali.c +63 -0
- package/cpp/core/src/languages/data/bosnian.c +73 -0
- package/cpp/core/src/languages/data/breton.c +52 -0
- package/cpp/core/src/languages/data/bulgarian.c +144 -0
- package/cpp/core/src/languages/data/burmese.c +48 -0
- package/cpp/core/src/languages/data/catalan.c +84 -0
- package/cpp/core/src/languages/data/chinese.c +70 -0
- package/cpp/core/src/languages/data/croatian.c +72 -0
- package/cpp/core/src/languages/data/czech.c +72 -0
- package/cpp/core/src/languages/data/dagbani.c +88 -0
- package/cpp/core/src/languages/data/danish.c +56 -0
- package/cpp/core/src/languages/data/declarations.h +114 -0
- package/cpp/core/src/languages/data/dutch.c +58 -0
- package/cpp/core/src/languages/data/english.c +172 -0
- package/cpp/core/src/languages/data/esperanto.c +71 -0
- package/cpp/core/src/languages/data/estonian.c +417 -0
- package/cpp/core/src/languages/data/ewe.c +89 -0
- package/cpp/core/src/languages/data/faroese.c +54 -0
- package/cpp/core/src/languages/data/fijian.c +44 -0
- package/cpp/core/src/languages/data/finnish.c +100 -0
- package/cpp/core/src/languages/data/french.c +74 -0
- package/cpp/core/src/languages/data/ga.c +83 -0
- package/cpp/core/src/languages/data/galician.c +70 -0
- package/cpp/core/src/languages/data/georgian.c +57 -0
- package/cpp/core/src/languages/data/german.c +96 -0
- package/cpp/core/src/languages/data/greek.c +202 -0
- package/cpp/core/src/languages/data/guarani.c +110 -0
- package/cpp/core/src/languages/data/gujarati.c +90 -0
- package/cpp/core/src/languages/data/haitian-creole.c +54 -0
- package/cpp/core/src/languages/data/hausa.c +85 -0
- package/cpp/core/src/languages/data/hawaiian.c +50 -0
- package/cpp/core/src/languages/data/hebrew.c +85 -0
- package/cpp/core/src/languages/data/hindi.c +91 -0
- package/cpp/core/src/languages/data/hungarian.c +98 -0
- package/cpp/core/src/languages/data/icelandic.c +73 -0
- package/cpp/core/src/languages/data/igbo.c +90 -0
- package/cpp/core/src/languages/data/indonesian.c +296 -0
- package/cpp/core/src/languages/data/irish.c +60 -0
- package/cpp/core/src/languages/data/italian.c +107 -0
- package/cpp/core/src/languages/data/japanese.c +52 -0
- package/cpp/core/src/languages/data/kannada.c +60 -0
- package/cpp/core/src/languages/data/kazakh.c +57 -0
- package/cpp/core/src/languages/data/khmer.c +71 -0
- package/cpp/core/src/languages/data/kinyarwanda.c +54 -0
- package/cpp/core/src/languages/data/kirundi.c +53 -0
- package/cpp/core/src/languages/data/korean.c +74 -0
- package/cpp/core/src/languages/data/kurmanji.c +43 -0
- package/cpp/core/src/languages/data/kyrgyz.c +65 -0
- package/cpp/core/src/languages/data/lao.c +46 -0
- package/cpp/core/src/languages/data/latin.c +46 -0
- package/cpp/core/src/languages/data/latvian.c +75 -0
- package/cpp/core/src/languages/data/lingala.c +88 -0
- package/cpp/core/src/languages/data/lithuanian.c +65 -0
- package/cpp/core/src/languages/data/luxembourgish.c +80 -0
- package/cpp/core/src/languages/data/macedonian.c +286 -0
- package/cpp/core/src/languages/data/malagasy.c +51 -0
- package/cpp/core/src/languages/data/malay.c +184 -0
- package/cpp/core/src/languages/data/malayalam.c +64 -0
- package/cpp/core/src/languages/data/maltese.c +102 -0
- package/cpp/core/src/languages/data/maori.c +52 -0
- package/cpp/core/src/languages/data/marathi.c +62 -0
- package/cpp/core/src/languages/data/nepali.c +72 -0
- package/cpp/core/src/languages/data/norwegian.c +74 -0
- package/cpp/core/src/languages/data/oromo.c +51 -0
- package/cpp/core/src/languages/data/persian.c +150 -0
- package/cpp/core/src/languages/data/polish.c +111 -0
- package/cpp/core/src/languages/data/portuguese.c +91 -0
- package/cpp/core/src/languages/data/punjabi.c +80 -0
- package/cpp/core/src/languages/data/registry.c +116 -0
- package/cpp/core/src/languages/data/romanian.c +93 -0
- package/cpp/core/src/languages/data/russian.c +76 -0
- package/cpp/core/src/languages/data/samoan.c +47 -0
- package/cpp/core/src/languages/data/sanskrit.c +53 -0
- package/cpp/core/src/languages/data/scottish-gaelic.c +119 -0
- package/cpp/core/src/languages/data/serbian.c +84 -0
- package/cpp/core/src/languages/data/shona.c +55 -0
- package/cpp/core/src/languages/data/sinhala.c +78 -0
- package/cpp/core/src/languages/data/slovak.c +135 -0
- package/cpp/core/src/languages/data/slovenian.c +139 -0
- package/cpp/core/src/languages/data/sorani.c +55 -0
- package/cpp/core/src/languages/data/spanish.c +84 -0
- package/cpp/core/src/languages/data/swahili.c +59 -0
- package/cpp/core/src/languages/data/swedish.c +60 -0
- package/cpp/core/src/languages/data/tagalog.c +75 -0
- package/cpp/core/src/languages/data/tamil.c +69 -0
- package/cpp/core/src/languages/data/tatar.c +548 -0
- package/cpp/core/src/languages/data/telugu.c +54 -0
- package/cpp/core/src/languages/data/thai.c +46 -0
- package/cpp/core/src/languages/data/tibetan.c +55 -0
- package/cpp/core/src/languages/data/tigrinya.c +73 -0
- package/cpp/core/src/languages/data/tongan.c +47 -0
- package/cpp/core/src/languages/data/turkish.c +100 -0
- package/cpp/core/src/languages/data/twi.c +88 -0
- package/cpp/core/src/languages/data/ukrainian.c +99 -0
- package/cpp/core/src/languages/data/urdu.c +174 -0
- package/cpp/core/src/languages/data/vietnamese.c +88 -0
- package/cpp/core/src/languages/data/welsh.c +143 -0
- package/cpp/core/src/languages/data/wolof.c +51 -0
- package/cpp/core/src/languages/data/xhosa.c +56 -0
- package/cpp/core/src/languages/data/yoruba.c +93 -0
- package/cpp/core/src/languages/data/zulu.c +58 -0
- package/cpp/core/src/languages/marks.c +64 -0
- package/cpp/core/src/languages/rules.c +63 -0
- package/cpp/core/src/languages/rules.h +68 -0
- package/cpp/core/src/languages/sanskrit_stemmer.c +50 -0
- package/cpp/core/src/languages/scripts.c +143 -0
- package/cpp/core/src/languages/slovenian_stemmer.c +46 -0
- package/cpp/core/src/languages/snowball/api.c +52 -0
- package/cpp/core/src/languages/snowball/api.h +43 -0
- package/cpp/core/src/languages/snowball/arabic_stemmer.c +1506 -0
- package/cpp/core/src/languages/snowball/arabic_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/armenian_stemmer.c +523 -0
- package/cpp/core/src/languages/snowball/armenian_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/basque_stemmer.c +1089 -0
- package/cpp/core/src/languages/snowball/basque_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/catalan_stemmer.c +1380 -0
- package/cpp/core/src/languages/snowball/catalan_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/czech_stemmer.c +582 -0
- package/cpp/core/src/languages/snowball/czech_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/danish_stemmer.c +352 -0
- package/cpp/core/src/languages/snowball/danish_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/dutch_stemmer.c +1657 -0
- package/cpp/core/src/languages/snowball/dutch_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/english_stemmer.c +1019 -0
- package/cpp/core/src/languages/snowball/english_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/esperanto_stemmer.c +716 -0
- package/cpp/core/src/languages/snowball/esperanto_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/estonian_stemmer.c +1361 -0
- package/cpp/core/src/languages/snowball/estonian_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/finnish_stemmer.c +800 -0
- package/cpp/core/src/languages/snowball/finnish_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/french_stemmer.c +1217 -0
- package/cpp/core/src/languages/snowball/french_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/german_stemmer.c +567 -0
- package/cpp/core/src/languages/snowball/german_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/greek_stemmer.c +3450 -0
- package/cpp/core/src/languages/snowball/greek_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/hindi_stemmer.c +325 -0
- package/cpp/core/src/languages/snowball/hindi_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/hungarian_stemmer.c +742 -0
- package/cpp/core/src/languages/snowball/hungarian_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/indonesian_stemmer.c +443 -0
- package/cpp/core/src/languages/snowball/indonesian_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/irish_stemmer.c +428 -0
- package/cpp/core/src/languages/snowball/irish_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/italian_stemmer.c +983 -0
- package/cpp/core/src/languages/snowball/italian_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/lithuanian_stemmer.c +794 -0
- package/cpp/core/src/languages/snowball/lithuanian_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/nepali_stemmer.c +361 -0
- package/cpp/core/src/languages/snowball/nepali_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/norwegian_stemmer.c +332 -0
- package/cpp/core/src/languages/snowball/norwegian_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/persian_stemmer.c +538 -0
- package/cpp/core/src/languages/snowball/persian_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/polish_stemmer.c +501 -0
- package/cpp/core/src/languages/snowball/polish_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/portuguese_stemmer.c +866 -0
- package/cpp/core/src/languages/snowball/portuguese_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/romanian_stemmer.c +940 -0
- package/cpp/core/src/languages/snowball/romanian_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/russian_stemmer.c +598 -0
- package/cpp/core/src/languages/snowball/russian_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/serbian_stemmer.c +6765 -0
- package/cpp/core/src/languages/snowball/serbian_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/snowball_runtime.h +122 -0
- package/cpp/core/src/languages/snowball/spanish_stemmer.c +955 -0
- package/cpp/core/src/languages/snowball/spanish_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/swedish_stemmer.c +364 -0
- package/cpp/core/src/languages/snowball/swedish_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/tamil_stemmer.c +1305 -0
- package/cpp/core/src/languages/snowball/tamil_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/turkish_stemmer.c +1700 -0
- package/cpp/core/src/languages/snowball/turkish_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/utilities.c +716 -0
- package/cpp/core/src/languages/substitutions.c +131 -0
- package/cpp/core/src/languages/swahili_stemmer.c +85 -0
- package/cpp/core/src/languages/ukrainian_stemmer.c +168 -0
- package/cpp/core/src/locks.c +2 -15
- package/cpp/core/src/math/apply.c +44 -0
- package/cpp/core/src/math/fdlibm.h +16 -0
- package/cpp/core/src/math/inverse.c +242 -0
- package/cpp/core/src/math/log.c +90 -0
- package/cpp/core/src/math/reduce.c +325 -0
- package/cpp/core/src/math/trig.c +188 -0
- package/cpp/core/src/math/words.h +45 -0
- package/cpp/core/src/thread_yield.h +18 -0
- package/cpp/core/src/unicode/decompositions.c +310 -0
- package/cpp/core/src/unicode/lowercase.c +172 -0
- package/cpp/core/src/unicode/properties.c +214 -0
- package/cpp/core/src/workspace.c +5 -1
- package/lib/engine/analysis.d.ts +3 -0
- package/lib/engine/analysis.js +76 -0
- package/lib/engine/core.js +5 -0
- package/lib/engine/keyword.d.ts +3 -0
- package/lib/engine/keyword.js +25 -0
- package/lib/specs/core.nitro.d.ts +1 -0
- package/nitrogen/generated/shared/c++/HybridNarsilCoreSpec.cpp +1 -0
- package/nitrogen/generated/shared/c++/HybridNarsilCoreSpec.hpp +1 -0
- package/package.json +4 -3
- package/src/engine/analysis.ts +87 -0
- package/src/engine/core.ts +6 -0
- package/src/engine/keyword.ts +26 -0
- package/src/specs/core.nitro.ts +1 -0
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
#include "../../analysis/languages.h"
|
|
2
|
+
#include "declarations.h"
|
|
3
|
+
#include "../rules.h"
|
|
4
|
+
|
|
5
|
+
#include <stddef.h>
|
|
6
|
+
#include <stdint.h>
|
|
7
|
+
|
|
8
|
+
static const uint16_t stop_word_units[731] = {
|
|
9
|
+
1488, 1489, 1500, 1488, 1493, 1488, 1493, 1500, 1497, 1488, 1493, 1514, 1492, 1488, 1493, 1514, 1493, 1488, 1493,
|
|
10
|
+
1514, 1497, 1488, 1493, 1514, 1498, 1488, 1493, 1514, 1501, 1488, 1493, 1514, 1503, 1488, 1493, 1514, 1504, 1493,
|
|
11
|
+
1488, 1494, 1488, 1495, 1512, 1488, 1495, 1512, 1493, 1514, 1488, 1495, 1512, 1497, 1488, 1495, 1512, 1497, 1499,
|
|
12
|
+
1503, 1488, 1495, 1512, 1497, 1501, 1488, 1495, 1512, 1514, 1488, 1497, 1488, 1497, 1494, 1492, 1488, 1497, 1498,
|
|
13
|
+
1488, 1497, 1503, 1488, 1497, 1508, 1492, 1488, 1497, 1514, 1492, 1488, 1497, 1514, 1493, 1488, 1497, 1514, 1497,
|
|
14
|
+
1488, 1497, 1514, 1498, 1488, 1497, 1514, 1499, 1501, 1488, 1497, 1514, 1499, 1503, 1488, 1497, 1514, 1501, 1488,
|
|
15
|
+
1497, 1514, 1503, 1488, 1497, 1514, 1504, 1493, 1488, 1498, 1488, 1500, 1488, 1500, 1492, 1488, 1500, 1493, 1488,
|
|
16
|
+
1501, 1488, 1504, 1495, 1504, 1493, 1488, 1504, 1497, 1488, 1505, 1488, 1507, 1488, 1510, 1500, 1488, 1513, 1512,
|
|
17
|
+
1488, 1514, 1488, 1514, 1492, 1488, 1514, 1499, 1501, 1488, 1514, 1499, 1503, 1488, 1514, 1501, 1488, 1514, 1503,
|
|
18
|
+
1489, 1488, 1497, 1494, 1493, 1502, 1497, 1491, 1492, 1489, 1488, 1502, 1510, 1506, 1489, 1488, 1502, 1510, 1506,
|
|
19
|
+
1493, 1514, 1489, 1490, 1500, 1500, 1489, 1497, 1503, 1489, 1500, 1497, 1489, 1502, 1497, 1491, 1492, 1489, 1502,
|
|
20
|
+
1511, 1493, 1501, 1513, 1489, 1493, 1489, 1512, 1501, 1489, 1513, 1489, 1497, 1500, 1489, 1513, 1506, 1492, 1513,
|
|
21
|
+
1489, 1514, 1493, 1498, 1490, 1501, 1491, 1512, 1498, 1492, 1493, 1488, 1492, 1497, 1488, 1492, 1497, 1492, 1492,
|
|
22
|
+
1497, 1499, 1503, 1492, 1497, 1514, 1492, 1492, 1497, 1514, 1497, 1492, 1501, 1492, 1503, 1492, 1504, 1492, 1492,
|
|
23
|
+
1505, 1497, 1489, 1492, 1513, 1489, 1490, 1500, 1500, 1492, 1492, 1512, 1497, 1493, 1488, 1497, 1500, 1493, 1493,
|
|
24
|
+
1488, 1514, 1494, 1488, 1514, 1494, 1492, 1494, 1493, 1514, 1497, 1492, 1497, 1492, 1497, 1493, 1499, 1500, 1497,
|
|
25
|
+
1493, 1499, 1500, 1493, 1497, 1493, 1514, 1512, 1502, 1491, 1497, 1497, 1499, 1493, 1500, 1497, 1499, 1493, 1500,
|
|
26
|
+
1492, 1497, 1499, 1493, 1500, 1493, 1514, 1497, 1499, 1493, 1500, 1497, 1501, 1497, 1499, 1500, 1497, 1499, 1500,
|
|
27
|
+
1492, 1497, 1499, 1500, 1493, 1497, 1513, 1499, 1488, 1503, 1499, 1488, 1513, 1512, 1499, 1493, 1500, 1501, 1499,
|
|
28
|
+
1493, 1500, 1503, 1499, 1494, 1492, 1499, 1497, 1499, 1497, 1510, 1491, 1499, 1498, 1499, 1499, 1492, 1499, 1500,
|
|
29
|
+
1499, 1500, 1500, 1499, 1502, 1493, 1499, 1503, 1499, 1508, 1497, 1499, 1513, 1500, 1488, 1500, 1488, 1493, 1500,
|
|
30
|
+
1488, 1497, 1494, 1493, 1514, 1499, 1500, 1497, 1514, 1500, 1488, 1503, 1500, 1489, 1497, 1503, 1500, 1492, 1500,
|
|
31
|
+
1492, 1497, 1493, 1514, 1500, 1492, 1501, 1500, 1492, 1503, 1500, 1493, 1500, 1497, 1500, 1499, 1501, 1500, 1499,
|
|
32
|
+
1503, 1500, 1502, 1492, 1500, 1502, 1496, 1492, 1500, 1502, 1506, 1500, 1492, 1500, 1502, 1511, 1493, 1501, 1513,
|
|
33
|
+
1489, 1493, 1500, 1502, 1512, 1493, 1514, 1500, 1504, 1493, 1500, 1506, 1489, 1512, 1500, 1506, 1497, 1499, 1503,
|
|
34
|
+
1500, 1508, 1497, 1499, 1498, 1500, 1508, 1504, 1497, 1502, 1488, 1491, 1502, 1488, 1495, 1493, 1512, 1497, 1502,
|
|
35
|
+
1488, 1497, 1494, 1493, 1505, 1497, 1489, 1492, 1502, 1488, 1497, 1503, 1502, 1488, 1497, 1508, 1492, 1502, 1489,
|
|
36
|
+
1500, 1497, 1502, 1489, 1506, 1491, 1502, 1491, 1493, 1506, 1502, 1492, 1502, 1492, 1497, 1499, 1503, 1502, 1493,
|
|
37
|
+
1500, 1502, 1495, 1493, 1509, 1502, 1497, 1502, 1499, 1488, 1503, 1502, 1499, 1497, 1493, 1493, 1503, 1502, 1500,
|
|
38
|
+
1489, 1491, 1502, 1503, 1502, 1504, 1497, 1503, 1502, 1505, 1493, 1490, 1500, 1502, 1506, 1496, 1502, 1506, 1496,
|
|
39
|
+
1497, 1501, 1502, 1506, 1500, 1502, 1510, 1491, 1502, 1511, 1493, 1501, 1489, 1493, 1502, 1514, 1495, 1514, 1502,
|
|
40
|
+
1514, 1497, 1504, 1490, 1491, 1504, 1490, 1512, 1504, 1493, 1506, 1491, 1506, 1494, 1506, 1500, 1506, 1500, 1497,
|
|
41
|
+
1506, 1500, 1497, 1492, 1506, 1500, 1497, 1492, 1501, 1506, 1500, 1497, 1492, 1503, 1506, 1500, 1497, 1493, 1506,
|
|
42
|
+
1500, 1497, 1498, 1506, 1500, 1497, 1499, 1501, 1506, 1500, 1497, 1504, 1493, 1506, 1501, 1506, 1510, 1502, 1492,
|
|
43
|
+
1506, 1510, 1502, 1492, 1501, 1506, 1510, 1502, 1492, 1503, 1506, 1510, 1502, 1493, 1506, 1510, 1502, 1497, 1506,
|
|
44
|
+
1510, 1502, 1501, 1506, 1510, 1502, 1503, 1506, 1510, 1502, 1504, 1493, 1508, 1492, 1512, 1511, 1513, 1493, 1489,
|
|
45
|
+
1513, 1500, 1513, 1500, 1492, 1513, 1500, 1492, 1501, 1513, 1500, 1492, 1503, 1513, 1500, 1493, 1513, 1500, 1497,
|
|
46
|
+
1513, 1500, 1498, 1513, 1500, 1499, 1492, 1513, 1500, 1499, 1501, 1513, 1500, 1499, 1503, 1513, 1500, 1504, 1493,
|
|
47
|
+
1513, 1501, 1514, 1492, 1497, 1492, 1514, 1495, 1514,
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
static const uint32_t stop_word_offsets[195] = {
|
|
51
|
+
0, 3, 5, 9, 13, 17, 21, 25, 29, 33, 38, 40, 43, 48, 52, 58, 63, 67, 69, 73, 76, 79, 83, 87, 91, 95, 99, 104, 109, 113,
|
|
52
|
+
117, 122, 124, 126, 129, 132, 134, 139, 142, 144, 146, 149, 152, 154, 157, 161, 165, 168, 171, 180, 185, 192, 196,
|
|
53
|
+
199, 202, 207, 215, 218, 223, 228, 232, 234, 237, 240, 243, 246, 250, 254, 258, 260, 262, 265, 276, 279, 284, 287,
|
|
54
|
+
290, 292, 295, 299, 303, 308, 315, 319, 324, 330, 336, 339, 343, 347, 349, 352, 356, 360, 364, 367, 369, 373, 375,
|
|
55
|
+
378, 380, 383, 386, 388, 391, 393, 395, 398, 408, 411, 415, 417, 422, 425, 428, 430, 432, 435, 438, 441, 445, 450,
|
|
56
|
+
458, 463, 466, 470, 475, 480, 484, 487, 493, 502, 506, 511, 515, 519, 523, 525, 530, 533, 537, 539, 543, 549, 553,
|
|
57
|
+
555, 559, 564, 567, 572, 575, 578, 584, 588, 591, 594, 597, 599, 601, 603, 605, 608, 612, 617, 622, 626, 630, 635,
|
|
58
|
+
640, 642, 646, 651, 656, 660, 664, 668, 672, 677, 679, 681, 684, 686, 689, 693, 697, 700, 703, 706, 710, 714, 718,
|
|
59
|
+
722, 724, 728, 731,
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
static const uint32_t keep_ranges[14] = {
|
|
63
|
+
48, 57, 65, 90, 97, 122, 1425, 1469, 1473, 1474, 1479, 1514, 1520, 1524,
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
const narsil_language narsil_language_hebrew = {
|
|
67
|
+
.name = "hebrew",
|
|
68
|
+
.revision = "1",
|
|
69
|
+
.stop_word_units = stop_word_units,
|
|
70
|
+
.stop_word_offsets = stop_word_offsets,
|
|
71
|
+
.stop_word_count = 194,
|
|
72
|
+
.ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
|
|
73
|
+
.keep_ranges = keep_ranges,
|
|
74
|
+
.keep_range_count = 7,
|
|
75
|
+
.splits_code_points = 0,
|
|
76
|
+
.strips_possessive = 0,
|
|
77
|
+
.normalises_diacritics = 0,
|
|
78
|
+
.min_token_length = 1,
|
|
79
|
+
.ngram_size = 0,
|
|
80
|
+
.normalise = narsil_hebrew_normalise,
|
|
81
|
+
.stem_by_rules = NULL,
|
|
82
|
+
.create_snowball = NULL,
|
|
83
|
+
.snowball_stem = NULL,
|
|
84
|
+
.close_snowball = NULL,
|
|
85
|
+
};
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
#include "../../analysis/languages.h"
|
|
2
|
+
#include "declarations.h"
|
|
3
|
+
#include "../rules.h"
|
|
4
|
+
#include "../snowball/hindi_stemmer.h"
|
|
5
|
+
|
|
6
|
+
#include <stddef.h>
|
|
7
|
+
#include <stdint.h>
|
|
8
|
+
|
|
9
|
+
static const uint16_t stop_word_units[815] = {
|
|
10
|
+
2309, 2306, 2342, 2352, 2309, 2340, 2309, 2342, 2367, 2309, 2346, 2309, 2346, 2344, 2366, 2309, 2346, 2344, 2367,
|
|
11
|
+
2309, 2346, 2344, 2368, 2309, 2346, 2344, 2375, 2309, 2349, 2367, 2309, 2349, 2368, 2310, 2342, 2367, 2310, 2346,
|
|
12
|
+
2311, 2306, 2361, 2367, 2306, 2311, 2306, 2361, 2375, 2306, 2311, 2306, 2361, 2379, 2306, 2311, 2340, 2351, 2366,
|
|
13
|
+
2342, 2367, 2311, 2340, 2381, 2351, 2366, 2342, 2367, 2311, 2344, 2311, 2344, 2325, 2366, 2311, 2344, 2381, 2361,
|
|
14
|
+
2368, 2306, 2311, 2344, 2381, 2361, 2375, 2306, 2311, 2344, 2381, 2361, 2379, 2306, 2311, 2360, 2311, 2360, 2325,
|
|
15
|
+
2366, 2311, 2360, 2325, 2367, 2311, 2360, 2325, 2368, 2311, 2360, 2325, 2375, 2311, 2360, 2350, 2375, 2306, 2311,
|
|
16
|
+
2360, 2367, 2311, 2360, 2368, 2311, 2360, 2375, 2313, 2306, 2361, 2367, 2306, 2313, 2306, 2361, 2375, 2306, 2313,
|
|
17
|
+
2306, 2361, 2379, 2306, 2313, 2344, 2313, 2344, 2325, 2366, 2313, 2344, 2325, 2367, 2313, 2344, 2325, 2368, 2313,
|
|
18
|
+
2344, 2325, 2375, 2313, 2344, 2325, 2379, 2313, 2344, 2381, 2361, 2368, 2306, 2313, 2344, 2381, 2361, 2375, 2306,
|
|
19
|
+
2313, 2344, 2381, 2361, 2379, 2306, 2313, 2360, 2313, 2360, 2325, 2375, 2313, 2360, 2367, 2313, 2360, 2368, 2313,
|
|
20
|
+
2360, 2375, 2319, 2325, 2319, 2357, 2306, 2319, 2360, 2319, 2360, 2375, 2320, 2360, 2375, 2323, 2352, 2324, 2352,
|
|
21
|
+
2325, 2311, 2325, 2312, 2325, 2352, 2325, 2352, 2340, 2366, 2325, 2352, 2340, 2375, 2325, 2352, 2344, 2366, 2325,
|
|
22
|
+
2352, 2344, 2375, 2325, 2352, 2375, 2306, 2325, 2361, 2340, 2375, 2325, 2361, 2366, 2325, 2366, 2325, 2366, 2347,
|
|
23
|
+
2364, 2368, 2325, 2366, 2347, 2367, 2325, 2367, 2325, 2367, 2306, 2361, 2375, 2306, 2325, 2367, 2306, 2361, 2379,
|
|
24
|
+
2306, 2325, 2367, 2340, 2344, 2366, 2325, 2367, 2344, 2381, 2361, 2375, 2306, 2325, 2367, 2344, 2381, 2361, 2379,
|
|
25
|
+
2306, 2325, 2367, 2351, 2366, 2325, 2367, 2352, 2325, 2367, 2360, 2325, 2367, 2360, 2367, 2325, 2367, 2360, 2368,
|
|
26
|
+
2325, 2367, 2360, 2375, 2325, 2368, 2325, 2369, 2331, 2325, 2369, 2354, 2325, 2375, 2325, 2379, 2325, 2379, 2311,
|
|
27
|
+
2325, 2379, 2312, 2325, 2379, 2344, 2325, 2379, 2344, 2360, 2366, 2325, 2380, 2344, 2325, 2380, 2344, 2360, 2366,
|
|
28
|
+
2327, 2351, 2366, 2328, 2352, 2332, 2348, 2332, 2361, 2366, 2305, 2332, 2361, 2366, 2306, 2332, 2366, 2332, 2367,
|
|
29
|
+
2306, 2361, 2375, 2306, 2332, 2367, 2306, 2361, 2379, 2306, 2332, 2367, 2340, 2344, 2366, 2332, 2367, 2343, 2352,
|
|
30
|
+
2332, 2367, 2344, 2332, 2367, 2344, 2381, 2361, 2375, 2306, 2332, 2367, 2344, 2381, 2361, 2379, 2306, 2332, 2367,
|
|
31
|
+
2360, 2332, 2367, 2360, 2375, 2332, 2368, 2343, 2352, 2332, 2375, 2360, 2366, 2332, 2375, 2360, 2375, 2332, 2376,
|
|
32
|
+
2360, 2366, 2332, 2376, 2360, 2375, 2332, 2379, 2340, 2325, 2340, 2348, 2340, 2352, 2361, 2340, 2367, 2306, 2361,
|
|
33
|
+
2375, 2306, 2340, 2367, 2306, 2361, 2379, 2306, 2340, 2367, 2344, 2340, 2367, 2344, 2381, 2361, 2375, 2306, 2340,
|
|
34
|
+
2367, 2344, 2381, 2361, 2379, 2306, 2340, 2367, 2360, 2340, 2367, 2360, 2375, 2340, 2379, 2341, 2366, 2341, 2367,
|
|
35
|
+
2341, 2368, 2341, 2375, 2342, 2348, 2366, 2352, 2366, 2342, 2357, 2366, 2352, 2366, 2342, 2367, 2351, 2366, 2342,
|
|
36
|
+
2369, 2360, 2352, 2366, 2342, 2369, 2360, 2352, 2375, 2342, 2370, 2360, 2352, 2375, 2342, 2379, 2342, 2381, 2357,
|
|
37
|
+
2366, 2352, 2366, 2344, 2344, 2361, 2367, 2306, 2344, 2361, 2368, 2306, 2344, 2366, 2344, 2367, 2330, 2375, 2344,
|
|
38
|
+
2367, 2361, 2366, 2351, 2340, 2344, 2368, 2330, 2375, 2344, 2375, 2346, 2352, 2346, 2361, 2354, 2375, 2346, 2369,
|
|
39
|
+
2352, 2366, 2346, 2370, 2352, 2366, 2346, 2375, 2347, 2367, 2352, 2348, 2344, 2367, 2348, 2344, 2368, 2348, 2361,
|
|
40
|
+
2367, 2348, 2361, 2368, 2348, 2361, 2369, 2340, 2348, 2366, 2342, 2348, 2366, 2354, 2366, 2348, 2367, 2354, 2325,
|
|
41
|
+
2369, 2354, 2349, 2367, 2349, 2367, 2340, 2352, 2349, 2368, 2349, 2368, 2340, 2352, 2350, 2327, 2352, 2350, 2366,
|
|
42
|
+
2344, 2379, 2350, 2375, 2350, 2375, 2306, 2351, 2342, 2367, 2351, 2361, 2351, 2361, 2366, 2305, 2351, 2361, 2366,
|
|
43
|
+
2306, 2351, 2361, 2367, 2351, 2361, 2368, 2351, 2366, 2351, 2367, 2361, 2351, 2375, 2352, 2326, 2375, 2306, 2352,
|
|
44
|
+
2357, 2366, 2360, 2366, 2352, 2361, 2366, 2352, 2361, 2375, 2353, 2381, 2357, 2366, 2360, 2366, 2354, 2367, 2319,
|
|
45
|
+
2354, 2367, 2351, 2375, 2354, 2375, 2325, 2367, 2344, 2357, 2357, 2327, 2364, 2376, 2352, 2361, 2357, 2327, 2375,
|
|
46
|
+
2352, 2361, 2357, 2352, 2327, 2357, 2352, 2381, 2327, 2357, 2361, 2357, 2361, 2366, 2305, 2357, 2361, 2366, 2306,
|
|
47
|
+
2357, 2361, 2367, 2306, 2357, 2361, 2368, 2306, 2357, 2366, 2354, 2375, 2357, 2369, 2361, 2357, 2375, 2360, 2306,
|
|
48
|
+
2327, 2360, 2325, 2340, 2366, 2360, 2325, 2340, 2375, 2360, 2348, 2360, 2375, 2360, 2349, 2367, 2360, 2349, 2368,
|
|
49
|
+
2360, 2366, 2341, 2360, 2366, 2348, 2369, 2340, 2360, 2366, 2349, 2360, 2366, 2352, 2366, 2360, 2375, 2360, 2379,
|
|
50
|
+
2361, 2367, 2361, 2368, 2361, 2369, 2309, 2361, 2369, 2310, 2361, 2369, 2311, 2361, 2369, 2312, 2361, 2369, 2319,
|
|
51
|
+
2361, 2375, 2361, 2375, 2306, 2361, 2376, 2361, 2376, 2306, 2361, 2379, 2361, 2379, 2340, 2366, 2361, 2379, 2340,
|
|
52
|
+
2367, 2361, 2379, 2340, 2368, 2361, 2379, 2340, 2375, 2361, 2379, 2344, 2366, 2361, 2379, 2344, 2375,
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
static const uint32_t stop_word_offsets[226] = {
|
|
56
|
+
0, 4, 6, 9, 11, 15, 19, 23, 27, 30, 33, 36, 38, 43, 48, 53, 59, 66, 68, 72, 78, 84, 90, 92, 96, 100, 104, 108, 113,
|
|
57
|
+
116, 119, 122, 127, 132, 137, 139, 143, 147, 151, 155, 159, 165, 171, 177, 179, 183, 186, 189, 192, 194, 197, 199,
|
|
58
|
+
202, 205, 207, 209, 211, 213, 215, 219, 223, 227, 231, 235, 239, 242, 244, 249, 253, 255, 261, 267, 272, 279, 286,
|
|
59
|
+
290, 293, 296, 300, 304, 308, 310, 313, 316, 318, 320, 323, 326, 329, 334, 337, 342, 345, 347, 349, 353, 357, 359,
|
|
60
|
+
365, 371, 376, 380, 383, 390, 397, 400, 404, 408, 412, 416, 420, 424, 426, 428, 430, 433, 439, 445, 448, 455, 462,
|
|
61
|
+
465, 469, 471, 473, 475, 477, 479, 484, 489, 493, 498, 503, 508, 510, 516, 517, 521, 525, 527, 531, 537, 541, 543,
|
|
62
|
+
545, 549, 553, 557, 559, 562, 565, 568, 571, 574, 578, 581, 585, 591, 593, 597, 599, 603, 606, 610, 612, 615, 618,
|
|
63
|
+
620, 624, 628, 631, 634, 636, 639, 641, 645, 650, 653, 656, 662, 665, 669, 674, 675, 681, 686, 689, 693, 695, 699,
|
|
64
|
+
703, 707, 711, 715, 718, 720, 723, 727, 731, 735, 738, 741, 744, 749, 752, 756, 758, 760, 762, 764, 767, 770, 773,
|
|
65
|
+
776, 779, 781, 784, 786, 789, 791, 795, 799, 803, 807, 811, 815,
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
static const uint32_t keep_ranges[10] = {
|
|
69
|
+
48, 57, 65, 90, 97, 122, 2304, 2403, 2406, 2431,
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
const narsil_language narsil_language_hindi = {
|
|
73
|
+
.name = "hindi",
|
|
74
|
+
.revision = "b7e1a82abe08c76a",
|
|
75
|
+
.stop_word_units = stop_word_units,
|
|
76
|
+
.stop_word_offsets = stop_word_offsets,
|
|
77
|
+
.stop_word_count = 225,
|
|
78
|
+
.ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
|
|
79
|
+
.keep_ranges = keep_ranges,
|
|
80
|
+
.keep_range_count = 5,
|
|
81
|
+
.splits_code_points = 0,
|
|
82
|
+
.strips_possessive = 0,
|
|
83
|
+
.normalises_diacritics = 0,
|
|
84
|
+
.min_token_length = 1,
|
|
85
|
+
.ngram_size = 0,
|
|
86
|
+
.normalise = narsil_hindi_normalise,
|
|
87
|
+
.stem_by_rules = NULL,
|
|
88
|
+
.create_snowball = hindi_create_env,
|
|
89
|
+
.snowball_stem = hindi_stem,
|
|
90
|
+
.close_snowball = hindi_close_env,
|
|
91
|
+
};
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
#include "../../analysis/languages.h"
|
|
2
|
+
#include "declarations.h"
|
|
3
|
+
#include "../snowball/hungarian_stemmer.h"
|
|
4
|
+
|
|
5
|
+
#include <stddef.h>
|
|
6
|
+
#include <stdint.h>
|
|
7
|
+
|
|
8
|
+
static const uint16_t stop_word_units[1164] = {
|
|
9
|
+
97, 97, 98, 98, 97, 97, 98, 98, 97, 110, 97, 98, 98, 243, 108, 97, 100, 100, 105, 103, 97, 104, 104, 111, 122, 97,
|
|
10
|
+
104, 111, 103, 121, 97, 104, 111, 108, 97, 107, 105, 97, 107, 105, 107, 97, 107, 107, 111, 114, 97, 107, 225, 114, 97,
|
|
11
|
+
108, 97, 112, 106, 225, 110, 97, 108, 97, 116, 116, 97, 108, 97, 116, 116, 97, 97, 108, 225, 97, 109, 101, 108, 121,
|
|
12
|
+
97, 109, 101, 108, 121, 101, 107, 97, 109, 101, 108, 121, 101, 116, 97, 109, 101, 108, 121, 105, 107, 97, 109, 101,
|
|
13
|
+
108, 121, 110, 101, 107, 97, 109, 105, 97, 109, 105, 107, 111, 114, 97, 109, 105, 116, 97, 109, 237, 103, 97, 110,
|
|
14
|
+
110, 97, 107, 97, 110, 110, 225, 108, 97, 114, 114, 97, 97, 114, 114, 243, 108, 97, 116, 116, 243, 108, 97, 122, 97,
|
|
15
|
+
122, 111, 107, 97, 122, 111, 107, 97, 116, 97, 122, 111, 110, 97, 122, 111, 110, 98, 97, 110, 97, 122, 116, 97, 122,
|
|
16
|
+
116, 225, 110, 97, 122, 117, 116, 225, 110, 97, 122, 122, 97, 108, 97, 122, 233, 114, 116, 98, 101, 98, 101, 108, 252,
|
|
17
|
+
108, 98, 101, 110, 110, 101, 98, 225, 114, 99, 115, 97, 107, 100, 101, 101, 101, 100, 100, 105, 103, 101, 103, 121,
|
|
18
|
+
101, 103, 121, 101, 100, 252, 108, 101, 103, 121, 101, 115, 101, 103, 121, 101, 116, 108, 101, 110, 101, 103, 121,
|
|
19
|
+
105, 107, 101, 103, 121, 109, 225, 115, 101, 103, 121, 114, 101, 101, 103, 121, 233, 98, 101, 103, 121, 252, 116, 116,
|
|
20
|
+
101, 103, 233, 115, 122, 101, 104, 104, 101, 122, 101, 107, 107, 111, 114, 101, 108, 101, 108, 108, 101, 110, 101,
|
|
21
|
+
108, 233, 103, 101, 108, 337, 101, 108, 337, 98, 98, 101, 108, 337, 115, 122, 246, 114, 101, 108, 337, 116, 116, 101,
|
|
22
|
+
108, 337, 122, 337, 101, 110, 103, 101, 109, 101, 110, 110, 101, 107, 101, 114, 114, 101, 101, 122, 101, 122, 101,
|
|
23
|
+
107, 101, 122, 101, 110, 101, 122, 116, 101, 122, 117, 116, 225, 110, 101, 122, 122, 101, 108, 101, 122, 233, 114,
|
|
24
|
+
116, 102, 101, 108, 102, 101, 108, 233, 104, 97, 104, 97, 110, 101, 109, 104, 105, 115, 122, 101, 110, 104, 111, 103,
|
|
25
|
+
121, 104, 111, 103, 121, 97, 110, 104, 111, 108, 104, 111, 108, 110, 97, 112, 104, 111, 110, 110, 97, 110, 104, 111,
|
|
26
|
+
118, 97, 104, 111, 122, 122, 225, 105, 100, 101, 105, 103, 101, 110, 105, 108, 108, 105, 108, 108, 101, 116, 118, 101,
|
|
27
|
+
105, 108, 121, 101, 110, 105, 109, 109, 225, 114, 105, 110, 107, 225, 98, 98, 105, 115, 105, 115, 109, 233, 116, 105,
|
|
28
|
+
116, 116, 106, 111, 98, 98, 97, 110, 106, 243, 108, 107, 101, 108, 108, 107, 101, 108, 108, 101, 116, 116, 107, 101,
|
|
29
|
+
114, 101, 115, 122, 116, 252, 108, 107, 101, 116, 116, 337, 107, 101, 118, 233, 115, 107, 105, 107, 105, 116, 107,
|
|
30
|
+
246, 122, 98, 101, 110, 107, 246, 122, 101, 108, 107, 246, 122, 233, 107, 246, 122, 246, 116, 116, 107, 246, 122, 252,
|
|
31
|
+
108, 107, 252, 108, 246, 110, 98, 101, 110, 107, 252, 108, 246, 110, 98, 246, 122, 337, 108, 101, 108, 101, 103, 97,
|
|
32
|
+
108, 225, 98, 98, 108, 101, 103, 121, 101, 110, 108, 101, 104, 101, 116, 108, 101, 110, 110, 101, 108, 101, 110, 110,
|
|
33
|
+
105, 108, 101, 115, 122, 108, 101, 116, 116, 109, 97, 109, 97, 103, 97, 109, 97, 103, 225, 116, 109, 97, 105, 109, 97,
|
|
34
|
+
106, 100, 109, 101, 103, 109, 101, 103, 105, 110, 116, 109, 101, 108, 108, 101, 116, 116, 109, 101, 108, 108, 233,
|
|
35
|
+
109, 101, 108, 121, 109, 101, 108, 121, 101, 107, 109, 101, 108, 121, 105, 107, 109, 101, 110, 110, 121, 105, 109,
|
|
36
|
+
101, 114, 116, 109, 105, 109, 105, 97, 116, 116, 109, 105, 107, 111, 114, 109, 105, 108, 121, 101, 110, 109, 105, 110,
|
|
37
|
+
100, 109, 105, 110, 100, 101, 110, 109, 105, 110, 100, 101, 110, 107, 105, 109, 105, 110, 100, 101, 110, 116, 109,
|
|
38
|
+
105, 110, 100, 105, 103, 109, 105, 110, 101, 107, 109, 105, 110, 107, 101, 116, 109, 105, 110, 116, 109, 105, 110,
|
|
39
|
+
116, 104, 97, 109, 105, 116, 109, 105, 118, 101, 108, 109, 105, 233, 114, 116, 109, 111, 110, 100, 116, 97, 109, 111,
|
|
40
|
+
115, 116, 109, 225, 114, 109, 225, 115, 109, 225, 115, 105, 107, 109, 225, 115, 111, 100, 105, 107, 109, 233, 103,
|
|
41
|
+
109, 233, 103, 105, 115, 109, 237, 103, 109, 250, 108, 118, 97, 110, 97, 103, 121, 110, 97, 103, 121, 111, 110, 110,
|
|
42
|
+
101, 110, 101, 107, 101, 100, 110, 101, 107, 101, 109, 110, 101, 107, 105, 110, 101, 107, 105, 107, 110, 101, 107,
|
|
43
|
+
252, 110, 107, 110, 101, 109, 110, 105, 110, 99, 115, 110, 225, 108, 117, 110, 107, 110, 233, 104, 97, 110, 233, 104,
|
|
44
|
+
225, 110, 121, 110, 233, 108, 107, 252, 108, 111, 100, 97, 111, 108, 121, 97, 110, 111, 110, 110, 97, 110, 111, 116,
|
|
45
|
+
116, 112, 101, 100, 105, 103, 112, 101, 114, 115, 122, 101, 114, 97, 106, 116, 97, 114, 243, 108, 97, 115, 97, 106,
|
|
46
|
+
225, 116, 115, 101, 115, 101, 109, 115, 101, 109, 109, 105, 115, 101, 110, 107, 105, 115, 111, 104, 97, 115, 111, 107,
|
|
47
|
+
115, 111, 107, 97, 116, 115, 111, 107, 107, 97, 108, 115, 111, 114, 225, 110, 115, 122, 101, 109, 98, 101, 110, 115,
|
|
48
|
+
122, 101, 114, 105, 110, 116, 115, 122, 105, 110, 116, 101, 115, 122, 225, 109, 225, 114, 97, 115, 337, 116, 116, 97,
|
|
49
|
+
108, 225, 110, 116, 101, 116, 101, 104, 225, 116, 116, 105, 116, 111, 118, 225, 98, 98, 116, 111, 118, 225, 98, 98,
|
|
50
|
+
105, 116, 111, 118, 225, 98, 98, 225, 116, 233, 103, 101, 100, 116, 337, 108, 101, 117, 103, 121, 97, 110, 105, 115,
|
|
51
|
+
117, 103, 121, 101, 117, 116, 225, 110, 117, 116, 225, 110, 97, 118, 97, 103, 121, 118, 97, 103, 121, 105, 115, 118,
|
|
52
|
+
97, 103, 121, 111, 107, 118, 97, 103, 121, 117, 110, 107, 118, 97, 106, 111, 110, 118, 97, 108, 97, 107, 105, 118, 97,
|
|
53
|
+
108, 97, 109, 105, 118, 97, 108, 97, 109, 105, 110, 116, 118, 97, 108, 243, 118, 97, 110, 118, 97, 110, 110, 97, 107,
|
|
54
|
+
118, 101, 108, 101, 118, 101, 108, 101, 109, 118, 101, 108, 252, 110, 107, 118, 105, 115, 115, 122, 97, 118, 105, 115,
|
|
55
|
+
122, 111, 110, 116, 118, 111, 108, 110, 97, 118, 111, 108, 116, 118, 111, 108, 116, 97, 107, 118, 233, 103, 114, 101,
|
|
56
|
+
118, 233, 103, 252, 108, 225, 108, 116, 97, 108, 225, 108, 116, 97, 108, 225, 98, 97, 110, 225, 116, 233, 110, 233,
|
|
57
|
+
115, 237, 103, 121, 337, 337, 107, 337, 107, 101, 116, 337, 116,
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
static const uint32_t stop_word_offsets[246] = {
|
|
61
|
+
0, 1, 5, 10, 15, 20, 25, 30, 34, 37, 41, 46, 50, 57, 62, 68, 71, 76, 83, 90, 97, 105, 108, 114, 118, 122, 127, 132,
|
|
62
|
+
136, 141, 146, 148, 152, 158, 162, 169, 172, 177, 183, 188, 193, 195, 200, 205, 208, 212, 214, 215, 220, 223, 230,
|
|
63
|
+
235, 243, 248, 254, 259, 264, 270, 275, 280, 285, 287, 292, 296, 299, 304, 311, 316, 321, 326, 331, 335, 337, 341,
|
|
64
|
+
345, 348, 354, 359, 364, 367, 371, 373, 378, 384, 388, 394, 397, 403, 409, 413, 418, 421, 425, 428, 435, 440, 445,
|
|
65
|
+
451, 453, 458, 461, 467, 470, 474, 481, 490, 495, 500, 502, 505, 511, 516, 520, 526, 531, 539, 548, 550, 558, 564,
|
|
66
|
+
569, 574, 579, 583, 587, 589, 593, 598, 601, 605, 608, 614, 621, 626, 630, 636, 642, 648, 652, 654, 659, 664, 670,
|
|
67
|
+
674, 680, 688, 695, 701, 706, 712, 716, 722, 725, 730, 735, 741, 745, 748, 751, 756, 763, 766, 771, 774, 779, 783,
|
|
68
|
+
789, 791, 796, 801, 805, 810, 816, 819, 824, 830, 834, 840, 846, 849, 854, 859, 862, 867, 873, 878, 882, 887, 889,
|
|
69
|
+
892, 897, 902, 906, 909, 914, 920, 925, 932, 939, 945, 952, 955, 960, 962, 967, 969, 975, 982, 989, 994, 998, 1005,
|
|
70
|
+
1009, 1013, 1018, 1022, 1028, 1034, 1041, 1046, 1052, 1058, 1066, 1070, 1073, 1079, 1083, 1088, 1094, 1100, 1107,
|
|
71
|
+
1112, 1116, 1122, 1127, 1132, 1137, 1146, 1148, 1150, 1152, 1155, 1156, 1158, 1162, 1164,
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
static const uint32_t keep_ranges[38] = {
|
|
75
|
+
48, 57, 65, 90, 97, 122, 193, 193, 201, 201, 205, 205, 211, 211, 214, 214, 218, 218, 220, 220, 225, 225, 233, 233,
|
|
76
|
+
237, 237, 243, 243, 246, 246, 250, 250, 252, 252, 336, 337, 368, 369,
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
const narsil_language narsil_language_hungarian = {
|
|
80
|
+
.name = "hungarian",
|
|
81
|
+
.revision = "7b9c2fc79181fc11",
|
|
82
|
+
.stop_word_units = stop_word_units,
|
|
83
|
+
.stop_word_offsets = stop_word_offsets,
|
|
84
|
+
.stop_word_count = 245,
|
|
85
|
+
.ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
|
|
86
|
+
.keep_ranges = keep_ranges,
|
|
87
|
+
.keep_range_count = 19,
|
|
88
|
+
.splits_code_points = 0,
|
|
89
|
+
.strips_possessive = 0,
|
|
90
|
+
.normalises_diacritics = 0,
|
|
91
|
+
.min_token_length = 1,
|
|
92
|
+
.ngram_size = 0,
|
|
93
|
+
.normalise = NULL,
|
|
94
|
+
.stem_by_rules = NULL,
|
|
95
|
+
.create_snowball = hungarian_create_env,
|
|
96
|
+
.snowball_stem = hungarian_stem,
|
|
97
|
+
.close_snowball = hungarian_close_env,
|
|
98
|
+
};
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
#include "../../analysis/languages.h"
|
|
2
|
+
#include "declarations.h"
|
|
3
|
+
|
|
4
|
+
#include <stddef.h>
|
|
5
|
+
#include <stdint.h>
|
|
6
|
+
|
|
7
|
+
static const uint16_t stop_word_units[681] = {
|
|
8
|
+
97, 102, 104, 118, 101, 114, 106, 117, 97, 102, 116, 97, 110, 97, 102, 116, 117, 114, 97, 102, 254, 118, 237, 97, 108,
|
|
9
|
+
100, 114, 101, 105, 97, 108, 108, 105, 114, 97, 108, 108, 116, 97, 108, 118, 101, 103, 97, 110, 110, 97, 114, 115, 97,
|
|
10
|
+
110, 110, 97, 240, 98, 97, 114, 97, 100, 97, 103, 101, 102, 116, 105, 114, 101, 105, 103, 97, 101, 105, 110, 104, 118,
|
|
11
|
+
101, 114, 101, 105, 110, 104, 118, 101, 114, 106, 105, 114, 101, 105, 110, 104, 118, 101, 114, 115, 101, 105, 110,
|
|
12
|
+
115, 101, 105, 110, 117, 101, 105, 116, 116, 104, 118, 97, 240, 101, 107, 107, 101, 114, 116, 101, 107, 107, 105, 101,
|
|
13
|
+
110, 110, 254, 225, 101, 114, 117, 101, 240, 97, 102, 97, 114, 97, 102, 101, 114, 102, 105, 110, 110, 97, 102, 106,
|
|
14
|
+
246, 108, 100, 105, 102, 114, 97, 109, 97, 110, 102, 114, 101, 107, 97, 114, 102, 114, 225, 102, 121, 114, 105, 114,
|
|
15
|
+
102, 243, 108, 107, 103, 101, 103, 110, 117, 109, 103, 101, 116, 97, 103, 101, 116, 117, 114, 103, 109, 103, 103, 111,
|
|
16
|
+
116, 116, 104, 97, 102, 97, 104, 97, 110, 110, 104, 101, 102, 104, 101, 102, 117, 114, 104, 101, 121, 114, 97, 104,
|
|
17
|
+
106, 225, 104, 118, 97, 114, 104, 118, 97, 240, 104, 118, 101, 114, 104, 118, 101, 114, 106, 105, 114, 104, 118, 101,
|
|
18
|
+
114, 106, 117, 109, 104, 118, 101, 114, 110, 105, 103, 104, 118, 111, 114, 104, 118, 111, 114, 116, 104, 230, 103,
|
|
19
|
+
116, 104, 233, 114, 104, 233, 114, 110, 97, 104, 250, 110, 105, 109, 103, 105, 110, 110, 107, 97, 110, 110, 115, 107,
|
|
20
|
+
105, 107, 111, 109, 97, 108, 111, 108, 108, 237, 107, 97, 109, 97, 240, 117, 114, 109, 101, 103, 97, 109, 101, 105,
|
|
21
|
+
114, 97, 109, 101, 240, 109, 105, 103, 109, 105, 107, 105, 240, 109, 105, 110, 110, 97, 109, 105, 110, 110, 105, 109,
|
|
22
|
+
105, 115, 115, 97, 109, 106, 246, 103, 109, 225, 116, 116, 109, 233, 114, 110, 101, 105, 110, 105, 240, 117, 114, 110,
|
|
23
|
+
250, 110, 97, 111, 102, 116, 111, 107, 107, 97, 114, 111, 107, 107, 117, 114, 112, 243, 115, 116, 112, 243, 115, 116,
|
|
24
|
+
117, 114, 114, 111, 102, 108, 115, 97, 109, 97, 110, 115, 101, 109, 115, 105, 103, 115, 105, 110, 110, 105, 115, 106,
|
|
25
|
+
225, 115, 109, 225, 115, 109, 225, 116, 116, 115, 112, 117, 114, 106, 97, 115, 112, 121, 114, 106, 97, 115, 116, 97,
|
|
26
|
+
240, 97, 114, 115, 116, 243, 114, 116, 115, 118, 111, 115, 118, 111, 110, 97, 115, 230, 108, 105, 114, 115, 230, 108,
|
|
27
|
+
108, 115, 233, 114, 115, 237, 240, 97, 110, 116, 97, 107, 97, 116, 97, 107, 107, 116, 105, 108, 116, 105, 108, 118,
|
|
28
|
+
105, 116, 110, 117, 110, 116, 105, 116, 108, 97, 114, 117, 112, 112, 118, 97, 114, 118, 101, 108, 118, 101, 108, 107,
|
|
29
|
+
111, 109, 105, 110, 118, 101, 108, 107, 111, 109, 105, 110, 110, 118, 101, 114, 97, 118, 101, 114, 105, 240, 118, 101,
|
|
30
|
+
114, 240, 117, 114, 118, 105, 108, 118, 105, 108, 106, 97, 118, 105, 108, 108, 118, 105, 116, 97, 118, 105, 240, 118,
|
|
31
|
+
230, 114, 105, 121, 102, 105, 114, 121, 107, 107, 97, 114, 230, 116, 116, 105, 254, 97, 107, 107, 97, 254, 97, 107,
|
|
32
|
+
107, 105, 114, 254, 97, 110, 110, 105, 103, 254, 97, 114, 254, 97, 114, 102, 254, 97, 117, 254, 97, 240, 254, 101,
|
|
33
|
+
103, 97, 114, 254, 101, 105, 109, 254, 101, 105, 114, 254, 101, 105, 114, 114, 97, 254, 101, 115, 115, 254, 101, 115,
|
|
34
|
+
115, 97, 254, 101, 115, 115, 105, 254, 101, 115, 115, 117, 254, 101, 115, 115, 117, 109, 254, 101, 116, 116, 97, 254,
|
|
35
|
+
105, 110, 110, 254, 105, 116, 116, 254, 105, 240, 254, 114, 225, 240, 254, 114, 225, 240, 117, 114, 254, 118, 237,
|
|
36
|
+
254, 230, 114, 254, 233, 114, 254, 237, 110,
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
static const uint32_t stop_word_offsets[150] = {
|
|
40
|
+
0, 8, 13, 18, 23, 29, 34, 38, 43, 49, 54, 58, 61, 66, 70, 77, 87, 95, 99, 103, 111, 117, 121, 126, 129, 132, 136, 139,
|
|
41
|
+
144, 150, 156, 162, 165, 170, 174, 180, 184, 189, 192, 196, 200, 204, 207, 212, 217, 220, 224, 228, 232, 239, 246,
|
|
42
|
+
253, 257, 262, 266, 269, 274, 277, 280, 283, 290, 294, 297, 301, 306, 310, 315, 318, 321, 326, 331, 336, 341, 345,
|
|
43
|
+
349, 352, 355, 360, 364, 367, 372, 377, 381, 387, 391, 396, 399, 402, 407, 410, 413, 418, 424, 430, 436, 441, 444,
|
|
44
|
+
449, 454, 458, 461, 466, 470, 474, 477, 486, 492, 495, 498, 501, 509, 518, 522, 527, 533, 536, 541, 545, 549, 552,
|
|
45
|
+
556, 560, 565, 569, 574, 580, 586, 589, 593, 596, 599, 604, 608, 612, 618, 622, 627, 632, 637, 643, 648, 652, 656,
|
|
46
|
+
659, 663, 669, 672, 675, 678, 681,
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
static const uint32_t keep_ranges[42] = {
|
|
50
|
+
48, 57, 65, 90, 97, 122, 193, 193, 198, 198, 201, 201, 205, 205, 208, 208, 211, 211, 214, 214, 218, 218, 221, 222,
|
|
51
|
+
225, 225, 230, 230, 233, 233, 237, 237, 240, 240, 243, 243, 246, 246, 250, 250, 253, 254,
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
const narsil_language narsil_language_icelandic = {
|
|
55
|
+
.name = "icelandic",
|
|
56
|
+
.revision = "1",
|
|
57
|
+
.stop_word_units = stop_word_units,
|
|
58
|
+
.stop_word_offsets = stop_word_offsets,
|
|
59
|
+
.stop_word_count = 149,
|
|
60
|
+
.ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
|
|
61
|
+
.keep_ranges = keep_ranges,
|
|
62
|
+
.keep_range_count = 21,
|
|
63
|
+
.splits_code_points = 0,
|
|
64
|
+
.strips_possessive = 0,
|
|
65
|
+
.normalises_diacritics = 0,
|
|
66
|
+
.min_token_length = 1,
|
|
67
|
+
.ngram_size = 0,
|
|
68
|
+
.normalise = NULL,
|
|
69
|
+
.stem_by_rules = NULL,
|
|
70
|
+
.create_snowball = NULL,
|
|
71
|
+
.snowball_stem = NULL,
|
|
72
|
+
.close_snowball = NULL,
|
|
73
|
+
};
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
#include "../../analysis/languages.h"
|
|
2
|
+
#include "declarations.h"
|
|
3
|
+
|
|
4
|
+
#include <stddef.h>
|
|
5
|
+
#include <stdint.h>
|
|
6
|
+
|
|
7
|
+
static const uint16_t stop_word_units[254] = {
|
|
8
|
+
97, 97, 103, 97, 97, 107, 97, 97, 108, 97, 97, 110, 121, 97, 97, 110, 121, 105, 97, 110, 121, 7883, 98, 97, 110, 121,
|
|
9
|
+
101, 114, 101, 98, 117, 98, 7909, 100, 105, 100, 7883, 101, 101, 98, 101, 101, 106, 105, 101, 108, 117, 101, 109, 101,
|
|
10
|
+
101, 116, 105, 116, 105, 103, 97, 103, 97, 114, 97, 103, 104, 105, 103, 104, 7883, 103, 105, 103, 119, 97, 103, 7883,
|
|
11
|
+
104, 97, 105, 105, 104, 101, 105, 107, 101, 105, 115, 105, 106, 105, 107, 97, 107, 119, 97, 107, 119, 97, 114, 97,
|
|
12
|
+
107, 119, 117, 107, 119, 117, 114, 117, 109, 109, 97, 109, 97, 107, 97, 109, 97, 110, 97, 109, 101, 114, 101, 109,
|
|
13
|
+
103, 98, 101, 110, 110, 97, 110, 97, 110, 105, 110, 97, 110, 7883, 110, 99, 104, 97, 110, 100, 105, 110, 100, 7883,
|
|
14
|
+
110, 105, 108, 101, 110, 105, 109, 101, 110, 107, 101, 110, 111, 110, 119, 97, 110, 119, 101, 114, 101, 110, 121, 101,
|
|
15
|
+
111, 111, 98, 105, 111, 98, 117, 108, 97, 111, 103, 101, 111, 107, 112, 117, 114, 117, 111, 107, 119, 117, 111, 110,
|
|
16
|
+
119, 101, 111, 110, 121, 101, 111, 116, 117, 111, 122, 111, 112, 117, 114, 117, 115, 105, 115, 105, 116, 101, 116, 97,
|
|
17
|
+
97, 116, 117, 112, 117, 117, 102, 111, 100, 117, 117, 103, 98, 117, 117, 110, 117, 119, 101, 121, 97, 7883, 7885,
|
|
18
|
+
7885, 98, 7909, 108, 97, 7885, 122, 7885, 7909, 102, 7885, 100, 7909,
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
static const uint32_t stop_word_offsets[82] = {
|
|
22
|
+
0, 1, 4, 7, 10, 14, 18, 22, 29, 31, 33, 35, 37, 38, 41, 44, 47, 50, 55, 57, 61, 64, 67, 69, 72, 74, 76, 77, 80, 83,
|
|
23
|
+
86, 88, 90, 93, 98, 101, 106, 107, 109, 113, 117, 121, 125, 126, 128, 132, 136, 140, 143, 146, 150, 154, 157, 159,
|
|
24
|
+
162, 167, 170, 171, 174, 179, 182, 188, 192, 196, 200, 203, 206, 210, 212, 216, 219, 223, 228, 232, 235, 237, 239,
|
|
25
|
+
240, 241, 246, 249, 254,
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
static const uint32_t keep_ranges[694] = {
|
|
29
|
+
48, 57, 65, 90, 97, 122, 192, 192, 200, 201, 210, 211, 224, 224, 232, 233, 242, 243, 296, 297, 360, 361, 383, 383,
|
|
30
|
+
558, 559, 768, 879, 921, 921, 953, 953, 1155, 1161, 1425, 1469, 1471, 1471, 1473, 1474, 1476, 1477, 1479, 1479, 1552,
|
|
31
|
+
1562, 1611, 1631, 1648, 1648, 1750, 1756, 1759, 1764, 1767, 1768, 1770, 1773, 1809, 1809, 1840, 1866, 1958, 1968,
|
|
32
|
+
2027, 2035, 2045, 2045, 2070, 2073, 2075, 2083, 2085, 2087, 2089, 2093, 2137, 2139, 2199, 2207, 2250, 2273, 2275,
|
|
33
|
+
2307, 2362, 2364, 2366, 2383, 2385, 2391, 2402, 2403, 2433, 2435, 2492, 2492, 2494, 2500, 2503, 2504, 2507, 2509,
|
|
34
|
+
2519, 2519, 2530, 2531, 2558, 2558, 2561, 2563, 2620, 2620, 2622, 2626, 2631, 2632, 2635, 2637, 2641, 2641, 2672,
|
|
35
|
+
2673, 2677, 2677, 2689, 2691, 2748, 2748, 2750, 2757, 2759, 2761, 2763, 2765, 2786, 2787, 2810, 2815, 2817, 2819,
|
|
36
|
+
2876, 2876, 2878, 2884, 2887, 2888, 2891, 2893, 2901, 2903, 2914, 2915, 2946, 2946, 3006, 3010, 3014, 3016, 3018,
|
|
37
|
+
3021, 3031, 3031, 3072, 3076, 3132, 3132, 3134, 3140, 3142, 3144, 3146, 3149, 3157, 3158, 3170, 3171, 3201, 3203,
|
|
38
|
+
3260, 3260, 3262, 3268, 3270, 3272, 3274, 3277, 3285, 3286, 3298, 3299, 3315, 3315, 3328, 3331, 3387, 3388, 3390,
|
|
39
|
+
3396, 3398, 3400, 3402, 3405, 3415, 3415, 3426, 3427, 3457, 3459, 3530, 3530, 3535, 3540, 3542, 3542, 3544, 3551,
|
|
40
|
+
3570, 3571, 3633, 3633, 3636, 3642, 3655, 3662, 3761, 3761, 3764, 3772, 3784, 3790, 3864, 3865, 3893, 3893, 3895,
|
|
41
|
+
3895, 3897, 3897, 3902, 3903, 3953, 3972, 3974, 3975, 3981, 3991, 3993, 4028, 4038, 4038, 4139, 4158, 4182, 4185,
|
|
42
|
+
4190, 4192, 4194, 4196, 4199, 4205, 4209, 4212, 4226, 4237, 4239, 4239, 4250, 4253, 4957, 4959, 5906, 5909, 5938,
|
|
43
|
+
5940, 5970, 5971, 6002, 6003, 6068, 6099, 6109, 6109, 6155, 6157, 6159, 6159, 6277, 6278, 6313, 6313, 6432, 6443,
|
|
44
|
+
6448, 6459, 6679, 6683, 6741, 6750, 6752, 6780, 6783, 6783, 6832, 6877, 6880, 6891, 6912, 6916, 6964, 6980, 7019,
|
|
45
|
+
7027, 7040, 7042, 7073, 7085, 7142, 7155, 7204, 7223, 7376, 7378, 7380, 7400, 7405, 7405, 7412, 7412, 7415, 7417,
|
|
46
|
+
7616, 7679, 7748, 7749, 7882, 7885, 7908, 7909, 8126, 8126, 8400, 8432, 8490, 8490, 11503, 11505, 11647, 11647, 11744,
|
|
47
|
+
11775, 12330, 12335, 12441, 12442, 42607, 42610, 42612, 42621, 42654, 42655, 42736, 42737, 43010, 43010, 43014, 43014,
|
|
48
|
+
43019, 43019, 43043, 43047, 43052, 43052, 43136, 43137, 43188, 43205, 43232, 43249, 43263, 43263, 43302, 43309, 43335,
|
|
49
|
+
43347, 43392, 43395, 43443, 43456, 43493, 43493, 43561, 43574, 43587, 43587, 43596, 43597, 43643, 43645, 43696, 43696,
|
|
50
|
+
43698, 43700, 43703, 43704, 43710, 43711, 43713, 43713, 43755, 43759, 43765, 43766, 44003, 44010, 44012, 44013, 64286,
|
|
51
|
+
64286, 65024, 65039, 65056, 65071, 66045, 66045, 66272, 66272, 66422, 66426, 68097, 68099, 68101, 68102, 68108, 68111,
|
|
52
|
+
68152, 68154, 68159, 68159, 68325, 68326, 68900, 68903, 68969, 68973, 69291, 69292, 69370, 69375, 69446, 69456, 69506,
|
|
53
|
+
69509, 69632, 69634, 69688, 69702, 69744, 69744, 69747, 69748, 69759, 69762, 69808, 69818, 69826, 69826, 69888, 69890,
|
|
54
|
+
69927, 69940, 69957, 69958, 70003, 70003, 70016, 70018, 70067, 70080, 70089, 70092, 70094, 70095, 70188, 70199, 70206,
|
|
55
|
+
70206, 70209, 70209, 70367, 70378, 70400, 70403, 70459, 70460, 70462, 70468, 70471, 70472, 70475, 70477, 70487, 70487,
|
|
56
|
+
70498, 70499, 70502, 70508, 70512, 70516, 70584, 70592, 70594, 70594, 70597, 70597, 70599, 70602, 70604, 70608, 70610,
|
|
57
|
+
70610, 70625, 70626, 70709, 70726, 70750, 70750, 70832, 70851, 71087, 71093, 71096, 71104, 71132, 71133, 71216, 71232,
|
|
58
|
+
71339, 71351, 71453, 71467, 71724, 71738, 71984, 71989, 71991, 71992, 71995, 71998, 72000, 72000, 72002, 72003, 72145,
|
|
59
|
+
72151, 72154, 72160, 72164, 72164, 72193, 72202, 72243, 72249, 72251, 72254, 72263, 72263, 72273, 72283, 72330, 72345,
|
|
60
|
+
72544, 72551, 72751, 72758, 72760, 72767, 72850, 72871, 72873, 72886, 73009, 73014, 73018, 73018, 73020, 73021, 73023,
|
|
61
|
+
73029, 73031, 73031, 73098, 73102, 73104, 73105, 73107, 73111, 73459, 73462, 73472, 73473, 73475, 73475, 73524, 73530,
|
|
62
|
+
73534, 73538, 73562, 73562, 78912, 78912, 78919, 78933, 90398, 90415, 92912, 92916, 92976, 92982, 94031, 94031, 94033,
|
|
63
|
+
94087, 94095, 94098, 94180, 94180, 94192, 94193, 113821, 113822, 118528, 118573, 118576, 118598, 119141, 119145,
|
|
64
|
+
119149, 119154, 119163, 119170, 119173, 119179, 119210, 119213, 119362, 119364, 121344, 121398, 121403, 121452,
|
|
65
|
+
121461, 121461, 121476, 121476, 121499, 121503, 121505, 121519, 122880, 122886, 122888, 122904, 122907, 122913,
|
|
66
|
+
122915, 122916, 122918, 122922, 123023, 123023, 123184, 123190, 123566, 123566, 123628, 123631, 124140, 124143,
|
|
67
|
+
124398, 124399, 124643, 124643, 124646, 124646, 124654, 124655, 124661, 124661, 125136, 125142, 125252, 125258,
|
|
68
|
+
917760, 917999,
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
const narsil_language narsil_language_igbo = {
|
|
72
|
+
.name = "igbo",
|
|
73
|
+
.revision = "1",
|
|
74
|
+
.stop_word_units = stop_word_units,
|
|
75
|
+
.stop_word_offsets = stop_word_offsets,
|
|
76
|
+
.stop_word_count = 81,
|
|
77
|
+
.ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
|
|
78
|
+
.keep_ranges = keep_ranges,
|
|
79
|
+
.keep_range_count = 347,
|
|
80
|
+
.splits_code_points = 1,
|
|
81
|
+
.strips_possessive = 0,
|
|
82
|
+
.normalises_diacritics = 0,
|
|
83
|
+
.min_token_length = 1,
|
|
84
|
+
.ngram_size = 0,
|
|
85
|
+
.normalise = NULL,
|
|
86
|
+
.stem_by_rules = NULL,
|
|
87
|
+
.create_snowball = NULL,
|
|
88
|
+
.snowball_stem = NULL,
|
|
89
|
+
.close_snowball = NULL,
|
|
90
|
+
};
|