@delali/narsil-react-native 0.3.1-next.155 → 0.3.1-next.156
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +144 -0
- package/NarsilReactNative.podspec +2 -2
- package/android/CMakeLists.txt +1 -1
- package/cpp/HybridNarsilCoreAnalysis.cpp +277 -0
- package/cpp/HybridNarsilCoreImpl.hpp +43 -0
- package/cpp/HybridNarsilCoreKeyword.cpp +253 -0
- package/cpp/HybridNarsilCoreMath.cpp +47 -0
- package/cpp/core/include/narsil_analysis.h +86 -0
- package/cpp/core/include/narsil_core.h +1 -1
- package/cpp/core/include/narsil_keyword.h +110 -0
- package/cpp/core/include/narsil_math.h +36 -0
- package/cpp/core/src/analysis/analyse.c +281 -0
- package/cpp/core/src/analysis/analyser.h +49 -0
- package/cpp/core/src/analysis/languages.c +79 -0
- package/cpp/core/src/analysis/languages.h +42 -0
- package/cpp/core/src/analysis/normalise.c +242 -0
- package/cpp/core/src/analysis/normalise.h +28 -0
- package/cpp/core/src/analysis/split.c +183 -0
- package/cpp/core/src/analysis/split.h +27 -0
- package/cpp/core/src/analysis/stop_words.c +82 -0
- package/cpp/core/src/analysis/stop_words.h +19 -0
- package/cpp/core/src/analysis/term_cache.c +88 -0
- package/cpp/core/src/analysis/term_cache.h +36 -0
- package/cpp/core/src/analysis/text.c +161 -0
- package/cpp/core/src/analysis/text.h +45 -0
- package/cpp/core/src/analysis/transform.c +103 -0
- package/cpp/core/src/analysis/unicode.c +252 -0
- package/cpp/core/src/analysis/unicode.h +54 -0
- package/cpp/core/src/internal.h +4 -0
- package/cpp/core/src/keyword/pass.h +117 -0
- package/cpp/core/src/keyword/prune.c +324 -0
- package/cpp/core/src/keyword/request.c +179 -0
- package/cpp/core/src/keyword/rows.c +83 -0
- package/cpp/core/src/keyword/rows.h +18 -0
- package/cpp/core/src/keyword/score.c +257 -0
- package/cpp/core/src/keyword/score.h +17 -0
- package/cpp/core/src/keyword/scratch.c +226 -0
- package/cpp/core/src/keyword/scratch.h +55 -0
- package/cpp/core/src/keyword/select.c +155 -0
- package/cpp/core/src/keyword/select.h +17 -0
- package/cpp/core/src/languages/bulgarian_stemmer.c +114 -0
- package/cpp/core/src/languages/data/albanian.c +54 -0
- package/cpp/core/src/languages/data/amharic.c +93 -0
- package/cpp/core/src/languages/data/arabic.c +146 -0
- package/cpp/core/src/languages/data/armenian.c +46 -0
- package/cpp/core/src/languages/data/azerbaijani.c +68 -0
- package/cpp/core/src/languages/data/bambara.c +48 -0
- package/cpp/core/src/languages/data/basque.c +62 -0
- package/cpp/core/src/languages/data/belarusian.c +56 -0
- package/cpp/core/src/languages/data/bengali.c +63 -0
- package/cpp/core/src/languages/data/bosnian.c +73 -0
- package/cpp/core/src/languages/data/breton.c +52 -0
- package/cpp/core/src/languages/data/bulgarian.c +144 -0
- package/cpp/core/src/languages/data/burmese.c +48 -0
- package/cpp/core/src/languages/data/catalan.c +84 -0
- package/cpp/core/src/languages/data/chinese.c +70 -0
- package/cpp/core/src/languages/data/croatian.c +72 -0
- package/cpp/core/src/languages/data/czech.c +72 -0
- package/cpp/core/src/languages/data/dagbani.c +88 -0
- package/cpp/core/src/languages/data/danish.c +56 -0
- package/cpp/core/src/languages/data/declarations.h +114 -0
- package/cpp/core/src/languages/data/dutch.c +58 -0
- package/cpp/core/src/languages/data/english.c +172 -0
- package/cpp/core/src/languages/data/esperanto.c +71 -0
- package/cpp/core/src/languages/data/estonian.c +417 -0
- package/cpp/core/src/languages/data/ewe.c +89 -0
- package/cpp/core/src/languages/data/faroese.c +54 -0
- package/cpp/core/src/languages/data/fijian.c +44 -0
- package/cpp/core/src/languages/data/finnish.c +100 -0
- package/cpp/core/src/languages/data/french.c +74 -0
- package/cpp/core/src/languages/data/ga.c +83 -0
- package/cpp/core/src/languages/data/galician.c +70 -0
- package/cpp/core/src/languages/data/georgian.c +57 -0
- package/cpp/core/src/languages/data/german.c +96 -0
- package/cpp/core/src/languages/data/greek.c +202 -0
- package/cpp/core/src/languages/data/guarani.c +110 -0
- package/cpp/core/src/languages/data/gujarati.c +90 -0
- package/cpp/core/src/languages/data/haitian-creole.c +54 -0
- package/cpp/core/src/languages/data/hausa.c +85 -0
- package/cpp/core/src/languages/data/hawaiian.c +50 -0
- package/cpp/core/src/languages/data/hebrew.c +85 -0
- package/cpp/core/src/languages/data/hindi.c +91 -0
- package/cpp/core/src/languages/data/hungarian.c +98 -0
- package/cpp/core/src/languages/data/icelandic.c +73 -0
- package/cpp/core/src/languages/data/igbo.c +90 -0
- package/cpp/core/src/languages/data/indonesian.c +296 -0
- package/cpp/core/src/languages/data/irish.c +60 -0
- package/cpp/core/src/languages/data/italian.c +107 -0
- package/cpp/core/src/languages/data/japanese.c +52 -0
- package/cpp/core/src/languages/data/kannada.c +60 -0
- package/cpp/core/src/languages/data/kazakh.c +57 -0
- package/cpp/core/src/languages/data/khmer.c +71 -0
- package/cpp/core/src/languages/data/kinyarwanda.c +54 -0
- package/cpp/core/src/languages/data/kirundi.c +53 -0
- package/cpp/core/src/languages/data/korean.c +74 -0
- package/cpp/core/src/languages/data/kurmanji.c +43 -0
- package/cpp/core/src/languages/data/kyrgyz.c +65 -0
- package/cpp/core/src/languages/data/lao.c +46 -0
- package/cpp/core/src/languages/data/latin.c +46 -0
- package/cpp/core/src/languages/data/latvian.c +75 -0
- package/cpp/core/src/languages/data/lingala.c +88 -0
- package/cpp/core/src/languages/data/lithuanian.c +65 -0
- package/cpp/core/src/languages/data/luxembourgish.c +80 -0
- package/cpp/core/src/languages/data/macedonian.c +286 -0
- package/cpp/core/src/languages/data/malagasy.c +51 -0
- package/cpp/core/src/languages/data/malay.c +184 -0
- package/cpp/core/src/languages/data/malayalam.c +64 -0
- package/cpp/core/src/languages/data/maltese.c +102 -0
- package/cpp/core/src/languages/data/maori.c +52 -0
- package/cpp/core/src/languages/data/marathi.c +62 -0
- package/cpp/core/src/languages/data/nepali.c +72 -0
- package/cpp/core/src/languages/data/norwegian.c +74 -0
- package/cpp/core/src/languages/data/oromo.c +51 -0
- package/cpp/core/src/languages/data/persian.c +150 -0
- package/cpp/core/src/languages/data/polish.c +111 -0
- package/cpp/core/src/languages/data/portuguese.c +91 -0
- package/cpp/core/src/languages/data/punjabi.c +80 -0
- package/cpp/core/src/languages/data/registry.c +116 -0
- package/cpp/core/src/languages/data/romanian.c +93 -0
- package/cpp/core/src/languages/data/russian.c +76 -0
- package/cpp/core/src/languages/data/samoan.c +47 -0
- package/cpp/core/src/languages/data/sanskrit.c +53 -0
- package/cpp/core/src/languages/data/scottish-gaelic.c +119 -0
- package/cpp/core/src/languages/data/serbian.c +84 -0
- package/cpp/core/src/languages/data/shona.c +55 -0
- package/cpp/core/src/languages/data/sinhala.c +78 -0
- package/cpp/core/src/languages/data/slovak.c +135 -0
- package/cpp/core/src/languages/data/slovenian.c +139 -0
- package/cpp/core/src/languages/data/sorani.c +55 -0
- package/cpp/core/src/languages/data/spanish.c +84 -0
- package/cpp/core/src/languages/data/swahili.c +59 -0
- package/cpp/core/src/languages/data/swedish.c +60 -0
- package/cpp/core/src/languages/data/tagalog.c +75 -0
- package/cpp/core/src/languages/data/tamil.c +69 -0
- package/cpp/core/src/languages/data/tatar.c +548 -0
- package/cpp/core/src/languages/data/telugu.c +54 -0
- package/cpp/core/src/languages/data/thai.c +46 -0
- package/cpp/core/src/languages/data/tibetan.c +55 -0
- package/cpp/core/src/languages/data/tigrinya.c +73 -0
- package/cpp/core/src/languages/data/tongan.c +47 -0
- package/cpp/core/src/languages/data/turkish.c +100 -0
- package/cpp/core/src/languages/data/twi.c +88 -0
- package/cpp/core/src/languages/data/ukrainian.c +99 -0
- package/cpp/core/src/languages/data/urdu.c +174 -0
- package/cpp/core/src/languages/data/vietnamese.c +88 -0
- package/cpp/core/src/languages/data/welsh.c +143 -0
- package/cpp/core/src/languages/data/wolof.c +51 -0
- package/cpp/core/src/languages/data/xhosa.c +56 -0
- package/cpp/core/src/languages/data/yoruba.c +93 -0
- package/cpp/core/src/languages/data/zulu.c +58 -0
- package/cpp/core/src/languages/marks.c +64 -0
- package/cpp/core/src/languages/rules.c +63 -0
- package/cpp/core/src/languages/rules.h +68 -0
- package/cpp/core/src/languages/sanskrit_stemmer.c +50 -0
- package/cpp/core/src/languages/scripts.c +143 -0
- package/cpp/core/src/languages/slovenian_stemmer.c +46 -0
- package/cpp/core/src/languages/snowball/api.c +52 -0
- package/cpp/core/src/languages/snowball/api.h +43 -0
- package/cpp/core/src/languages/snowball/arabic_stemmer.c +1506 -0
- package/cpp/core/src/languages/snowball/arabic_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/armenian_stemmer.c +523 -0
- package/cpp/core/src/languages/snowball/armenian_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/basque_stemmer.c +1089 -0
- package/cpp/core/src/languages/snowball/basque_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/catalan_stemmer.c +1380 -0
- package/cpp/core/src/languages/snowball/catalan_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/czech_stemmer.c +582 -0
- package/cpp/core/src/languages/snowball/czech_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/danish_stemmer.c +352 -0
- package/cpp/core/src/languages/snowball/danish_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/dutch_stemmer.c +1657 -0
- package/cpp/core/src/languages/snowball/dutch_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/english_stemmer.c +1019 -0
- package/cpp/core/src/languages/snowball/english_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/esperanto_stemmer.c +716 -0
- package/cpp/core/src/languages/snowball/esperanto_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/estonian_stemmer.c +1361 -0
- package/cpp/core/src/languages/snowball/estonian_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/finnish_stemmer.c +800 -0
- package/cpp/core/src/languages/snowball/finnish_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/french_stemmer.c +1217 -0
- package/cpp/core/src/languages/snowball/french_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/german_stemmer.c +567 -0
- package/cpp/core/src/languages/snowball/german_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/greek_stemmer.c +3450 -0
- package/cpp/core/src/languages/snowball/greek_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/hindi_stemmer.c +325 -0
- package/cpp/core/src/languages/snowball/hindi_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/hungarian_stemmer.c +742 -0
- package/cpp/core/src/languages/snowball/hungarian_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/indonesian_stemmer.c +443 -0
- package/cpp/core/src/languages/snowball/indonesian_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/irish_stemmer.c +428 -0
- package/cpp/core/src/languages/snowball/irish_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/italian_stemmer.c +983 -0
- package/cpp/core/src/languages/snowball/italian_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/lithuanian_stemmer.c +794 -0
- package/cpp/core/src/languages/snowball/lithuanian_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/nepali_stemmer.c +361 -0
- package/cpp/core/src/languages/snowball/nepali_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/norwegian_stemmer.c +332 -0
- package/cpp/core/src/languages/snowball/norwegian_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/persian_stemmer.c +538 -0
- package/cpp/core/src/languages/snowball/persian_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/polish_stemmer.c +501 -0
- package/cpp/core/src/languages/snowball/polish_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/portuguese_stemmer.c +866 -0
- package/cpp/core/src/languages/snowball/portuguese_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/romanian_stemmer.c +940 -0
- package/cpp/core/src/languages/snowball/romanian_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/russian_stemmer.c +598 -0
- package/cpp/core/src/languages/snowball/russian_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/serbian_stemmer.c +6765 -0
- package/cpp/core/src/languages/snowball/serbian_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/snowball_runtime.h +122 -0
- package/cpp/core/src/languages/snowball/spanish_stemmer.c +955 -0
- package/cpp/core/src/languages/snowball/spanish_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/swedish_stemmer.c +364 -0
- package/cpp/core/src/languages/snowball/swedish_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/tamil_stemmer.c +1305 -0
- package/cpp/core/src/languages/snowball/tamil_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/turkish_stemmer.c +1700 -0
- package/cpp/core/src/languages/snowball/turkish_stemmer.h +24 -0
- package/cpp/core/src/languages/snowball/utilities.c +716 -0
- package/cpp/core/src/languages/substitutions.c +131 -0
- package/cpp/core/src/languages/swahili_stemmer.c +85 -0
- package/cpp/core/src/languages/ukrainian_stemmer.c +168 -0
- package/cpp/core/src/locks.c +2 -15
- package/cpp/core/src/math/apply.c +44 -0
- package/cpp/core/src/math/fdlibm.h +16 -0
- package/cpp/core/src/math/inverse.c +242 -0
- package/cpp/core/src/math/log.c +90 -0
- package/cpp/core/src/math/reduce.c +325 -0
- package/cpp/core/src/math/trig.c +188 -0
- package/cpp/core/src/math/words.h +45 -0
- package/cpp/core/src/thread_yield.h +18 -0
- package/cpp/core/src/unicode/decompositions.c +310 -0
- package/cpp/core/src/unicode/lowercase.c +172 -0
- package/cpp/core/src/unicode/properties.c +214 -0
- package/cpp/core/src/workspace.c +5 -1
- package/lib/engine/analysis.d.ts +3 -0
- package/lib/engine/analysis.js +76 -0
- package/lib/engine/core.js +5 -0
- package/lib/engine/keyword.d.ts +3 -0
- package/lib/engine/keyword.js +25 -0
- package/lib/specs/core.nitro.d.ts +1 -0
- package/nitrogen/generated/shared/c++/HybridNarsilCoreSpec.cpp +1 -0
- package/nitrogen/generated/shared/c++/HybridNarsilCoreSpec.hpp +1 -0
- package/package.json +4 -3
- package/src/engine/analysis.ts +87 -0
- package/src/engine/core.ts +6 -0
- package/src/engine/keyword.ts +26 -0
- package/src/specs/core.nitro.ts +1 -0
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
#include "../../analysis/languages.h"
|
|
2
|
+
#include "declarations.h"
|
|
3
|
+
#include "../snowball/finnish_stemmer.h"
|
|
4
|
+
|
|
5
|
+
#include <stddef.h>
|
|
6
|
+
#include <stdint.h>
|
|
7
|
+
|
|
8
|
+
static const uint16_t stop_word_units[1201] = {
|
|
9
|
+
101, 105, 101, 105, 118, 228, 116, 101, 109, 109, 101, 101, 110, 101, 116, 101, 116, 116, 101, 101, 116, 116, 228,
|
|
10
|
+
104, 101, 104, 101, 105, 100, 228, 110, 104, 101, 105, 100, 228, 116, 104, 101, 105, 104, 105, 110, 104, 101, 105,
|
|
11
|
+
108, 108, 101, 104, 101, 105, 108, 108, 228, 104, 101, 105, 108, 116, 228, 104, 101, 105, 115, 115, 228, 104, 101,
|
|
12
|
+
105, 115, 116, 228, 104, 101, 105, 116, 228, 104, 228, 110, 104, 228, 110, 101, 101, 110, 104, 228, 110, 101, 108,
|
|
13
|
+
108, 101, 104, 228, 110, 101, 108, 108, 228, 104, 228, 110, 101, 108, 116, 228, 104, 228, 110, 101, 110, 104, 228,
|
|
14
|
+
110, 101, 115, 115, 228, 104, 228, 110, 101, 115, 116, 228, 104, 228, 110, 101, 116, 104, 228, 110, 116, 228, 105,
|
|
15
|
+
116, 115, 101, 106, 97, 106, 111, 104, 111, 110, 106, 111, 105, 100, 101, 110, 106, 111, 105, 104, 105, 110, 106, 111,
|
|
16
|
+
105, 107, 115, 105, 106, 111, 105, 108, 108, 97, 106, 111, 105, 108, 108, 101, 106, 111, 105, 108, 116, 97, 106, 111,
|
|
17
|
+
105, 110, 97, 106, 111, 105, 115, 115, 97, 106, 111, 105, 115, 116, 97, 106, 111, 105, 116, 97, 106, 111, 107, 97,
|
|
18
|
+
106, 111, 107, 115, 105, 106, 111, 108, 108, 97, 106, 111, 108, 108, 101, 106, 111, 108, 116, 97, 106, 111, 110, 97,
|
|
19
|
+
106, 111, 110, 107, 97, 106, 111, 115, 106, 111, 115, 115, 97, 106, 111, 115, 116, 97, 106, 111, 116, 97, 106, 111,
|
|
20
|
+
116, 107, 97, 107, 97, 110, 115, 115, 97, 107, 101, 105, 100, 101, 110, 107, 101, 105, 104, 105, 110, 107, 101, 105,
|
|
21
|
+
107, 115, 105, 107, 101, 105, 108, 108, 101, 107, 101, 105, 108, 108, 228, 107, 101, 105, 108, 116, 228, 107, 101,
|
|
22
|
+
105, 110, 228, 107, 101, 105, 115, 115, 228, 107, 101, 105, 115, 116, 228, 107, 101, 105, 116, 228, 107, 101, 110,
|
|
23
|
+
101, 101, 110, 107, 101, 110, 101, 107, 115, 105, 107, 101, 110, 101, 108, 108, 101, 107, 101, 110, 101, 108, 108,
|
|
24
|
+
228, 107, 101, 110, 101, 108, 116, 228, 107, 101, 110, 101, 110, 107, 101, 110, 101, 110, 228, 107, 101, 110, 101,
|
|
25
|
+
115, 115, 228, 107, 101, 110, 101, 115, 116, 228, 107, 101, 110, 101, 116, 107, 101, 116, 107, 228, 107, 101, 116,
|
|
26
|
+
228, 107, 111, 115, 107, 97, 107, 117, 105, 110, 107, 117, 107, 97, 107, 117, 110, 109, 101, 109, 101, 105, 100, 228,
|
|
27
|
+
110, 109, 101, 105, 100, 228, 116, 109, 101, 105, 104, 105, 110, 109, 101, 105, 108, 108, 101, 109, 101, 105, 108,
|
|
28
|
+
108, 228, 109, 101, 105, 108, 116, 228, 109, 101, 105, 115, 115, 228, 109, 101, 105, 115, 116, 228, 109, 101, 105,
|
|
29
|
+
116, 228, 109, 105, 104, 105, 110, 109, 105, 107, 115, 105, 109, 105, 107, 228, 109, 105, 108, 108, 101, 109, 105,
|
|
30
|
+
108, 108, 228, 109, 105, 108, 116, 228, 109, 105, 110, 107, 228, 109, 105, 110, 117, 97, 109, 105, 110, 117, 108, 108,
|
|
31
|
+
97, 109, 105, 110, 117, 108, 108, 101, 109, 105, 110, 117, 108, 116, 97, 109, 105, 110, 117, 110, 109, 105, 110, 117,
|
|
32
|
+
115, 115, 97, 109, 105, 110, 117, 115, 116, 97, 109, 105, 110, 117, 116, 109, 105, 110, 117, 117, 110, 109, 105, 110,
|
|
33
|
+
228, 109, 105, 115, 115, 228, 109, 105, 115, 116, 228, 109, 105, 116, 107, 228, 109, 105, 116, 228, 109, 117, 107, 97,
|
|
34
|
+
97, 110, 109, 117, 116, 116, 97, 110, 101, 110, 105, 105, 100, 101, 110, 110, 105, 105, 104, 105, 110, 110, 105, 105,
|
|
35
|
+
107, 115, 105, 110, 105, 105, 108, 108, 101, 110, 105, 105, 108, 108, 228, 110, 105, 105, 108, 116, 228, 110, 105,
|
|
36
|
+
105, 110, 110, 105, 105, 110, 228, 110, 105, 105, 115, 115, 228, 110, 105, 105, 115, 116, 228, 110, 105, 105, 116,
|
|
37
|
+
228, 110, 111, 105, 100, 101, 110, 110, 111, 105, 104, 105, 110, 110, 111, 105, 107, 115, 105, 110, 111, 105, 108,
|
|
38
|
+
108, 97, 110, 111, 105, 108, 108, 101, 110, 111, 105, 108, 116, 97, 110, 111, 105, 110, 110, 111, 105, 110, 97, 110,
|
|
39
|
+
111, 105, 115, 115, 97, 110, 111, 105, 115, 116, 97, 110, 111, 105, 116, 97, 110, 117, 111, 110, 121, 116, 110, 228,
|
|
40
|
+
105, 100, 101, 110, 110, 228, 105, 104, 105, 110, 110, 228, 105, 107, 115, 105, 110, 228, 105, 108, 108, 101, 110,
|
|
41
|
+
228, 105, 108, 108, 228, 110, 228, 105, 108, 116, 228, 110, 228, 105, 110, 228, 110, 228, 105, 115, 115, 228, 110,
|
|
42
|
+
228, 105, 115, 116, 228, 110, 228, 105, 116, 228, 110, 228, 109, 228, 111, 108, 101, 111, 108, 101, 109, 109, 101,
|
|
43
|
+
111, 108, 101, 110, 111, 108, 101, 116, 111, 108, 101, 116, 116, 101, 111, 108, 105, 111, 108, 105, 109, 109, 101,
|
|
44
|
+
111, 108, 105, 110, 111, 108, 105, 115, 105, 111, 108, 105, 115, 105, 109, 109, 101, 111, 108, 105, 115, 105, 110,
|
|
45
|
+
111, 108, 105, 115, 105, 116, 111, 108, 105, 115, 105, 116, 116, 101, 111, 108, 105, 115, 105, 118, 97, 116, 111, 108,
|
|
46
|
+
105, 116, 111, 108, 105, 116, 116, 101, 111, 108, 105, 118, 97, 116, 111, 108, 108, 97, 111, 108, 108, 101, 101, 116,
|
|
47
|
+
111, 108, 108, 117, 116, 111, 110, 111, 118, 97, 116, 112, 111, 105, 107, 107, 105, 115, 101, 115, 101, 107, 228, 115,
|
|
48
|
+
101, 110, 115, 105, 105, 104, 101, 110, 115, 105, 105, 110, 228, 115, 105, 105, 116, 228, 115, 105, 107, 115, 105,
|
|
49
|
+
115, 105, 108, 108, 101, 115, 105, 108, 108, 228, 115, 105, 108, 116, 228, 115, 105, 110, 117, 97, 115, 105, 110, 117,
|
|
50
|
+
108, 108, 97, 115, 105, 110, 117, 108, 108, 101, 115, 105, 110, 117, 108, 116, 97, 115, 105, 110, 117, 110, 115, 105,
|
|
51
|
+
110, 117, 115, 115, 97, 115, 105, 110, 117, 115, 116, 97, 115, 105, 110, 117, 116, 115, 105, 110, 117, 117, 110, 115,
|
|
52
|
+
105, 110, 228, 115, 105, 116, 228, 116, 97, 105, 116, 101, 116, 101, 105, 100, 228, 110, 116, 101, 105, 100, 228, 116,
|
|
53
|
+
116, 101, 105, 104, 105, 110, 116, 101, 105, 108, 108, 101, 116, 101, 105, 108, 108, 228, 116, 101, 105, 108, 116,
|
|
54
|
+
228, 116, 101, 105, 115, 115, 228, 116, 101, 105, 115, 116, 228, 116, 101, 105, 116, 228, 116, 117, 111, 116, 117,
|
|
55
|
+
111, 104, 111, 110, 116, 117, 111, 107, 115, 105, 116, 117, 111, 108, 108, 97, 116, 117, 111, 108, 108, 101, 116, 117,
|
|
56
|
+
111, 108, 116, 97, 116, 117, 111, 110, 116, 117, 111, 110, 97, 116, 117, 111, 115, 115, 97, 116, 117, 111, 115, 116,
|
|
57
|
+
97, 116, 117, 111, 116, 97, 116, 228, 104, 228, 110, 116, 228, 107, 115, 105, 116, 228, 108, 108, 101, 116, 228, 108,
|
|
58
|
+
108, 228, 116, 228, 108, 116, 228, 116, 228, 109, 228, 116, 228, 109, 228, 110, 116, 228, 110, 228, 116, 228, 115,
|
|
59
|
+
115, 228, 116, 228, 115, 116, 228, 116, 228, 116, 228, 118, 97, 97, 110, 118, 97, 105, 118, 97, 105, 107, 107, 97,
|
|
60
|
+
121, 108, 105,
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
static const uint32_t stop_word_offsets[230] = {
|
|
64
|
+
0, 2, 7, 11, 13, 15, 19, 23, 25, 31, 37, 43, 49, 55, 61, 67, 73, 78, 81, 87, 94, 101, 108, 113, 120, 127, 132, 137,
|
|
65
|
+
141, 143, 148, 154, 160, 166, 172, 178, 184, 189, 195, 201, 206, 210, 215, 220, 225, 230, 234, 239, 242, 247, 252,
|
|
66
|
+
256, 261, 267, 273, 279, 285, 291, 297, 303, 308, 314, 320, 325, 331, 338, 345, 352, 359, 364, 370, 377, 384, 389,
|
|
67
|
+
394, 398, 403, 407, 411, 414, 416, 422, 428, 434, 440, 446, 452, 458, 464, 469, 474, 479, 483, 488, 493, 498, 503,
|
|
68
|
+
508, 515, 522, 529, 534, 541, 548, 553, 559, 563, 568, 573, 578, 582, 588, 593, 595, 601, 607, 613, 619, 625, 631,
|
|
69
|
+
635, 640, 646, 652, 657, 663, 669, 675, 681, 687, 693, 697, 702, 708, 714, 719, 722, 725, 731, 737, 743, 749, 755,
|
|
70
|
+
761, 766, 772, 778, 783, 787, 790, 796, 800, 804, 810, 813, 819, 823, 828, 836, 842, 848, 856, 864, 868, 874, 880,
|
|
71
|
+
884, 890, 895, 897, 901, 907, 909, 913, 916, 922, 927, 932, 937, 942, 947, 952, 957, 964, 971, 978, 983, 990, 997,
|
|
72
|
+
1002, 1008, 1012, 1016, 1019, 1021, 1027, 1033, 1039, 1045, 1051, 1057, 1063, 1069, 1074, 1077, 1083, 1089, 1095,
|
|
73
|
+
1101, 1107, 1111, 1116, 1122, 1128, 1133, 1138, 1143, 1148, 1153, 1158, 1162, 1167, 1171, 1176, 1181, 1185, 1189,
|
|
74
|
+
1192, 1198, 1201,
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
static const uint32_t keep_ranges[16] = {
|
|
78
|
+
48, 57, 65, 90, 97, 122, 196, 196, 214, 214, 228, 228, 246, 246, 352, 353,
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
const narsil_language narsil_language_finnish = {
|
|
82
|
+
.name = "finnish",
|
|
83
|
+
.revision = "cd95cb543801a620",
|
|
84
|
+
.stop_word_units = stop_word_units,
|
|
85
|
+
.stop_word_offsets = stop_word_offsets,
|
|
86
|
+
.stop_word_count = 229,
|
|
87
|
+
.ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
|
|
88
|
+
.keep_ranges = keep_ranges,
|
|
89
|
+
.keep_range_count = 8,
|
|
90
|
+
.splits_code_points = 0,
|
|
91
|
+
.strips_possessive = 0,
|
|
92
|
+
.normalises_diacritics = 0,
|
|
93
|
+
.min_token_length = 1,
|
|
94
|
+
.ngram_size = 0,
|
|
95
|
+
.normalise = NULL,
|
|
96
|
+
.stem_by_rules = NULL,
|
|
97
|
+
.create_snowball = finnish_create_env,
|
|
98
|
+
.snowball_stem = finnish_stem,
|
|
99
|
+
.close_snowball = finnish_close_env,
|
|
100
|
+
};
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
#include "../../analysis/languages.h"
|
|
2
|
+
#include "declarations.h"
|
|
3
|
+
#include "../snowball/french_stemmer.h"
|
|
4
|
+
|
|
5
|
+
#include <stddef.h>
|
|
6
|
+
#include <stdint.h>
|
|
7
|
+
|
|
8
|
+
static const uint16_t stop_word_units[649] = {
|
|
9
|
+
97, 105, 97, 105, 101, 97, 105, 101, 110, 116, 97, 105, 101, 115, 97, 105, 116, 97, 115, 97, 117, 97, 117, 114, 97,
|
|
10
|
+
97, 117, 114, 97, 105, 97, 117, 114, 97, 105, 101, 110, 116, 97, 117, 114, 97, 105, 115, 97, 117, 114, 97, 105, 116,
|
|
11
|
+
97, 117, 114, 97, 115, 97, 117, 114, 101, 122, 97, 117, 114, 105, 101, 122, 97, 117, 114, 105, 111, 110, 115, 97, 117,
|
|
12
|
+
114, 111, 110, 115, 97, 117, 114, 111, 110, 116, 97, 117, 120, 97, 118, 97, 105, 101, 110, 116, 97, 118, 97, 105, 115,
|
|
13
|
+
97, 118, 97, 105, 116, 97, 118, 101, 99, 97, 118, 101, 122, 97, 118, 105, 101, 122, 97, 118, 105, 111, 110, 115, 97,
|
|
14
|
+
118, 111, 110, 115, 97, 121, 97, 110, 116, 97, 121, 101, 122, 97, 121, 111, 110, 115, 99, 99, 101, 99, 101, 99, 105,
|
|
15
|
+
99, 101, 108, 97, 99, 101, 108, 224, 99, 101, 115, 99, 101, 116, 99, 101, 116, 116, 101, 100, 100, 97, 110, 115, 100,
|
|
16
|
+
101, 100, 101, 115, 100, 117, 101, 108, 108, 101, 101, 110, 101, 115, 101, 115, 116, 101, 116, 101, 117, 101, 117,
|
|
17
|
+
101, 101, 117, 101, 115, 101, 117, 114, 101, 110, 116, 101, 117, 115, 101, 117, 115, 115, 101, 101, 117, 115, 115,
|
|
18
|
+
101, 110, 116, 101, 117, 115, 115, 101, 115, 101, 117, 115, 115, 105, 101, 122, 101, 117, 115, 115, 105, 111, 110,
|
|
19
|
+
115, 101, 117, 116, 101, 117, 120, 101, 251, 109, 101, 115, 101, 251, 116, 101, 251, 116, 101, 115, 102, 117, 114,
|
|
20
|
+
101, 110, 116, 102, 117, 115, 102, 117, 115, 115, 101, 102, 117, 115, 115, 101, 110, 116, 102, 117, 115, 115, 101,
|
|
21
|
+
115, 102, 117, 115, 115, 105, 101, 122, 102, 117, 115, 115, 105, 111, 110, 115, 102, 117, 116, 102, 251, 109, 101,
|
|
22
|
+
115, 102, 251, 116, 102, 251, 116, 101, 115, 105, 99, 105, 105, 108, 105, 108, 115, 106, 106, 101, 108, 108, 97, 108,
|
|
23
|
+
101, 108, 101, 115, 108, 101, 117, 114, 108, 101, 117, 114, 115, 108, 117, 105, 109, 109, 97, 109, 97, 105, 115, 109,
|
|
24
|
+
101, 109, 101, 115, 109, 111, 105, 109, 111, 110, 109, 234, 109, 101, 110, 110, 101, 110, 111, 115, 110, 111, 116,
|
|
25
|
+
114, 101, 110, 111, 117, 115, 111, 110, 111, 110, 116, 111, 117, 112, 97, 114, 112, 97, 115, 112, 111, 117, 114, 113,
|
|
26
|
+
117, 113, 117, 101, 113, 117, 101, 108, 113, 117, 101, 108, 108, 101, 113, 117, 101, 108, 108, 101, 115, 113, 117,
|
|
27
|
+
101, 108, 115, 113, 117, 105, 115, 115, 97, 115, 97, 110, 115, 115, 101, 115, 101, 114, 97, 115, 101, 114, 97, 105,
|
|
28
|
+
115, 101, 114, 97, 105, 101, 110, 116, 115, 101, 114, 97, 105, 115, 115, 101, 114, 97, 105, 116, 115, 101, 114, 97,
|
|
29
|
+
115, 115, 101, 114, 101, 122, 115, 101, 114, 105, 101, 122, 115, 101, 114, 105, 111, 110, 115, 115, 101, 114, 111,
|
|
30
|
+
110, 115, 115, 101, 114, 111, 110, 116, 115, 101, 115, 115, 111, 105, 115, 111, 105, 101, 110, 116, 115, 111, 105,
|
|
31
|
+
115, 115, 111, 105, 116, 115, 111, 109, 109, 101, 115, 115, 111, 110, 115, 111, 110, 116, 115, 111, 121, 101, 122,
|
|
32
|
+
115, 111, 121, 111, 110, 115, 115, 117, 105, 115, 115, 117, 114, 116, 116, 97, 116, 101, 116, 101, 115, 116, 111, 105,
|
|
33
|
+
116, 111, 110, 116, 117, 117, 110, 117, 110, 101, 118, 111, 115, 118, 111, 116, 114, 101, 118, 111, 117, 115, 121,
|
|
34
|
+
224, 233, 116, 97, 105, 101, 110, 116, 233, 116, 97, 105, 115, 233, 116, 97, 105, 116, 233, 116, 97, 110, 116, 233,
|
|
35
|
+
116, 105, 101, 122, 233, 116, 105, 111, 110, 115, 233, 116, 233, 233, 116, 233, 101, 233, 116, 233, 101, 115, 233,
|
|
36
|
+
116, 233, 115, 234, 116, 101, 115,
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
static const uint32_t stop_word_offsets[165] = {
|
|
40
|
+
0, 2, 5, 10, 14, 17, 19, 21, 25, 30, 38, 44, 50, 55, 60, 66, 73, 79, 85, 88, 95, 100, 105, 109, 113, 118, 124, 129,
|
|
41
|
+
134, 138, 143, 144, 146, 150, 154, 158, 161, 164, 169, 170, 174, 176, 179, 181, 185, 187, 189, 192, 194, 196, 199,
|
|
42
|
+
203, 209, 212, 217, 224, 230, 237, 245, 248, 251, 256, 259, 264, 270, 273, 278, 285, 291, 298, 306, 309, 314, 317,
|
|
43
|
+
322, 325, 327, 330, 331, 333, 334, 336, 338, 341, 345, 350, 353, 354, 356, 360, 362, 365, 368, 371, 375, 376, 378,
|
|
44
|
+
381, 386, 390, 392, 395, 397, 400, 403, 407, 409, 412, 416, 422, 429, 434, 437, 438, 440, 444, 446, 450, 455, 463,
|
|
45
|
+
469, 475, 480, 485, 491, 498, 504, 510, 513, 516, 522, 526, 530, 536, 539, 543, 548, 554, 558, 561, 562, 564, 566,
|
|
46
|
+
569, 572, 575, 577, 579, 582, 585, 590, 594, 595, 596, 603, 608, 613, 618, 623, 629, 632, 636, 641, 645, 649,
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
static const uint32_t keep_ranges[50] = {
|
|
50
|
+
45, 45, 48, 57, 65, 90, 97, 122, 192, 192, 194, 194, 196, 196, 198, 203, 206, 207, 212, 212, 214, 214, 217, 217, 219,
|
|
51
|
+
220, 224, 224, 226, 226, 228, 228, 230, 235, 238, 239, 244, 244, 246, 246, 249, 249, 251, 252, 255, 255, 338, 339,
|
|
52
|
+
376, 376,
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
const narsil_language narsil_language_french = {
|
|
56
|
+
.name = "french",
|
|
57
|
+
.revision = "240a93ad04b96326",
|
|
58
|
+
.stop_word_units = stop_word_units,
|
|
59
|
+
.stop_word_offsets = stop_word_offsets,
|
|
60
|
+
.stop_word_count = 164,
|
|
61
|
+
.ascii_keep = {0x03FF200000000000ULL, 0x07FFFFFE07FFFFFEULL},
|
|
62
|
+
.keep_ranges = keep_ranges,
|
|
63
|
+
.keep_range_count = 25,
|
|
64
|
+
.splits_code_points = 0,
|
|
65
|
+
.strips_possessive = 0,
|
|
66
|
+
.normalises_diacritics = 0,
|
|
67
|
+
.min_token_length = 1,
|
|
68
|
+
.ngram_size = 0,
|
|
69
|
+
.normalise = NULL,
|
|
70
|
+
.stem_by_rules = NULL,
|
|
71
|
+
.create_snowball = french_create_env,
|
|
72
|
+
.snowball_stem = french_stem,
|
|
73
|
+
.close_snowball = french_close_env,
|
|
74
|
+
};
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
#include "../../analysis/languages.h"
|
|
2
|
+
#include "declarations.h"
|
|
3
|
+
|
|
4
|
+
#include <stddef.h>
|
|
5
|
+
#include <stdint.h>
|
|
6
|
+
|
|
7
|
+
static const uint16_t stop_word_units[148] = {
|
|
8
|
+
97, 104, 105, 97, 107, 603, 97, 109, 603, 97, 110, 97, 98, 97, 98, 101, 101, 98, 105, 105, 98, 111, 101, 101, 101,
|
|
9
|
+
101, 101, 107, 111, 109, 101, 101, 110, 105, 101, 110, 111, 101, 115, 97, 102, 101, 101, 102, 111, 102, 603, 603, 104,
|
|
10
|
+
101, 106, 105, 107, 97, 110, 101, 107, 111, 107, 111, 109, 101, 107, 603, 108, 97, 108, 97, 97, 108, 101, 108, 111,
|
|
11
|
+
108, 603, 109, 105, 109, 108, 105, 109, 111, 109, 111, 107, 111, 109, 596, 110, 97, 110, 97, 97, 110, 101, 110, 101,
|
|
12
|
+
101, 110, 105, 110, 105, 105, 110, 111, 110, 111, 107, 111, 110, 117, 110, 596, 110, 603, 603, 115, 97, 97, 115, 104,
|
|
13
|
+
105, 115, 104, 105, 115, 104, 105, 116, 97, 116, 101, 101, 116, 115, 111, 116, 115, 596, 119, 596, 121, 97, 97, 110,
|
|
14
|
+
596, 121, 603,
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
static const uint32_t stop_word_offsets[55] = {
|
|
18
|
+
0, 3, 6, 9, 12, 14, 17, 20, 22, 23, 26, 31, 34, 37, 40, 43, 45, 48, 50, 52, 56, 58, 62, 64, 66, 69, 71, 73, 75, 77,
|
|
19
|
+
80, 82, 86, 88, 90, 93, 95, 98, 100, 103, 105, 109, 111, 113, 116, 119, 122, 128, 130, 133, 136, 139, 141, 146, 148,
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
static const uint32_t keep_ranges[684] = {
|
|
23
|
+
48, 57, 65, 90, 97, 122, 195, 195, 227, 227, 330, 331, 383, 383, 390, 390, 400, 400, 596, 596, 603, 603, 768, 879,
|
|
24
|
+
921, 921, 953, 953, 1155, 1161, 1425, 1469, 1471, 1471, 1473, 1474, 1476, 1477, 1479, 1479, 1552, 1562, 1611, 1631,
|
|
25
|
+
1648, 1648, 1750, 1756, 1759, 1764, 1767, 1768, 1770, 1773, 1809, 1809, 1840, 1866, 1958, 1968, 2027, 2035, 2045,
|
|
26
|
+
2045, 2070, 2073, 2075, 2083, 2085, 2087, 2089, 2093, 2137, 2139, 2199, 2207, 2250, 2273, 2275, 2307, 2362, 2364,
|
|
27
|
+
2366, 2383, 2385, 2391, 2402, 2403, 2433, 2435, 2492, 2492, 2494, 2500, 2503, 2504, 2507, 2509, 2519, 2519, 2530,
|
|
28
|
+
2531, 2558, 2558, 2561, 2563, 2620, 2620, 2622, 2626, 2631, 2632, 2635, 2637, 2641, 2641, 2672, 2673, 2677, 2677,
|
|
29
|
+
2689, 2691, 2748, 2748, 2750, 2757, 2759, 2761, 2763, 2765, 2786, 2787, 2810, 2815, 2817, 2819, 2876, 2876, 2878,
|
|
30
|
+
2884, 2887, 2888, 2891, 2893, 2901, 2903, 2914, 2915, 2946, 2946, 3006, 3010, 3014, 3016, 3018, 3021, 3031, 3031,
|
|
31
|
+
3072, 3076, 3132, 3132, 3134, 3140, 3142, 3144, 3146, 3149, 3157, 3158, 3170, 3171, 3201, 3203, 3260, 3260, 3262,
|
|
32
|
+
3268, 3270, 3272, 3274, 3277, 3285, 3286, 3298, 3299, 3315, 3315, 3328, 3331, 3387, 3388, 3390, 3396, 3398, 3400,
|
|
33
|
+
3402, 3405, 3415, 3415, 3426, 3427, 3457, 3459, 3530, 3530, 3535, 3540, 3542, 3542, 3544, 3551, 3570, 3571, 3633,
|
|
34
|
+
3633, 3636, 3642, 3655, 3662, 3761, 3761, 3764, 3772, 3784, 3790, 3864, 3865, 3893, 3893, 3895, 3895, 3897, 3897,
|
|
35
|
+
3902, 3903, 3953, 3972, 3974, 3975, 3981, 3991, 3993, 4028, 4038, 4038, 4139, 4158, 4182, 4185, 4190, 4192, 4194,
|
|
36
|
+
4196, 4199, 4205, 4209, 4212, 4226, 4237, 4239, 4239, 4250, 4253, 4957, 4959, 5906, 5909, 5938, 5940, 5970, 5971,
|
|
37
|
+
6002, 6003, 6068, 6099, 6109, 6109, 6155, 6157, 6159, 6159, 6277, 6278, 6313, 6313, 6432, 6443, 6448, 6459, 6679,
|
|
38
|
+
6683, 6741, 6750, 6752, 6780, 6783, 6783, 6832, 6877, 6880, 6891, 6912, 6916, 6964, 6980, 7019, 7027, 7040, 7042,
|
|
39
|
+
7073, 7085, 7142, 7155, 7204, 7223, 7376, 7378, 7380, 7400, 7405, 7405, 7412, 7412, 7415, 7417, 7616, 7679, 8126,
|
|
40
|
+
8126, 8400, 8432, 8490, 8490, 11503, 11505, 11647, 11647, 11744, 11775, 12330, 12335, 12441, 12442, 42607, 42610,
|
|
41
|
+
42612, 42621, 42654, 42655, 42736, 42737, 43010, 43010, 43014, 43014, 43019, 43019, 43043, 43047, 43052, 43052, 43136,
|
|
42
|
+
43137, 43188, 43205, 43232, 43249, 43263, 43263, 43302, 43309, 43335, 43347, 43392, 43395, 43443, 43456, 43493, 43493,
|
|
43
|
+
43561, 43574, 43587, 43587, 43596, 43597, 43643, 43645, 43696, 43696, 43698, 43700, 43703, 43704, 43710, 43711, 43713,
|
|
44
|
+
43713, 43755, 43759, 43765, 43766, 44003, 44010, 44012, 44013, 64286, 64286, 65024, 65039, 65056, 65071, 66045, 66045,
|
|
45
|
+
66272, 66272, 66422, 66426, 68097, 68099, 68101, 68102, 68108, 68111, 68152, 68154, 68159, 68159, 68325, 68326, 68900,
|
|
46
|
+
68903, 68969, 68973, 69291, 69292, 69370, 69375, 69446, 69456, 69506, 69509, 69632, 69634, 69688, 69702, 69744, 69744,
|
|
47
|
+
69747, 69748, 69759, 69762, 69808, 69818, 69826, 69826, 69888, 69890, 69927, 69940, 69957, 69958, 70003, 70003, 70016,
|
|
48
|
+
70018, 70067, 70080, 70089, 70092, 70094, 70095, 70188, 70199, 70206, 70206, 70209, 70209, 70367, 70378, 70400, 70403,
|
|
49
|
+
70459, 70460, 70462, 70468, 70471, 70472, 70475, 70477, 70487, 70487, 70498, 70499, 70502, 70508, 70512, 70516, 70584,
|
|
50
|
+
70592, 70594, 70594, 70597, 70597, 70599, 70602, 70604, 70608, 70610, 70610, 70625, 70626, 70709, 70726, 70750, 70750,
|
|
51
|
+
70832, 70851, 71087, 71093, 71096, 71104, 71132, 71133, 71216, 71232, 71339, 71351, 71453, 71467, 71724, 71738, 71984,
|
|
52
|
+
71989, 71991, 71992, 71995, 71998, 72000, 72000, 72002, 72003, 72145, 72151, 72154, 72160, 72164, 72164, 72193, 72202,
|
|
53
|
+
72243, 72249, 72251, 72254, 72263, 72263, 72273, 72283, 72330, 72345, 72544, 72551, 72751, 72758, 72760, 72767, 72850,
|
|
54
|
+
72871, 72873, 72886, 73009, 73014, 73018, 73018, 73020, 73021, 73023, 73029, 73031, 73031, 73098, 73102, 73104, 73105,
|
|
55
|
+
73107, 73111, 73459, 73462, 73472, 73473, 73475, 73475, 73524, 73530, 73534, 73538, 73562, 73562, 78912, 78912, 78919,
|
|
56
|
+
78933, 90398, 90415, 92912, 92916, 92976, 92982, 94031, 94031, 94033, 94087, 94095, 94098, 94180, 94180, 94192, 94193,
|
|
57
|
+
113821, 113822, 118528, 118573, 118576, 118598, 119141, 119145, 119149, 119154, 119163, 119170, 119173, 119179,
|
|
58
|
+
119210, 119213, 119362, 119364, 121344, 121398, 121403, 121452, 121461, 121461, 121476, 121476, 121499, 121503,
|
|
59
|
+
121505, 121519, 122880, 122886, 122888, 122904, 122907, 122913, 122915, 122916, 122918, 122922, 123023, 123023,
|
|
60
|
+
123184, 123190, 123566, 123566, 123628, 123631, 124140, 124143, 124398, 124399, 124643, 124643, 124646, 124646,
|
|
61
|
+
124654, 124655, 124661, 124661, 125136, 125142, 125252, 125258, 917760, 917999,
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
const narsil_language narsil_language_ga = {
|
|
65
|
+
.name = "ga",
|
|
66
|
+
.revision = "1",
|
|
67
|
+
.stop_word_units = stop_word_units,
|
|
68
|
+
.stop_word_offsets = stop_word_offsets,
|
|
69
|
+
.stop_word_count = 54,
|
|
70
|
+
.ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
|
|
71
|
+
.keep_ranges = keep_ranges,
|
|
72
|
+
.keep_range_count = 342,
|
|
73
|
+
.splits_code_points = 1,
|
|
74
|
+
.strips_possessive = 0,
|
|
75
|
+
.normalises_diacritics = 0,
|
|
76
|
+
.min_token_length = 1,
|
|
77
|
+
.ngram_size = 0,
|
|
78
|
+
.normalise = NULL,
|
|
79
|
+
.stem_by_rules = NULL,
|
|
80
|
+
.create_snowball = NULL,
|
|
81
|
+
.snowball_stem = NULL,
|
|
82
|
+
.close_snowball = NULL,
|
|
83
|
+
};
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
#include "../../analysis/languages.h"
|
|
2
|
+
#include "declarations.h"
|
|
3
|
+
|
|
4
|
+
#include <stddef.h>
|
|
5
|
+
#include <stdint.h>
|
|
6
|
+
|
|
7
|
+
static const uint16_t stop_word_units[613] = {
|
|
8
|
+
97, 97, 108, 237, 97, 111, 97, 111, 115, 97, 113, 117, 101, 108, 97, 113, 117, 101, 108, 97, 97, 113, 117, 101, 108,
|
|
9
|
+
97, 115, 97, 113, 117, 101, 108, 101, 115, 97, 113, 117, 105, 108, 111, 97, 113, 117, 237, 97, 115, 97, 115, 237, 97,
|
|
10
|
+
237, 110, 100, 97, 98, 101, 110, 99, 97, 110, 100, 111, 99, 104, 101, 99, 111, 99, 111, 97, 99, 111, 97, 115, 99, 111,
|
|
11
|
+
109, 105, 103, 111, 99, 111, 110, 99, 111, 110, 110, 111, 115, 99, 111, 99, 111, 110, 116, 105, 103, 111, 99, 111,
|
|
12
|
+
110, 118, 111, 115, 99, 111, 99, 111, 115, 99, 117, 110, 99, 117, 110, 104, 97, 99, 117, 110, 104, 97, 115, 99, 117,
|
|
13
|
+
110, 115, 100, 97, 100, 97, 108, 103, 117, 110, 104, 97, 100, 97, 108, 103, 117, 110, 104, 97, 115, 100, 97, 108, 103,
|
|
14
|
+
250, 110, 100, 97, 108, 103, 250, 110, 115, 100, 97, 115, 100, 101, 100, 101, 108, 100, 101, 108, 97, 100, 101, 108,
|
|
15
|
+
97, 115, 100, 101, 108, 101, 115, 100, 101, 115, 100, 101, 100, 101, 115, 116, 101, 100, 111, 100, 111, 115, 100, 117,
|
|
16
|
+
110, 100, 117, 110, 104, 97, 100, 117, 110, 104, 97, 115, 100, 117, 110, 115, 101, 101, 108, 101, 108, 97, 101, 108,
|
|
17
|
+
97, 115, 101, 108, 101, 115, 101, 110, 101, 114, 97, 101, 114, 97, 110, 101, 115, 97, 101, 115, 97, 115, 101, 115,
|
|
18
|
+
101, 101, 115, 101, 115, 101, 115, 116, 97, 101, 115, 116, 97, 98, 97, 101, 115, 116, 97, 114, 101, 115, 116, 101,
|
|
19
|
+
101, 115, 116, 101, 115, 101, 115, 116, 105, 118, 101, 110, 101, 115, 116, 111, 117, 101, 115, 116, 225, 101, 115,
|
|
20
|
+
116, 225, 110, 101, 117, 102, 97, 99, 101, 114, 102, 111, 105, 102, 111, 114, 111, 110, 102, 117, 110, 104, 97, 98,
|
|
21
|
+
237, 97, 104, 97, 105, 105, 115, 111, 105, 115, 116, 111, 108, 97, 108, 97, 115, 108, 108, 101, 108, 108, 101, 115,
|
|
22
|
+
108, 111, 108, 111, 115, 109, 97, 105, 115, 109, 101, 109, 101, 117, 109, 101, 117, 115, 109, 105, 110, 109, 105, 241,
|
|
23
|
+
97, 109, 105, 241, 97, 115, 109, 111, 105, 110, 97, 110, 97, 115, 110, 101, 115, 116, 101, 110, 105, 110, 110, 111,
|
|
24
|
+
110, 111, 110, 110, 111, 115, 110, 111, 115, 97, 110, 111, 115, 97, 115, 110, 111, 115, 111, 110, 111, 115, 111, 115,
|
|
25
|
+
110, 117, 110, 110, 117, 110, 104, 97, 110, 117, 110, 104, 97, 115, 110, 117, 110, 115, 110, 243, 115, 111, 111, 115,
|
|
26
|
+
111, 117, 112, 97, 114, 97, 112, 101, 114, 111, 112, 111, 100, 101, 112, 111, 105, 115, 112, 111, 108, 97, 112, 111,
|
|
27
|
+
108, 97, 115, 112, 111, 108, 111, 112, 111, 108, 111, 115, 112, 111, 114, 113, 117, 101, 115, 101, 115, 101, 110, 243,
|
|
28
|
+
110, 115, 101, 114, 115, 101, 117, 115, 101, 117, 115, 115, 101, 120, 97, 115, 105, 100, 111, 115, 111, 98, 114, 101,
|
|
29
|
+
115, 250, 97, 115, 250, 97, 115, 116, 97, 109, 233, 110, 116, 97, 110, 116, 101, 116, 101, 110, 116, 101, 114, 116,
|
|
30
|
+
101, 117, 116, 101, 117, 115, 116, 101, 241, 101, 110, 116, 101, 241, 111, 116, 105, 116, 105, 100, 111, 116, 105,
|
|
31
|
+
118, 101, 110, 116, 105, 241, 97, 116, 250, 97, 116, 250, 97, 115, 117, 110, 117, 110, 104, 97, 117, 110, 104, 97,
|
|
32
|
+
115, 117, 110, 115, 118, 111, 115, 118, 111, 115, 97, 118, 111, 115, 97, 115, 118, 111, 115, 111, 118, 111, 115, 111,
|
|
33
|
+
115, 118, 243, 115, 225, 233, 243, 243, 115,
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
static const uint32_t stop_word_offsets[161] = {
|
|
37
|
+
0, 1, 4, 6, 9, 14, 20, 27, 34, 40, 44, 46, 49, 54, 57, 62, 65, 67, 70, 74, 80, 83, 91, 98, 106, 109, 112, 117, 123,
|
|
38
|
+
127, 129, 137, 146, 152, 159, 162, 164, 167, 171, 176, 181, 186, 191, 193, 196, 199, 204, 210, 214, 215, 217, 220,
|
|
39
|
+
224, 228, 230, 233, 237, 240, 244, 247, 251, 255, 261, 266, 270, 275, 282, 287, 291, 296, 298, 303, 306, 311, 314,
|
|
40
|
+
319, 322, 325, 329, 331, 334, 337, 341, 343, 346, 350, 352, 355, 359, 362, 366, 371, 374, 376, 379, 384, 387, 389,
|
|
41
|
+
392, 395, 399, 404, 408, 413, 416, 421, 427, 431, 434, 435, 437, 439, 443, 447, 451, 455, 459, 464, 468, 473, 476,
|
|
42
|
+
479, 481, 486, 489, 492, 496, 500, 504, 509, 512, 516, 521, 524, 526, 529, 532, 535, 539, 544, 548, 550, 554, 559,
|
|
43
|
+
563, 566, 570, 572, 576, 581, 584, 587, 591, 596, 600, 605, 608, 609, 610, 611, 613,
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
static const uint32_t keep_ranges[34] = {
|
|
47
|
+
48, 57, 65, 90, 97, 122, 193, 193, 201, 201, 205, 205, 209, 209, 211, 211, 218, 218, 220, 220, 225, 225, 233, 233,
|
|
48
|
+
237, 237, 241, 241, 243, 243, 250, 250, 252, 252,
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
const narsil_language narsil_language_galician = {
|
|
52
|
+
.name = "galician",
|
|
53
|
+
.revision = "1",
|
|
54
|
+
.stop_word_units = stop_word_units,
|
|
55
|
+
.stop_word_offsets = stop_word_offsets,
|
|
56
|
+
.stop_word_count = 160,
|
|
57
|
+
.ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
|
|
58
|
+
.keep_ranges = keep_ranges,
|
|
59
|
+
.keep_range_count = 17,
|
|
60
|
+
.splits_code_points = 0,
|
|
61
|
+
.strips_possessive = 0,
|
|
62
|
+
.normalises_diacritics = 0,
|
|
63
|
+
.min_token_length = 1,
|
|
64
|
+
.ngram_size = 0,
|
|
65
|
+
.normalise = NULL,
|
|
66
|
+
.stem_by_rules = NULL,
|
|
67
|
+
.create_snowball = NULL,
|
|
68
|
+
.snowball_stem = NULL,
|
|
69
|
+
.close_snowball = NULL,
|
|
70
|
+
};
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
#include "../../analysis/languages.h"
|
|
2
|
+
#include "declarations.h"
|
|
3
|
+
|
|
4
|
+
#include <stddef.h>
|
|
5
|
+
#include <stdint.h>
|
|
6
|
+
|
|
7
|
+
static const uint16_t stop_word_units[313] = {
|
|
8
|
+
4304, 4315, 4304, 4315, 4312, 4322, 4317, 4315, 4304, 4316, 4304, 4316, 4323, 4304, 4320, 4304, 4320, 4304, 4304,
|
|
9
|
+
4320, 4312, 4304, 4316, 4304, 4320, 4312, 4321, 4304, 4321, 4308, 4304, 4321, 4308, 4309, 4308, 4304, 4325, 4306,
|
|
10
|
+
4304, 4315, 4317, 4306, 4304, 4320, 4307, 4304, 4307, 4304, 4307, 4304, 4316, 4307, 4320, 4317, 4321, 4308, 4320,
|
|
11
|
+
4311, 4308, 4320, 4311, 4312, 4308, 4321, 4309, 4308, 4320, 4309, 4312, 4307, 4320, 4308, 4309, 4312, 4316, 4310,
|
|
12
|
+
4308, 4311, 4304, 4309, 4312, 4321, 4312, 4311, 4304, 4316, 4311, 4323, 4311, 4323, 4315, 4330, 4304, 4311, 4325,
|
|
13
|
+
4309, 4308, 4316, 4311, 4325, 4309, 4308, 4316, 4312, 4312, 4306, 4312, 4312, 4315, 4312, 4321, 4312, 4321, 4308,
|
|
14
|
+
4312, 4321, 4312, 4316, 4312, 4312, 4325, 4312, 4325, 4316, 4308, 4305, 4304, 4312, 4327, 4317, 4313, 4312, 4315,
|
|
15
|
+
4304, 4306, 4320, 4304, 4315, 4315, 4304, 4311, 4315, 4304, 4311, 4312, 4315, 4304, 4316, 4315, 4304, 4321, 4315,
|
|
16
|
+
4308, 4315, 4312, 4308, 4320, 4315, 4312, 4315, 4304, 4320, 4311, 4315, 4312, 4321, 4312, 4315, 4334, 4317, 4314,
|
|
17
|
+
4317, 4307, 4316, 4323, 4320, 4304, 4320, 4304, 4307, 4306, 4304, 4316, 4320, 4304, 4315, 4307, 4308, 4316, 4312,
|
|
18
|
+
4320, 4304, 4322, 4317, 4315, 4320, 4317, 4306, 4317, 4320, 4330, 4320, 4317, 4307, 4312, 4321, 4320, 4317, 4315,
|
|
19
|
+
4320, 4317, 4315, 4308, 4314, 4312, 4320, 4317, 4315, 4308, 4314, 4312, 4330, 4320, 4317, 4330, 4304, 4321, 4304,
|
|
20
|
+
4307, 4321, 4334, 4309, 4304, 4323, 4313, 4309, 4308, 4323, 4324, 4320, 4317, 4327, 4309, 4308, 4314, 4304, 4327,
|
|
21
|
+
4317, 4309, 4308, 4314, 4327, 4317, 4324, 4312, 4314, 4304, 4328, 4308, 4315, 4307, 4308, 4306, 4328, 4308, 4316,
|
|
22
|
+
4328, 4308, 4316, 4312, 4328, 4308, 4321, 4304, 4334, 4308, 4305, 4328, 4312, 4328, 4317, 4320, 4312, 4321, 4329,
|
|
23
|
+
4308, 4315, 4312, 4329, 4309, 4308, 4316, 4329, 4309, 4308, 4316, 4312, 4331, 4304, 4314, 4312, 4304, 4316, 4332,
|
|
24
|
+
4312, 4316, 4334, 4317, 4314, 4317, 4334, 4317, 4315,
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
static const uint32_t stop_word_offsets[80] = {
|
|
28
|
+
0, 2, 8, 10, 13, 15, 18, 23, 27, 30, 35, 37, 41, 46, 48, 51, 55, 58, 62, 64, 67, 72, 75, 77, 83, 86, 88, 93, 98, 104,
|
|
29
|
+
107, 109, 111, 114, 119, 121, 127, 130, 132, 138, 141, 145, 148, 151, 153, 157, 163, 167, 173, 175, 177, 183, 190,
|
|
30
|
+
195, 201, 206, 209, 215, 222, 226, 229, 233, 237, 241, 246, 251, 257, 263, 266, 270, 277, 279, 284, 288, 292, 297,
|
|
31
|
+
303, 306, 310, 313,
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
static const uint32_t keep_ranges[8] = {
|
|
35
|
+
48, 57, 4256, 4351, 7312, 7359, 11520, 11567,
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
const narsil_language narsil_language_georgian = {
|
|
39
|
+
.name = "georgian",
|
|
40
|
+
.revision = "1",
|
|
41
|
+
.stop_word_units = stop_word_units,
|
|
42
|
+
.stop_word_offsets = stop_word_offsets,
|
|
43
|
+
.stop_word_count = 79,
|
|
44
|
+
.ascii_keep = {0x03FF000000000000ULL, 0x0000000000000000ULL},
|
|
45
|
+
.keep_ranges = keep_ranges,
|
|
46
|
+
.keep_range_count = 4,
|
|
47
|
+
.splits_code_points = 0,
|
|
48
|
+
.strips_possessive = 0,
|
|
49
|
+
.normalises_diacritics = 0,
|
|
50
|
+
.min_token_length = 1,
|
|
51
|
+
.ngram_size = 0,
|
|
52
|
+
.normalise = NULL,
|
|
53
|
+
.stem_by_rules = NULL,
|
|
54
|
+
.create_snowball = NULL,
|
|
55
|
+
.snowball_stem = NULL,
|
|
56
|
+
.close_snowball = NULL,
|
|
57
|
+
};
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
#include "../../analysis/languages.h"
|
|
2
|
+
#include "declarations.h"
|
|
3
|
+
#include "../rules.h"
|
|
4
|
+
#include "../snowball/german_stemmer.h"
|
|
5
|
+
|
|
6
|
+
#include <stddef.h>
|
|
7
|
+
#include <stdint.h>
|
|
8
|
+
|
|
9
|
+
static const uint16_t stop_word_units[1114] = {
|
|
10
|
+
97, 98, 101, 114, 97, 108, 108, 101, 97, 108, 108, 101, 109, 97, 108, 108, 101, 110, 97, 108, 108, 101, 114, 97, 108,
|
|
11
|
+
108, 101, 115, 97, 108, 115, 97, 108, 115, 111, 97, 109, 97, 110, 97, 110, 100, 101, 114, 97, 110, 100, 101, 114, 101,
|
|
12
|
+
97, 110, 100, 101, 114, 101, 109, 97, 110, 100, 101, 114, 101, 110, 97, 110, 100, 101, 114, 101, 114, 97, 110, 100,
|
|
13
|
+
101, 114, 101, 115, 97, 110, 100, 101, 114, 109, 97, 110, 100, 101, 114, 110, 97, 110, 100, 101, 114, 114, 97, 110,
|
|
14
|
+
100, 101, 114, 115, 97, 117, 99, 104, 97, 117, 102, 97, 117, 115, 98, 101, 105, 98, 105, 110, 98, 105, 115, 98, 105,
|
|
15
|
+
115, 116, 100, 97, 100, 97, 109, 105, 116, 100, 97, 110, 110, 100, 97, 115, 100, 97, 115, 115, 100, 97, 115, 115, 101,
|
|
16
|
+
108, 98, 101, 100, 97, 122, 117, 100, 97, 223, 100, 101, 105, 110, 100, 101, 105, 110, 101, 100, 101, 105, 110, 101,
|
|
17
|
+
109, 100, 101, 105, 110, 101, 110, 100, 101, 105, 110, 101, 114, 100, 101, 105, 110, 101, 115, 100, 101, 109, 100,
|
|
18
|
+
101, 109, 115, 101, 108, 98, 101, 110, 100, 101, 110, 100, 101, 110, 110, 100, 101, 110, 115, 101, 108, 98, 101, 110,
|
|
19
|
+
100, 101, 114, 100, 101, 114, 101, 114, 100, 101, 114, 115, 101, 108, 98, 101, 100, 101, 114, 115, 101, 108, 98, 101,
|
|
20
|
+
110, 100, 101, 115, 100, 101, 115, 115, 101, 108, 98, 101, 110, 100, 101, 115, 115, 101, 110, 100, 105, 99, 104, 100,
|
|
21
|
+
105, 101, 100, 105, 101, 115, 100, 105, 101, 115, 101, 100, 105, 101, 115, 101, 108, 98, 101, 100, 105, 101, 115, 101,
|
|
22
|
+
108, 98, 101, 110, 100, 105, 101, 115, 101, 109, 100, 105, 101, 115, 101, 110, 100, 105, 101, 115, 101, 114, 100, 105,
|
|
23
|
+
101, 115, 101, 115, 100, 105, 114, 100, 111, 99, 104, 100, 111, 114, 116, 100, 117, 100, 117, 114, 99, 104, 101, 105,
|
|
24
|
+
110, 101, 105, 110, 101, 101, 105, 110, 101, 109, 101, 105, 110, 101, 110, 101, 105, 110, 101, 114, 101, 105, 110,
|
|
25
|
+
101, 115, 101, 105, 110, 105, 103, 101, 105, 110, 105, 103, 101, 101, 105, 110, 105, 103, 101, 109, 101, 105, 110,
|
|
26
|
+
105, 103, 101, 110, 101, 105, 110, 105, 103, 101, 114, 101, 105, 110, 105, 103, 101, 115, 101, 105, 110, 109, 97, 108,
|
|
27
|
+
101, 114, 101, 115, 101, 116, 119, 97, 115, 101, 117, 99, 104, 101, 117, 101, 114, 101, 117, 114, 101, 101, 117, 114,
|
|
28
|
+
101, 109, 101, 117, 114, 101, 110, 101, 117, 114, 101, 114, 101, 117, 114, 101, 115, 102, 252, 114, 103, 101, 103,
|
|
29
|
+
101, 110, 103, 101, 119, 101, 115, 101, 110, 104, 97, 98, 104, 97, 98, 101, 104, 97, 98, 101, 110, 104, 97, 116, 104,
|
|
30
|
+
97, 116, 116, 101, 104, 97, 116, 116, 101, 110, 104, 105, 101, 114, 104, 105, 110, 104, 105, 110, 116, 101, 114, 105,
|
|
31
|
+
99, 104, 105, 104, 109, 105, 104, 110, 105, 104, 110, 101, 110, 105, 104, 114, 105, 104, 114, 101, 105, 104, 114, 101,
|
|
32
|
+
109, 105, 104, 114, 101, 110, 105, 104, 114, 101, 114, 105, 104, 114, 101, 115, 105, 109, 105, 110, 105, 110, 100,
|
|
33
|
+
101, 109, 105, 110, 115, 105, 115, 116, 106, 101, 100, 101, 106, 101, 100, 101, 109, 106, 101, 100, 101, 110, 106,
|
|
34
|
+
101, 100, 101, 114, 106, 101, 100, 101, 115, 106, 101, 110, 101, 106, 101, 110, 101, 109, 106, 101, 110, 101, 110,
|
|
35
|
+
106, 101, 110, 101, 114, 106, 101, 110, 101, 115, 106, 101, 116, 122, 116, 107, 97, 110, 110, 107, 101, 105, 110, 107,
|
|
36
|
+
101, 105, 110, 101, 107, 101, 105, 110, 101, 109, 107, 101, 105, 110, 101, 110, 107, 101, 105, 110, 101, 114, 107,
|
|
37
|
+
101, 105, 110, 101, 115, 107, 246, 110, 110, 101, 110, 107, 246, 110, 110, 116, 101, 109, 97, 99, 104, 101, 110, 109,
|
|
38
|
+
97, 110, 109, 97, 110, 99, 104, 101, 109, 97, 110, 99, 104, 101, 109, 109, 97, 110, 99, 104, 101, 110, 109, 97, 110,
|
|
39
|
+
99, 104, 101, 114, 109, 97, 110, 99, 104, 101, 115, 109, 101, 105, 110, 109, 101, 105, 110, 101, 109, 101, 105, 110,
|
|
40
|
+
101, 109, 109, 101, 105, 110, 101, 110, 109, 101, 105, 110, 101, 114, 109, 101, 105, 110, 101, 115, 109, 105, 99, 104,
|
|
41
|
+
109, 105, 114, 109, 105, 116, 109, 117, 115, 115, 109, 117, 115, 115, 116, 101, 110, 97, 99, 104, 110, 105, 99, 104,
|
|
42
|
+
116, 110, 105, 99, 104, 116, 115, 110, 111, 99, 104, 110, 117, 110, 110, 117, 114, 111, 98, 111, 100, 101, 114, 111,
|
|
43
|
+
104, 110, 101, 115, 101, 104, 114, 115, 101, 105, 110, 115, 101, 105, 110, 101, 115, 101, 105, 110, 101, 109, 115,
|
|
44
|
+
101, 105, 110, 101, 110, 115, 101, 105, 110, 101, 114, 115, 101, 105, 110, 101, 115, 115, 101, 108, 98, 115, 116, 115,
|
|
45
|
+
105, 99, 104, 115, 105, 101, 115, 105, 110, 100, 115, 111, 115, 111, 108, 99, 104, 101, 115, 111, 108, 99, 104, 101,
|
|
46
|
+
109, 115, 111, 108, 99, 104, 101, 110, 115, 111, 108, 99, 104, 101, 114, 115, 111, 108, 99, 104, 101, 115, 115, 111,
|
|
47
|
+
108, 108, 115, 111, 108, 108, 116, 101, 115, 111, 110, 100, 101, 114, 110, 115, 111, 110, 115, 116, 117, 109, 117,
|
|
48
|
+
110, 100, 117, 110, 115, 117, 110, 115, 101, 117, 110, 115, 101, 109, 117, 110, 115, 101, 110, 117, 110, 115, 101,
|
|
49
|
+
114, 117, 110, 115, 101, 115, 117, 110, 116, 101, 114, 118, 105, 101, 108, 118, 111, 109, 118, 111, 110, 118, 111,
|
|
50
|
+
114, 119, 97, 114, 119, 97, 114, 101, 110, 119, 97, 114, 115, 116, 119, 97, 115, 119, 101, 103, 119, 101, 105, 108,
|
|
51
|
+
119, 101, 105, 116, 101, 114, 119, 101, 108, 99, 104, 101, 119, 101, 108, 99, 104, 101, 109, 119, 101, 108, 99, 104,
|
|
52
|
+
101, 110, 119, 101, 108, 99, 104, 101, 114, 119, 101, 108, 99, 104, 101, 115, 119, 101, 110, 110, 119, 101, 114, 100,
|
|
53
|
+
101, 119, 101, 114, 100, 101, 110, 119, 105, 101, 119, 105, 101, 100, 101, 114, 119, 105, 108, 108, 119, 105, 114,
|
|
54
|
+
119, 105, 114, 100, 119, 105, 114, 115, 116, 119, 111, 119, 111, 108, 108, 101, 110, 119, 111, 108, 108, 116, 101,
|
|
55
|
+
119, 228, 104, 114, 101, 110, 100, 119, 252, 114, 100, 101, 119, 252, 114, 100, 101, 110, 122, 117, 122, 117, 109,
|
|
56
|
+
122, 117, 114, 122, 119, 97, 114, 122, 119, 105, 115, 99, 104, 101, 110, 252, 98, 101, 114,
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
static const uint32_t stop_word_offsets[233] = {
|
|
60
|
+
0, 4, 8, 13, 18, 23, 28, 31, 35, 37, 39, 44, 50, 57, 64, 71, 78, 84, 90, 96, 102, 106, 109, 112, 115, 118, 121, 125,
|
|
61
|
+
127, 132, 136, 139, 143, 151, 155, 158, 162, 167, 173, 179, 185, 191, 194, 203, 206, 210, 219, 222, 227, 235, 244,
|
|
62
|
+
247, 256, 262, 266, 269, 273, 278, 286, 295, 301, 307, 313, 319, 322, 326, 330, 332, 337, 340, 344, 349, 354, 359,
|
|
63
|
+
364, 369, 375, 382, 389, 396, 403, 409, 411, 413, 418, 422, 426, 430, 435, 440, 445, 450, 453, 458, 465, 468, 472,
|
|
64
|
+
477, 480, 485, 491, 495, 498, 504, 507, 510, 513, 518, 521, 525, 530, 535, 540, 545, 547, 549, 554, 557, 560, 564,
|
|
65
|
+
569, 574, 579, 584, 588, 593, 598, 603, 608, 613, 617, 621, 626, 632, 638, 644, 650, 656, 662, 668, 671, 677, 684,
|
|
66
|
+
691, 698, 705, 709, 714, 720, 726, 732, 738, 742, 745, 748, 752, 758, 762, 767, 773, 777, 780, 783, 785, 789, 793,
|
|
67
|
+
797, 801, 806, 812, 818, 824, 830, 836, 840, 843, 847, 849, 855, 862, 869, 876, 883, 887, 893, 900, 905, 907, 910,
|
|
68
|
+
913, 917, 922, 927, 932, 937, 942, 946, 949, 952, 955, 958, 963, 968, 971, 974, 978, 984, 990, 997, 1004, 1011, 1018,
|
|
69
|
+
1022, 1027, 1033, 1036, 1042, 1046, 1049, 1053, 1058, 1060, 1066, 1072, 1079, 1084, 1090, 1092, 1095, 1098, 1102,
|
|
70
|
+
1110, 1114,
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
static const uint32_t keep_ranges[24] = {
|
|
74
|
+
48, 57, 65, 90, 97, 122, 196, 196, 201, 201, 214, 214, 220, 220, 223, 223, 228, 228, 233, 233, 246, 246, 252, 252,
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
const narsil_language narsil_language_german = {
|
|
78
|
+
.name = "german",
|
|
79
|
+
.revision = "ff10b0dd9feeb7cb",
|
|
80
|
+
.stop_word_units = stop_word_units,
|
|
81
|
+
.stop_word_offsets = stop_word_offsets,
|
|
82
|
+
.stop_word_count = 232,
|
|
83
|
+
.ascii_keep = {0x03FF000000000000ULL, 0x07FFFFFE07FFFFFEULL},
|
|
84
|
+
.keep_ranges = keep_ranges,
|
|
85
|
+
.keep_range_count = 12,
|
|
86
|
+
.splits_code_points = 0,
|
|
87
|
+
.strips_possessive = 0,
|
|
88
|
+
.normalises_diacritics = 0,
|
|
89
|
+
.min_token_length = 1,
|
|
90
|
+
.ngram_size = 0,
|
|
91
|
+
.normalise = narsil_german_normalise,
|
|
92
|
+
.stem_by_rules = NULL,
|
|
93
|
+
.create_snowball = german_create_env,
|
|
94
|
+
.snowball_stem = german_stem,
|
|
95
|
+
.close_snowball = german_close_env,
|
|
96
|
+
};
|