ruby-stemmer 0.9.3-x86-mswin32-60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/MIT-LICENSE +21 -0
- data/README.rdoc +113 -0
- data/Rakefile +70 -0
- data/TODO +0 -0
- data/VERSION +1 -0
- data/ext/lingua/extconf.rb +40 -0
- data/ext/lingua/stemmer.c +115 -0
- data/lib/lingua/1.8/stemmer_native.so +0 -0
- data/lib/lingua/1.9/stemmer_native.so +0 -0
- data/lib/lingua/stemmer.rb +60 -0
- data/libstemmer_c/MANIFEST +72 -0
- data/libstemmer_c/Makefile +9 -0
- data/libstemmer_c/Makefile.windows +15 -0
- data/libstemmer_c/README +125 -0
- data/libstemmer_c/examples/stemwords.c +209 -0
- data/libstemmer_c/include/libstemmer.h +79 -0
- data/libstemmer_c/libstemmer/libstemmer.c +93 -0
- data/libstemmer_c/libstemmer/libstemmer_utf8.c +93 -0
- data/libstemmer_c/libstemmer/modules.h +195 -0
- data/libstemmer_c/libstemmer/modules.txt +51 -0
- data/libstemmer_c/libstemmer/modules_utf8.h +123 -0
- data/libstemmer_c/libstemmer/modules_utf8.txt +50 -0
- data/libstemmer_c/mkinc.mak +86 -0
- data/libstemmer_c/mkinc_utf8.mak +54 -0
- data/libstemmer_c/runtime/api.c +66 -0
- data/libstemmer_c/runtime/api.h +26 -0
- data/libstemmer_c/runtime/header.h +58 -0
- data/libstemmer_c/runtime/utilities.c +478 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_danish.c +337 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_danish.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_dutch.c +624 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_dutch.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_english.c +1117 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_english.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_finnish.c +762 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_finnish.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_french.c +1246 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_french.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_german.c +503 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_german.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_hungarian.c +1230 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_hungarian.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_italian.c +1065 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_italian.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_latin.c +443 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_latin.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_norwegian.c +297 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_norwegian.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_porter.c +749 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_porter.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_portuguese.c +1017 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_portuguese.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_spanish.c +1093 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_spanish.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_swedish.c +307 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_swedish.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_2_romanian.c +998 -0
- data/libstemmer_c/src_c/stem_ISO_8859_2_romanian.h +16 -0
- data/libstemmer_c/src_c/stem_KOI8_R_russian.c +700 -0
- data/libstemmer_c/src_c/stem_KOI8_R_russian.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_danish.c +339 -0
- data/libstemmer_c/src_c/stem_UTF_8_danish.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_dutch.c +634 -0
- data/libstemmer_c/src_c/stem_UTF_8_dutch.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_english.c +1125 -0
- data/libstemmer_c/src_c/stem_UTF_8_english.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_finnish.c +768 -0
- data/libstemmer_c/src_c/stem_UTF_8_finnish.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_french.c +1256 -0
- data/libstemmer_c/src_c/stem_UTF_8_french.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_german.c +509 -0
- data/libstemmer_c/src_c/stem_UTF_8_german.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_hungarian.c +1234 -0
- data/libstemmer_c/src_c/stem_UTF_8_hungarian.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_italian.c +1073 -0
- data/libstemmer_c/src_c/stem_UTF_8_italian.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_latin.c +443 -0
- data/libstemmer_c/src_c/stem_UTF_8_latin.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_norwegian.c +299 -0
- data/libstemmer_c/src_c/stem_UTF_8_norwegian.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_porter.c +755 -0
- data/libstemmer_c/src_c/stem_UTF_8_porter.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_portuguese.c +1023 -0
- data/libstemmer_c/src_c/stem_UTF_8_portuguese.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_romanian.c +1004 -0
- data/libstemmer_c/src_c/stem_UTF_8_romanian.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_russian.c +694 -0
- data/libstemmer_c/src_c/stem_UTF_8_russian.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_spanish.c +1097 -0
- data/libstemmer_c/src_c/stem_UTF_8_spanish.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_swedish.c +309 -0
- data/libstemmer_c/src_c/stem_UTF_8_swedish.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_turkish.c +2205 -0
- data/libstemmer_c/src_c/stem_UTF_8_turkish.h +16 -0
- data/test/helper.rb +3 -0
- data/test/lingua/test_stemmer.rb +99 -0
- metadata +162 -0
@@ -0,0 +1,51 @@
|
|
1
|
+
# This file contains a list of stemmers to include in the distribution.
|
2
|
+
# The format is a set of space separated lines - on each line:
|
3
|
+
# First item is name of stemmer.
|
4
|
+
# Second item is comma separated list of character sets.
|
5
|
+
# Third item is comma separated list of names to refer to the stemmer by.
|
6
|
+
#
|
7
|
+
# Lines starting with a #, or blank lines, are ignored.
|
8
|
+
|
9
|
+
# List all the main algorithms for each language, in UTF-8, and also with
|
10
|
+
# the most commonly used encoding.
|
11
|
+
|
12
|
+
latin UTF_8,ISO_8859_1 latin
|
13
|
+
danish UTF_8,ISO_8859_1 danish,da,dan
|
14
|
+
dutch UTF_8,ISO_8859_1 dutch,nl,dut,nld
|
15
|
+
english UTF_8,ISO_8859_1 english,en,eng
|
16
|
+
finnish UTF_8,ISO_8859_1 finnish,fi,fin
|
17
|
+
french UTF_8,ISO_8859_1 french,fr,fre,fra
|
18
|
+
german UTF_8,ISO_8859_1 german,de,ger,deu
|
19
|
+
hungarian UTF_8,ISO_8859_1 hungarian,hu,hun
|
20
|
+
italian UTF_8,ISO_8859_1 italian,it,ita
|
21
|
+
norwegian UTF_8,ISO_8859_1 norwegian,no,nor
|
22
|
+
portuguese UTF_8,ISO_8859_1 portuguese,pt,por
|
23
|
+
romanian UTF_8,ISO_8859_2 romanian,ro,rum,ron
|
24
|
+
russian UTF_8,KOI8_R russian,ru,rus
|
25
|
+
spanish UTF_8,ISO_8859_1 spanish,es,esl,spa
|
26
|
+
swedish UTF_8,ISO_8859_1 swedish,sv,swe
|
27
|
+
turkish UTF_8 turkish,tr,tur
|
28
|
+
|
29
|
+
# Also include the traditional porter algorithm for english.
|
30
|
+
# The porter algorithm is included in the libstemmer distribution to assist
|
31
|
+
# with backwards compatibility, but for new systems the english algorithm
|
32
|
+
# should be used in preference.
|
33
|
+
porter UTF_8,ISO_8859_1 porter
|
34
|
+
|
35
|
+
# Some other stemmers in the snowball project are not included in the standard
|
36
|
+
# distribution. To compile a libstemmer with them in, add them to this list,
|
37
|
+
# and regenerate the distribution. (You will need a full source checkout for
|
38
|
+
# this.) They are included in the snowball website as curiosities, but are not
|
39
|
+
# intended for general use, and use of them is is not fully supported. These
|
40
|
+
# algorithms are:
|
41
|
+
#
|
42
|
+
# german2 - This is a slight modification of the german stemmer.
|
43
|
+
#german2 UTF_8,ISO_8859_1 german2
|
44
|
+
#
|
45
|
+
# kraaij_pohlmann - This is a different dutch stemmer.
|
46
|
+
#kraaij_pohlmann UTF_8,ISO_8859_1 kraaij_pohlmann
|
47
|
+
#
|
48
|
+
# lovins - This is an english stemmer, but fairly outdated, and
|
49
|
+
# only really applicable to a restricted type of input text
|
50
|
+
# (keywords in academic publications).
|
51
|
+
#lovins UTF_8,ISO_8859_1 lovins
|
@@ -0,0 +1,123 @@
|
|
1
|
+
/* libstemmer/modules_utf8.h: List of stemming modules.
|
2
|
+
*
|
3
|
+
* This file is generated by mkmodules.pl from a list of module names.
|
4
|
+
* Do not edit manually.
|
5
|
+
*
|
6
|
+
* Modules included by this file are: latin, danish, dutch, english, finnish, french,
|
7
|
+
* german, hungarian, italian, norwegian, porter, portuguese, romanian,
|
8
|
+
* russian, spanish, swedish, turkish
|
9
|
+
*/
|
10
|
+
|
11
|
+
#include "../src_c/stem_UTF_8_latin.h"
|
12
|
+
#include "../src_c/stem_UTF_8_danish.h"
|
13
|
+
#include "../src_c/stem_UTF_8_dutch.h"
|
14
|
+
#include "../src_c/stem_UTF_8_english.h"
|
15
|
+
#include "../src_c/stem_UTF_8_finnish.h"
|
16
|
+
#include "../src_c/stem_UTF_8_french.h"
|
17
|
+
#include "../src_c/stem_UTF_8_german.h"
|
18
|
+
#include "../src_c/stem_UTF_8_hungarian.h"
|
19
|
+
#include "../src_c/stem_UTF_8_italian.h"
|
20
|
+
#include "../src_c/stem_UTF_8_norwegian.h"
|
21
|
+
#include "../src_c/stem_UTF_8_porter.h"
|
22
|
+
#include "../src_c/stem_UTF_8_portuguese.h"
|
23
|
+
#include "../src_c/stem_UTF_8_romanian.h"
|
24
|
+
#include "../src_c/stem_UTF_8_russian.h"
|
25
|
+
#include "../src_c/stem_UTF_8_spanish.h"
|
26
|
+
#include "../src_c/stem_UTF_8_swedish.h"
|
27
|
+
#include "../src_c/stem_UTF_8_turkish.h"
|
28
|
+
|
29
|
+
typedef enum {
|
30
|
+
ENC_UNKNOWN=0,
|
31
|
+
ENC_UTF_8
|
32
|
+
} stemmer_encoding_t;
|
33
|
+
|
34
|
+
struct stemmer_encoding {
|
35
|
+
const char * name;
|
36
|
+
stemmer_encoding_t enc;
|
37
|
+
};
|
38
|
+
static struct stemmer_encoding encodings[] = {
|
39
|
+
{"UTF_8", ENC_UTF_8},
|
40
|
+
{0,ENC_UNKNOWN}
|
41
|
+
};
|
42
|
+
|
43
|
+
struct stemmer_modules {
|
44
|
+
const char * name;
|
45
|
+
stemmer_encoding_t enc;
|
46
|
+
struct SN_env * (*create)(void);
|
47
|
+
void (*close)(struct SN_env *);
|
48
|
+
int (*stem)(struct SN_env *);
|
49
|
+
};
|
50
|
+
static struct stemmer_modules modules[] = {
|
51
|
+
{"latin", ENC_UTF_8, latin_UTF_8_create_env, latin_UTF_8_close_env, latin_UTF_8_stem},
|
52
|
+
{"da", ENC_UTF_8, danish_UTF_8_create_env, danish_UTF_8_close_env, danish_UTF_8_stem},
|
53
|
+
{"dan", ENC_UTF_8, danish_UTF_8_create_env, danish_UTF_8_close_env, danish_UTF_8_stem},
|
54
|
+
{"danish", ENC_UTF_8, danish_UTF_8_create_env, danish_UTF_8_close_env, danish_UTF_8_stem},
|
55
|
+
{"de", ENC_UTF_8, german_UTF_8_create_env, german_UTF_8_close_env, german_UTF_8_stem},
|
56
|
+
{"deu", ENC_UTF_8, german_UTF_8_create_env, german_UTF_8_close_env, german_UTF_8_stem},
|
57
|
+
{"dut", ENC_UTF_8, dutch_UTF_8_create_env, dutch_UTF_8_close_env, dutch_UTF_8_stem},
|
58
|
+
{"dutch", ENC_UTF_8, dutch_UTF_8_create_env, dutch_UTF_8_close_env, dutch_UTF_8_stem},
|
59
|
+
{"en", ENC_UTF_8, english_UTF_8_create_env, english_UTF_8_close_env, english_UTF_8_stem},
|
60
|
+
{"eng", ENC_UTF_8, english_UTF_8_create_env, english_UTF_8_close_env, english_UTF_8_stem},
|
61
|
+
{"english", ENC_UTF_8, english_UTF_8_create_env, english_UTF_8_close_env, english_UTF_8_stem},
|
62
|
+
{"es", ENC_UTF_8, spanish_UTF_8_create_env, spanish_UTF_8_close_env, spanish_UTF_8_stem},
|
63
|
+
{"esl", ENC_UTF_8, spanish_UTF_8_create_env, spanish_UTF_8_close_env, spanish_UTF_8_stem},
|
64
|
+
{"fi", ENC_UTF_8, finnish_UTF_8_create_env, finnish_UTF_8_close_env, finnish_UTF_8_stem},
|
65
|
+
{"fin", ENC_UTF_8, finnish_UTF_8_create_env, finnish_UTF_8_close_env, finnish_UTF_8_stem},
|
66
|
+
{"finnish", ENC_UTF_8, finnish_UTF_8_create_env, finnish_UTF_8_close_env, finnish_UTF_8_stem},
|
67
|
+
{"fr", ENC_UTF_8, french_UTF_8_create_env, french_UTF_8_close_env, french_UTF_8_stem},
|
68
|
+
{"fra", ENC_UTF_8, french_UTF_8_create_env, french_UTF_8_close_env, french_UTF_8_stem},
|
69
|
+
{"fre", ENC_UTF_8, french_UTF_8_create_env, french_UTF_8_close_env, french_UTF_8_stem},
|
70
|
+
{"french", ENC_UTF_8, french_UTF_8_create_env, french_UTF_8_close_env, french_UTF_8_stem},
|
71
|
+
{"ger", ENC_UTF_8, german_UTF_8_create_env, german_UTF_8_close_env, german_UTF_8_stem},
|
72
|
+
{"german", ENC_UTF_8, german_UTF_8_create_env, german_UTF_8_close_env, german_UTF_8_stem},
|
73
|
+
{"hu", ENC_UTF_8, hungarian_UTF_8_create_env, hungarian_UTF_8_close_env, hungarian_UTF_8_stem},
|
74
|
+
{"hun", ENC_UTF_8, hungarian_UTF_8_create_env, hungarian_UTF_8_close_env, hungarian_UTF_8_stem},
|
75
|
+
{"hungarian", ENC_UTF_8, hungarian_UTF_8_create_env, hungarian_UTF_8_close_env, hungarian_UTF_8_stem},
|
76
|
+
{"it", ENC_UTF_8, italian_UTF_8_create_env, italian_UTF_8_close_env, italian_UTF_8_stem},
|
77
|
+
{"ita", ENC_UTF_8, italian_UTF_8_create_env, italian_UTF_8_close_env, italian_UTF_8_stem},
|
78
|
+
{"italian", ENC_UTF_8, italian_UTF_8_create_env, italian_UTF_8_close_env, italian_UTF_8_stem},
|
79
|
+
{"nl", ENC_UTF_8, dutch_UTF_8_create_env, dutch_UTF_8_close_env, dutch_UTF_8_stem},
|
80
|
+
{"nld", ENC_UTF_8, dutch_UTF_8_create_env, dutch_UTF_8_close_env, dutch_UTF_8_stem},
|
81
|
+
{"no", ENC_UTF_8, norwegian_UTF_8_create_env, norwegian_UTF_8_close_env, norwegian_UTF_8_stem},
|
82
|
+
{"nor", ENC_UTF_8, norwegian_UTF_8_create_env, norwegian_UTF_8_close_env, norwegian_UTF_8_stem},
|
83
|
+
{"norwegian", ENC_UTF_8, norwegian_UTF_8_create_env, norwegian_UTF_8_close_env, norwegian_UTF_8_stem},
|
84
|
+
{"por", ENC_UTF_8, portuguese_UTF_8_create_env, portuguese_UTF_8_close_env, portuguese_UTF_8_stem},
|
85
|
+
{"porter", ENC_UTF_8, porter_UTF_8_create_env, porter_UTF_8_close_env, porter_UTF_8_stem},
|
86
|
+
{"portuguese", ENC_UTF_8, portuguese_UTF_8_create_env, portuguese_UTF_8_close_env, portuguese_UTF_8_stem},
|
87
|
+
{"pt", ENC_UTF_8, portuguese_UTF_8_create_env, portuguese_UTF_8_close_env, portuguese_UTF_8_stem},
|
88
|
+
{"ro", ENC_UTF_8, romanian_UTF_8_create_env, romanian_UTF_8_close_env, romanian_UTF_8_stem},
|
89
|
+
{"romanian", ENC_UTF_8, romanian_UTF_8_create_env, romanian_UTF_8_close_env, romanian_UTF_8_stem},
|
90
|
+
{"ron", ENC_UTF_8, romanian_UTF_8_create_env, romanian_UTF_8_close_env, romanian_UTF_8_stem},
|
91
|
+
{"ru", ENC_UTF_8, russian_UTF_8_create_env, russian_UTF_8_close_env, russian_UTF_8_stem},
|
92
|
+
{"rum", ENC_UTF_8, romanian_UTF_8_create_env, romanian_UTF_8_close_env, romanian_UTF_8_stem},
|
93
|
+
{"rus", ENC_UTF_8, russian_UTF_8_create_env, russian_UTF_8_close_env, russian_UTF_8_stem},
|
94
|
+
{"russian", ENC_UTF_8, russian_UTF_8_create_env, russian_UTF_8_close_env, russian_UTF_8_stem},
|
95
|
+
{"spa", ENC_UTF_8, spanish_UTF_8_create_env, spanish_UTF_8_close_env, spanish_UTF_8_stem},
|
96
|
+
{"spanish", ENC_UTF_8, spanish_UTF_8_create_env, spanish_UTF_8_close_env, spanish_UTF_8_stem},
|
97
|
+
{"sv", ENC_UTF_8, swedish_UTF_8_create_env, swedish_UTF_8_close_env, swedish_UTF_8_stem},
|
98
|
+
{"swe", ENC_UTF_8, swedish_UTF_8_create_env, swedish_UTF_8_close_env, swedish_UTF_8_stem},
|
99
|
+
{"swedish", ENC_UTF_8, swedish_UTF_8_create_env, swedish_UTF_8_close_env, swedish_UTF_8_stem},
|
100
|
+
{"tr", ENC_UTF_8, turkish_UTF_8_create_env, turkish_UTF_8_close_env, turkish_UTF_8_stem},
|
101
|
+
{"tur", ENC_UTF_8, turkish_UTF_8_create_env, turkish_UTF_8_close_env, turkish_UTF_8_stem},
|
102
|
+
{"turkish", ENC_UTF_8, turkish_UTF_8_create_env, turkish_UTF_8_close_env, turkish_UTF_8_stem},
|
103
|
+
{0,ENC_UNKNOWN,0,0,0}
|
104
|
+
};
|
105
|
+
static const char * algorithm_names[] = {
|
106
|
+
"danish",
|
107
|
+
"dutch",
|
108
|
+
"english",
|
109
|
+
"finnish",
|
110
|
+
"french",
|
111
|
+
"german",
|
112
|
+
"hungarian",
|
113
|
+
"italian",
|
114
|
+
"norwegian",
|
115
|
+
"porter",
|
116
|
+
"portuguese",
|
117
|
+
"romanian",
|
118
|
+
"russian",
|
119
|
+
"spanish",
|
120
|
+
"swedish",
|
121
|
+
"turkish",
|
122
|
+
0
|
123
|
+
};
|
@@ -0,0 +1,50 @@
|
|
1
|
+
# This file contains a list of stemmers to include in the distribution.
|
2
|
+
# The format is a set of space separated lines - on each line:
|
3
|
+
# First item is name of stemmer.
|
4
|
+
# Second item is comma separated list of character sets.
|
5
|
+
# Third item is comma separated list of names to refer to the stemmer by.
|
6
|
+
#
|
7
|
+
# Lines starting with a #, or blank lines, are ignored.
|
8
|
+
|
9
|
+
# List all the main algorithms for each language, in UTF-8.
|
10
|
+
|
11
|
+
latin UTF_8 latin
|
12
|
+
danish UTF_8 danish,da,dan
|
13
|
+
dutch UTF_8 dutch,nl,dut,nld
|
14
|
+
english UTF_8 english,en,eng
|
15
|
+
finnish UTF_8 finnish,fi,fin
|
16
|
+
french UTF_8 french,fr,fre,fra
|
17
|
+
german UTF_8 german,de,ger,deu
|
18
|
+
hungarian UTF_8 hungarian,hu,hun
|
19
|
+
italian UTF_8 italian,it,ita
|
20
|
+
norwegian UTF_8 norwegian,no,nor
|
21
|
+
portuguese UTF_8 portuguese,pt,por
|
22
|
+
romanian UTF_8 romanian,ro,rum,ron
|
23
|
+
russian UTF_8 russian,ru,rus
|
24
|
+
spanish UTF_8 spanish,es,esl,spa
|
25
|
+
swedish UTF_8 swedish,sv,swe
|
26
|
+
turkish UTF_8 turkish,tr,tur
|
27
|
+
|
28
|
+
# Also include the traditional porter algorithm for english.
|
29
|
+
# The porter algorithm is included in the libstemmer distribution to assist
|
30
|
+
# with backwards compatibility, but for new systems the english algorithm
|
31
|
+
# should be used in preference.
|
32
|
+
porter UTF_8 porter
|
33
|
+
|
34
|
+
# Some other stemmers in the snowball project are not included in the standard
|
35
|
+
# distribution. To compile a libstemmer with them in, add them to this list,
|
36
|
+
# and regenerate the distribution. (You will need a full source checkout for
|
37
|
+
# this.) They are included in the snowball website as curiosities, but are not
|
38
|
+
# intended for general use, and use of them is is not fully supported. These
|
39
|
+
# algorithms are:
|
40
|
+
#
|
41
|
+
# german2 - This is a slight modification of the german stemmer.
|
42
|
+
#german2 UTF_8 german2
|
43
|
+
#
|
44
|
+
# kraaij_pohlmann - This is a different dutch stemmer.
|
45
|
+
#kraaij_pohlmann UTF_8 kraaij_pohlmann
|
46
|
+
#
|
47
|
+
# lovins - This is an english stemmer, but fairly outdated, and
|
48
|
+
# only really applicable to a restricted type of input text
|
49
|
+
# (keywords in academic publications).
|
50
|
+
#lovins UTF_8 lovins
|
@@ -0,0 +1,86 @@
|
|
1
|
+
# libstemmer/mkinc.mak: List of stemming module source files
|
2
|
+
#
|
3
|
+
# This file is generated by mkmodules.pl from a list of module names.
|
4
|
+
# Do not edit manually.
|
5
|
+
#
|
6
|
+
# Modules included by this file are: latin, danish, dutch, english, finnish, french,
|
7
|
+
# german, hungarian, italian, norwegian, porter, portuguese, romanian,
|
8
|
+
# russian, spanish, swedish, turkish
|
9
|
+
|
10
|
+
snowball_sources= \
|
11
|
+
src_c/stem_ISO_8859_1_latin.c \
|
12
|
+
src_c/stem_UTF_8_latin.c \
|
13
|
+
src_c/stem_ISO_8859_1_danish.c \
|
14
|
+
src_c/stem_UTF_8_danish.c \
|
15
|
+
src_c/stem_ISO_8859_1_dutch.c \
|
16
|
+
src_c/stem_UTF_8_dutch.c \
|
17
|
+
src_c/stem_ISO_8859_1_english.c \
|
18
|
+
src_c/stem_UTF_8_english.c \
|
19
|
+
src_c/stem_ISO_8859_1_finnish.c \
|
20
|
+
src_c/stem_UTF_8_finnish.c \
|
21
|
+
src_c/stem_ISO_8859_1_french.c \
|
22
|
+
src_c/stem_UTF_8_french.c \
|
23
|
+
src_c/stem_ISO_8859_1_german.c \
|
24
|
+
src_c/stem_UTF_8_german.c \
|
25
|
+
src_c/stem_ISO_8859_1_hungarian.c \
|
26
|
+
src_c/stem_UTF_8_hungarian.c \
|
27
|
+
src_c/stem_ISO_8859_1_italian.c \
|
28
|
+
src_c/stem_UTF_8_italian.c \
|
29
|
+
src_c/stem_ISO_8859_1_norwegian.c \
|
30
|
+
src_c/stem_UTF_8_norwegian.c \
|
31
|
+
src_c/stem_ISO_8859_1_porter.c \
|
32
|
+
src_c/stem_UTF_8_porter.c \
|
33
|
+
src_c/stem_ISO_8859_1_portuguese.c \
|
34
|
+
src_c/stem_UTF_8_portuguese.c \
|
35
|
+
src_c/stem_ISO_8859_2_romanian.c \
|
36
|
+
src_c/stem_UTF_8_romanian.c \
|
37
|
+
src_c/stem_KOI8_R_russian.c \
|
38
|
+
src_c/stem_UTF_8_russian.c \
|
39
|
+
src_c/stem_ISO_8859_1_spanish.c \
|
40
|
+
src_c/stem_UTF_8_spanish.c \
|
41
|
+
src_c/stem_ISO_8859_1_swedish.c \
|
42
|
+
src_c/stem_UTF_8_swedish.c \
|
43
|
+
src_c/stem_UTF_8_turkish.c \
|
44
|
+
runtime/api.c \
|
45
|
+
runtime/utilities.c \
|
46
|
+
libstemmer/libstemmer.c
|
47
|
+
|
48
|
+
snowball_headers= \
|
49
|
+
src_c/stem_ISO_8859_1_latin.h \
|
50
|
+
src_c/stem_UTF_8_latin.h \
|
51
|
+
src_c/stem_ISO_8859_1_danish.h \
|
52
|
+
src_c/stem_UTF_8_danish.h \
|
53
|
+
src_c/stem_ISO_8859_1_dutch.h \
|
54
|
+
src_c/stem_UTF_8_dutch.h \
|
55
|
+
src_c/stem_ISO_8859_1_english.h \
|
56
|
+
src_c/stem_UTF_8_english.h \
|
57
|
+
src_c/stem_ISO_8859_1_finnish.h \
|
58
|
+
src_c/stem_UTF_8_finnish.h \
|
59
|
+
src_c/stem_ISO_8859_1_french.h \
|
60
|
+
src_c/stem_UTF_8_french.h \
|
61
|
+
src_c/stem_ISO_8859_1_german.h \
|
62
|
+
src_c/stem_UTF_8_german.h \
|
63
|
+
src_c/stem_ISO_8859_1_hungarian.h \
|
64
|
+
src_c/stem_UTF_8_hungarian.h \
|
65
|
+
src_c/stem_ISO_8859_1_italian.h \
|
66
|
+
src_c/stem_UTF_8_italian.h \
|
67
|
+
src_c/stem_ISO_8859_1_norwegian.h \
|
68
|
+
src_c/stem_UTF_8_norwegian.h \
|
69
|
+
src_c/stem_ISO_8859_1_porter.h \
|
70
|
+
src_c/stem_UTF_8_porter.h \
|
71
|
+
src_c/stem_ISO_8859_1_portuguese.h \
|
72
|
+
src_c/stem_UTF_8_portuguese.h \
|
73
|
+
src_c/stem_ISO_8859_2_romanian.h \
|
74
|
+
src_c/stem_UTF_8_romanian.h \
|
75
|
+
src_c/stem_KOI8_R_russian.h \
|
76
|
+
src_c/stem_UTF_8_russian.h \
|
77
|
+
src_c/stem_ISO_8859_1_spanish.h \
|
78
|
+
src_c/stem_UTF_8_spanish.h \
|
79
|
+
src_c/stem_ISO_8859_1_swedish.h \
|
80
|
+
src_c/stem_UTF_8_swedish.h \
|
81
|
+
src_c/stem_UTF_8_turkish.h \
|
82
|
+
include/libstemmer.h \
|
83
|
+
libstemmer/modules.h \
|
84
|
+
runtime/api.h \
|
85
|
+
runtime/header.h
|
86
|
+
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# libstemmer/mkinc_utf8.mak: List of stemming module source files
|
2
|
+
#
|
3
|
+
# This file is generated by mkmodules.pl from a list of module names.
|
4
|
+
# Do not edit manually.
|
5
|
+
#
|
6
|
+
# Modules included by this file are: danish, dutch, english, finnish, french,
|
7
|
+
# german, hungarian, italian, norwegian, porter, portuguese, romanian,
|
8
|
+
# russian, spanish, swedish, turkish
|
9
|
+
|
10
|
+
snowball_sources= \
|
11
|
+
src_c/stem_UTF_8_latin.c \
|
12
|
+
src_c/stem_UTF_8_danish.c \
|
13
|
+
src_c/stem_UTF_8_dutch.c \
|
14
|
+
src_c/stem_UTF_8_english.c \
|
15
|
+
src_c/stem_UTF_8_finnish.c \
|
16
|
+
src_c/stem_UTF_8_french.c \
|
17
|
+
src_c/stem_UTF_8_german.c \
|
18
|
+
src_c/stem_UTF_8_hungarian.c \
|
19
|
+
src_c/stem_UTF_8_italian.c \
|
20
|
+
src_c/stem_UTF_8_norwegian.c \
|
21
|
+
src_c/stem_UTF_8_porter.c \
|
22
|
+
src_c/stem_UTF_8_portuguese.c \
|
23
|
+
src_c/stem_UTF_8_romanian.c \
|
24
|
+
src_c/stem_UTF_8_russian.c \
|
25
|
+
src_c/stem_UTF_8_spanish.c \
|
26
|
+
src_c/stem_UTF_8_swedish.c \
|
27
|
+
src_c/stem_UTF_8_turkish.c \
|
28
|
+
runtime/api.c \
|
29
|
+
runtime/utilities.c \
|
30
|
+
libstemmer/libstemmer_utf8.c
|
31
|
+
|
32
|
+
snowball_headers= \
|
33
|
+
src_c/stem_UTF_8_latin.h \
|
34
|
+
src_c/stem_UTF_8_danish.h \
|
35
|
+
src_c/stem_UTF_8_dutch.h \
|
36
|
+
src_c/stem_UTF_8_english.h \
|
37
|
+
src_c/stem_UTF_8_finnish.h \
|
38
|
+
src_c/stem_UTF_8_french.h \
|
39
|
+
src_c/stem_UTF_8_german.h \
|
40
|
+
src_c/stem_UTF_8_hungarian.h \
|
41
|
+
src_c/stem_UTF_8_italian.h \
|
42
|
+
src_c/stem_UTF_8_norwegian.h \
|
43
|
+
src_c/stem_UTF_8_porter.h \
|
44
|
+
src_c/stem_UTF_8_portuguese.h \
|
45
|
+
src_c/stem_UTF_8_romanian.h \
|
46
|
+
src_c/stem_UTF_8_russian.h \
|
47
|
+
src_c/stem_UTF_8_spanish.h \
|
48
|
+
src_c/stem_UTF_8_swedish.h \
|
49
|
+
src_c/stem_UTF_8_turkish.h \
|
50
|
+
include/libstemmer.h \
|
51
|
+
libstemmer/modules_utf8.h \
|
52
|
+
runtime/api.h \
|
53
|
+
runtime/header.h
|
54
|
+
|
@@ -0,0 +1,66 @@
|
|
1
|
+
|
2
|
+
#include <stdlib.h> /* for calloc, free */
|
3
|
+
#include "header.h"
|
4
|
+
|
5
|
+
extern struct SN_env * SN_create_env(int S_size, int I_size, int B_size)
|
6
|
+
{
|
7
|
+
struct SN_env * z = (struct SN_env *) calloc(1, sizeof(struct SN_env));
|
8
|
+
if (z == NULL) return NULL;
|
9
|
+
z->p = create_s();
|
10
|
+
if (z->p == NULL) goto error;
|
11
|
+
if (S_size)
|
12
|
+
{
|
13
|
+
int i;
|
14
|
+
z->S = (symbol * *) calloc(S_size, sizeof(symbol *));
|
15
|
+
if (z->S == NULL) goto error;
|
16
|
+
|
17
|
+
for (i = 0; i < S_size; i++)
|
18
|
+
{
|
19
|
+
z->S[i] = create_s();
|
20
|
+
if (z->S[i] == NULL) goto error;
|
21
|
+
}
|
22
|
+
}
|
23
|
+
|
24
|
+
if (I_size)
|
25
|
+
{
|
26
|
+
z->I = (int *) calloc(I_size, sizeof(int));
|
27
|
+
if (z->I == NULL) goto error;
|
28
|
+
}
|
29
|
+
|
30
|
+
if (B_size)
|
31
|
+
{
|
32
|
+
z->B = (unsigned char *) calloc(B_size, sizeof(unsigned char));
|
33
|
+
if (z->B == NULL) goto error;
|
34
|
+
}
|
35
|
+
|
36
|
+
return z;
|
37
|
+
error:
|
38
|
+
SN_close_env(z, S_size);
|
39
|
+
return NULL;
|
40
|
+
}
|
41
|
+
|
42
|
+
extern void SN_close_env(struct SN_env * z, int S_size)
|
43
|
+
{
|
44
|
+
if (z == NULL) return;
|
45
|
+
if (S_size)
|
46
|
+
{
|
47
|
+
int i;
|
48
|
+
for (i = 0; i < S_size; i++)
|
49
|
+
{
|
50
|
+
lose_s(z->S[i]);
|
51
|
+
}
|
52
|
+
free(z->S);
|
53
|
+
}
|
54
|
+
free(z->I);
|
55
|
+
free(z->B);
|
56
|
+
if (z->p) lose_s(z->p);
|
57
|
+
free(z);
|
58
|
+
}
|
59
|
+
|
60
|
+
extern int SN_set_current(struct SN_env * z, int size, const symbol * s)
|
61
|
+
{
|
62
|
+
int err = replace_s(z, 0, z->l, size, s, NULL);
|
63
|
+
z->c = 0;
|
64
|
+
return err;
|
65
|
+
}
|
66
|
+
|
@@ -0,0 +1,26 @@
|
|
1
|
+
|
2
|
+
typedef unsigned char symbol;
|
3
|
+
|
4
|
+
/* Or replace 'char' above with 'short' for 16 bit characters.
|
5
|
+
|
6
|
+
More precisely, replace 'char' with whatever type guarantees the
|
7
|
+
character width you need. Note however that sizeof(symbol) should divide
|
8
|
+
HEAD, defined in header.h as 2*sizeof(int), without remainder, otherwise
|
9
|
+
there is an alignment problem. In the unlikely event of a problem here,
|
10
|
+
consult Martin Porter.
|
11
|
+
|
12
|
+
*/
|
13
|
+
|
14
|
+
struct SN_env {
|
15
|
+
symbol * p;
|
16
|
+
int c; int l; int lb; int bra; int ket;
|
17
|
+
symbol * * S;
|
18
|
+
int * I;
|
19
|
+
unsigned char * B;
|
20
|
+
};
|
21
|
+
|
22
|
+
extern struct SN_env * SN_create_env(int S_size, int I_size, int B_size);
|
23
|
+
extern void SN_close_env(struct SN_env * z, int S_size);
|
24
|
+
|
25
|
+
extern int SN_set_current(struct SN_env * z, int size, const symbol * s);
|
26
|
+
|
@@ -0,0 +1,58 @@
|
|
1
|
+
|
2
|
+
#include <limits.h>
|
3
|
+
|
4
|
+
#include "api.h"
|
5
|
+
|
6
|
+
#define MAXINT INT_MAX
|
7
|
+
#define MININT INT_MIN
|
8
|
+
|
9
|
+
#define HEAD 2*sizeof(int)
|
10
|
+
|
11
|
+
#define SIZE(p) ((int *)(p))[-1]
|
12
|
+
#define SET_SIZE(p, n) ((int *)(p))[-1] = n
|
13
|
+
#define CAPACITY(p) ((int *)(p))[-2]
|
14
|
+
|
15
|
+
struct among
|
16
|
+
{ int s_size; /* number of chars in string */
|
17
|
+
const symbol * s; /* search string */
|
18
|
+
int substring_i;/* index to longest matching substring */
|
19
|
+
int result; /* result of the lookup */
|
20
|
+
int (* function)(struct SN_env *);
|
21
|
+
};
|
22
|
+
|
23
|
+
extern symbol * create_s(void);
|
24
|
+
extern void lose_s(symbol * p);
|
25
|
+
|
26
|
+
extern int skip_utf8(const symbol * p, int c, int lb, int l, int n);
|
27
|
+
|
28
|
+
extern int in_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
|
29
|
+
extern int in_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
|
30
|
+
extern int out_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
|
31
|
+
extern int out_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
|
32
|
+
|
33
|
+
extern int in_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
|
34
|
+
extern int in_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
|
35
|
+
extern int out_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
|
36
|
+
extern int out_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
|
37
|
+
|
38
|
+
extern int eq_s(struct SN_env * z, int s_size, const symbol * s);
|
39
|
+
extern int eq_s_b(struct SN_env * z, int s_size, const symbol * s);
|
40
|
+
extern int eq_v(struct SN_env * z, const symbol * p);
|
41
|
+
extern int eq_v_b(struct SN_env * z, const symbol * p);
|
42
|
+
|
43
|
+
extern int find_among(struct SN_env * z, const struct among * v, int v_size);
|
44
|
+
extern int find_among_b(struct SN_env * z, const struct among * v, int v_size);
|
45
|
+
|
46
|
+
extern int replace_s(struct SN_env * z, int c_bra, int c_ket, int s_size, const symbol * s, int * adjustment);
|
47
|
+
extern int slice_from_s(struct SN_env * z, int s_size, const symbol * s);
|
48
|
+
extern int slice_from_v(struct SN_env * z, const symbol * p);
|
49
|
+
extern int slice_del(struct SN_env * z);
|
50
|
+
|
51
|
+
extern int insert_s(struct SN_env * z, int bra, int ket, int s_size, const symbol * s);
|
52
|
+
extern int insert_v(struct SN_env * z, int bra, int ket, const symbol * p);
|
53
|
+
|
54
|
+
extern symbol * slice_to(struct SN_env * z, symbol * p);
|
55
|
+
extern symbol * assign_to(struct SN_env * z, symbol * p);
|
56
|
+
|
57
|
+
extern void debug(struct SN_env * z, int number, int line_count);
|
58
|
+
|