ruby-stemmer 0.9.3-x86-mswin32-60

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. data/MIT-LICENSE +21 -0
  2. data/README.rdoc +113 -0
  3. data/Rakefile +70 -0
  4. data/TODO +0 -0
  5. data/VERSION +1 -0
  6. data/ext/lingua/extconf.rb +40 -0
  7. data/ext/lingua/stemmer.c +115 -0
  8. data/lib/lingua/1.8/stemmer_native.so +0 -0
  9. data/lib/lingua/1.9/stemmer_native.so +0 -0
  10. data/lib/lingua/stemmer.rb +60 -0
  11. data/libstemmer_c/MANIFEST +72 -0
  12. data/libstemmer_c/Makefile +9 -0
  13. data/libstemmer_c/Makefile.windows +15 -0
  14. data/libstemmer_c/README +125 -0
  15. data/libstemmer_c/examples/stemwords.c +209 -0
  16. data/libstemmer_c/include/libstemmer.h +79 -0
  17. data/libstemmer_c/libstemmer/libstemmer.c +93 -0
  18. data/libstemmer_c/libstemmer/libstemmer_utf8.c +93 -0
  19. data/libstemmer_c/libstemmer/modules.h +195 -0
  20. data/libstemmer_c/libstemmer/modules.txt +51 -0
  21. data/libstemmer_c/libstemmer/modules_utf8.h +123 -0
  22. data/libstemmer_c/libstemmer/modules_utf8.txt +50 -0
  23. data/libstemmer_c/mkinc.mak +86 -0
  24. data/libstemmer_c/mkinc_utf8.mak +54 -0
  25. data/libstemmer_c/runtime/api.c +66 -0
  26. data/libstemmer_c/runtime/api.h +26 -0
  27. data/libstemmer_c/runtime/header.h +58 -0
  28. data/libstemmer_c/runtime/utilities.c +478 -0
  29. data/libstemmer_c/src_c/stem_ISO_8859_1_danish.c +337 -0
  30. data/libstemmer_c/src_c/stem_ISO_8859_1_danish.h +16 -0
  31. data/libstemmer_c/src_c/stem_ISO_8859_1_dutch.c +624 -0
  32. data/libstemmer_c/src_c/stem_ISO_8859_1_dutch.h +16 -0
  33. data/libstemmer_c/src_c/stem_ISO_8859_1_english.c +1117 -0
  34. data/libstemmer_c/src_c/stem_ISO_8859_1_english.h +16 -0
  35. data/libstemmer_c/src_c/stem_ISO_8859_1_finnish.c +762 -0
  36. data/libstemmer_c/src_c/stem_ISO_8859_1_finnish.h +16 -0
  37. data/libstemmer_c/src_c/stem_ISO_8859_1_french.c +1246 -0
  38. data/libstemmer_c/src_c/stem_ISO_8859_1_french.h +16 -0
  39. data/libstemmer_c/src_c/stem_ISO_8859_1_german.c +503 -0
  40. data/libstemmer_c/src_c/stem_ISO_8859_1_german.h +16 -0
  41. data/libstemmer_c/src_c/stem_ISO_8859_1_hungarian.c +1230 -0
  42. data/libstemmer_c/src_c/stem_ISO_8859_1_hungarian.h +16 -0
  43. data/libstemmer_c/src_c/stem_ISO_8859_1_italian.c +1065 -0
  44. data/libstemmer_c/src_c/stem_ISO_8859_1_italian.h +16 -0
  45. data/libstemmer_c/src_c/stem_ISO_8859_1_latin.c +443 -0
  46. data/libstemmer_c/src_c/stem_ISO_8859_1_latin.h +16 -0
  47. data/libstemmer_c/src_c/stem_ISO_8859_1_norwegian.c +297 -0
  48. data/libstemmer_c/src_c/stem_ISO_8859_1_norwegian.h +16 -0
  49. data/libstemmer_c/src_c/stem_ISO_8859_1_porter.c +749 -0
  50. data/libstemmer_c/src_c/stem_ISO_8859_1_porter.h +16 -0
  51. data/libstemmer_c/src_c/stem_ISO_8859_1_portuguese.c +1017 -0
  52. data/libstemmer_c/src_c/stem_ISO_8859_1_portuguese.h +16 -0
  53. data/libstemmer_c/src_c/stem_ISO_8859_1_spanish.c +1093 -0
  54. data/libstemmer_c/src_c/stem_ISO_8859_1_spanish.h +16 -0
  55. data/libstemmer_c/src_c/stem_ISO_8859_1_swedish.c +307 -0
  56. data/libstemmer_c/src_c/stem_ISO_8859_1_swedish.h +16 -0
  57. data/libstemmer_c/src_c/stem_ISO_8859_2_romanian.c +998 -0
  58. data/libstemmer_c/src_c/stem_ISO_8859_2_romanian.h +16 -0
  59. data/libstemmer_c/src_c/stem_KOI8_R_russian.c +700 -0
  60. data/libstemmer_c/src_c/stem_KOI8_R_russian.h +16 -0
  61. data/libstemmer_c/src_c/stem_UTF_8_danish.c +339 -0
  62. data/libstemmer_c/src_c/stem_UTF_8_danish.h +16 -0
  63. data/libstemmer_c/src_c/stem_UTF_8_dutch.c +634 -0
  64. data/libstemmer_c/src_c/stem_UTF_8_dutch.h +16 -0
  65. data/libstemmer_c/src_c/stem_UTF_8_english.c +1125 -0
  66. data/libstemmer_c/src_c/stem_UTF_8_english.h +16 -0
  67. data/libstemmer_c/src_c/stem_UTF_8_finnish.c +768 -0
  68. data/libstemmer_c/src_c/stem_UTF_8_finnish.h +16 -0
  69. data/libstemmer_c/src_c/stem_UTF_8_french.c +1256 -0
  70. data/libstemmer_c/src_c/stem_UTF_8_french.h +16 -0
  71. data/libstemmer_c/src_c/stem_UTF_8_german.c +509 -0
  72. data/libstemmer_c/src_c/stem_UTF_8_german.h +16 -0
  73. data/libstemmer_c/src_c/stem_UTF_8_hungarian.c +1234 -0
  74. data/libstemmer_c/src_c/stem_UTF_8_hungarian.h +16 -0
  75. data/libstemmer_c/src_c/stem_UTF_8_italian.c +1073 -0
  76. data/libstemmer_c/src_c/stem_UTF_8_italian.h +16 -0
  77. data/libstemmer_c/src_c/stem_UTF_8_latin.c +443 -0
  78. data/libstemmer_c/src_c/stem_UTF_8_latin.h +16 -0
  79. data/libstemmer_c/src_c/stem_UTF_8_norwegian.c +299 -0
  80. data/libstemmer_c/src_c/stem_UTF_8_norwegian.h +16 -0
  81. data/libstemmer_c/src_c/stem_UTF_8_porter.c +755 -0
  82. data/libstemmer_c/src_c/stem_UTF_8_porter.h +16 -0
  83. data/libstemmer_c/src_c/stem_UTF_8_portuguese.c +1023 -0
  84. data/libstemmer_c/src_c/stem_UTF_8_portuguese.h +16 -0
  85. data/libstemmer_c/src_c/stem_UTF_8_romanian.c +1004 -0
  86. data/libstemmer_c/src_c/stem_UTF_8_romanian.h +16 -0
  87. data/libstemmer_c/src_c/stem_UTF_8_russian.c +694 -0
  88. data/libstemmer_c/src_c/stem_UTF_8_russian.h +16 -0
  89. data/libstemmer_c/src_c/stem_UTF_8_spanish.c +1097 -0
  90. data/libstemmer_c/src_c/stem_UTF_8_spanish.h +16 -0
  91. data/libstemmer_c/src_c/stem_UTF_8_swedish.c +309 -0
  92. data/libstemmer_c/src_c/stem_UTF_8_swedish.h +16 -0
  93. data/libstemmer_c/src_c/stem_UTF_8_turkish.c +2205 -0
  94. data/libstemmer_c/src_c/stem_UTF_8_turkish.h +16 -0
  95. data/test/helper.rb +3 -0
  96. data/test/lingua/test_stemmer.rb +99 -0
  97. metadata +162 -0
@@ -0,0 +1,51 @@
1
+ # This file contains a list of stemmers to include in the distribution.
2
+ # The format is a set of space separated lines - on each line:
3
+ # First item is name of stemmer.
4
+ # Second item is comma separated list of character sets.
5
+ # Third item is comma separated list of names to refer to the stemmer by.
6
+ #
7
+ # Lines starting with a #, or blank lines, are ignored.
8
+
9
+ # List all the main algorithms for each language, in UTF-8, and also with
10
+ # the most commonly used encoding.
11
+
12
+ latin UTF_8,ISO_8859_1 latin
13
+ danish UTF_8,ISO_8859_1 danish,da,dan
14
+ dutch UTF_8,ISO_8859_1 dutch,nl,dut,nld
15
+ english UTF_8,ISO_8859_1 english,en,eng
16
+ finnish UTF_8,ISO_8859_1 finnish,fi,fin
17
+ french UTF_8,ISO_8859_1 french,fr,fre,fra
18
+ german UTF_8,ISO_8859_1 german,de,ger,deu
19
+ hungarian UTF_8,ISO_8859_1 hungarian,hu,hun
20
+ italian UTF_8,ISO_8859_1 italian,it,ita
21
+ norwegian UTF_8,ISO_8859_1 norwegian,no,nor
22
+ portuguese UTF_8,ISO_8859_1 portuguese,pt,por
23
+ romanian UTF_8,ISO_8859_2 romanian,ro,rum,ron
24
+ russian UTF_8,KOI8_R russian,ru,rus
25
+ spanish UTF_8,ISO_8859_1 spanish,es,esl,spa
26
+ swedish UTF_8,ISO_8859_1 swedish,sv,swe
27
+ turkish UTF_8 turkish,tr,tur
28
+
29
+ # Also include the traditional porter algorithm for english.
30
+ # The porter algorithm is included in the libstemmer distribution to assist
31
+ # with backwards compatibility, but for new systems the english algorithm
32
+ # should be used in preference.
33
+ porter UTF_8,ISO_8859_1 porter
34
+
35
+ # Some other stemmers in the snowball project are not included in the standard
36
+ # distribution. To compile a libstemmer with them in, add them to this list,
37
+ # and regenerate the distribution. (You will need a full source checkout for
38
+ # this.) They are included in the snowball website as curiosities, but are not
39
+ # intended for general use, and use of them is is not fully supported. These
40
+ # algorithms are:
41
+ #
42
+ # german2 - This is a slight modification of the german stemmer.
43
+ #german2 UTF_8,ISO_8859_1 german2
44
+ #
45
+ # kraaij_pohlmann - This is a different dutch stemmer.
46
+ #kraaij_pohlmann UTF_8,ISO_8859_1 kraaij_pohlmann
47
+ #
48
+ # lovins - This is an english stemmer, but fairly outdated, and
49
+ # only really applicable to a restricted type of input text
50
+ # (keywords in academic publications).
51
+ #lovins UTF_8,ISO_8859_1 lovins
@@ -0,0 +1,123 @@
1
+ /* libstemmer/modules_utf8.h: List of stemming modules.
2
+ *
3
+ * This file is generated by mkmodules.pl from a list of module names.
4
+ * Do not edit manually.
5
+ *
6
+ * Modules included by this file are: latin, danish, dutch, english, finnish, french,
7
+ * german, hungarian, italian, norwegian, porter, portuguese, romanian,
8
+ * russian, spanish, swedish, turkish
9
+ */
10
+
11
+ #include "../src_c/stem_UTF_8_latin.h"
12
+ #include "../src_c/stem_UTF_8_danish.h"
13
+ #include "../src_c/stem_UTF_8_dutch.h"
14
+ #include "../src_c/stem_UTF_8_english.h"
15
+ #include "../src_c/stem_UTF_8_finnish.h"
16
+ #include "../src_c/stem_UTF_8_french.h"
17
+ #include "../src_c/stem_UTF_8_german.h"
18
+ #include "../src_c/stem_UTF_8_hungarian.h"
19
+ #include "../src_c/stem_UTF_8_italian.h"
20
+ #include "../src_c/stem_UTF_8_norwegian.h"
21
+ #include "../src_c/stem_UTF_8_porter.h"
22
+ #include "../src_c/stem_UTF_8_portuguese.h"
23
+ #include "../src_c/stem_UTF_8_romanian.h"
24
+ #include "../src_c/stem_UTF_8_russian.h"
25
+ #include "../src_c/stem_UTF_8_spanish.h"
26
+ #include "../src_c/stem_UTF_8_swedish.h"
27
+ #include "../src_c/stem_UTF_8_turkish.h"
28
+
29
+ typedef enum {
30
+ ENC_UNKNOWN=0,
31
+ ENC_UTF_8
32
+ } stemmer_encoding_t;
33
+
34
+ struct stemmer_encoding {
35
+ const char * name;
36
+ stemmer_encoding_t enc;
37
+ };
38
+ static struct stemmer_encoding encodings[] = {
39
+ {"UTF_8", ENC_UTF_8},
40
+ {0,ENC_UNKNOWN}
41
+ };
42
+
43
+ struct stemmer_modules {
44
+ const char * name;
45
+ stemmer_encoding_t enc;
46
+ struct SN_env * (*create)(void);
47
+ void (*close)(struct SN_env *);
48
+ int (*stem)(struct SN_env *);
49
+ };
50
+ static struct stemmer_modules modules[] = {
51
+ {"latin", ENC_UTF_8, latin_UTF_8_create_env, latin_UTF_8_close_env, latin_UTF_8_stem},
52
+ {"da", ENC_UTF_8, danish_UTF_8_create_env, danish_UTF_8_close_env, danish_UTF_8_stem},
53
+ {"dan", ENC_UTF_8, danish_UTF_8_create_env, danish_UTF_8_close_env, danish_UTF_8_stem},
54
+ {"danish", ENC_UTF_8, danish_UTF_8_create_env, danish_UTF_8_close_env, danish_UTF_8_stem},
55
+ {"de", ENC_UTF_8, german_UTF_8_create_env, german_UTF_8_close_env, german_UTF_8_stem},
56
+ {"deu", ENC_UTF_8, german_UTF_8_create_env, german_UTF_8_close_env, german_UTF_8_stem},
57
+ {"dut", ENC_UTF_8, dutch_UTF_8_create_env, dutch_UTF_8_close_env, dutch_UTF_8_stem},
58
+ {"dutch", ENC_UTF_8, dutch_UTF_8_create_env, dutch_UTF_8_close_env, dutch_UTF_8_stem},
59
+ {"en", ENC_UTF_8, english_UTF_8_create_env, english_UTF_8_close_env, english_UTF_8_stem},
60
+ {"eng", ENC_UTF_8, english_UTF_8_create_env, english_UTF_8_close_env, english_UTF_8_stem},
61
+ {"english", ENC_UTF_8, english_UTF_8_create_env, english_UTF_8_close_env, english_UTF_8_stem},
62
+ {"es", ENC_UTF_8, spanish_UTF_8_create_env, spanish_UTF_8_close_env, spanish_UTF_8_stem},
63
+ {"esl", ENC_UTF_8, spanish_UTF_8_create_env, spanish_UTF_8_close_env, spanish_UTF_8_stem},
64
+ {"fi", ENC_UTF_8, finnish_UTF_8_create_env, finnish_UTF_8_close_env, finnish_UTF_8_stem},
65
+ {"fin", ENC_UTF_8, finnish_UTF_8_create_env, finnish_UTF_8_close_env, finnish_UTF_8_stem},
66
+ {"finnish", ENC_UTF_8, finnish_UTF_8_create_env, finnish_UTF_8_close_env, finnish_UTF_8_stem},
67
+ {"fr", ENC_UTF_8, french_UTF_8_create_env, french_UTF_8_close_env, french_UTF_8_stem},
68
+ {"fra", ENC_UTF_8, french_UTF_8_create_env, french_UTF_8_close_env, french_UTF_8_stem},
69
+ {"fre", ENC_UTF_8, french_UTF_8_create_env, french_UTF_8_close_env, french_UTF_8_stem},
70
+ {"french", ENC_UTF_8, french_UTF_8_create_env, french_UTF_8_close_env, french_UTF_8_stem},
71
+ {"ger", ENC_UTF_8, german_UTF_8_create_env, german_UTF_8_close_env, german_UTF_8_stem},
72
+ {"german", ENC_UTF_8, german_UTF_8_create_env, german_UTF_8_close_env, german_UTF_8_stem},
73
+ {"hu", ENC_UTF_8, hungarian_UTF_8_create_env, hungarian_UTF_8_close_env, hungarian_UTF_8_stem},
74
+ {"hun", ENC_UTF_8, hungarian_UTF_8_create_env, hungarian_UTF_8_close_env, hungarian_UTF_8_stem},
75
+ {"hungarian", ENC_UTF_8, hungarian_UTF_8_create_env, hungarian_UTF_8_close_env, hungarian_UTF_8_stem},
76
+ {"it", ENC_UTF_8, italian_UTF_8_create_env, italian_UTF_8_close_env, italian_UTF_8_stem},
77
+ {"ita", ENC_UTF_8, italian_UTF_8_create_env, italian_UTF_8_close_env, italian_UTF_8_stem},
78
+ {"italian", ENC_UTF_8, italian_UTF_8_create_env, italian_UTF_8_close_env, italian_UTF_8_stem},
79
+ {"nl", ENC_UTF_8, dutch_UTF_8_create_env, dutch_UTF_8_close_env, dutch_UTF_8_stem},
80
+ {"nld", ENC_UTF_8, dutch_UTF_8_create_env, dutch_UTF_8_close_env, dutch_UTF_8_stem},
81
+ {"no", ENC_UTF_8, norwegian_UTF_8_create_env, norwegian_UTF_8_close_env, norwegian_UTF_8_stem},
82
+ {"nor", ENC_UTF_8, norwegian_UTF_8_create_env, norwegian_UTF_8_close_env, norwegian_UTF_8_stem},
83
+ {"norwegian", ENC_UTF_8, norwegian_UTF_8_create_env, norwegian_UTF_8_close_env, norwegian_UTF_8_stem},
84
+ {"por", ENC_UTF_8, portuguese_UTF_8_create_env, portuguese_UTF_8_close_env, portuguese_UTF_8_stem},
85
+ {"porter", ENC_UTF_8, porter_UTF_8_create_env, porter_UTF_8_close_env, porter_UTF_8_stem},
86
+ {"portuguese", ENC_UTF_8, portuguese_UTF_8_create_env, portuguese_UTF_8_close_env, portuguese_UTF_8_stem},
87
+ {"pt", ENC_UTF_8, portuguese_UTF_8_create_env, portuguese_UTF_8_close_env, portuguese_UTF_8_stem},
88
+ {"ro", ENC_UTF_8, romanian_UTF_8_create_env, romanian_UTF_8_close_env, romanian_UTF_8_stem},
89
+ {"romanian", ENC_UTF_8, romanian_UTF_8_create_env, romanian_UTF_8_close_env, romanian_UTF_8_stem},
90
+ {"ron", ENC_UTF_8, romanian_UTF_8_create_env, romanian_UTF_8_close_env, romanian_UTF_8_stem},
91
+ {"ru", ENC_UTF_8, russian_UTF_8_create_env, russian_UTF_8_close_env, russian_UTF_8_stem},
92
+ {"rum", ENC_UTF_8, romanian_UTF_8_create_env, romanian_UTF_8_close_env, romanian_UTF_8_stem},
93
+ {"rus", ENC_UTF_8, russian_UTF_8_create_env, russian_UTF_8_close_env, russian_UTF_8_stem},
94
+ {"russian", ENC_UTF_8, russian_UTF_8_create_env, russian_UTF_8_close_env, russian_UTF_8_stem},
95
+ {"spa", ENC_UTF_8, spanish_UTF_8_create_env, spanish_UTF_8_close_env, spanish_UTF_8_stem},
96
+ {"spanish", ENC_UTF_8, spanish_UTF_8_create_env, spanish_UTF_8_close_env, spanish_UTF_8_stem},
97
+ {"sv", ENC_UTF_8, swedish_UTF_8_create_env, swedish_UTF_8_close_env, swedish_UTF_8_stem},
98
+ {"swe", ENC_UTF_8, swedish_UTF_8_create_env, swedish_UTF_8_close_env, swedish_UTF_8_stem},
99
+ {"swedish", ENC_UTF_8, swedish_UTF_8_create_env, swedish_UTF_8_close_env, swedish_UTF_8_stem},
100
+ {"tr", ENC_UTF_8, turkish_UTF_8_create_env, turkish_UTF_8_close_env, turkish_UTF_8_stem},
101
+ {"tur", ENC_UTF_8, turkish_UTF_8_create_env, turkish_UTF_8_close_env, turkish_UTF_8_stem},
102
+ {"turkish", ENC_UTF_8, turkish_UTF_8_create_env, turkish_UTF_8_close_env, turkish_UTF_8_stem},
103
+ {0,ENC_UNKNOWN,0,0,0}
104
+ };
105
+ static const char * algorithm_names[] = {
106
+ "danish",
107
+ "dutch",
108
+ "english",
109
+ "finnish",
110
+ "french",
111
+ "german",
112
+ "hungarian",
113
+ "italian",
114
+ "norwegian",
115
+ "porter",
116
+ "portuguese",
117
+ "romanian",
118
+ "russian",
119
+ "spanish",
120
+ "swedish",
121
+ "turkish",
122
+ 0
123
+ };
@@ -0,0 +1,50 @@
1
+ # This file contains a list of stemmers to include in the distribution.
2
+ # The format is a set of space separated lines - on each line:
3
+ # First item is name of stemmer.
4
+ # Second item is comma separated list of character sets.
5
+ # Third item is comma separated list of names to refer to the stemmer by.
6
+ #
7
+ # Lines starting with a #, or blank lines, are ignored.
8
+
9
+ # List all the main algorithms for each language, in UTF-8.
10
+
11
+ latin UTF_8 latin
12
+ danish UTF_8 danish,da,dan
13
+ dutch UTF_8 dutch,nl,dut,nld
14
+ english UTF_8 english,en,eng
15
+ finnish UTF_8 finnish,fi,fin
16
+ french UTF_8 french,fr,fre,fra
17
+ german UTF_8 german,de,ger,deu
18
+ hungarian UTF_8 hungarian,hu,hun
19
+ italian UTF_8 italian,it,ita
20
+ norwegian UTF_8 norwegian,no,nor
21
+ portuguese UTF_8 portuguese,pt,por
22
+ romanian UTF_8 romanian,ro,rum,ron
23
+ russian UTF_8 russian,ru,rus
24
+ spanish UTF_8 spanish,es,esl,spa
25
+ swedish UTF_8 swedish,sv,swe
26
+ turkish UTF_8 turkish,tr,tur
27
+
28
+ # Also include the traditional porter algorithm for english.
29
+ # The porter algorithm is included in the libstemmer distribution to assist
30
+ # with backwards compatibility, but for new systems the english algorithm
31
+ # should be used in preference.
32
+ porter UTF_8 porter
33
+
34
+ # Some other stemmers in the snowball project are not included in the standard
35
+ # distribution. To compile a libstemmer with them in, add them to this list,
36
+ # and regenerate the distribution. (You will need a full source checkout for
37
+ # this.) They are included in the snowball website as curiosities, but are not
38
+ # intended for general use, and use of them is is not fully supported. These
39
+ # algorithms are:
40
+ #
41
+ # german2 - This is a slight modification of the german stemmer.
42
+ #german2 UTF_8 german2
43
+ #
44
+ # kraaij_pohlmann - This is a different dutch stemmer.
45
+ #kraaij_pohlmann UTF_8 kraaij_pohlmann
46
+ #
47
+ # lovins - This is an english stemmer, but fairly outdated, and
48
+ # only really applicable to a restricted type of input text
49
+ # (keywords in academic publications).
50
+ #lovins UTF_8 lovins
@@ -0,0 +1,86 @@
1
+ # libstemmer/mkinc.mak: List of stemming module source files
2
+ #
3
+ # This file is generated by mkmodules.pl from a list of module names.
4
+ # Do not edit manually.
5
+ #
6
+ # Modules included by this file are: latin, danish, dutch, english, finnish, french,
7
+ # german, hungarian, italian, norwegian, porter, portuguese, romanian,
8
+ # russian, spanish, swedish, turkish
9
+
10
+ snowball_sources= \
11
+ src_c/stem_ISO_8859_1_latin.c \
12
+ src_c/stem_UTF_8_latin.c \
13
+ src_c/stem_ISO_8859_1_danish.c \
14
+ src_c/stem_UTF_8_danish.c \
15
+ src_c/stem_ISO_8859_1_dutch.c \
16
+ src_c/stem_UTF_8_dutch.c \
17
+ src_c/stem_ISO_8859_1_english.c \
18
+ src_c/stem_UTF_8_english.c \
19
+ src_c/stem_ISO_8859_1_finnish.c \
20
+ src_c/stem_UTF_8_finnish.c \
21
+ src_c/stem_ISO_8859_1_french.c \
22
+ src_c/stem_UTF_8_french.c \
23
+ src_c/stem_ISO_8859_1_german.c \
24
+ src_c/stem_UTF_8_german.c \
25
+ src_c/stem_ISO_8859_1_hungarian.c \
26
+ src_c/stem_UTF_8_hungarian.c \
27
+ src_c/stem_ISO_8859_1_italian.c \
28
+ src_c/stem_UTF_8_italian.c \
29
+ src_c/stem_ISO_8859_1_norwegian.c \
30
+ src_c/stem_UTF_8_norwegian.c \
31
+ src_c/stem_ISO_8859_1_porter.c \
32
+ src_c/stem_UTF_8_porter.c \
33
+ src_c/stem_ISO_8859_1_portuguese.c \
34
+ src_c/stem_UTF_8_portuguese.c \
35
+ src_c/stem_ISO_8859_2_romanian.c \
36
+ src_c/stem_UTF_8_romanian.c \
37
+ src_c/stem_KOI8_R_russian.c \
38
+ src_c/stem_UTF_8_russian.c \
39
+ src_c/stem_ISO_8859_1_spanish.c \
40
+ src_c/stem_UTF_8_spanish.c \
41
+ src_c/stem_ISO_8859_1_swedish.c \
42
+ src_c/stem_UTF_8_swedish.c \
43
+ src_c/stem_UTF_8_turkish.c \
44
+ runtime/api.c \
45
+ runtime/utilities.c \
46
+ libstemmer/libstemmer.c
47
+
48
+ snowball_headers= \
49
+ src_c/stem_ISO_8859_1_latin.h \
50
+ src_c/stem_UTF_8_latin.h \
51
+ src_c/stem_ISO_8859_1_danish.h \
52
+ src_c/stem_UTF_8_danish.h \
53
+ src_c/stem_ISO_8859_1_dutch.h \
54
+ src_c/stem_UTF_8_dutch.h \
55
+ src_c/stem_ISO_8859_1_english.h \
56
+ src_c/stem_UTF_8_english.h \
57
+ src_c/stem_ISO_8859_1_finnish.h \
58
+ src_c/stem_UTF_8_finnish.h \
59
+ src_c/stem_ISO_8859_1_french.h \
60
+ src_c/stem_UTF_8_french.h \
61
+ src_c/stem_ISO_8859_1_german.h \
62
+ src_c/stem_UTF_8_german.h \
63
+ src_c/stem_ISO_8859_1_hungarian.h \
64
+ src_c/stem_UTF_8_hungarian.h \
65
+ src_c/stem_ISO_8859_1_italian.h \
66
+ src_c/stem_UTF_8_italian.h \
67
+ src_c/stem_ISO_8859_1_norwegian.h \
68
+ src_c/stem_UTF_8_norwegian.h \
69
+ src_c/stem_ISO_8859_1_porter.h \
70
+ src_c/stem_UTF_8_porter.h \
71
+ src_c/stem_ISO_8859_1_portuguese.h \
72
+ src_c/stem_UTF_8_portuguese.h \
73
+ src_c/stem_ISO_8859_2_romanian.h \
74
+ src_c/stem_UTF_8_romanian.h \
75
+ src_c/stem_KOI8_R_russian.h \
76
+ src_c/stem_UTF_8_russian.h \
77
+ src_c/stem_ISO_8859_1_spanish.h \
78
+ src_c/stem_UTF_8_spanish.h \
79
+ src_c/stem_ISO_8859_1_swedish.h \
80
+ src_c/stem_UTF_8_swedish.h \
81
+ src_c/stem_UTF_8_turkish.h \
82
+ include/libstemmer.h \
83
+ libstemmer/modules.h \
84
+ runtime/api.h \
85
+ runtime/header.h
86
+
@@ -0,0 +1,54 @@
1
+ # libstemmer/mkinc_utf8.mak: List of stemming module source files
2
+ #
3
+ # This file is generated by mkmodules.pl from a list of module names.
4
+ # Do not edit manually.
5
+ #
6
+ # Modules included by this file are: danish, dutch, english, finnish, french,
7
+ # german, hungarian, italian, norwegian, porter, portuguese, romanian,
8
+ # russian, spanish, swedish, turkish
9
+
10
+ snowball_sources= \
11
+ src_c/stem_UTF_8_latin.c \
12
+ src_c/stem_UTF_8_danish.c \
13
+ src_c/stem_UTF_8_dutch.c \
14
+ src_c/stem_UTF_8_english.c \
15
+ src_c/stem_UTF_8_finnish.c \
16
+ src_c/stem_UTF_8_french.c \
17
+ src_c/stem_UTF_8_german.c \
18
+ src_c/stem_UTF_8_hungarian.c \
19
+ src_c/stem_UTF_8_italian.c \
20
+ src_c/stem_UTF_8_norwegian.c \
21
+ src_c/stem_UTF_8_porter.c \
22
+ src_c/stem_UTF_8_portuguese.c \
23
+ src_c/stem_UTF_8_romanian.c \
24
+ src_c/stem_UTF_8_russian.c \
25
+ src_c/stem_UTF_8_spanish.c \
26
+ src_c/stem_UTF_8_swedish.c \
27
+ src_c/stem_UTF_8_turkish.c \
28
+ runtime/api.c \
29
+ runtime/utilities.c \
30
+ libstemmer/libstemmer_utf8.c
31
+
32
+ snowball_headers= \
33
+ src_c/stem_UTF_8_latin.h \
34
+ src_c/stem_UTF_8_danish.h \
35
+ src_c/stem_UTF_8_dutch.h \
36
+ src_c/stem_UTF_8_english.h \
37
+ src_c/stem_UTF_8_finnish.h \
38
+ src_c/stem_UTF_8_french.h \
39
+ src_c/stem_UTF_8_german.h \
40
+ src_c/stem_UTF_8_hungarian.h \
41
+ src_c/stem_UTF_8_italian.h \
42
+ src_c/stem_UTF_8_norwegian.h \
43
+ src_c/stem_UTF_8_porter.h \
44
+ src_c/stem_UTF_8_portuguese.h \
45
+ src_c/stem_UTF_8_romanian.h \
46
+ src_c/stem_UTF_8_russian.h \
47
+ src_c/stem_UTF_8_spanish.h \
48
+ src_c/stem_UTF_8_swedish.h \
49
+ src_c/stem_UTF_8_turkish.h \
50
+ include/libstemmer.h \
51
+ libstemmer/modules_utf8.h \
52
+ runtime/api.h \
53
+ runtime/header.h
54
+
@@ -0,0 +1,66 @@
1
+
2
+ #include <stdlib.h> /* for calloc, free */
3
+ #include "header.h"
4
+
5
+ extern struct SN_env * SN_create_env(int S_size, int I_size, int B_size)
6
+ {
7
+ struct SN_env * z = (struct SN_env *) calloc(1, sizeof(struct SN_env));
8
+ if (z == NULL) return NULL;
9
+ z->p = create_s();
10
+ if (z->p == NULL) goto error;
11
+ if (S_size)
12
+ {
13
+ int i;
14
+ z->S = (symbol * *) calloc(S_size, sizeof(symbol *));
15
+ if (z->S == NULL) goto error;
16
+
17
+ for (i = 0; i < S_size; i++)
18
+ {
19
+ z->S[i] = create_s();
20
+ if (z->S[i] == NULL) goto error;
21
+ }
22
+ }
23
+
24
+ if (I_size)
25
+ {
26
+ z->I = (int *) calloc(I_size, sizeof(int));
27
+ if (z->I == NULL) goto error;
28
+ }
29
+
30
+ if (B_size)
31
+ {
32
+ z->B = (unsigned char *) calloc(B_size, sizeof(unsigned char));
33
+ if (z->B == NULL) goto error;
34
+ }
35
+
36
+ return z;
37
+ error:
38
+ SN_close_env(z, S_size);
39
+ return NULL;
40
+ }
41
+
42
+ extern void SN_close_env(struct SN_env * z, int S_size)
43
+ {
44
+ if (z == NULL) return;
45
+ if (S_size)
46
+ {
47
+ int i;
48
+ for (i = 0; i < S_size; i++)
49
+ {
50
+ lose_s(z->S[i]);
51
+ }
52
+ free(z->S);
53
+ }
54
+ free(z->I);
55
+ free(z->B);
56
+ if (z->p) lose_s(z->p);
57
+ free(z);
58
+ }
59
+
60
+ extern int SN_set_current(struct SN_env * z, int size, const symbol * s)
61
+ {
62
+ int err = replace_s(z, 0, z->l, size, s, NULL);
63
+ z->c = 0;
64
+ return err;
65
+ }
66
+
@@ -0,0 +1,26 @@
1
+
2
+ typedef unsigned char symbol;
3
+
4
+ /* Or replace 'char' above with 'short' for 16 bit characters.
5
+
6
+ More precisely, replace 'char' with whatever type guarantees the
7
+ character width you need. Note however that sizeof(symbol) should divide
8
+ HEAD, defined in header.h as 2*sizeof(int), without remainder, otherwise
9
+ there is an alignment problem. In the unlikely event of a problem here,
10
+ consult Martin Porter.
11
+
12
+ */
13
+
14
+ struct SN_env {
15
+ symbol * p;
16
+ int c; int l; int lb; int bra; int ket;
17
+ symbol * * S;
18
+ int * I;
19
+ unsigned char * B;
20
+ };
21
+
22
+ extern struct SN_env * SN_create_env(int S_size, int I_size, int B_size);
23
+ extern void SN_close_env(struct SN_env * z, int S_size);
24
+
25
+ extern int SN_set_current(struct SN_env * z, int size, const symbol * s);
26
+
@@ -0,0 +1,58 @@
1
+
2
+ #include <limits.h>
3
+
4
+ #include "api.h"
5
+
6
+ #define MAXINT INT_MAX
7
+ #define MININT INT_MIN
8
+
9
+ #define HEAD 2*sizeof(int)
10
+
11
+ #define SIZE(p) ((int *)(p))[-1]
12
+ #define SET_SIZE(p, n) ((int *)(p))[-1] = n
13
+ #define CAPACITY(p) ((int *)(p))[-2]
14
+
15
+ struct among
16
+ { int s_size; /* number of chars in string */
17
+ const symbol * s; /* search string */
18
+ int substring_i;/* index to longest matching substring */
19
+ int result; /* result of the lookup */
20
+ int (* function)(struct SN_env *);
21
+ };
22
+
23
+ extern symbol * create_s(void);
24
+ extern void lose_s(symbol * p);
25
+
26
+ extern int skip_utf8(const symbol * p, int c, int lb, int l, int n);
27
+
28
+ extern int in_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
29
+ extern int in_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
30
+ extern int out_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
31
+ extern int out_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
32
+
33
+ extern int in_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
34
+ extern int in_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
35
+ extern int out_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
36
+ extern int out_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
37
+
38
+ extern int eq_s(struct SN_env * z, int s_size, const symbol * s);
39
+ extern int eq_s_b(struct SN_env * z, int s_size, const symbol * s);
40
+ extern int eq_v(struct SN_env * z, const symbol * p);
41
+ extern int eq_v_b(struct SN_env * z, const symbol * p);
42
+
43
+ extern int find_among(struct SN_env * z, const struct among * v, int v_size);
44
+ extern int find_among_b(struct SN_env * z, const struct among * v, int v_size);
45
+
46
+ extern int replace_s(struct SN_env * z, int c_bra, int c_ket, int s_size, const symbol * s, int * adjustment);
47
+ extern int slice_from_s(struct SN_env * z, int s_size, const symbol * s);
48
+ extern int slice_from_v(struct SN_env * z, const symbol * p);
49
+ extern int slice_del(struct SN_env * z);
50
+
51
+ extern int insert_s(struct SN_env * z, int bra, int ket, int s_size, const symbol * s);
52
+ extern int insert_v(struct SN_env * z, int bra, int ket, const symbol * p);
53
+
54
+ extern symbol * slice_to(struct SN_env * z, symbol * p);
55
+ extern symbol * assign_to(struct SN_env * z, symbol * p);
56
+
57
+ extern void debug(struct SN_env * z, int number, int line_count);
58
+