ruby-stemmer 0.9.3-x86-mswin32-60

Sign up to get free protection for your applications and to get access to all the features.
Files changed (97) hide show
  1. data/MIT-LICENSE +21 -0
  2. data/README.rdoc +113 -0
  3. data/Rakefile +70 -0
  4. data/TODO +0 -0
  5. data/VERSION +1 -0
  6. data/ext/lingua/extconf.rb +40 -0
  7. data/ext/lingua/stemmer.c +115 -0
  8. data/lib/lingua/1.8/stemmer_native.so +0 -0
  9. data/lib/lingua/1.9/stemmer_native.so +0 -0
  10. data/lib/lingua/stemmer.rb +60 -0
  11. data/libstemmer_c/MANIFEST +72 -0
  12. data/libstemmer_c/Makefile +9 -0
  13. data/libstemmer_c/Makefile.windows +15 -0
  14. data/libstemmer_c/README +125 -0
  15. data/libstemmer_c/examples/stemwords.c +209 -0
  16. data/libstemmer_c/include/libstemmer.h +79 -0
  17. data/libstemmer_c/libstemmer/libstemmer.c +93 -0
  18. data/libstemmer_c/libstemmer/libstemmer_utf8.c +93 -0
  19. data/libstemmer_c/libstemmer/modules.h +195 -0
  20. data/libstemmer_c/libstemmer/modules.txt +51 -0
  21. data/libstemmer_c/libstemmer/modules_utf8.h +123 -0
  22. data/libstemmer_c/libstemmer/modules_utf8.txt +50 -0
  23. data/libstemmer_c/mkinc.mak +86 -0
  24. data/libstemmer_c/mkinc_utf8.mak +54 -0
  25. data/libstemmer_c/runtime/api.c +66 -0
  26. data/libstemmer_c/runtime/api.h +26 -0
  27. data/libstemmer_c/runtime/header.h +58 -0
  28. data/libstemmer_c/runtime/utilities.c +478 -0
  29. data/libstemmer_c/src_c/stem_ISO_8859_1_danish.c +337 -0
  30. data/libstemmer_c/src_c/stem_ISO_8859_1_danish.h +16 -0
  31. data/libstemmer_c/src_c/stem_ISO_8859_1_dutch.c +624 -0
  32. data/libstemmer_c/src_c/stem_ISO_8859_1_dutch.h +16 -0
  33. data/libstemmer_c/src_c/stem_ISO_8859_1_english.c +1117 -0
  34. data/libstemmer_c/src_c/stem_ISO_8859_1_english.h +16 -0
  35. data/libstemmer_c/src_c/stem_ISO_8859_1_finnish.c +762 -0
  36. data/libstemmer_c/src_c/stem_ISO_8859_1_finnish.h +16 -0
  37. data/libstemmer_c/src_c/stem_ISO_8859_1_french.c +1246 -0
  38. data/libstemmer_c/src_c/stem_ISO_8859_1_french.h +16 -0
  39. data/libstemmer_c/src_c/stem_ISO_8859_1_german.c +503 -0
  40. data/libstemmer_c/src_c/stem_ISO_8859_1_german.h +16 -0
  41. data/libstemmer_c/src_c/stem_ISO_8859_1_hungarian.c +1230 -0
  42. data/libstemmer_c/src_c/stem_ISO_8859_1_hungarian.h +16 -0
  43. data/libstemmer_c/src_c/stem_ISO_8859_1_italian.c +1065 -0
  44. data/libstemmer_c/src_c/stem_ISO_8859_1_italian.h +16 -0
  45. data/libstemmer_c/src_c/stem_ISO_8859_1_latin.c +443 -0
  46. data/libstemmer_c/src_c/stem_ISO_8859_1_latin.h +16 -0
  47. data/libstemmer_c/src_c/stem_ISO_8859_1_norwegian.c +297 -0
  48. data/libstemmer_c/src_c/stem_ISO_8859_1_norwegian.h +16 -0
  49. data/libstemmer_c/src_c/stem_ISO_8859_1_porter.c +749 -0
  50. data/libstemmer_c/src_c/stem_ISO_8859_1_porter.h +16 -0
  51. data/libstemmer_c/src_c/stem_ISO_8859_1_portuguese.c +1017 -0
  52. data/libstemmer_c/src_c/stem_ISO_8859_1_portuguese.h +16 -0
  53. data/libstemmer_c/src_c/stem_ISO_8859_1_spanish.c +1093 -0
  54. data/libstemmer_c/src_c/stem_ISO_8859_1_spanish.h +16 -0
  55. data/libstemmer_c/src_c/stem_ISO_8859_1_swedish.c +307 -0
  56. data/libstemmer_c/src_c/stem_ISO_8859_1_swedish.h +16 -0
  57. data/libstemmer_c/src_c/stem_ISO_8859_2_romanian.c +998 -0
  58. data/libstemmer_c/src_c/stem_ISO_8859_2_romanian.h +16 -0
  59. data/libstemmer_c/src_c/stem_KOI8_R_russian.c +700 -0
  60. data/libstemmer_c/src_c/stem_KOI8_R_russian.h +16 -0
  61. data/libstemmer_c/src_c/stem_UTF_8_danish.c +339 -0
  62. data/libstemmer_c/src_c/stem_UTF_8_danish.h +16 -0
  63. data/libstemmer_c/src_c/stem_UTF_8_dutch.c +634 -0
  64. data/libstemmer_c/src_c/stem_UTF_8_dutch.h +16 -0
  65. data/libstemmer_c/src_c/stem_UTF_8_english.c +1125 -0
  66. data/libstemmer_c/src_c/stem_UTF_8_english.h +16 -0
  67. data/libstemmer_c/src_c/stem_UTF_8_finnish.c +768 -0
  68. data/libstemmer_c/src_c/stem_UTF_8_finnish.h +16 -0
  69. data/libstemmer_c/src_c/stem_UTF_8_french.c +1256 -0
  70. data/libstemmer_c/src_c/stem_UTF_8_french.h +16 -0
  71. data/libstemmer_c/src_c/stem_UTF_8_german.c +509 -0
  72. data/libstemmer_c/src_c/stem_UTF_8_german.h +16 -0
  73. data/libstemmer_c/src_c/stem_UTF_8_hungarian.c +1234 -0
  74. data/libstemmer_c/src_c/stem_UTF_8_hungarian.h +16 -0
  75. data/libstemmer_c/src_c/stem_UTF_8_italian.c +1073 -0
  76. data/libstemmer_c/src_c/stem_UTF_8_italian.h +16 -0
  77. data/libstemmer_c/src_c/stem_UTF_8_latin.c +443 -0
  78. data/libstemmer_c/src_c/stem_UTF_8_latin.h +16 -0
  79. data/libstemmer_c/src_c/stem_UTF_8_norwegian.c +299 -0
  80. data/libstemmer_c/src_c/stem_UTF_8_norwegian.h +16 -0
  81. data/libstemmer_c/src_c/stem_UTF_8_porter.c +755 -0
  82. data/libstemmer_c/src_c/stem_UTF_8_porter.h +16 -0
  83. data/libstemmer_c/src_c/stem_UTF_8_portuguese.c +1023 -0
  84. data/libstemmer_c/src_c/stem_UTF_8_portuguese.h +16 -0
  85. data/libstemmer_c/src_c/stem_UTF_8_romanian.c +1004 -0
  86. data/libstemmer_c/src_c/stem_UTF_8_romanian.h +16 -0
  87. data/libstemmer_c/src_c/stem_UTF_8_russian.c +694 -0
  88. data/libstemmer_c/src_c/stem_UTF_8_russian.h +16 -0
  89. data/libstemmer_c/src_c/stem_UTF_8_spanish.c +1097 -0
  90. data/libstemmer_c/src_c/stem_UTF_8_spanish.h +16 -0
  91. data/libstemmer_c/src_c/stem_UTF_8_swedish.c +309 -0
  92. data/libstemmer_c/src_c/stem_UTF_8_swedish.h +16 -0
  93. data/libstemmer_c/src_c/stem_UTF_8_turkish.c +2205 -0
  94. data/libstemmer_c/src_c/stem_UTF_8_turkish.h +16 -0
  95. data/test/helper.rb +3 -0
  96. data/test/lingua/test_stemmer.rb +99 -0
  97. metadata +162 -0
@@ -0,0 +1,209 @@
1
+ /* This is a simple program which uses libstemmer to provide a command
2
+ * line interface for stemming using any of the algorithms provided.
3
+ */
4
+
5
+ #include <stdio.h>
6
+ #include <stdlib.h> /* for malloc, free */
7
+ #include <string.h> /* for memmove */
8
+ #include <ctype.h> /* for isupper, tolower */
9
+
10
+ #include "libstemmer.h"
11
+
12
+ const char * progname;
13
+ static int pretty = 1;
14
+
15
+ static void
16
+ stem_file(struct sb_stemmer * stemmer, FILE * f_in, FILE * f_out)
17
+ {
18
+ #define INC 10
19
+ int lim = INC;
20
+ sb_symbol * b = (sb_symbol *) malloc(lim * sizeof(sb_symbol));
21
+
22
+ while(1) {
23
+ int ch = getc(f_in);
24
+ if (ch == EOF) {
25
+ free(b); return;
26
+ }
27
+ {
28
+ int i = 0;
29
+ int inlen = 0;
30
+ while(1) {
31
+ if (ch == '\n' || ch == EOF) break;
32
+ if (i == lim) {
33
+ sb_symbol * newb;
34
+ newb = (sb_symbol *)
35
+ realloc(b, (lim + INC) * sizeof(sb_symbol));
36
+ if (newb == 0) goto error;
37
+ b = newb;
38
+ lim = lim + INC;
39
+ }
40
+ /* Update count of utf-8 characters. */
41
+ if (ch < 0x80 || ch > 0xBF) inlen += 1;
42
+ /* force lower case: */
43
+ if (isupper(ch)) ch = tolower(ch);
44
+
45
+ b[i] = ch;
46
+ i++;
47
+ ch = getc(f_in);
48
+ }
49
+
50
+ {
51
+ const sb_symbol * stemmed = sb_stemmer_stem(stemmer, b, i);
52
+ if (stemmed == NULL)
53
+ {
54
+ fprintf(stderr, "Out of memory");
55
+ exit(1);
56
+ }
57
+ else
58
+ {
59
+ if (pretty == 1) {
60
+ fwrite(b, i, 1, f_out);
61
+ fputs(" -> ", f_out);
62
+ } else if (pretty == 2) {
63
+ fwrite(b, i, 1, f_out);
64
+ if (sb_stemmer_length(stemmer) > 0) {
65
+ int j;
66
+ if (inlen < 30) {
67
+ for (j = 30 - inlen; j > 0; j--)
68
+ fputs(" ", f_out);
69
+ } else {
70
+ fputs("\n", f_out);
71
+ for (j = 30; j > 0; j--)
72
+ fputs(" ", f_out);
73
+ }
74
+ }
75
+ }
76
+
77
+ fputs((char *)stemmed, f_out);
78
+ putc('\n', f_out);
79
+ }
80
+ }
81
+ }
82
+ }
83
+ error:
84
+ if (b != 0) free(b);
85
+ return;
86
+ }
87
+
88
+ /** Display the command line syntax, and then exit.
89
+ * @param n The value to exit with.
90
+ */
91
+ static void
92
+ usage(int n)
93
+ {
94
+ printf("usage: %s [-l <language>] [-i <input file>] [-o <output file>] [-c <character encoding>] [-p[2]] [-h]\n"
95
+ "\n"
96
+ "The input file consists of a list of words to be stemmed, one per\n"
97
+ "line. Words should be in lower case, but (for English) A-Z letters\n"
98
+ "are mapped to their a-z equivalents anyway. If omitted, stdin is\n"
99
+ "used.\n"
100
+ "\n"
101
+ "If -c is given, the argument is the character encoding of the input\n"
102
+ "and output files. If it is omitted, the UTF-8 encoding is used.\n"
103
+ "\n"
104
+ "If -p is given the output file consists of each word of the input\n"
105
+ "file followed by \"->\" followed by its stemmed equivalent.\n"
106
+ "If -p2 is given the output file is a two column layout containing\n"
107
+ "the input words in the first column and the stemmed eqivalents in\n"
108
+ "the second column.\n"
109
+ "Otherwise, the output file consists of the stemmed words, one per\n"
110
+ "line.\n"
111
+ "\n"
112
+ "-h displays this help\n",
113
+ progname);
114
+ exit(n);
115
+ }
116
+
117
+ int
118
+ main(int argc, char * argv[])
119
+ {
120
+ char * in = 0;
121
+ char * out = 0;
122
+ FILE * f_in;
123
+ FILE * f_out;
124
+ struct sb_stemmer * stemmer;
125
+
126
+ char * language = "english";
127
+ char * charenc = NULL;
128
+
129
+ char * s;
130
+ int i = 1;
131
+ pretty = 0;
132
+
133
+ progname = argv[0];
134
+
135
+ while(i < argc) {
136
+ s = argv[i++];
137
+ if (s[0] == '-') {
138
+ if (strcmp(s, "-o") == 0) {
139
+ if (i >= argc) {
140
+ fprintf(stderr, "%s requires an argument\n", s);
141
+ exit(1);
142
+ }
143
+ out = argv[i++];
144
+ } else if (strcmp(s, "-i") == 0) {
145
+ if (i >= argc) {
146
+ fprintf(stderr, "%s requires an argument\n", s);
147
+ exit(1);
148
+ }
149
+ in = argv[i++];
150
+ } else if (strcmp(s, "-l") == 0) {
151
+ if (i >= argc) {
152
+ fprintf(stderr, "%s requires an argument\n", s);
153
+ exit(1);
154
+ }
155
+ language = argv[i++];
156
+ } else if (strcmp(s, "-c") == 0) {
157
+ if (i >= argc) {
158
+ fprintf(stderr, "%s requires an argument\n", s);
159
+ exit(1);
160
+ }
161
+ charenc = argv[i++];
162
+ } else if (strcmp(s, "-p2") == 0) {
163
+ pretty = 2;
164
+ } else if (strcmp(s, "-p") == 0) {
165
+ pretty = 1;
166
+ } else if (strcmp(s, "-h") == 0) {
167
+ usage(0);
168
+ } else {
169
+ fprintf(stderr, "option %s unknown\n", s);
170
+ usage(1);
171
+ }
172
+ } else {
173
+ fprintf(stderr, "unexpected parameter %s\n", s);
174
+ usage(1);
175
+ }
176
+ }
177
+
178
+ /* prepare the files */
179
+ f_in = (in == 0) ? stdin : fopen(in, "r");
180
+ if (f_in == 0) {
181
+ fprintf(stderr, "file %s not found\n", in);
182
+ exit(1);
183
+ }
184
+ f_out = (out == 0) ? stdout : fopen(out, "w");
185
+ if (f_out == 0) {
186
+ fprintf(stderr, "file %s cannot be opened\n", out);
187
+ exit(1);
188
+ }
189
+
190
+ /* do the stemming process: */
191
+ stemmer = sb_stemmer_new(language, charenc);
192
+ if (stemmer == 0) {
193
+ if (charenc == NULL) {
194
+ fprintf(stderr, "language `%s' not available for stemming\n", language);
195
+ exit(1);
196
+ } else {
197
+ fprintf(stderr, "language `%s' not available for stemming in encoding `%s'\n", language, charenc);
198
+ exit(1);
199
+ }
200
+ }
201
+ stem_file(stemmer, f_in, f_out);
202
+ sb_stemmer_delete(stemmer);
203
+
204
+ if (in != 0) (void) fclose(f_in);
205
+ if (out != 0) (void) fclose(f_out);
206
+
207
+ return 0;
208
+ }
209
+
@@ -0,0 +1,79 @@
1
+
2
+ /* Make header file work when included from C++ */
3
+ #ifdef __cplusplus
4
+ extern "C" {
5
+ #endif
6
+
7
+ struct sb_stemmer;
8
+ typedef unsigned char sb_symbol;
9
+
10
+ /* FIXME - should be able to get a version number for each stemming
11
+ * algorithm (which will be incremented each time the output changes). */
12
+
13
+ /** Returns an array of the names of the available stemming algorithms.
14
+ * Note that these are the canonical names - aliases (ie, other names for
15
+ * the same algorithm) will not be included in the list.
16
+ * The list is terminated with a null pointer.
17
+ *
18
+ * The list must not be modified in any way.
19
+ */
20
+ const char ** sb_stemmer_list(void);
21
+
22
+ /** Create a new stemmer object, using the specified algorithm, for the
23
+ * specified character encoding.
24
+ *
25
+ * All algorithms will usually be available in UTF-8, but may also be
26
+ * available in other character encodings.
27
+ *
28
+ * @param algorithm The algorithm name. This is either the english
29
+ * name of the algorithm, or the 2 or 3 letter ISO 639 codes for the
30
+ * language. Note that case is significant in this parameter - the
31
+ * value should be supplied in lower case.
32
+ *
33
+ * @param charenc The character encoding. NULL may be passed as
34
+ * this value, in which case UTF-8 encoding will be assumed. Otherwise,
35
+ * the argument may be one of "UTF_8", "ISO_8859_1" (ie, Latin 1),
36
+ * "CP850" (ie, MS-DOS Latin 1) or "KOI8_R" (Russian). Note that
37
+ * case is significant in this parameter.
38
+ *
39
+ * @return NULL if the specified algorithm is not recognised, or the
40
+ * algorithm is not available for the requested encoding. Otherwise,
41
+ * returns a pointer to a newly created stemmer for the requested algorithm.
42
+ * The returned pointer must be deleted by calling sb_stemmer_delete().
43
+ *
44
+ * @note NULL will also be returned if an out of memory error occurs.
45
+ */
46
+ struct sb_stemmer * sb_stemmer_new(const char * algorithm, const char * charenc);
47
+
48
+ /** Delete a stemmer object.
49
+ *
50
+ * This frees all resources allocated for the stemmer. After calling
51
+ * this function, the supplied stemmer may no longer be used in any way.
52
+ *
53
+ * It is safe to pass a null pointer to this function - this will have
54
+ * no effect.
55
+ */
56
+ void sb_stemmer_delete(struct sb_stemmer * stemmer);
57
+
58
+ /** Stem a word.
59
+ *
60
+ * The return value is owned by the stemmer - it must not be freed or
61
+ * modified, and it will become invalid when the stemmer is called again,
62
+ * or if the stemmer is freed.
63
+ *
64
+ * The length of the return value can be obtained using sb_stemmer_length().
65
+ *
66
+ * If an out-of-memory error occurs, this will return NULL.
67
+ */
68
+ const sb_symbol * sb_stemmer_stem(struct sb_stemmer * stemmer,
69
+ const sb_symbol * word, int size);
70
+
71
+ /** Get the length of the result of the last stemmed word.
72
+ * This should not be called before sb_stemmer_stem() has been called.
73
+ */
74
+ int sb_stemmer_length(struct sb_stemmer * stemmer);
75
+
76
+ #ifdef __cplusplus
77
+ }
78
+ #endif
79
+
@@ -0,0 +1,93 @@
1
+
2
+ #include <stdlib.h>
3
+ #include <string.h>
4
+ #include "../include/libstemmer.h"
5
+ #include "../runtime/api.h"
6
+ #include "modules.h"
7
+
8
+ struct sb_stemmer {
9
+ struct SN_env * (*create)(void);
10
+ void (*close)(struct SN_env *);
11
+ int (*stem)(struct SN_env *);
12
+
13
+ struct SN_env * env;
14
+ };
15
+
16
+ extern const char **
17
+ sb_stemmer_list(void)
18
+ {
19
+ return algorithm_names;
20
+ }
21
+
22
+ static stemmer_encoding_t
23
+ sb_getenc(const char * charenc)
24
+ {
25
+ struct stemmer_encoding * encoding;
26
+ if (charenc == NULL) return ENC_UTF_8;
27
+ for (encoding = encodings; encoding->name != 0; encoding++) {
28
+ if (strcmp(encoding->name, charenc) == 0) break;
29
+ }
30
+ if (encoding->name == NULL) return ENC_UNKNOWN;
31
+ return encoding->enc;
32
+ }
33
+
34
+ extern struct sb_stemmer *
35
+ sb_stemmer_new(const char * algorithm, const char * charenc)
36
+ {
37
+ stemmer_encoding_t enc;
38
+ struct stemmer_modules * module;
39
+ struct sb_stemmer * stemmer =
40
+ (struct sb_stemmer *) malloc(sizeof(struct sb_stemmer));
41
+ if (stemmer == NULL) return NULL;
42
+ enc = sb_getenc(charenc);
43
+ if (enc == ENC_UNKNOWN) return NULL;
44
+
45
+ for (module = modules; module->name != 0; module++) {
46
+ if (strcmp(module->name, algorithm) == 0 && module->enc == enc) break;
47
+ }
48
+ if (module->name == NULL) return NULL;
49
+
50
+ stemmer->create = module->create;
51
+ stemmer->close = module->close;
52
+ stemmer->stem = module->stem;
53
+
54
+ stemmer->env = stemmer->create();
55
+ if (stemmer->env == NULL)
56
+ {
57
+ sb_stemmer_delete(stemmer);
58
+ return NULL;
59
+ }
60
+
61
+ return stemmer;
62
+ }
63
+
64
+ void
65
+ sb_stemmer_delete(struct sb_stemmer * stemmer)
66
+ {
67
+ if (stemmer == 0) return;
68
+ if (stemmer->close == 0) return;
69
+ stemmer->close(stemmer->env);
70
+ stemmer->close = 0;
71
+ free(stemmer);
72
+ }
73
+
74
+ const sb_symbol *
75
+ sb_stemmer_stem(struct sb_stemmer * stemmer, const sb_symbol * word, int size)
76
+ {
77
+ int ret;
78
+ if (SN_set_current(stemmer->env, size, (const symbol *)(word)))
79
+ {
80
+ stemmer->env->l = 0;
81
+ return NULL;
82
+ }
83
+ ret = stemmer->stem(stemmer->env);
84
+ if (ret < 0) return NULL;
85
+ stemmer->env->p[stemmer->env->l] = 0;
86
+ return (const sb_symbol *)(stemmer->env->p);
87
+ }
88
+
89
+ int
90
+ sb_stemmer_length(struct sb_stemmer * stemmer)
91
+ {
92
+ return stemmer->env->l;
93
+ }
@@ -0,0 +1,93 @@
1
+
2
+ #include <stdlib.h>
3
+ #include <string.h>
4
+ #include "../include/libstemmer.h"
5
+ #include "../runtime/api.h"
6
+ #include "modules_utf8.h"
7
+
8
+ struct sb_stemmer {
9
+ struct SN_env * (*create)(void);
10
+ void (*close)(struct SN_env *);
11
+ int (*stem)(struct SN_env *);
12
+
13
+ struct SN_env * env;
14
+ };
15
+
16
+ extern const char **
17
+ sb_stemmer_list(void)
18
+ {
19
+ return algorithm_names;
20
+ }
21
+
22
+ static stemmer_encoding_t
23
+ sb_getenc(const char * charenc)
24
+ {
25
+ struct stemmer_encoding * encoding;
26
+ if (charenc == NULL) return ENC_UTF_8;
27
+ for (encoding = encodings; encoding->name != 0; encoding++) {
28
+ if (strcmp(encoding->name, charenc) == 0) break;
29
+ }
30
+ if (encoding->name == NULL) return ENC_UNKNOWN;
31
+ return encoding->enc;
32
+ }
33
+
34
+ extern struct sb_stemmer *
35
+ sb_stemmer_new(const char * algorithm, const char * charenc)
36
+ {
37
+ stemmer_encoding_t enc;
38
+ struct stemmer_modules * module;
39
+ struct sb_stemmer * stemmer =
40
+ (struct sb_stemmer *) malloc(sizeof(struct sb_stemmer));
41
+ if (stemmer == NULL) return NULL;
42
+ enc = sb_getenc(charenc);
43
+ if (enc == ENC_UNKNOWN) return NULL;
44
+
45
+ for (module = modules; module->name != 0; module++) {
46
+ if (strcmp(module->name, algorithm) == 0 && module->enc == enc) break;
47
+ }
48
+ if (module->name == NULL) return NULL;
49
+
50
+ stemmer->create = module->create;
51
+ stemmer->close = module->close;
52
+ stemmer->stem = module->stem;
53
+
54
+ stemmer->env = stemmer->create();
55
+ if (stemmer->env == NULL)
56
+ {
57
+ sb_stemmer_delete(stemmer);
58
+ return NULL;
59
+ }
60
+
61
+ return stemmer;
62
+ }
63
+
64
+ void
65
+ sb_stemmer_delete(struct sb_stemmer * stemmer)
66
+ {
67
+ if (stemmer == 0) return;
68
+ if (stemmer->close == 0) return;
69
+ stemmer->close(stemmer->env);
70
+ stemmer->close = 0;
71
+ free(stemmer);
72
+ }
73
+
74
+ const sb_symbol *
75
+ sb_stemmer_stem(struct sb_stemmer * stemmer, const sb_symbol * word, int size)
76
+ {
77
+ int ret;
78
+ if (SN_set_current(stemmer->env, size, (const symbol *)(word)))
79
+ {
80
+ stemmer->env->l = 0;
81
+ return NULL;
82
+ }
83
+ ret = stemmer->stem(stemmer->env);
84
+ if (ret < 0) return NULL;
85
+ stemmer->env->p[stemmer->env->l] = 0;
86
+ return (const sb_symbol *)(stemmer->env->p);
87
+ }
88
+
89
+ int
90
+ sb_stemmer_length(struct sb_stemmer * stemmer)
91
+ {
92
+ return stemmer->env->l;
93
+ }
@@ -0,0 +1,195 @@
1
+ /* libstemmer/modules.h: List of stemming modules.
2
+ *
3
+ * This file is generated by mkmodules.pl from a list of module names.
4
+ * Do not edit manually.
5
+ *
6
+ * Modules included by this file are: latin, danish, dutch, english, finnish, french,
7
+ * german, hungarian, italian, norwegian, porter, portuguese, romanian,
8
+ * russian, spanish, swedish, turkish
9
+ */
10
+
11
+ #include "../src_c/stem_ISO_8859_1_latin.h"
12
+ #include "../src_c/stem_UTF_8_latin.h"
13
+ #include "../src_c/stem_ISO_8859_1_danish.h"
14
+ #include "../src_c/stem_UTF_8_danish.h"
15
+ #include "../src_c/stem_ISO_8859_1_dutch.h"
16
+ #include "../src_c/stem_UTF_8_dutch.h"
17
+ #include "../src_c/stem_ISO_8859_1_english.h"
18
+ #include "../src_c/stem_UTF_8_english.h"
19
+ #include "../src_c/stem_ISO_8859_1_finnish.h"
20
+ #include "../src_c/stem_UTF_8_finnish.h"
21
+ #include "../src_c/stem_ISO_8859_1_french.h"
22
+ #include "../src_c/stem_UTF_8_french.h"
23
+ #include "../src_c/stem_ISO_8859_1_german.h"
24
+ #include "../src_c/stem_UTF_8_german.h"
25
+ #include "../src_c/stem_ISO_8859_1_hungarian.h"
26
+ #include "../src_c/stem_UTF_8_hungarian.h"
27
+ #include "../src_c/stem_ISO_8859_1_italian.h"
28
+ #include "../src_c/stem_UTF_8_italian.h"
29
+ #include "../src_c/stem_ISO_8859_1_norwegian.h"
30
+ #include "../src_c/stem_UTF_8_norwegian.h"
31
+ #include "../src_c/stem_ISO_8859_1_porter.h"
32
+ #include "../src_c/stem_UTF_8_porter.h"
33
+ #include "../src_c/stem_ISO_8859_1_portuguese.h"
34
+ #include "../src_c/stem_UTF_8_portuguese.h"
35
+ #include "../src_c/stem_ISO_8859_2_romanian.h"
36
+ #include "../src_c/stem_UTF_8_romanian.h"
37
+ #include "../src_c/stem_KOI8_R_russian.h"
38
+ #include "../src_c/stem_UTF_8_russian.h"
39
+ #include "../src_c/stem_ISO_8859_1_spanish.h"
40
+ #include "../src_c/stem_UTF_8_spanish.h"
41
+ #include "../src_c/stem_ISO_8859_1_swedish.h"
42
+ #include "../src_c/stem_UTF_8_swedish.h"
43
+ #include "../src_c/stem_UTF_8_turkish.h"
44
+
45
+ typedef enum {
46
+ ENC_UNKNOWN=0,
47
+ ENC_ISO_8859_1,
48
+ ENC_ISO_8859_2,
49
+ ENC_KOI8_R,
50
+ ENC_UTF_8
51
+ } stemmer_encoding_t;
52
+
53
+ struct stemmer_encoding {
54
+ const char * name;
55
+ stemmer_encoding_t enc;
56
+ };
57
+ static struct stemmer_encoding encodings[] = {
58
+ {"ISO_8859_1", ENC_ISO_8859_1},
59
+ {"ISO_8859_2", ENC_ISO_8859_2},
60
+ {"KOI8_R", ENC_KOI8_R},
61
+ {"UTF_8", ENC_UTF_8},
62
+ {0,ENC_UNKNOWN}
63
+ };
64
+
65
+ struct stemmer_modules {
66
+ const char * name;
67
+ stemmer_encoding_t enc;
68
+ struct SN_env * (*create)(void);
69
+ void (*close)(struct SN_env *);
70
+ int (*stem)(struct SN_env *);
71
+ };
72
+ static struct stemmer_modules modules[] = {
73
+ {"latin", ENC_ISO_8859_1, latin_ISO_8859_1_create_env, latin_ISO_8859_1_close_env, latin_ISO_8859_1_stem},
74
+ {"latin", ENC_UTF_8, latin_UTF_8_create_env, latin_UTF_8_close_env, latin_UTF_8_stem},
75
+ {"da", ENC_ISO_8859_1, danish_ISO_8859_1_create_env, danish_ISO_8859_1_close_env, danish_ISO_8859_1_stem},
76
+ {"da", ENC_UTF_8, danish_UTF_8_create_env, danish_UTF_8_close_env, danish_UTF_8_stem},
77
+ {"dan", ENC_ISO_8859_1, danish_ISO_8859_1_create_env, danish_ISO_8859_1_close_env, danish_ISO_8859_1_stem},
78
+ {"dan", ENC_UTF_8, danish_UTF_8_create_env, danish_UTF_8_close_env, danish_UTF_8_stem},
79
+ {"danish", ENC_ISO_8859_1, danish_ISO_8859_1_create_env, danish_ISO_8859_1_close_env, danish_ISO_8859_1_stem},
80
+ {"danish", ENC_UTF_8, danish_UTF_8_create_env, danish_UTF_8_close_env, danish_UTF_8_stem},
81
+ {"de", ENC_ISO_8859_1, german_ISO_8859_1_create_env, german_ISO_8859_1_close_env, german_ISO_8859_1_stem},
82
+ {"de", ENC_UTF_8, german_UTF_8_create_env, german_UTF_8_close_env, german_UTF_8_stem},
83
+ {"deu", ENC_ISO_8859_1, german_ISO_8859_1_create_env, german_ISO_8859_1_close_env, german_ISO_8859_1_stem},
84
+ {"deu", ENC_UTF_8, german_UTF_8_create_env, german_UTF_8_close_env, german_UTF_8_stem},
85
+ {"dut", ENC_ISO_8859_1, dutch_ISO_8859_1_create_env, dutch_ISO_8859_1_close_env, dutch_ISO_8859_1_stem},
86
+ {"dut", ENC_UTF_8, dutch_UTF_8_create_env, dutch_UTF_8_close_env, dutch_UTF_8_stem},
87
+ {"dutch", ENC_ISO_8859_1, dutch_ISO_8859_1_create_env, dutch_ISO_8859_1_close_env, dutch_ISO_8859_1_stem},
88
+ {"dutch", ENC_UTF_8, dutch_UTF_8_create_env, dutch_UTF_8_close_env, dutch_UTF_8_stem},
89
+ {"en", ENC_ISO_8859_1, english_ISO_8859_1_create_env, english_ISO_8859_1_close_env, english_ISO_8859_1_stem},
90
+ {"en", ENC_UTF_8, english_UTF_8_create_env, english_UTF_8_close_env, english_UTF_8_stem},
91
+ {"eng", ENC_ISO_8859_1, english_ISO_8859_1_create_env, english_ISO_8859_1_close_env, english_ISO_8859_1_stem},
92
+ {"eng", ENC_UTF_8, english_UTF_8_create_env, english_UTF_8_close_env, english_UTF_8_stem},
93
+ {"english", ENC_ISO_8859_1, english_ISO_8859_1_create_env, english_ISO_8859_1_close_env, english_ISO_8859_1_stem},
94
+ {"english", ENC_UTF_8, english_UTF_8_create_env, english_UTF_8_close_env, english_UTF_8_stem},
95
+ {"es", ENC_ISO_8859_1, spanish_ISO_8859_1_create_env, spanish_ISO_8859_1_close_env, spanish_ISO_8859_1_stem},
96
+ {"es", ENC_UTF_8, spanish_UTF_8_create_env, spanish_UTF_8_close_env, spanish_UTF_8_stem},
97
+ {"esl", ENC_ISO_8859_1, spanish_ISO_8859_1_create_env, spanish_ISO_8859_1_close_env, spanish_ISO_8859_1_stem},
98
+ {"esl", ENC_UTF_8, spanish_UTF_8_create_env, spanish_UTF_8_close_env, spanish_UTF_8_stem},
99
+ {"fi", ENC_ISO_8859_1, finnish_ISO_8859_1_create_env, finnish_ISO_8859_1_close_env, finnish_ISO_8859_1_stem},
100
+ {"fi", ENC_UTF_8, finnish_UTF_8_create_env, finnish_UTF_8_close_env, finnish_UTF_8_stem},
101
+ {"fin", ENC_ISO_8859_1, finnish_ISO_8859_1_create_env, finnish_ISO_8859_1_close_env, finnish_ISO_8859_1_stem},
102
+ {"fin", ENC_UTF_8, finnish_UTF_8_create_env, finnish_UTF_8_close_env, finnish_UTF_8_stem},
103
+ {"finnish", ENC_ISO_8859_1, finnish_ISO_8859_1_create_env, finnish_ISO_8859_1_close_env, finnish_ISO_8859_1_stem},
104
+ {"finnish", ENC_UTF_8, finnish_UTF_8_create_env, finnish_UTF_8_close_env, finnish_UTF_8_stem},
105
+ {"fr", ENC_ISO_8859_1, french_ISO_8859_1_create_env, french_ISO_8859_1_close_env, french_ISO_8859_1_stem},
106
+ {"fr", ENC_UTF_8, french_UTF_8_create_env, french_UTF_8_close_env, french_UTF_8_stem},
107
+ {"fra", ENC_ISO_8859_1, french_ISO_8859_1_create_env, french_ISO_8859_1_close_env, french_ISO_8859_1_stem},
108
+ {"fra", ENC_UTF_8, french_UTF_8_create_env, french_UTF_8_close_env, french_UTF_8_stem},
109
+ {"fre", ENC_ISO_8859_1, french_ISO_8859_1_create_env, french_ISO_8859_1_close_env, french_ISO_8859_1_stem},
110
+ {"fre", ENC_UTF_8, french_UTF_8_create_env, french_UTF_8_close_env, french_UTF_8_stem},
111
+ {"french", ENC_ISO_8859_1, french_ISO_8859_1_create_env, french_ISO_8859_1_close_env, french_ISO_8859_1_stem},
112
+ {"french", ENC_UTF_8, french_UTF_8_create_env, french_UTF_8_close_env, french_UTF_8_stem},
113
+ {"ger", ENC_ISO_8859_1, german_ISO_8859_1_create_env, german_ISO_8859_1_close_env, german_ISO_8859_1_stem},
114
+ {"ger", ENC_UTF_8, german_UTF_8_create_env, german_UTF_8_close_env, german_UTF_8_stem},
115
+ {"german", ENC_ISO_8859_1, german_ISO_8859_1_create_env, german_ISO_8859_1_close_env, german_ISO_8859_1_stem},
116
+ {"german", ENC_UTF_8, german_UTF_8_create_env, german_UTF_8_close_env, german_UTF_8_stem},
117
+ {"hu", ENC_ISO_8859_1, hungarian_ISO_8859_1_create_env, hungarian_ISO_8859_1_close_env, hungarian_ISO_8859_1_stem},
118
+ {"hu", ENC_UTF_8, hungarian_UTF_8_create_env, hungarian_UTF_8_close_env, hungarian_UTF_8_stem},
119
+ {"hun", ENC_ISO_8859_1, hungarian_ISO_8859_1_create_env, hungarian_ISO_8859_1_close_env, hungarian_ISO_8859_1_stem},
120
+ {"hun", ENC_UTF_8, hungarian_UTF_8_create_env, hungarian_UTF_8_close_env, hungarian_UTF_8_stem},
121
+ {"hungarian", ENC_ISO_8859_1, hungarian_ISO_8859_1_create_env, hungarian_ISO_8859_1_close_env, hungarian_ISO_8859_1_stem},
122
+ {"hungarian", ENC_UTF_8, hungarian_UTF_8_create_env, hungarian_UTF_8_close_env, hungarian_UTF_8_stem},
123
+ {"it", ENC_ISO_8859_1, italian_ISO_8859_1_create_env, italian_ISO_8859_1_close_env, italian_ISO_8859_1_stem},
124
+ {"it", ENC_UTF_8, italian_UTF_8_create_env, italian_UTF_8_close_env, italian_UTF_8_stem},
125
+ {"ita", ENC_ISO_8859_1, italian_ISO_8859_1_create_env, italian_ISO_8859_1_close_env, italian_ISO_8859_1_stem},
126
+ {"ita", ENC_UTF_8, italian_UTF_8_create_env, italian_UTF_8_close_env, italian_UTF_8_stem},
127
+ {"italian", ENC_ISO_8859_1, italian_ISO_8859_1_create_env, italian_ISO_8859_1_close_env, italian_ISO_8859_1_stem},
128
+ {"italian", ENC_UTF_8, italian_UTF_8_create_env, italian_UTF_8_close_env, italian_UTF_8_stem},
129
+ {"nl", ENC_ISO_8859_1, dutch_ISO_8859_1_create_env, dutch_ISO_8859_1_close_env, dutch_ISO_8859_1_stem},
130
+ {"nl", ENC_UTF_8, dutch_UTF_8_create_env, dutch_UTF_8_close_env, dutch_UTF_8_stem},
131
+ {"nld", ENC_ISO_8859_1, dutch_ISO_8859_1_create_env, dutch_ISO_8859_1_close_env, dutch_ISO_8859_1_stem},
132
+ {"nld", ENC_UTF_8, dutch_UTF_8_create_env, dutch_UTF_8_close_env, dutch_UTF_8_stem},
133
+ {"no", ENC_ISO_8859_1, norwegian_ISO_8859_1_create_env, norwegian_ISO_8859_1_close_env, norwegian_ISO_8859_1_stem},
134
+ {"no", ENC_UTF_8, norwegian_UTF_8_create_env, norwegian_UTF_8_close_env, norwegian_UTF_8_stem},
135
+ {"nor", ENC_ISO_8859_1, norwegian_ISO_8859_1_create_env, norwegian_ISO_8859_1_close_env, norwegian_ISO_8859_1_stem},
136
+ {"nor", ENC_UTF_8, norwegian_UTF_8_create_env, norwegian_UTF_8_close_env, norwegian_UTF_8_stem},
137
+ {"norwegian", ENC_ISO_8859_1, norwegian_ISO_8859_1_create_env, norwegian_ISO_8859_1_close_env, norwegian_ISO_8859_1_stem},
138
+ {"norwegian", ENC_UTF_8, norwegian_UTF_8_create_env, norwegian_UTF_8_close_env, norwegian_UTF_8_stem},
139
+ {"por", ENC_ISO_8859_1, portuguese_ISO_8859_1_create_env, portuguese_ISO_8859_1_close_env, portuguese_ISO_8859_1_stem},
140
+ {"por", ENC_UTF_8, portuguese_UTF_8_create_env, portuguese_UTF_8_close_env, portuguese_UTF_8_stem},
141
+ {"porter", ENC_ISO_8859_1, porter_ISO_8859_1_create_env, porter_ISO_8859_1_close_env, porter_ISO_8859_1_stem},
142
+ {"porter", ENC_UTF_8, porter_UTF_8_create_env, porter_UTF_8_close_env, porter_UTF_8_stem},
143
+ {"portuguese", ENC_ISO_8859_1, portuguese_ISO_8859_1_create_env, portuguese_ISO_8859_1_close_env, portuguese_ISO_8859_1_stem},
144
+ {"portuguese", ENC_UTF_8, portuguese_UTF_8_create_env, portuguese_UTF_8_close_env, portuguese_UTF_8_stem},
145
+ {"pt", ENC_ISO_8859_1, portuguese_ISO_8859_1_create_env, portuguese_ISO_8859_1_close_env, portuguese_ISO_8859_1_stem},
146
+ {"pt", ENC_UTF_8, portuguese_UTF_8_create_env, portuguese_UTF_8_close_env, portuguese_UTF_8_stem},
147
+ {"ro", ENC_ISO_8859_2, romanian_ISO_8859_2_create_env, romanian_ISO_8859_2_close_env, romanian_ISO_8859_2_stem},
148
+ {"ro", ENC_UTF_8, romanian_UTF_8_create_env, romanian_UTF_8_close_env, romanian_UTF_8_stem},
149
+ {"romanian", ENC_ISO_8859_2, romanian_ISO_8859_2_create_env, romanian_ISO_8859_2_close_env, romanian_ISO_8859_2_stem},
150
+ {"romanian", ENC_UTF_8, romanian_UTF_8_create_env, romanian_UTF_8_close_env, romanian_UTF_8_stem},
151
+ {"ron", ENC_ISO_8859_2, romanian_ISO_8859_2_create_env, romanian_ISO_8859_2_close_env, romanian_ISO_8859_2_stem},
152
+ {"ron", ENC_UTF_8, romanian_UTF_8_create_env, romanian_UTF_8_close_env, romanian_UTF_8_stem},
153
+ {"ru", ENC_KOI8_R, russian_KOI8_R_create_env, russian_KOI8_R_close_env, russian_KOI8_R_stem},
154
+ {"ru", ENC_UTF_8, russian_UTF_8_create_env, russian_UTF_8_close_env, russian_UTF_8_stem},
155
+ {"rum", ENC_ISO_8859_2, romanian_ISO_8859_2_create_env, romanian_ISO_8859_2_close_env, romanian_ISO_8859_2_stem},
156
+ {"rum", ENC_UTF_8, romanian_UTF_8_create_env, romanian_UTF_8_close_env, romanian_UTF_8_stem},
157
+ {"rus", ENC_KOI8_R, russian_KOI8_R_create_env, russian_KOI8_R_close_env, russian_KOI8_R_stem},
158
+ {"rus", ENC_UTF_8, russian_UTF_8_create_env, russian_UTF_8_close_env, russian_UTF_8_stem},
159
+ {"russian", ENC_KOI8_R, russian_KOI8_R_create_env, russian_KOI8_R_close_env, russian_KOI8_R_stem},
160
+ {"russian", ENC_UTF_8, russian_UTF_8_create_env, russian_UTF_8_close_env, russian_UTF_8_stem},
161
+ {"spa", ENC_ISO_8859_1, spanish_ISO_8859_1_create_env, spanish_ISO_8859_1_close_env, spanish_ISO_8859_1_stem},
162
+ {"spa", ENC_UTF_8, spanish_UTF_8_create_env, spanish_UTF_8_close_env, spanish_UTF_8_stem},
163
+ {"spanish", ENC_ISO_8859_1, spanish_ISO_8859_1_create_env, spanish_ISO_8859_1_close_env, spanish_ISO_8859_1_stem},
164
+ {"spanish", ENC_UTF_8, spanish_UTF_8_create_env, spanish_UTF_8_close_env, spanish_UTF_8_stem},
165
+ {"sv", ENC_ISO_8859_1, swedish_ISO_8859_1_create_env, swedish_ISO_8859_1_close_env, swedish_ISO_8859_1_stem},
166
+ {"sv", ENC_UTF_8, swedish_UTF_8_create_env, swedish_UTF_8_close_env, swedish_UTF_8_stem},
167
+ {"swe", ENC_ISO_8859_1, swedish_ISO_8859_1_create_env, swedish_ISO_8859_1_close_env, swedish_ISO_8859_1_stem},
168
+ {"swe", ENC_UTF_8, swedish_UTF_8_create_env, swedish_UTF_8_close_env, swedish_UTF_8_stem},
169
+ {"swedish", ENC_ISO_8859_1, swedish_ISO_8859_1_create_env, swedish_ISO_8859_1_close_env, swedish_ISO_8859_1_stem},
170
+ {"swedish", ENC_UTF_8, swedish_UTF_8_create_env, swedish_UTF_8_close_env, swedish_UTF_8_stem},
171
+ {"tr", ENC_UTF_8, turkish_UTF_8_create_env, turkish_UTF_8_close_env, turkish_UTF_8_stem},
172
+ {"tur", ENC_UTF_8, turkish_UTF_8_create_env, turkish_UTF_8_close_env, turkish_UTF_8_stem},
173
+ {"turkish", ENC_UTF_8, turkish_UTF_8_create_env, turkish_UTF_8_close_env, turkish_UTF_8_stem},
174
+ {0,ENC_UNKNOWN,0,0,0}
175
+ };
176
+ static const char * algorithm_names[] = {
177
+ "latin",
178
+ "danish",
179
+ "dutch",
180
+ "english",
181
+ "finnish",
182
+ "french",
183
+ "german",
184
+ "hungarian",
185
+ "italian",
186
+ "norwegian",
187
+ "porter",
188
+ "portuguese",
189
+ "romanian",
190
+ "russian",
191
+ "spanish",
192
+ "swedish",
193
+ "turkish",
194
+ 0
195
+ };