ruby-stemmer-dimelo 0.9.3.dimelo1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (95) hide show
  1. checksums.yaml +7 -0
  2. data/MIT-LICENSE +21 -0
  3. data/README.rdoc +117 -0
  4. data/Rakefile +70 -0
  5. data/VERSION +1 -0
  6. data/ext/lingua/extconf.rb +40 -0
  7. data/ext/lingua/stemmer.c +115 -0
  8. data/lib/lingua/stemmer.rb +60 -0
  9. data/libstemmer_c/MANIFEST +72 -0
  10. data/libstemmer_c/Makefile +9 -0
  11. data/libstemmer_c/Makefile.windows +15 -0
  12. data/libstemmer_c/README +125 -0
  13. data/libstemmer_c/examples/stemwords.c +209 -0
  14. data/libstemmer_c/include/libstemmer.h +79 -0
  15. data/libstemmer_c/libstemmer/libstemmer.c +93 -0
  16. data/libstemmer_c/libstemmer/libstemmer_utf8.c +93 -0
  17. data/libstemmer_c/libstemmer/modules.h +195 -0
  18. data/libstemmer_c/libstemmer/modules.txt +51 -0
  19. data/libstemmer_c/libstemmer/modules_utf8.h +123 -0
  20. data/libstemmer_c/libstemmer/modules_utf8.txt +50 -0
  21. data/libstemmer_c/mkinc.mak +86 -0
  22. data/libstemmer_c/mkinc_utf8.mak +54 -0
  23. data/libstemmer_c/runtime/api.c +66 -0
  24. data/libstemmer_c/runtime/api.h +26 -0
  25. data/libstemmer_c/runtime/header.h +58 -0
  26. data/libstemmer_c/runtime/utilities.c +478 -0
  27. data/libstemmer_c/src_c/stem_ISO_8859_1_danish.c +337 -0
  28. data/libstemmer_c/src_c/stem_ISO_8859_1_danish.h +16 -0
  29. data/libstemmer_c/src_c/stem_ISO_8859_1_dutch.c +624 -0
  30. data/libstemmer_c/src_c/stem_ISO_8859_1_dutch.h +16 -0
  31. data/libstemmer_c/src_c/stem_ISO_8859_1_english.c +1117 -0
  32. data/libstemmer_c/src_c/stem_ISO_8859_1_english.h +16 -0
  33. data/libstemmer_c/src_c/stem_ISO_8859_1_finnish.c +762 -0
  34. data/libstemmer_c/src_c/stem_ISO_8859_1_finnish.h +16 -0
  35. data/libstemmer_c/src_c/stem_ISO_8859_1_french.c +1230 -0
  36. data/libstemmer_c/src_c/stem_ISO_8859_1_french.h +16 -0
  37. data/libstemmer_c/src_c/stem_ISO_8859_1_german.c +503 -0
  38. data/libstemmer_c/src_c/stem_ISO_8859_1_german.h +16 -0
  39. data/libstemmer_c/src_c/stem_ISO_8859_1_hungarian.c +1230 -0
  40. data/libstemmer_c/src_c/stem_ISO_8859_1_hungarian.h +16 -0
  41. data/libstemmer_c/src_c/stem_ISO_8859_1_italian.c +1065 -0
  42. data/libstemmer_c/src_c/stem_ISO_8859_1_italian.h +16 -0
  43. data/libstemmer_c/src_c/stem_ISO_8859_1_latin.c +443 -0
  44. data/libstemmer_c/src_c/stem_ISO_8859_1_latin.h +16 -0
  45. data/libstemmer_c/src_c/stem_ISO_8859_1_norwegian.c +297 -0
  46. data/libstemmer_c/src_c/stem_ISO_8859_1_norwegian.h +16 -0
  47. data/libstemmer_c/src_c/stem_ISO_8859_1_porter.c +749 -0
  48. data/libstemmer_c/src_c/stem_ISO_8859_1_porter.h +16 -0
  49. data/libstemmer_c/src_c/stem_ISO_8859_1_portuguese.c +1017 -0
  50. data/libstemmer_c/src_c/stem_ISO_8859_1_portuguese.h +16 -0
  51. data/libstemmer_c/src_c/stem_ISO_8859_1_spanish.c +1093 -0
  52. data/libstemmer_c/src_c/stem_ISO_8859_1_spanish.h +16 -0
  53. data/libstemmer_c/src_c/stem_ISO_8859_1_swedish.c +307 -0
  54. data/libstemmer_c/src_c/stem_ISO_8859_1_swedish.h +16 -0
  55. data/libstemmer_c/src_c/stem_ISO_8859_2_romanian.c +998 -0
  56. data/libstemmer_c/src_c/stem_ISO_8859_2_romanian.h +16 -0
  57. data/libstemmer_c/src_c/stem_KOI8_R_russian.c +700 -0
  58. data/libstemmer_c/src_c/stem_KOI8_R_russian.h +16 -0
  59. data/libstemmer_c/src_c/stem_UTF_8_danish.c +339 -0
  60. data/libstemmer_c/src_c/stem_UTF_8_danish.h +16 -0
  61. data/libstemmer_c/src_c/stem_UTF_8_dutch.c +634 -0
  62. data/libstemmer_c/src_c/stem_UTF_8_dutch.h +16 -0
  63. data/libstemmer_c/src_c/stem_UTF_8_english.c +1125 -0
  64. data/libstemmer_c/src_c/stem_UTF_8_english.h +16 -0
  65. data/libstemmer_c/src_c/stem_UTF_8_finnish.c +768 -0
  66. data/libstemmer_c/src_c/stem_UTF_8_finnish.h +16 -0
  67. data/libstemmer_c/src_c/stem_UTF_8_french.c +1230 -0
  68. data/libstemmer_c/src_c/stem_UTF_8_french.h +16 -0
  69. data/libstemmer_c/src_c/stem_UTF_8_german.c +509 -0
  70. data/libstemmer_c/src_c/stem_UTF_8_german.h +16 -0
  71. data/libstemmer_c/src_c/stem_UTF_8_hungarian.c +1234 -0
  72. data/libstemmer_c/src_c/stem_UTF_8_hungarian.h +16 -0
  73. data/libstemmer_c/src_c/stem_UTF_8_italian.c +1073 -0
  74. data/libstemmer_c/src_c/stem_UTF_8_italian.h +16 -0
  75. data/libstemmer_c/src_c/stem_UTF_8_latin.c +443 -0
  76. data/libstemmer_c/src_c/stem_UTF_8_latin.h +16 -0
  77. data/libstemmer_c/src_c/stem_UTF_8_norwegian.c +299 -0
  78. data/libstemmer_c/src_c/stem_UTF_8_norwegian.h +16 -0
  79. data/libstemmer_c/src_c/stem_UTF_8_porter.c +755 -0
  80. data/libstemmer_c/src_c/stem_UTF_8_porter.h +16 -0
  81. data/libstemmer_c/src_c/stem_UTF_8_portuguese.c +1023 -0
  82. data/libstemmer_c/src_c/stem_UTF_8_portuguese.h +16 -0
  83. data/libstemmer_c/src_c/stem_UTF_8_romanian.c +1004 -0
  84. data/libstemmer_c/src_c/stem_UTF_8_romanian.h +16 -0
  85. data/libstemmer_c/src_c/stem_UTF_8_russian.c +694 -0
  86. data/libstemmer_c/src_c/stem_UTF_8_russian.h +16 -0
  87. data/libstemmer_c/src_c/stem_UTF_8_spanish.c +1097 -0
  88. data/libstemmer_c/src_c/stem_UTF_8_spanish.h +16 -0
  89. data/libstemmer_c/src_c/stem_UTF_8_swedish.c +309 -0
  90. data/libstemmer_c/src_c/stem_UTF_8_swedish.h +16 -0
  91. data/libstemmer_c/src_c/stem_UTF_8_turkish.c +2205 -0
  92. data/libstemmer_c/src_c/stem_UTF_8_turkish.h +16 -0
  93. data/test/helper.rb +3 -0
  94. data/test/lingua/test_stemmer.rb +99 -0
  95. metadata +141 -0
@@ -0,0 +1,9 @@
1
+ include mkinc.mak
2
+ CFLAGS=-Iinclude -fPIC $(ARCHFLAGS)
3
+ all: libstemmer.o stemwords
4
+ libstemmer.o: $(snowball_sources:.c=.o)
5
+ $(AR) -cru $@ $^
6
+ stemwords: examples/stemwords.o libstemmer.o
7
+ $(CC) -o $@ $^
8
+ clean:
9
+ rm -f stemwords *.o src_c/*.o runtime/*.o libstemmer/*.o
@@ -0,0 +1,15 @@
1
+ include mkinc.mak
2
+
3
+ AR=i686-w64-mingw32-ar
4
+ CC=i686-w64-mingw32-gcc
5
+ LD=i686-w64-mingw32-ld
6
+
7
+ CFLAGS=-Iinclude
8
+
9
+ all: libstemmer.o stemwords
10
+ libstemmer.o: $(snowball_sources:.c=.o)
11
+ $(AR) -cru $@ $^
12
+ stemwords: examples/stemwords.o libstemmer.o
13
+ $(CC) -o $@ $^
14
+ clean:
15
+ rm -f stemwords *.o src_c/*.o runtime/*.o libstemmer/*.o
@@ -0,0 +1,125 @@
1
+ libstemmer_c
2
+ ============
3
+
4
+ This document pertains to the C version of the libstemmer distribution,
5
+ available for download from:
6
+
7
+ http://snowball.tartarus.org/dist/libstemmer_c.tgz
8
+
9
+
10
+ Compiling the library
11
+ =====================
12
+
13
+ A simple makefile is provided for Unix style systems. On such systems, it
14
+ should be possible simply to run "make", and the file "libstemmer.o"
15
+ and the example program "stemwords" will be generated.
16
+
17
+ If this doesn't work on your system, you need to write your own build
18
+ system (or call the compiler directly). The files to compile are
19
+ all contained in the "libstemmer", "runtime" and "src_c" directories,
20
+ and the public header file is contained in the "include" directory.
21
+
22
+ The library comes in two flavours; UTF-8 only, and UTF-8 plus other character
23
+ sets. To use the utf-8 only flavour, compile "libstemmer_utf8.c" instead of
24
+ "libstemmer.c".
25
+
26
+ For convenience "mkinc.mak" is a makefile fragment listing the source files and
27
+ header files used to compile the standard version of the library.
28
+ "mkinc_utf8.mak" is a comparable makefile fragment listing just the source
29
+ files for the UTF-8 only version of the library.
30
+
31
+
32
+ Using the library
33
+ =================
34
+
35
+ The library provides a simple C API. Essentially, a new stemmer can
36
+ be obtained by using "sb_stemmer_new". "sb_stemmer_stem" is then
37
+ used to stem a word, "sb_stemmer_length" returns the stemmed
38
+ length of the last word processed, and "sb_stemmer_delete" is
39
+ used to delete a stemmer.
40
+
41
+ Creating a stemmer is a relatively expensive operation - the expected
42
+ usage pattern is that a new stemmer is created when needed, used
43
+ to stem many words, and deleted after some time.
44
+
45
+ Stemmers are re-entrant, but not threadsafe. In other words, if
46
+ you wish to access the same stemmer object from multiple threads,
47
+ you must ensure that all access is protected by a mutex or similar
48
+ device.
49
+
50
+ libstemmer does not currently incorporate any mechanism for caching the results
51
+ of stemming operations. Such caching can greatly increase the performance of a
52
+ stemmer under certain situations, so suitable patches will be considered for
53
+ inclusion.
54
+
55
+ The standard libstemmer sources contain an algorithm for each of the supported
56
+ languages. The algorithm may be selected using the english name of the
57
+ language, or using the 2 or 3 letter ISO 639 language codes. In addition,
58
+ the traditional "Porter" stemming algorithm for english is included for
59
+ backwards compatibility purposes, but we recommend use of the "English"
60
+ stemmer in preference for new projects.
61
+
62
+ (Some minor algorithms which are included only as curiosities in the snowball
63
+ website, such as the Lovins stemmer and the Kraaij Pohlmann stemmer, are not
64
+ included in the standard libstemmer sources. These are not really supported by
65
+ the snowball project, but it would be possible to compile a modified libstemmer
66
+ library containing these if desired.)
67
+
68
+
69
+ The stemwords example
70
+ =====================
71
+
72
+ The stemwords example program allows you to run any of the stemmers
73
+ compiled into the libstemmer library on a sample vocabulary. For
74
+ details on how to use it, run it with the "-h" command line option.
75
+
76
+
77
+ Using the library in a larger system
78
+ ====================================
79
+
80
+ If you are incorporating the library into the build system of a larger
81
+ program, I recommend copying the unpacked tarball without modification into
82
+ a subdirectory of the sources of your program. Future versions of the
83
+ library are intended to keep the same structure, so this will keep the
84
+ work required to move to a new version of the library to a minimum.
85
+
86
+ As an additional convenience, the list of source and header files used
87
+ in the library is detailed in mkinc.mak - a file which is in a suitable
88
+ format for inclusion by a Makefile. By including this file in your build
89
+ system, you can link the snowball system into your program with a few
90
+ extra rules.
91
+
92
+ Using the library in a system using GNU autotools
93
+ =================================================
94
+
95
+ The libstemmer_c library can be integrated into a larger system which uses the
96
+ GNU autotool framework (and in particular, automake and autoconf) as follows:
97
+
98
+ 1) Unpack libstemmer_c.tgz in the top level project directory so that there is
99
+ a libstemmer_c subdirectory of the top level directory of the project.
100
+
101
+ 2) Add a file "Makefile.am" to the unpacked libstemmer_c folder, containing:
102
+
103
+ noinst_LTLIBRARIES = libstemmer.la
104
+ include $(srcdir)/mkinc.mak
105
+ noinst_HEADERS = $(snowball_headers)
106
+ libstemmer_la_SOURCES = $(snowball_sources)
107
+
108
+ (You may also need to add other lines to this, for example, if you are using
109
+ compiler options which are not compatible with compiling the libstemmer
110
+ library.)
111
+
112
+ 3) Add libstemmer_c to the AC_CONFIG_FILES declaration in the project's
113
+ configure.ac file.
114
+
115
+ 4) Add to the top level makefile the following lines (or modify existing
116
+ assignments to these variables appropriately):
117
+
118
+ AUTOMAKE_OPTIONS = subdir-objects
119
+ AM_CPPFLAGS = -I$(top_srcdir)/libstemmer_c/include
120
+ SUBDIRS=libstemmer_c
121
+ <name>_LIBADD = libstemmer_c/libstemmer.la
122
+
123
+ (Where <name> is the name of the library or executable which links against
124
+ libstemmer.)
125
+
@@ -0,0 +1,209 @@
1
+ /* This is a simple program which uses libstemmer to provide a command
2
+ * line interface for stemming using any of the algorithms provided.
3
+ */
4
+
5
+ #include <stdio.h>
6
+ #include <stdlib.h> /* for malloc, free */
7
+ #include <string.h> /* for memmove */
8
+ #include <ctype.h> /* for isupper, tolower */
9
+
10
+ #include "libstemmer.h"
11
+
12
+ const char * progname;
13
+ static int pretty = 1;
14
+
15
+ static void
16
+ stem_file(struct sb_stemmer * stemmer, FILE * f_in, FILE * f_out)
17
+ {
18
+ #define INC 10
19
+ int lim = INC;
20
+ sb_symbol * b = (sb_symbol *) malloc(lim * sizeof(sb_symbol));
21
+
22
+ while(1) {
23
+ int ch = getc(f_in);
24
+ if (ch == EOF) {
25
+ free(b); return;
26
+ }
27
+ {
28
+ int i = 0;
29
+ int inlen = 0;
30
+ while(1) {
31
+ if (ch == '\n' || ch == EOF) break;
32
+ if (i == lim) {
33
+ sb_symbol * newb;
34
+ newb = (sb_symbol *)
35
+ realloc(b, (lim + INC) * sizeof(sb_symbol));
36
+ if (newb == 0) goto error;
37
+ b = newb;
38
+ lim = lim + INC;
39
+ }
40
+ /* Update count of utf-8 characters. */
41
+ if (ch < 0x80 || ch > 0xBF) inlen += 1;
42
+ /* force lower case: */
43
+ if (isupper(ch)) ch = tolower(ch);
44
+
45
+ b[i] = ch;
46
+ i++;
47
+ ch = getc(f_in);
48
+ }
49
+
50
+ {
51
+ const sb_symbol * stemmed = sb_stemmer_stem(stemmer, b, i);
52
+ if (stemmed == NULL)
53
+ {
54
+ fprintf(stderr, "Out of memory");
55
+ exit(1);
56
+ }
57
+ else
58
+ {
59
+ if (pretty == 1) {
60
+ fwrite(b, i, 1, f_out);
61
+ fputs(" -> ", f_out);
62
+ } else if (pretty == 2) {
63
+ fwrite(b, i, 1, f_out);
64
+ if (sb_stemmer_length(stemmer) > 0) {
65
+ int j;
66
+ if (inlen < 30) {
67
+ for (j = 30 - inlen; j > 0; j--)
68
+ fputs(" ", f_out);
69
+ } else {
70
+ fputs("\n", f_out);
71
+ for (j = 30; j > 0; j--)
72
+ fputs(" ", f_out);
73
+ }
74
+ }
75
+ }
76
+
77
+ fputs((char *)stemmed, f_out);
78
+ putc('\n', f_out);
79
+ }
80
+ }
81
+ }
82
+ }
83
+ error:
84
+ if (b != 0) free(b);
85
+ return;
86
+ }
87
+
88
+ /** Display the command line syntax, and then exit.
89
+ * @param n The value to exit with.
90
+ */
91
+ static void
92
+ usage(int n)
93
+ {
94
+ printf("usage: %s [-l <language>] [-i <input file>] [-o <output file>] [-c <character encoding>] [-p[2]] [-h]\n"
95
+ "\n"
96
+ "The input file consists of a list of words to be stemmed, one per\n"
97
+ "line. Words should be in lower case, but (for English) A-Z letters\n"
98
+ "are mapped to their a-z equivalents anyway. If omitted, stdin is\n"
99
+ "used.\n"
100
+ "\n"
101
+ "If -c is given, the argument is the character encoding of the input\n"
102
+ "and output files. If it is omitted, the UTF-8 encoding is used.\n"
103
+ "\n"
104
+ "If -p is given the output file consists of each word of the input\n"
105
+ "file followed by \"->\" followed by its stemmed equivalent.\n"
106
+ "If -p2 is given the output file is a two column layout containing\n"
107
+ "the input words in the first column and the stemmed eqivalents in\n"
108
+ "the second column.\n"
109
+ "Otherwise, the output file consists of the stemmed words, one per\n"
110
+ "line.\n"
111
+ "\n"
112
+ "-h displays this help\n",
113
+ progname);
114
+ exit(n);
115
+ }
116
+
117
+ int
118
+ main(int argc, char * argv[])
119
+ {
120
+ char * in = 0;
121
+ char * out = 0;
122
+ FILE * f_in;
123
+ FILE * f_out;
124
+ struct sb_stemmer * stemmer;
125
+
126
+ char * language = "english";
127
+ char * charenc = NULL;
128
+
129
+ char * s;
130
+ int i = 1;
131
+ pretty = 0;
132
+
133
+ progname = argv[0];
134
+
135
+ while(i < argc) {
136
+ s = argv[i++];
137
+ if (s[0] == '-') {
138
+ if (strcmp(s, "-o") == 0) {
139
+ if (i >= argc) {
140
+ fprintf(stderr, "%s requires an argument\n", s);
141
+ exit(1);
142
+ }
143
+ out = argv[i++];
144
+ } else if (strcmp(s, "-i") == 0) {
145
+ if (i >= argc) {
146
+ fprintf(stderr, "%s requires an argument\n", s);
147
+ exit(1);
148
+ }
149
+ in = argv[i++];
150
+ } else if (strcmp(s, "-l") == 0) {
151
+ if (i >= argc) {
152
+ fprintf(stderr, "%s requires an argument\n", s);
153
+ exit(1);
154
+ }
155
+ language = argv[i++];
156
+ } else if (strcmp(s, "-c") == 0) {
157
+ if (i >= argc) {
158
+ fprintf(stderr, "%s requires an argument\n", s);
159
+ exit(1);
160
+ }
161
+ charenc = argv[i++];
162
+ } else if (strcmp(s, "-p2") == 0) {
163
+ pretty = 2;
164
+ } else if (strcmp(s, "-p") == 0) {
165
+ pretty = 1;
166
+ } else if (strcmp(s, "-h") == 0) {
167
+ usage(0);
168
+ } else {
169
+ fprintf(stderr, "option %s unknown\n", s);
170
+ usage(1);
171
+ }
172
+ } else {
173
+ fprintf(stderr, "unexpected parameter %s\n", s);
174
+ usage(1);
175
+ }
176
+ }
177
+
178
+ /* prepare the files */
179
+ f_in = (in == 0) ? stdin : fopen(in, "r");
180
+ if (f_in == 0) {
181
+ fprintf(stderr, "file %s not found\n", in);
182
+ exit(1);
183
+ }
184
+ f_out = (out == 0) ? stdout : fopen(out, "w");
185
+ if (f_out == 0) {
186
+ fprintf(stderr, "file %s cannot be opened\n", out);
187
+ exit(1);
188
+ }
189
+
190
+ /* do the stemming process: */
191
+ stemmer = sb_stemmer_new(language, charenc);
192
+ if (stemmer == 0) {
193
+ if (charenc == NULL) {
194
+ fprintf(stderr, "language `%s' not available for stemming\n", language);
195
+ exit(1);
196
+ } else {
197
+ fprintf(stderr, "language `%s' not available for stemming in encoding `%s'\n", language, charenc);
198
+ exit(1);
199
+ }
200
+ }
201
+ stem_file(stemmer, f_in, f_out);
202
+ sb_stemmer_delete(stemmer);
203
+
204
+ if (in != 0) (void) fclose(f_in);
205
+ if (out != 0) (void) fclose(f_out);
206
+
207
+ return 0;
208
+ }
209
+
@@ -0,0 +1,79 @@
1
+
2
+ /* Make header file work when included from C++ */
3
+ #ifdef __cplusplus
4
+ extern "C" {
5
+ #endif
6
+
7
+ struct sb_stemmer;
8
+ typedef unsigned char sb_symbol;
9
+
10
+ /* FIXME - should be able to get a version number for each stemming
11
+ * algorithm (which will be incremented each time the output changes). */
12
+
13
+ /** Returns an array of the names of the available stemming algorithms.
14
+ * Note that these are the canonical names - aliases (ie, other names for
15
+ * the same algorithm) will not be included in the list.
16
+ * The list is terminated with a null pointer.
17
+ *
18
+ * The list must not be modified in any way.
19
+ */
20
+ const char ** sb_stemmer_list(void);
21
+
22
+ /** Create a new stemmer object, using the specified algorithm, for the
23
+ * specified character encoding.
24
+ *
25
+ * All algorithms will usually be available in UTF-8, but may also be
26
+ * available in other character encodings.
27
+ *
28
+ * @param algorithm The algorithm name. This is either the english
29
+ * name of the algorithm, or the 2 or 3 letter ISO 639 codes for the
30
+ * language. Note that case is significant in this parameter - the
31
+ * value should be supplied in lower case.
32
+ *
33
+ * @param charenc The character encoding. NULL may be passed as
34
+ * this value, in which case UTF-8 encoding will be assumed. Otherwise,
35
+ * the argument may be one of "UTF_8", "ISO_8859_1" (ie, Latin 1),
36
+ * "CP850" (ie, MS-DOS Latin 1) or "KOI8_R" (Russian). Note that
37
+ * case is significant in this parameter.
38
+ *
39
+ * @return NULL if the specified algorithm is not recognised, or the
40
+ * algorithm is not available for the requested encoding. Otherwise,
41
+ * returns a pointer to a newly created stemmer for the requested algorithm.
42
+ * The returned pointer must be deleted by calling sb_stemmer_delete().
43
+ *
44
+ * @note NULL will also be returned if an out of memory error occurs.
45
+ */
46
+ struct sb_stemmer * sb_stemmer_new(const char * algorithm, const char * charenc);
47
+
48
+ /** Delete a stemmer object.
49
+ *
50
+ * This frees all resources allocated for the stemmer. After calling
51
+ * this function, the supplied stemmer may no longer be used in any way.
52
+ *
53
+ * It is safe to pass a null pointer to this function - this will have
54
+ * no effect.
55
+ */
56
+ void sb_stemmer_delete(struct sb_stemmer * stemmer);
57
+
58
+ /** Stem a word.
59
+ *
60
+ * The return value is owned by the stemmer - it must not be freed or
61
+ * modified, and it will become invalid when the stemmer is called again,
62
+ * or if the stemmer is freed.
63
+ *
64
+ * The length of the return value can be obtained using sb_stemmer_length().
65
+ *
66
+ * If an out-of-memory error occurs, this will return NULL.
67
+ */
68
+ const sb_symbol * sb_stemmer_stem(struct sb_stemmer * stemmer,
69
+ const sb_symbol * word, int size);
70
+
71
+ /** Get the length of the result of the last stemmed word.
72
+ * This should not be called before sb_stemmer_stem() has been called.
73
+ */
74
+ int sb_stemmer_length(struct sb_stemmer * stemmer);
75
+
76
+ #ifdef __cplusplus
77
+ }
78
+ #endif
79
+
@@ -0,0 +1,93 @@
1
+
2
+ #include <stdlib.h>
3
+ #include <string.h>
4
+ #include "../include/libstemmer.h"
5
+ #include "../runtime/api.h"
6
+ #include "modules.h"
7
+
8
+ struct sb_stemmer {
9
+ struct SN_env * (*create)(void);
10
+ void (*close)(struct SN_env *);
11
+ int (*stem)(struct SN_env *);
12
+
13
+ struct SN_env * env;
14
+ };
15
+
16
+ extern const char **
17
+ sb_stemmer_list(void)
18
+ {
19
+ return algorithm_names;
20
+ }
21
+
22
+ static stemmer_encoding_t
23
+ sb_getenc(const char * charenc)
24
+ {
25
+ struct stemmer_encoding * encoding;
26
+ if (charenc == NULL) return ENC_UTF_8;
27
+ for (encoding = encodings; encoding->name != 0; encoding++) {
28
+ if (strcmp(encoding->name, charenc) == 0) break;
29
+ }
30
+ if (encoding->name == NULL) return ENC_UNKNOWN;
31
+ return encoding->enc;
32
+ }
33
+
34
+ extern struct sb_stemmer *
35
+ sb_stemmer_new(const char * algorithm, const char * charenc)
36
+ {
37
+ stemmer_encoding_t enc;
38
+ struct stemmer_modules * module;
39
+ struct sb_stemmer * stemmer =
40
+ (struct sb_stemmer *) malloc(sizeof(struct sb_stemmer));
41
+ if (stemmer == NULL) return NULL;
42
+ enc = sb_getenc(charenc);
43
+ if (enc == ENC_UNKNOWN) return NULL;
44
+
45
+ for (module = modules; module->name != 0; module++) {
46
+ if (strcmp(module->name, algorithm) == 0 && module->enc == enc) break;
47
+ }
48
+ if (module->name == NULL) return NULL;
49
+
50
+ stemmer->create = module->create;
51
+ stemmer->close = module->close;
52
+ stemmer->stem = module->stem;
53
+
54
+ stemmer->env = stemmer->create();
55
+ if (stemmer->env == NULL)
56
+ {
57
+ sb_stemmer_delete(stemmer);
58
+ return NULL;
59
+ }
60
+
61
+ return stemmer;
62
+ }
63
+
64
+ void
65
+ sb_stemmer_delete(struct sb_stemmer * stemmer)
66
+ {
67
+ if (stemmer == 0) return;
68
+ if (stemmer->close == 0) return;
69
+ stemmer->close(stemmer->env);
70
+ stemmer->close = 0;
71
+ free(stemmer);
72
+ }
73
+
74
+ const sb_symbol *
75
+ sb_stemmer_stem(struct sb_stemmer * stemmer, const sb_symbol * word, int size)
76
+ {
77
+ int ret;
78
+ if (SN_set_current(stemmer->env, size, (const symbol *)(word)))
79
+ {
80
+ stemmer->env->l = 0;
81
+ return NULL;
82
+ }
83
+ ret = stemmer->stem(stemmer->env);
84
+ if (ret < 0) return NULL;
85
+ stemmer->env->p[stemmer->env->l] = 0;
86
+ return (const sb_symbol *)(stemmer->env->p);
87
+ }
88
+
89
+ int
90
+ sb_stemmer_length(struct sb_stemmer * stemmer)
91
+ {
92
+ return stemmer->env->l;
93
+ }