ruby-stemmer-dimelo 0.9.3.dimelo1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/MIT-LICENSE +21 -0
- data/README.rdoc +117 -0
- data/Rakefile +70 -0
- data/VERSION +1 -0
- data/ext/lingua/extconf.rb +40 -0
- data/ext/lingua/stemmer.c +115 -0
- data/lib/lingua/stemmer.rb +60 -0
- data/libstemmer_c/MANIFEST +72 -0
- data/libstemmer_c/Makefile +9 -0
- data/libstemmer_c/Makefile.windows +15 -0
- data/libstemmer_c/README +125 -0
- data/libstemmer_c/examples/stemwords.c +209 -0
- data/libstemmer_c/include/libstemmer.h +79 -0
- data/libstemmer_c/libstemmer/libstemmer.c +93 -0
- data/libstemmer_c/libstemmer/libstemmer_utf8.c +93 -0
- data/libstemmer_c/libstemmer/modules.h +195 -0
- data/libstemmer_c/libstemmer/modules.txt +51 -0
- data/libstemmer_c/libstemmer/modules_utf8.h +123 -0
- data/libstemmer_c/libstemmer/modules_utf8.txt +50 -0
- data/libstemmer_c/mkinc.mak +86 -0
- data/libstemmer_c/mkinc_utf8.mak +54 -0
- data/libstemmer_c/runtime/api.c +66 -0
- data/libstemmer_c/runtime/api.h +26 -0
- data/libstemmer_c/runtime/header.h +58 -0
- data/libstemmer_c/runtime/utilities.c +478 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_danish.c +337 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_danish.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_dutch.c +624 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_dutch.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_english.c +1117 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_english.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_finnish.c +762 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_finnish.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_french.c +1230 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_french.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_german.c +503 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_german.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_hungarian.c +1230 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_hungarian.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_italian.c +1065 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_italian.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_latin.c +443 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_latin.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_norwegian.c +297 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_norwegian.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_porter.c +749 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_porter.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_portuguese.c +1017 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_portuguese.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_spanish.c +1093 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_spanish.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_swedish.c +307 -0
- data/libstemmer_c/src_c/stem_ISO_8859_1_swedish.h +16 -0
- data/libstemmer_c/src_c/stem_ISO_8859_2_romanian.c +998 -0
- data/libstemmer_c/src_c/stem_ISO_8859_2_romanian.h +16 -0
- data/libstemmer_c/src_c/stem_KOI8_R_russian.c +700 -0
- data/libstemmer_c/src_c/stem_KOI8_R_russian.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_danish.c +339 -0
- data/libstemmer_c/src_c/stem_UTF_8_danish.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_dutch.c +634 -0
- data/libstemmer_c/src_c/stem_UTF_8_dutch.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_english.c +1125 -0
- data/libstemmer_c/src_c/stem_UTF_8_english.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_finnish.c +768 -0
- data/libstemmer_c/src_c/stem_UTF_8_finnish.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_french.c +1230 -0
- data/libstemmer_c/src_c/stem_UTF_8_french.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_german.c +509 -0
- data/libstemmer_c/src_c/stem_UTF_8_german.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_hungarian.c +1234 -0
- data/libstemmer_c/src_c/stem_UTF_8_hungarian.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_italian.c +1073 -0
- data/libstemmer_c/src_c/stem_UTF_8_italian.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_latin.c +443 -0
- data/libstemmer_c/src_c/stem_UTF_8_latin.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_norwegian.c +299 -0
- data/libstemmer_c/src_c/stem_UTF_8_norwegian.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_porter.c +755 -0
- data/libstemmer_c/src_c/stem_UTF_8_porter.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_portuguese.c +1023 -0
- data/libstemmer_c/src_c/stem_UTF_8_portuguese.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_romanian.c +1004 -0
- data/libstemmer_c/src_c/stem_UTF_8_romanian.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_russian.c +694 -0
- data/libstemmer_c/src_c/stem_UTF_8_russian.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_spanish.c +1097 -0
- data/libstemmer_c/src_c/stem_UTF_8_spanish.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_swedish.c +309 -0
- data/libstemmer_c/src_c/stem_UTF_8_swedish.h +16 -0
- data/libstemmer_c/src_c/stem_UTF_8_turkish.c +2205 -0
- data/libstemmer_c/src_c/stem_UTF_8_turkish.h +16 -0
- data/test/helper.rb +3 -0
- data/test/lingua/test_stemmer.rb +99 -0
- metadata +141 -0
@@ -0,0 +1,9 @@
|
|
1
|
+
include mkinc.mak
|
2
|
+
CFLAGS=-Iinclude -fPIC $(ARCHFLAGS)
|
3
|
+
all: libstemmer.o stemwords
|
4
|
+
libstemmer.o: $(snowball_sources:.c=.o)
|
5
|
+
$(AR) -cru $@ $^
|
6
|
+
stemwords: examples/stemwords.o libstemmer.o
|
7
|
+
$(CC) -o $@ $^
|
8
|
+
clean:
|
9
|
+
rm -f stemwords *.o src_c/*.o runtime/*.o libstemmer/*.o
|
@@ -0,0 +1,15 @@
|
|
1
|
+
include mkinc.mak
|
2
|
+
|
3
|
+
AR=i686-w64-mingw32-ar
|
4
|
+
CC=i686-w64-mingw32-gcc
|
5
|
+
LD=i686-w64-mingw32-ld
|
6
|
+
|
7
|
+
CFLAGS=-Iinclude
|
8
|
+
|
9
|
+
all: libstemmer.o stemwords
|
10
|
+
libstemmer.o: $(snowball_sources:.c=.o)
|
11
|
+
$(AR) -cru $@ $^
|
12
|
+
stemwords: examples/stemwords.o libstemmer.o
|
13
|
+
$(CC) -o $@ $^
|
14
|
+
clean:
|
15
|
+
rm -f stemwords *.o src_c/*.o runtime/*.o libstemmer/*.o
|
data/libstemmer_c/README
ADDED
@@ -0,0 +1,125 @@
|
|
1
|
+
libstemmer_c
|
2
|
+
============
|
3
|
+
|
4
|
+
This document pertains to the C version of the libstemmer distribution,
|
5
|
+
available for download from:
|
6
|
+
|
7
|
+
http://snowball.tartarus.org/dist/libstemmer_c.tgz
|
8
|
+
|
9
|
+
|
10
|
+
Compiling the library
|
11
|
+
=====================
|
12
|
+
|
13
|
+
A simple makefile is provided for Unix style systems. On such systems, it
|
14
|
+
should be possible simply to run "make", and the file "libstemmer.o"
|
15
|
+
and the example program "stemwords" will be generated.
|
16
|
+
|
17
|
+
If this doesn't work on your system, you need to write your own build
|
18
|
+
system (or call the compiler directly). The files to compile are
|
19
|
+
all contained in the "libstemmer", "runtime" and "src_c" directories,
|
20
|
+
and the public header file is contained in the "include" directory.
|
21
|
+
|
22
|
+
The library comes in two flavours; UTF-8 only, and UTF-8 plus other character
|
23
|
+
sets. To use the utf-8 only flavour, compile "libstemmer_utf8.c" instead of
|
24
|
+
"libstemmer.c".
|
25
|
+
|
26
|
+
For convenience "mkinc.mak" is a makefile fragment listing the source files and
|
27
|
+
header files used to compile the standard version of the library.
|
28
|
+
"mkinc_utf8.mak" is a comparable makefile fragment listing just the source
|
29
|
+
files for the UTF-8 only version of the library.
|
30
|
+
|
31
|
+
|
32
|
+
Using the library
|
33
|
+
=================
|
34
|
+
|
35
|
+
The library provides a simple C API. Essentially, a new stemmer can
|
36
|
+
be obtained by using "sb_stemmer_new". "sb_stemmer_stem" is then
|
37
|
+
used to stem a word, "sb_stemmer_length" returns the stemmed
|
38
|
+
length of the last word processed, and "sb_stemmer_delete" is
|
39
|
+
used to delete a stemmer.
|
40
|
+
|
41
|
+
Creating a stemmer is a relatively expensive operation - the expected
|
42
|
+
usage pattern is that a new stemmer is created when needed, used
|
43
|
+
to stem many words, and deleted after some time.
|
44
|
+
|
45
|
+
Stemmers are re-entrant, but not threadsafe. In other words, if
|
46
|
+
you wish to access the same stemmer object from multiple threads,
|
47
|
+
you must ensure that all access is protected by a mutex or similar
|
48
|
+
device.
|
49
|
+
|
50
|
+
libstemmer does not currently incorporate any mechanism for caching the results
|
51
|
+
of stemming operations. Such caching can greatly increase the performance of a
|
52
|
+
stemmer under certain situations, so suitable patches will be considered for
|
53
|
+
inclusion.
|
54
|
+
|
55
|
+
The standard libstemmer sources contain an algorithm for each of the supported
|
56
|
+
languages. The algorithm may be selected using the english name of the
|
57
|
+
language, or using the 2 or 3 letter ISO 639 language codes. In addition,
|
58
|
+
the traditional "Porter" stemming algorithm for english is included for
|
59
|
+
backwards compatibility purposes, but we recommend use of the "English"
|
60
|
+
stemmer in preference for new projects.
|
61
|
+
|
62
|
+
(Some minor algorithms which are included only as curiosities in the snowball
|
63
|
+
website, such as the Lovins stemmer and the Kraaij Pohlmann stemmer, are not
|
64
|
+
included in the standard libstemmer sources. These are not really supported by
|
65
|
+
the snowball project, but it would be possible to compile a modified libstemmer
|
66
|
+
library containing these if desired.)
|
67
|
+
|
68
|
+
|
69
|
+
The stemwords example
|
70
|
+
=====================
|
71
|
+
|
72
|
+
The stemwords example program allows you to run any of the stemmers
|
73
|
+
compiled into the libstemmer library on a sample vocabulary. For
|
74
|
+
details on how to use it, run it with the "-h" command line option.
|
75
|
+
|
76
|
+
|
77
|
+
Using the library in a larger system
|
78
|
+
====================================
|
79
|
+
|
80
|
+
If you are incorporating the library into the build system of a larger
|
81
|
+
program, I recommend copying the unpacked tarball without modification into
|
82
|
+
a subdirectory of the sources of your program. Future versions of the
|
83
|
+
library are intended to keep the same structure, so this will keep the
|
84
|
+
work required to move to a new version of the library to a minimum.
|
85
|
+
|
86
|
+
As an additional convenience, the list of source and header files used
|
87
|
+
in the library is detailed in mkinc.mak - a file which is in a suitable
|
88
|
+
format for inclusion by a Makefile. By including this file in your build
|
89
|
+
system, you can link the snowball system into your program with a few
|
90
|
+
extra rules.
|
91
|
+
|
92
|
+
Using the library in a system using GNU autotools
|
93
|
+
=================================================
|
94
|
+
|
95
|
+
The libstemmer_c library can be integrated into a larger system which uses the
|
96
|
+
GNU autotool framework (and in particular, automake and autoconf) as follows:
|
97
|
+
|
98
|
+
1) Unpack libstemmer_c.tgz in the top level project directory so that there is
|
99
|
+
a libstemmer_c subdirectory of the top level directory of the project.
|
100
|
+
|
101
|
+
2) Add a file "Makefile.am" to the unpacked libstemmer_c folder, containing:
|
102
|
+
|
103
|
+
noinst_LTLIBRARIES = libstemmer.la
|
104
|
+
include $(srcdir)/mkinc.mak
|
105
|
+
noinst_HEADERS = $(snowball_headers)
|
106
|
+
libstemmer_la_SOURCES = $(snowball_sources)
|
107
|
+
|
108
|
+
(You may also need to add other lines to this, for example, if you are using
|
109
|
+
compiler options which are not compatible with compiling the libstemmer
|
110
|
+
library.)
|
111
|
+
|
112
|
+
3) Add libstemmer_c to the AC_CONFIG_FILES declaration in the project's
|
113
|
+
configure.ac file.
|
114
|
+
|
115
|
+
4) Add to the top level makefile the following lines (or modify existing
|
116
|
+
assignments to these variables appropriately):
|
117
|
+
|
118
|
+
AUTOMAKE_OPTIONS = subdir-objects
|
119
|
+
AM_CPPFLAGS = -I$(top_srcdir)/libstemmer_c/include
|
120
|
+
SUBDIRS=libstemmer_c
|
121
|
+
<name>_LIBADD = libstemmer_c/libstemmer.la
|
122
|
+
|
123
|
+
(Where <name> is the name of the library or executable which links against
|
124
|
+
libstemmer.)
|
125
|
+
|
@@ -0,0 +1,209 @@
|
|
1
|
+
/* This is a simple program which uses libstemmer to provide a command
|
2
|
+
* line interface for stemming using any of the algorithms provided.
|
3
|
+
*/
|
4
|
+
|
5
|
+
#include <stdio.h>
|
6
|
+
#include <stdlib.h> /* for malloc, free */
|
7
|
+
#include <string.h> /* for memmove */
|
8
|
+
#include <ctype.h> /* for isupper, tolower */
|
9
|
+
|
10
|
+
#include "libstemmer.h"
|
11
|
+
|
12
|
+
const char * progname;
|
13
|
+
static int pretty = 1;
|
14
|
+
|
15
|
+
static void
|
16
|
+
stem_file(struct sb_stemmer * stemmer, FILE * f_in, FILE * f_out)
|
17
|
+
{
|
18
|
+
#define INC 10
|
19
|
+
int lim = INC;
|
20
|
+
sb_symbol * b = (sb_symbol *) malloc(lim * sizeof(sb_symbol));
|
21
|
+
|
22
|
+
while(1) {
|
23
|
+
int ch = getc(f_in);
|
24
|
+
if (ch == EOF) {
|
25
|
+
free(b); return;
|
26
|
+
}
|
27
|
+
{
|
28
|
+
int i = 0;
|
29
|
+
int inlen = 0;
|
30
|
+
while(1) {
|
31
|
+
if (ch == '\n' || ch == EOF) break;
|
32
|
+
if (i == lim) {
|
33
|
+
sb_symbol * newb;
|
34
|
+
newb = (sb_symbol *)
|
35
|
+
realloc(b, (lim + INC) * sizeof(sb_symbol));
|
36
|
+
if (newb == 0) goto error;
|
37
|
+
b = newb;
|
38
|
+
lim = lim + INC;
|
39
|
+
}
|
40
|
+
/* Update count of utf-8 characters. */
|
41
|
+
if (ch < 0x80 || ch > 0xBF) inlen += 1;
|
42
|
+
/* force lower case: */
|
43
|
+
if (isupper(ch)) ch = tolower(ch);
|
44
|
+
|
45
|
+
b[i] = ch;
|
46
|
+
i++;
|
47
|
+
ch = getc(f_in);
|
48
|
+
}
|
49
|
+
|
50
|
+
{
|
51
|
+
const sb_symbol * stemmed = sb_stemmer_stem(stemmer, b, i);
|
52
|
+
if (stemmed == NULL)
|
53
|
+
{
|
54
|
+
fprintf(stderr, "Out of memory");
|
55
|
+
exit(1);
|
56
|
+
}
|
57
|
+
else
|
58
|
+
{
|
59
|
+
if (pretty == 1) {
|
60
|
+
fwrite(b, i, 1, f_out);
|
61
|
+
fputs(" -> ", f_out);
|
62
|
+
} else if (pretty == 2) {
|
63
|
+
fwrite(b, i, 1, f_out);
|
64
|
+
if (sb_stemmer_length(stemmer) > 0) {
|
65
|
+
int j;
|
66
|
+
if (inlen < 30) {
|
67
|
+
for (j = 30 - inlen; j > 0; j--)
|
68
|
+
fputs(" ", f_out);
|
69
|
+
} else {
|
70
|
+
fputs("\n", f_out);
|
71
|
+
for (j = 30; j > 0; j--)
|
72
|
+
fputs(" ", f_out);
|
73
|
+
}
|
74
|
+
}
|
75
|
+
}
|
76
|
+
|
77
|
+
fputs((char *)stemmed, f_out);
|
78
|
+
putc('\n', f_out);
|
79
|
+
}
|
80
|
+
}
|
81
|
+
}
|
82
|
+
}
|
83
|
+
error:
|
84
|
+
if (b != 0) free(b);
|
85
|
+
return;
|
86
|
+
}
|
87
|
+
|
88
|
+
/** Display the command line syntax, and then exit.
|
89
|
+
* @param n The value to exit with.
|
90
|
+
*/
|
91
|
+
static void
|
92
|
+
usage(int n)
|
93
|
+
{
|
94
|
+
printf("usage: %s [-l <language>] [-i <input file>] [-o <output file>] [-c <character encoding>] [-p[2]] [-h]\n"
|
95
|
+
"\n"
|
96
|
+
"The input file consists of a list of words to be stemmed, one per\n"
|
97
|
+
"line. Words should be in lower case, but (for English) A-Z letters\n"
|
98
|
+
"are mapped to their a-z equivalents anyway. If omitted, stdin is\n"
|
99
|
+
"used.\n"
|
100
|
+
"\n"
|
101
|
+
"If -c is given, the argument is the character encoding of the input\n"
|
102
|
+
"and output files. If it is omitted, the UTF-8 encoding is used.\n"
|
103
|
+
"\n"
|
104
|
+
"If -p is given the output file consists of each word of the input\n"
|
105
|
+
"file followed by \"->\" followed by its stemmed equivalent.\n"
|
106
|
+
"If -p2 is given the output file is a two column layout containing\n"
|
107
|
+
"the input words in the first column and the stemmed eqivalents in\n"
|
108
|
+
"the second column.\n"
|
109
|
+
"Otherwise, the output file consists of the stemmed words, one per\n"
|
110
|
+
"line.\n"
|
111
|
+
"\n"
|
112
|
+
"-h displays this help\n",
|
113
|
+
progname);
|
114
|
+
exit(n);
|
115
|
+
}
|
116
|
+
|
117
|
+
int
|
118
|
+
main(int argc, char * argv[])
|
119
|
+
{
|
120
|
+
char * in = 0;
|
121
|
+
char * out = 0;
|
122
|
+
FILE * f_in;
|
123
|
+
FILE * f_out;
|
124
|
+
struct sb_stemmer * stemmer;
|
125
|
+
|
126
|
+
char * language = "english";
|
127
|
+
char * charenc = NULL;
|
128
|
+
|
129
|
+
char * s;
|
130
|
+
int i = 1;
|
131
|
+
pretty = 0;
|
132
|
+
|
133
|
+
progname = argv[0];
|
134
|
+
|
135
|
+
while(i < argc) {
|
136
|
+
s = argv[i++];
|
137
|
+
if (s[0] == '-') {
|
138
|
+
if (strcmp(s, "-o") == 0) {
|
139
|
+
if (i >= argc) {
|
140
|
+
fprintf(stderr, "%s requires an argument\n", s);
|
141
|
+
exit(1);
|
142
|
+
}
|
143
|
+
out = argv[i++];
|
144
|
+
} else if (strcmp(s, "-i") == 0) {
|
145
|
+
if (i >= argc) {
|
146
|
+
fprintf(stderr, "%s requires an argument\n", s);
|
147
|
+
exit(1);
|
148
|
+
}
|
149
|
+
in = argv[i++];
|
150
|
+
} else if (strcmp(s, "-l") == 0) {
|
151
|
+
if (i >= argc) {
|
152
|
+
fprintf(stderr, "%s requires an argument\n", s);
|
153
|
+
exit(1);
|
154
|
+
}
|
155
|
+
language = argv[i++];
|
156
|
+
} else if (strcmp(s, "-c") == 0) {
|
157
|
+
if (i >= argc) {
|
158
|
+
fprintf(stderr, "%s requires an argument\n", s);
|
159
|
+
exit(1);
|
160
|
+
}
|
161
|
+
charenc = argv[i++];
|
162
|
+
} else if (strcmp(s, "-p2") == 0) {
|
163
|
+
pretty = 2;
|
164
|
+
} else if (strcmp(s, "-p") == 0) {
|
165
|
+
pretty = 1;
|
166
|
+
} else if (strcmp(s, "-h") == 0) {
|
167
|
+
usage(0);
|
168
|
+
} else {
|
169
|
+
fprintf(stderr, "option %s unknown\n", s);
|
170
|
+
usage(1);
|
171
|
+
}
|
172
|
+
} else {
|
173
|
+
fprintf(stderr, "unexpected parameter %s\n", s);
|
174
|
+
usage(1);
|
175
|
+
}
|
176
|
+
}
|
177
|
+
|
178
|
+
/* prepare the files */
|
179
|
+
f_in = (in == 0) ? stdin : fopen(in, "r");
|
180
|
+
if (f_in == 0) {
|
181
|
+
fprintf(stderr, "file %s not found\n", in);
|
182
|
+
exit(1);
|
183
|
+
}
|
184
|
+
f_out = (out == 0) ? stdout : fopen(out, "w");
|
185
|
+
if (f_out == 0) {
|
186
|
+
fprintf(stderr, "file %s cannot be opened\n", out);
|
187
|
+
exit(1);
|
188
|
+
}
|
189
|
+
|
190
|
+
/* do the stemming process: */
|
191
|
+
stemmer = sb_stemmer_new(language, charenc);
|
192
|
+
if (stemmer == 0) {
|
193
|
+
if (charenc == NULL) {
|
194
|
+
fprintf(stderr, "language `%s' not available for stemming\n", language);
|
195
|
+
exit(1);
|
196
|
+
} else {
|
197
|
+
fprintf(stderr, "language `%s' not available for stemming in encoding `%s'\n", language, charenc);
|
198
|
+
exit(1);
|
199
|
+
}
|
200
|
+
}
|
201
|
+
stem_file(stemmer, f_in, f_out);
|
202
|
+
sb_stemmer_delete(stemmer);
|
203
|
+
|
204
|
+
if (in != 0) (void) fclose(f_in);
|
205
|
+
if (out != 0) (void) fclose(f_out);
|
206
|
+
|
207
|
+
return 0;
|
208
|
+
}
|
209
|
+
|
@@ -0,0 +1,79 @@
|
|
1
|
+
|
2
|
+
/* Make header file work when included from C++ */
|
3
|
+
#ifdef __cplusplus
|
4
|
+
extern "C" {
|
5
|
+
#endif
|
6
|
+
|
7
|
+
struct sb_stemmer;
|
8
|
+
typedef unsigned char sb_symbol;
|
9
|
+
|
10
|
+
/* FIXME - should be able to get a version number for each stemming
|
11
|
+
* algorithm (which will be incremented each time the output changes). */
|
12
|
+
|
13
|
+
/** Returns an array of the names of the available stemming algorithms.
|
14
|
+
* Note that these are the canonical names - aliases (ie, other names for
|
15
|
+
* the same algorithm) will not be included in the list.
|
16
|
+
* The list is terminated with a null pointer.
|
17
|
+
*
|
18
|
+
* The list must not be modified in any way.
|
19
|
+
*/
|
20
|
+
const char ** sb_stemmer_list(void);
|
21
|
+
|
22
|
+
/** Create a new stemmer object, using the specified algorithm, for the
|
23
|
+
* specified character encoding.
|
24
|
+
*
|
25
|
+
* All algorithms will usually be available in UTF-8, but may also be
|
26
|
+
* available in other character encodings.
|
27
|
+
*
|
28
|
+
* @param algorithm The algorithm name. This is either the english
|
29
|
+
* name of the algorithm, or the 2 or 3 letter ISO 639 codes for the
|
30
|
+
* language. Note that case is significant in this parameter - the
|
31
|
+
* value should be supplied in lower case.
|
32
|
+
*
|
33
|
+
* @param charenc The character encoding. NULL may be passed as
|
34
|
+
* this value, in which case UTF-8 encoding will be assumed. Otherwise,
|
35
|
+
* the argument may be one of "UTF_8", "ISO_8859_1" (ie, Latin 1),
|
36
|
+
* "CP850" (ie, MS-DOS Latin 1) or "KOI8_R" (Russian). Note that
|
37
|
+
* case is significant in this parameter.
|
38
|
+
*
|
39
|
+
* @return NULL if the specified algorithm is not recognised, or the
|
40
|
+
* algorithm is not available for the requested encoding. Otherwise,
|
41
|
+
* returns a pointer to a newly created stemmer for the requested algorithm.
|
42
|
+
* The returned pointer must be deleted by calling sb_stemmer_delete().
|
43
|
+
*
|
44
|
+
* @note NULL will also be returned if an out of memory error occurs.
|
45
|
+
*/
|
46
|
+
struct sb_stemmer * sb_stemmer_new(const char * algorithm, const char * charenc);
|
47
|
+
|
48
|
+
/** Delete a stemmer object.
|
49
|
+
*
|
50
|
+
* This frees all resources allocated for the stemmer. After calling
|
51
|
+
* this function, the supplied stemmer may no longer be used in any way.
|
52
|
+
*
|
53
|
+
* It is safe to pass a null pointer to this function - this will have
|
54
|
+
* no effect.
|
55
|
+
*/
|
56
|
+
void sb_stemmer_delete(struct sb_stemmer * stemmer);
|
57
|
+
|
58
|
+
/** Stem a word.
|
59
|
+
*
|
60
|
+
* The return value is owned by the stemmer - it must not be freed or
|
61
|
+
* modified, and it will become invalid when the stemmer is called again,
|
62
|
+
* or if the stemmer is freed.
|
63
|
+
*
|
64
|
+
* The length of the return value can be obtained using sb_stemmer_length().
|
65
|
+
*
|
66
|
+
* If an out-of-memory error occurs, this will return NULL.
|
67
|
+
*/
|
68
|
+
const sb_symbol * sb_stemmer_stem(struct sb_stemmer * stemmer,
|
69
|
+
const sb_symbol * word, int size);
|
70
|
+
|
71
|
+
/** Get the length of the result of the last stemmed word.
|
72
|
+
* This should not be called before sb_stemmer_stem() has been called.
|
73
|
+
*/
|
74
|
+
int sb_stemmer_length(struct sb_stemmer * stemmer);
|
75
|
+
|
76
|
+
#ifdef __cplusplus
|
77
|
+
}
|
78
|
+
#endif
|
79
|
+
|
@@ -0,0 +1,93 @@
|
|
1
|
+
|
2
|
+
#include <stdlib.h>
|
3
|
+
#include <string.h>
|
4
|
+
#include "../include/libstemmer.h"
|
5
|
+
#include "../runtime/api.h"
|
6
|
+
#include "modules.h"
|
7
|
+
|
8
|
+
struct sb_stemmer {
|
9
|
+
struct SN_env * (*create)(void);
|
10
|
+
void (*close)(struct SN_env *);
|
11
|
+
int (*stem)(struct SN_env *);
|
12
|
+
|
13
|
+
struct SN_env * env;
|
14
|
+
};
|
15
|
+
|
16
|
+
extern const char **
|
17
|
+
sb_stemmer_list(void)
|
18
|
+
{
|
19
|
+
return algorithm_names;
|
20
|
+
}
|
21
|
+
|
22
|
+
static stemmer_encoding_t
|
23
|
+
sb_getenc(const char * charenc)
|
24
|
+
{
|
25
|
+
struct stemmer_encoding * encoding;
|
26
|
+
if (charenc == NULL) return ENC_UTF_8;
|
27
|
+
for (encoding = encodings; encoding->name != 0; encoding++) {
|
28
|
+
if (strcmp(encoding->name, charenc) == 0) break;
|
29
|
+
}
|
30
|
+
if (encoding->name == NULL) return ENC_UNKNOWN;
|
31
|
+
return encoding->enc;
|
32
|
+
}
|
33
|
+
|
34
|
+
extern struct sb_stemmer *
|
35
|
+
sb_stemmer_new(const char * algorithm, const char * charenc)
|
36
|
+
{
|
37
|
+
stemmer_encoding_t enc;
|
38
|
+
struct stemmer_modules * module;
|
39
|
+
struct sb_stemmer * stemmer =
|
40
|
+
(struct sb_stemmer *) malloc(sizeof(struct sb_stemmer));
|
41
|
+
if (stemmer == NULL) return NULL;
|
42
|
+
enc = sb_getenc(charenc);
|
43
|
+
if (enc == ENC_UNKNOWN) return NULL;
|
44
|
+
|
45
|
+
for (module = modules; module->name != 0; module++) {
|
46
|
+
if (strcmp(module->name, algorithm) == 0 && module->enc == enc) break;
|
47
|
+
}
|
48
|
+
if (module->name == NULL) return NULL;
|
49
|
+
|
50
|
+
stemmer->create = module->create;
|
51
|
+
stemmer->close = module->close;
|
52
|
+
stemmer->stem = module->stem;
|
53
|
+
|
54
|
+
stemmer->env = stemmer->create();
|
55
|
+
if (stemmer->env == NULL)
|
56
|
+
{
|
57
|
+
sb_stemmer_delete(stemmer);
|
58
|
+
return NULL;
|
59
|
+
}
|
60
|
+
|
61
|
+
return stemmer;
|
62
|
+
}
|
63
|
+
|
64
|
+
void
|
65
|
+
sb_stemmer_delete(struct sb_stemmer * stemmer)
|
66
|
+
{
|
67
|
+
if (stemmer == 0) return;
|
68
|
+
if (stemmer->close == 0) return;
|
69
|
+
stemmer->close(stemmer->env);
|
70
|
+
stemmer->close = 0;
|
71
|
+
free(stemmer);
|
72
|
+
}
|
73
|
+
|
74
|
+
const sb_symbol *
|
75
|
+
sb_stemmer_stem(struct sb_stemmer * stemmer, const sb_symbol * word, int size)
|
76
|
+
{
|
77
|
+
int ret;
|
78
|
+
if (SN_set_current(stemmer->env, size, (const symbol *)(word)))
|
79
|
+
{
|
80
|
+
stemmer->env->l = 0;
|
81
|
+
return NULL;
|
82
|
+
}
|
83
|
+
ret = stemmer->stem(stemmer->env);
|
84
|
+
if (ret < 0) return NULL;
|
85
|
+
stemmer->env->p[stemmer->env->l] = 0;
|
86
|
+
return (const sb_symbol *)(stemmer->env->p);
|
87
|
+
}
|
88
|
+
|
89
|
+
int
|
90
|
+
sb_stemmer_length(struct sb_stemmer * stemmer)
|
91
|
+
{
|
92
|
+
return stemmer->env->l;
|
93
|
+
}
|