jk-ferret 0.11.8.2
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG +24 -0
- data/MIT-LICENSE +20 -0
- data/README +90 -0
- data/RELEASE_CHANGES +137 -0
- data/RELEASE_NOTES +60 -0
- data/Rakefile +443 -0
- data/TODO +109 -0
- data/TUTORIAL +231 -0
- data/bin/ferret-browser +79 -0
- data/ext/BZLIB_blocksort.c +1094 -0
- data/ext/BZLIB_bzlib.c +1578 -0
- data/ext/BZLIB_compress.c +672 -0
- data/ext/BZLIB_crctable.c +104 -0
- data/ext/BZLIB_decompress.c +626 -0
- data/ext/BZLIB_huffman.c +205 -0
- data/ext/BZLIB_randtable.c +84 -0
- data/ext/STEMMER_api.c +66 -0
- data/ext/STEMMER_libstemmer.c +93 -0
- data/ext/STEMMER_stem_ISO_8859_1_danish.c +337 -0
- data/ext/STEMMER_stem_ISO_8859_1_dutch.c +624 -0
- data/ext/STEMMER_stem_ISO_8859_1_english.c +1117 -0
- data/ext/STEMMER_stem_ISO_8859_1_finnish.c +762 -0
- data/ext/STEMMER_stem_ISO_8859_1_french.c +1246 -0
- data/ext/STEMMER_stem_ISO_8859_1_german.c +503 -0
- data/ext/STEMMER_stem_ISO_8859_1_hungarian.c +1230 -0
- data/ext/STEMMER_stem_ISO_8859_1_italian.c +1065 -0
- data/ext/STEMMER_stem_ISO_8859_1_norwegian.c +297 -0
- data/ext/STEMMER_stem_ISO_8859_1_porter.c +749 -0
- data/ext/STEMMER_stem_ISO_8859_1_portuguese.c +1017 -0
- data/ext/STEMMER_stem_ISO_8859_1_spanish.c +1093 -0
- data/ext/STEMMER_stem_ISO_8859_1_swedish.c +307 -0
- data/ext/STEMMER_stem_ISO_8859_2_romanian.c +998 -0
- data/ext/STEMMER_stem_KOI8_R_russian.c +700 -0
- data/ext/STEMMER_stem_UTF_8_danish.c +339 -0
- data/ext/STEMMER_stem_UTF_8_dutch.c +634 -0
- data/ext/STEMMER_stem_UTF_8_english.c +1125 -0
- data/ext/STEMMER_stem_UTF_8_finnish.c +768 -0
- data/ext/STEMMER_stem_UTF_8_french.c +1256 -0
- data/ext/STEMMER_stem_UTF_8_german.c +509 -0
- data/ext/STEMMER_stem_UTF_8_hungarian.c +1234 -0
- data/ext/STEMMER_stem_UTF_8_italian.c +1073 -0
- data/ext/STEMMER_stem_UTF_8_norwegian.c +299 -0
- data/ext/STEMMER_stem_UTF_8_porter.c +755 -0
- data/ext/STEMMER_stem_UTF_8_portuguese.c +1023 -0
- data/ext/STEMMER_stem_UTF_8_romanian.c +1004 -0
- data/ext/STEMMER_stem_UTF_8_russian.c +694 -0
- data/ext/STEMMER_stem_UTF_8_spanish.c +1097 -0
- data/ext/STEMMER_stem_UTF_8_swedish.c +309 -0
- data/ext/STEMMER_stem_UTF_8_turkish.c +2205 -0
- data/ext/STEMMER_utilities.c +478 -0
- data/ext/analysis.c +1710 -0
- data/ext/analysis.h +266 -0
- data/ext/api.h +26 -0
- data/ext/array.c +125 -0
- data/ext/array.h +62 -0
- data/ext/bitvector.c +96 -0
- data/ext/bitvector.h +594 -0
- data/ext/bzlib.h +282 -0
- data/ext/bzlib_private.h +503 -0
- data/ext/compound_io.c +384 -0
- data/ext/config.h +52 -0
- data/ext/document.c +159 -0
- data/ext/document.h +63 -0
- data/ext/except.c +102 -0
- data/ext/except.h +176 -0
- data/ext/extconf.rb +15 -0
- data/ext/ferret.c +416 -0
- data/ext/ferret.h +94 -0
- data/ext/field_index.c +262 -0
- data/ext/field_index.h +52 -0
- data/ext/filter.c +157 -0
- data/ext/fs_store.c +493 -0
- data/ext/global.c +458 -0
- data/ext/global.h +302 -0
- data/ext/hash.c +524 -0
- data/ext/hash.h +515 -0
- data/ext/hashset.c +192 -0
- data/ext/hashset.h +215 -0
- data/ext/header.h +58 -0
- data/ext/helper.c +63 -0
- data/ext/helper.h +21 -0
- data/ext/index.c +6804 -0
- data/ext/index.h +935 -0
- data/ext/internal.h +1019 -0
- data/ext/lang.c +10 -0
- data/ext/lang.h +68 -0
- data/ext/libstemmer.h +79 -0
- data/ext/mempool.c +88 -0
- data/ext/mempool.h +43 -0
- data/ext/modules.h +190 -0
- data/ext/multimapper.c +351 -0
- data/ext/multimapper.h +60 -0
- data/ext/posh.c +1006 -0
- data/ext/posh.h +973 -0
- data/ext/priorityqueue.c +149 -0
- data/ext/priorityqueue.h +155 -0
- data/ext/q_boolean.c +1621 -0
- data/ext/q_const_score.c +162 -0
- data/ext/q_filtered_query.c +212 -0
- data/ext/q_fuzzy.c +280 -0
- data/ext/q_match_all.c +149 -0
- data/ext/q_multi_term.c +673 -0
- data/ext/q_parser.c +3103 -0
- data/ext/q_phrase.c +1206 -0
- data/ext/q_prefix.c +98 -0
- data/ext/q_range.c +682 -0
- data/ext/q_span.c +2390 -0
- data/ext/q_term.c +337 -0
- data/ext/q_wildcard.c +167 -0
- data/ext/r_analysis.c +2626 -0
- data/ext/r_index.c +3468 -0
- data/ext/r_qparser.c +635 -0
- data/ext/r_search.c +4490 -0
- data/ext/r_store.c +513 -0
- data/ext/r_utils.c +1131 -0
- data/ext/ram_store.c +476 -0
- data/ext/scanner.c +895 -0
- data/ext/scanner.h +36 -0
- data/ext/scanner_mb.c +6701 -0
- data/ext/scanner_utf8.c +4415 -0
- data/ext/search.c +1864 -0
- data/ext/search.h +953 -0
- data/ext/similarity.c +151 -0
- data/ext/similarity.h +89 -0
- data/ext/sort.c +786 -0
- data/ext/stem_ISO_8859_1_danish.h +16 -0
- data/ext/stem_ISO_8859_1_dutch.h +16 -0
- data/ext/stem_ISO_8859_1_english.h +16 -0
- data/ext/stem_ISO_8859_1_finnish.h +16 -0
- data/ext/stem_ISO_8859_1_french.h +16 -0
- data/ext/stem_ISO_8859_1_german.h +16 -0
- data/ext/stem_ISO_8859_1_hungarian.h +16 -0
- data/ext/stem_ISO_8859_1_italian.h +16 -0
- data/ext/stem_ISO_8859_1_norwegian.h +16 -0
- data/ext/stem_ISO_8859_1_porter.h +16 -0
- data/ext/stem_ISO_8859_1_portuguese.h +16 -0
- data/ext/stem_ISO_8859_1_spanish.h +16 -0
- data/ext/stem_ISO_8859_1_swedish.h +16 -0
- data/ext/stem_ISO_8859_2_romanian.h +16 -0
- data/ext/stem_KOI8_R_russian.h +16 -0
- data/ext/stem_UTF_8_danish.h +16 -0
- data/ext/stem_UTF_8_dutch.h +16 -0
- data/ext/stem_UTF_8_english.h +16 -0
- data/ext/stem_UTF_8_finnish.h +16 -0
- data/ext/stem_UTF_8_french.h +16 -0
- data/ext/stem_UTF_8_german.h +16 -0
- data/ext/stem_UTF_8_hungarian.h +16 -0
- data/ext/stem_UTF_8_italian.h +16 -0
- data/ext/stem_UTF_8_norwegian.h +16 -0
- data/ext/stem_UTF_8_porter.h +16 -0
- data/ext/stem_UTF_8_portuguese.h +16 -0
- data/ext/stem_UTF_8_romanian.h +16 -0
- data/ext/stem_UTF_8_russian.h +16 -0
- data/ext/stem_UTF_8_spanish.h +16 -0
- data/ext/stem_UTF_8_swedish.h +16 -0
- data/ext/stem_UTF_8_turkish.h +16 -0
- data/ext/stopwords.c +410 -0
- data/ext/store.c +698 -0
- data/ext/store.h +799 -0
- data/ext/symbol.c +10 -0
- data/ext/symbol.h +23 -0
- data/ext/term_vectors.c +73 -0
- data/ext/threading.h +31 -0
- data/ext/win32.h +62 -0
- data/lib/ferret.rb +30 -0
- data/lib/ferret/browser.rb +246 -0
- data/lib/ferret/browser/s/global.js +192 -0
- data/lib/ferret/browser/s/style.css +148 -0
- data/lib/ferret/browser/views/document/list.rhtml +49 -0
- data/lib/ferret/browser/views/document/show.rhtml +27 -0
- data/lib/ferret/browser/views/error/index.rhtml +7 -0
- data/lib/ferret/browser/views/help/index.rhtml +8 -0
- data/lib/ferret/browser/views/home/index.rhtml +29 -0
- data/lib/ferret/browser/views/layout.rhtml +22 -0
- data/lib/ferret/browser/views/term-vector/index.rhtml +4 -0
- data/lib/ferret/browser/views/term/index.rhtml +199 -0
- data/lib/ferret/browser/views/term/termdocs.rhtml +1 -0
- data/lib/ferret/browser/webrick.rb +14 -0
- data/lib/ferret/document.rb +130 -0
- data/lib/ferret/field_infos.rb +44 -0
- data/lib/ferret/field_symbol.rb +87 -0
- data/lib/ferret/index.rb +973 -0
- data/lib/ferret/number_tools.rb +157 -0
- data/lib/ferret/version.rb +3 -0
- data/setup.rb +1555 -0
- data/test/long_running/largefile/tc_largefile.rb +46 -0
- data/test/test_all.rb +5 -0
- data/test/test_helper.rb +29 -0
- data/test/test_installed.rb +1 -0
- data/test/threading/number_to_spoken.rb +132 -0
- data/test/threading/thread_safety_index_test.rb +88 -0
- data/test/threading/thread_safety_read_write_test.rb +73 -0
- data/test/threading/thread_safety_test.rb +133 -0
- data/test/unit/analysis/tc_analyzer.rb +550 -0
- data/test/unit/analysis/tc_token_stream.rb +653 -0
- data/test/unit/index/tc_index.rb +867 -0
- data/test/unit/index/tc_index_reader.rb +699 -0
- data/test/unit/index/tc_index_writer.rb +447 -0
- data/test/unit/index/th_doc.rb +332 -0
- data/test/unit/query_parser/tc_query_parser.rb +238 -0
- data/test/unit/search/tc_filter.rb +156 -0
- data/test/unit/search/tc_fuzzy_query.rb +147 -0
- data/test/unit/search/tc_index_searcher.rb +67 -0
- data/test/unit/search/tc_multi_searcher.rb +128 -0
- data/test/unit/search/tc_multiple_search_requests.rb +58 -0
- data/test/unit/search/tc_search_and_sort.rb +179 -0
- data/test/unit/search/tc_sort.rb +49 -0
- data/test/unit/search/tc_sort_field.rb +27 -0
- data/test/unit/search/tc_spans.rb +190 -0
- data/test/unit/search/tm_searcher.rb +436 -0
- data/test/unit/store/tc_fs_store.rb +115 -0
- data/test/unit/store/tc_ram_store.rb +35 -0
- data/test/unit/store/tm_store.rb +34 -0
- data/test/unit/store/tm_store_lock.rb +68 -0
- data/test/unit/tc_document.rb +81 -0
- data/test/unit/tc_field_symbol.rb +26 -0
- data/test/unit/ts_analysis.rb +2 -0
- data/test/unit/ts_index.rb +2 -0
- data/test/unit/ts_largefile.rb +4 -0
- data/test/unit/ts_query_parser.rb +2 -0
- data/test/unit/ts_search.rb +2 -0
- data/test/unit/ts_store.rb +2 -0
- data/test/unit/ts_utils.rb +2 -0
- data/test/unit/utils/tc_bit_vector.rb +295 -0
- data/test/unit/utils/tc_number_tools.rb +117 -0
- data/test/unit/utils/tc_priority_queue.rb +106 -0
- data/test/utils/content_generator.rb +226 -0
- metadata +319 -0
data/ext/BZLIB_huffman.c
ADDED
@@ -0,0 +1,205 @@
|
|
1
|
+
|
2
|
+
/*-------------------------------------------------------------*/
|
3
|
+
/*--- Huffman coding low-level stuff ---*/
|
4
|
+
/*--- huffman.c ---*/
|
5
|
+
/*-------------------------------------------------------------*/
|
6
|
+
|
7
|
+
/* ------------------------------------------------------------------
|
8
|
+
This file is part of bzip2/libbzip2, a program and library for
|
9
|
+
lossless, block-sorting data compression.
|
10
|
+
|
11
|
+
bzip2/libbzip2 version 1.0.4 of 20 December 2006
|
12
|
+
Copyright (C) 1996-2006 Julian Seward <jseward@bzip.org>
|
13
|
+
|
14
|
+
Please read the WARNING, DISCLAIMER and PATENTS sections in the
|
15
|
+
README file.
|
16
|
+
|
17
|
+
This program is released under the terms of the license contained
|
18
|
+
in the file LICENSE.
|
19
|
+
------------------------------------------------------------------ */
|
20
|
+
|
21
|
+
|
22
|
+
#include "bzlib_private.h"
|
23
|
+
|
24
|
+
/*---------------------------------------------------*/
|
25
|
+
#define WEIGHTOF(zz0) ((zz0) & 0xffffff00)
|
26
|
+
#define DEPTHOF(zz1) ((zz1) & 0x000000ff)
|
27
|
+
#define MYMAX(zz2,zz3) ((zz2) > (zz3) ? (zz2) : (zz3))
|
28
|
+
|
29
|
+
#define ADDWEIGHTS(zw1,zw2) \
|
30
|
+
(WEIGHTOF(zw1)+WEIGHTOF(zw2)) | \
|
31
|
+
(1 + MYMAX(DEPTHOF(zw1),DEPTHOF(zw2)))
|
32
|
+
|
33
|
+
#define UPHEAP(z) \
|
34
|
+
{ \
|
35
|
+
Int32 zz, tmp; \
|
36
|
+
zz = z; tmp = heap[zz]; \
|
37
|
+
while (weight[tmp] < weight[heap[zz >> 1]]) { \
|
38
|
+
heap[zz] = heap[zz >> 1]; \
|
39
|
+
zz >>= 1; \
|
40
|
+
} \
|
41
|
+
heap[zz] = tmp; \
|
42
|
+
}
|
43
|
+
|
44
|
+
#define DOWNHEAP(z) \
|
45
|
+
{ \
|
46
|
+
Int32 zz, yy, tmp; \
|
47
|
+
zz = z; tmp = heap[zz]; \
|
48
|
+
while (True) { \
|
49
|
+
yy = zz << 1; \
|
50
|
+
if (yy > nHeap) break; \
|
51
|
+
if (yy < nHeap && \
|
52
|
+
weight[heap[yy+1]] < weight[heap[yy]]) \
|
53
|
+
yy++; \
|
54
|
+
if (weight[tmp] < weight[heap[yy]]) break; \
|
55
|
+
heap[zz] = heap[yy]; \
|
56
|
+
zz = yy; \
|
57
|
+
} \
|
58
|
+
heap[zz] = tmp; \
|
59
|
+
}
|
60
|
+
|
61
|
+
|
62
|
+
/*---------------------------------------------------*/
|
63
|
+
void BZ2_hbMakeCodeLengths ( UChar *len,
|
64
|
+
Int32 *freq,
|
65
|
+
Int32 alphaSize,
|
66
|
+
Int32 maxLen )
|
67
|
+
{
|
68
|
+
/*--
|
69
|
+
Nodes and heap entries run from 1. Entry 0
|
70
|
+
for both the heap and nodes is a sentinel.
|
71
|
+
--*/
|
72
|
+
Int32 nNodes, nHeap, n1, n2, i, j, k;
|
73
|
+
Bool tooLong;
|
74
|
+
|
75
|
+
Int32 heap [ BZ_MAX_ALPHA_SIZE + 2 ];
|
76
|
+
Int32 weight [ BZ_MAX_ALPHA_SIZE * 2 ];
|
77
|
+
Int32 parent [ BZ_MAX_ALPHA_SIZE * 2 ];
|
78
|
+
|
79
|
+
for (i = 0; i < alphaSize; i++)
|
80
|
+
weight[i+1] = (freq[i] == 0 ? 1 : freq[i]) << 8;
|
81
|
+
|
82
|
+
while (True) {
|
83
|
+
|
84
|
+
nNodes = alphaSize;
|
85
|
+
nHeap = 0;
|
86
|
+
|
87
|
+
heap[0] = 0;
|
88
|
+
weight[0] = 0;
|
89
|
+
parent[0] = -2;
|
90
|
+
|
91
|
+
for (i = 1; i <= alphaSize; i++) {
|
92
|
+
parent[i] = -1;
|
93
|
+
nHeap++;
|
94
|
+
heap[nHeap] = i;
|
95
|
+
UPHEAP(nHeap);
|
96
|
+
}
|
97
|
+
|
98
|
+
AssertH( nHeap < (BZ_MAX_ALPHA_SIZE+2), 2001 );
|
99
|
+
|
100
|
+
while (nHeap > 1) {
|
101
|
+
n1 = heap[1]; heap[1] = heap[nHeap]; nHeap--; DOWNHEAP(1);
|
102
|
+
n2 = heap[1]; heap[1] = heap[nHeap]; nHeap--; DOWNHEAP(1);
|
103
|
+
nNodes++;
|
104
|
+
parent[n1] = parent[n2] = nNodes;
|
105
|
+
weight[nNodes] = ADDWEIGHTS(weight[n1], weight[n2]);
|
106
|
+
parent[nNodes] = -1;
|
107
|
+
nHeap++;
|
108
|
+
heap[nHeap] = nNodes;
|
109
|
+
UPHEAP(nHeap);
|
110
|
+
}
|
111
|
+
|
112
|
+
AssertH( nNodes < (BZ_MAX_ALPHA_SIZE * 2), 2002 );
|
113
|
+
|
114
|
+
tooLong = False;
|
115
|
+
for (i = 1; i <= alphaSize; i++) {
|
116
|
+
j = 0;
|
117
|
+
k = i;
|
118
|
+
while (parent[k] >= 0) { k = parent[k]; j++; }
|
119
|
+
len[i-1] = j;
|
120
|
+
if (j > maxLen) tooLong = True;
|
121
|
+
}
|
122
|
+
|
123
|
+
if (! tooLong) break;
|
124
|
+
|
125
|
+
/* 17 Oct 04: keep-going condition for the following loop used
|
126
|
+
to be 'i < alphaSize', which missed the last element,
|
127
|
+
theoretically leading to the possibility of the compressor
|
128
|
+
looping. However, this count-scaling step is only needed if
|
129
|
+
one of the generated Huffman code words is longer than
|
130
|
+
maxLen, which up to and including version 1.0.2 was 20 bits,
|
131
|
+
which is extremely unlikely. In version 1.0.3 maxLen was
|
132
|
+
changed to 17 bits, which has minimal effect on compression
|
133
|
+
ratio, but does mean this scaling step is used from time to
|
134
|
+
time, enough to verify that it works.
|
135
|
+
|
136
|
+
This means that bzip2-1.0.3 and later will only produce
|
137
|
+
Huffman codes with a maximum length of 17 bits. However, in
|
138
|
+
order to preserve backwards compatibility with bitstreams
|
139
|
+
produced by versions pre-1.0.3, the decompressor must still
|
140
|
+
handle lengths of up to 20. */
|
141
|
+
|
142
|
+
for (i = 1; i <= alphaSize; i++) {
|
143
|
+
j = weight[i] >> 8;
|
144
|
+
j = 1 + (j / 2);
|
145
|
+
weight[i] = j << 8;
|
146
|
+
}
|
147
|
+
}
|
148
|
+
}
|
149
|
+
|
150
|
+
|
151
|
+
/*---------------------------------------------------*/
|
152
|
+
void BZ2_hbAssignCodes ( Int32 *code,
|
153
|
+
UChar *length,
|
154
|
+
Int32 minLen,
|
155
|
+
Int32 maxLen,
|
156
|
+
Int32 alphaSize )
|
157
|
+
{
|
158
|
+
Int32 n, vec, i;
|
159
|
+
|
160
|
+
vec = 0;
|
161
|
+
for (n = minLen; n <= maxLen; n++) {
|
162
|
+
for (i = 0; i < alphaSize; i++)
|
163
|
+
if (length[i] == n) { code[i] = vec; vec++; };
|
164
|
+
vec <<= 1;
|
165
|
+
}
|
166
|
+
}
|
167
|
+
|
168
|
+
|
169
|
+
/*---------------------------------------------------*/
|
170
|
+
void BZ2_hbCreateDecodeTables ( Int32 *limit,
|
171
|
+
Int32 *base,
|
172
|
+
Int32 *perm,
|
173
|
+
UChar *length,
|
174
|
+
Int32 minLen,
|
175
|
+
Int32 maxLen,
|
176
|
+
Int32 alphaSize )
|
177
|
+
{
|
178
|
+
Int32 pp, i, j, vec;
|
179
|
+
|
180
|
+
pp = 0;
|
181
|
+
for (i = minLen; i <= maxLen; i++)
|
182
|
+
for (j = 0; j < alphaSize; j++)
|
183
|
+
if (length[j] == i) { perm[pp] = j; pp++; };
|
184
|
+
|
185
|
+
for (i = 0; i < BZ_MAX_CODE_LEN; i++) base[i] = 0;
|
186
|
+
for (i = 0; i < alphaSize; i++) base[length[i]+1]++;
|
187
|
+
|
188
|
+
for (i = 1; i < BZ_MAX_CODE_LEN; i++) base[i] += base[i-1];
|
189
|
+
|
190
|
+
for (i = 0; i < BZ_MAX_CODE_LEN; i++) limit[i] = 0;
|
191
|
+
vec = 0;
|
192
|
+
|
193
|
+
for (i = minLen; i <= maxLen; i++) {
|
194
|
+
vec += (base[i+1] - base[i]);
|
195
|
+
limit[i] = vec-1;
|
196
|
+
vec <<= 1;
|
197
|
+
}
|
198
|
+
for (i = minLen + 1; i <= maxLen; i++)
|
199
|
+
base[i] = ((limit[i-1] + 1) << 1) - base[i];
|
200
|
+
}
|
201
|
+
|
202
|
+
|
203
|
+
/*-------------------------------------------------------------*/
|
204
|
+
/*--- end huffman.c ---*/
|
205
|
+
/*-------------------------------------------------------------*/
|
@@ -0,0 +1,84 @@
|
|
1
|
+
|
2
|
+
/*-------------------------------------------------------------*/
|
3
|
+
/*--- Table for randomising repetitive blocks ---*/
|
4
|
+
/*--- randtable.c ---*/
|
5
|
+
/*-------------------------------------------------------------*/
|
6
|
+
|
7
|
+
/* ------------------------------------------------------------------
|
8
|
+
This file is part of bzip2/libbzip2, a program and library for
|
9
|
+
lossless, block-sorting data compression.
|
10
|
+
|
11
|
+
bzip2/libbzip2 version 1.0.4 of 20 December 2006
|
12
|
+
Copyright (C) 1996-2006 Julian Seward <jseward@bzip.org>
|
13
|
+
|
14
|
+
Please read the WARNING, DISCLAIMER and PATENTS sections in the
|
15
|
+
README file.
|
16
|
+
|
17
|
+
This program is released under the terms of the license contained
|
18
|
+
in the file LICENSE.
|
19
|
+
------------------------------------------------------------------ */
|
20
|
+
|
21
|
+
|
22
|
+
#include "bzlib_private.h"
|
23
|
+
|
24
|
+
|
25
|
+
/*---------------------------------------------*/
|
26
|
+
Int32 BZ2_rNums[512] = {
|
27
|
+
619, 720, 127, 481, 931, 816, 813, 233, 566, 247,
|
28
|
+
985, 724, 205, 454, 863, 491, 741, 242, 949, 214,
|
29
|
+
733, 859, 335, 708, 621, 574, 73, 654, 730, 472,
|
30
|
+
419, 436, 278, 496, 867, 210, 399, 680, 480, 51,
|
31
|
+
878, 465, 811, 169, 869, 675, 611, 697, 867, 561,
|
32
|
+
862, 687, 507, 283, 482, 129, 807, 591, 733, 623,
|
33
|
+
150, 238, 59, 379, 684, 877, 625, 169, 643, 105,
|
34
|
+
170, 607, 520, 932, 727, 476, 693, 425, 174, 647,
|
35
|
+
73, 122, 335, 530, 442, 853, 695, 249, 445, 515,
|
36
|
+
909, 545, 703, 919, 874, 474, 882, 500, 594, 612,
|
37
|
+
641, 801, 220, 162, 819, 984, 589, 513, 495, 799,
|
38
|
+
161, 604, 958, 533, 221, 400, 386, 867, 600, 782,
|
39
|
+
382, 596, 414, 171, 516, 375, 682, 485, 911, 276,
|
40
|
+
98, 553, 163, 354, 666, 933, 424, 341, 533, 870,
|
41
|
+
227, 730, 475, 186, 263, 647, 537, 686, 600, 224,
|
42
|
+
469, 68, 770, 919, 190, 373, 294, 822, 808, 206,
|
43
|
+
184, 943, 795, 384, 383, 461, 404, 758, 839, 887,
|
44
|
+
715, 67, 618, 276, 204, 918, 873, 777, 604, 560,
|
45
|
+
951, 160, 578, 722, 79, 804, 96, 409, 713, 940,
|
46
|
+
652, 934, 970, 447, 318, 353, 859, 672, 112, 785,
|
47
|
+
645, 863, 803, 350, 139, 93, 354, 99, 820, 908,
|
48
|
+
609, 772, 154, 274, 580, 184, 79, 626, 630, 742,
|
49
|
+
653, 282, 762, 623, 680, 81, 927, 626, 789, 125,
|
50
|
+
411, 521, 938, 300, 821, 78, 343, 175, 128, 250,
|
51
|
+
170, 774, 972, 275, 999, 639, 495, 78, 352, 126,
|
52
|
+
857, 956, 358, 619, 580, 124, 737, 594, 701, 612,
|
53
|
+
669, 112, 134, 694, 363, 992, 809, 743, 168, 974,
|
54
|
+
944, 375, 748, 52, 600, 747, 642, 182, 862, 81,
|
55
|
+
344, 805, 988, 739, 511, 655, 814, 334, 249, 515,
|
56
|
+
897, 955, 664, 981, 649, 113, 974, 459, 893, 228,
|
57
|
+
433, 837, 553, 268, 926, 240, 102, 654, 459, 51,
|
58
|
+
686, 754, 806, 760, 493, 403, 415, 394, 687, 700,
|
59
|
+
946, 670, 656, 610, 738, 392, 760, 799, 887, 653,
|
60
|
+
978, 321, 576, 617, 626, 502, 894, 679, 243, 440,
|
61
|
+
680, 879, 194, 572, 640, 724, 926, 56, 204, 700,
|
62
|
+
707, 151, 457, 449, 797, 195, 791, 558, 945, 679,
|
63
|
+
297, 59, 87, 824, 713, 663, 412, 693, 342, 606,
|
64
|
+
134, 108, 571, 364, 631, 212, 174, 643, 304, 329,
|
65
|
+
343, 97, 430, 751, 497, 314, 983, 374, 822, 928,
|
66
|
+
140, 206, 73, 263, 980, 736, 876, 478, 430, 305,
|
67
|
+
170, 514, 364, 692, 829, 82, 855, 953, 676, 246,
|
68
|
+
369, 970, 294, 750, 807, 827, 150, 790, 288, 923,
|
69
|
+
804, 378, 215, 828, 592, 281, 565, 555, 710, 82,
|
70
|
+
896, 831, 547, 261, 524, 462, 293, 465, 502, 56,
|
71
|
+
661, 821, 976, 991, 658, 869, 905, 758, 745, 193,
|
72
|
+
768, 550, 608, 933, 378, 286, 215, 979, 792, 961,
|
73
|
+
61, 688, 793, 644, 986, 403, 106, 366, 905, 644,
|
74
|
+
372, 567, 466, 434, 645, 210, 389, 550, 919, 135,
|
75
|
+
780, 773, 635, 389, 707, 100, 626, 958, 165, 504,
|
76
|
+
920, 176, 193, 713, 857, 265, 203, 50, 668, 108,
|
77
|
+
645, 990, 626, 197, 510, 357, 358, 850, 858, 364,
|
78
|
+
936, 638
|
79
|
+
};
|
80
|
+
|
81
|
+
|
82
|
+
/*-------------------------------------------------------------*/
|
83
|
+
/*--- end randtable.c ---*/
|
84
|
+
/*-------------------------------------------------------------*/
|
data/ext/STEMMER_api.c
ADDED
@@ -0,0 +1,66 @@
|
|
1
|
+
|
2
|
+
#include <stdlib.h> /* for calloc, free */
|
3
|
+
#include "header.h"
|
4
|
+
|
5
|
+
extern struct SN_env * SN_create_env(int S_size, int I_size, int B_size)
|
6
|
+
{
|
7
|
+
struct SN_env * z = (struct SN_env *) calloc(1, sizeof(struct SN_env));
|
8
|
+
if (z == NULL) return NULL;
|
9
|
+
z->p = create_s();
|
10
|
+
if (z->p == NULL) goto error;
|
11
|
+
if (S_size)
|
12
|
+
{
|
13
|
+
int i;
|
14
|
+
z->S = (symbol * *) calloc(S_size, sizeof(symbol *));
|
15
|
+
if (z->S == NULL) goto error;
|
16
|
+
|
17
|
+
for (i = 0; i < S_size; i++)
|
18
|
+
{
|
19
|
+
z->S[i] = create_s();
|
20
|
+
if (z->S[i] == NULL) goto error;
|
21
|
+
}
|
22
|
+
}
|
23
|
+
|
24
|
+
if (I_size)
|
25
|
+
{
|
26
|
+
z->I = (int *) calloc(I_size, sizeof(int));
|
27
|
+
if (z->I == NULL) goto error;
|
28
|
+
}
|
29
|
+
|
30
|
+
if (B_size)
|
31
|
+
{
|
32
|
+
z->B = (unsigned char *) calloc(B_size, sizeof(unsigned char));
|
33
|
+
if (z->B == NULL) goto error;
|
34
|
+
}
|
35
|
+
|
36
|
+
return z;
|
37
|
+
error:
|
38
|
+
SN_close_env(z, S_size);
|
39
|
+
return NULL;
|
40
|
+
}
|
41
|
+
|
42
|
+
extern void SN_close_env(struct SN_env * z, int S_size)
|
43
|
+
{
|
44
|
+
if (z == NULL) return;
|
45
|
+
if (S_size)
|
46
|
+
{
|
47
|
+
int i;
|
48
|
+
for (i = 0; i < S_size; i++)
|
49
|
+
{
|
50
|
+
lose_s(z->S[i]);
|
51
|
+
}
|
52
|
+
free(z->S);
|
53
|
+
}
|
54
|
+
free(z->I);
|
55
|
+
free(z->B);
|
56
|
+
if (z->p) lose_s(z->p);
|
57
|
+
free(z);
|
58
|
+
}
|
59
|
+
|
60
|
+
extern int SN_set_current(struct SN_env * z, int size, const symbol * s)
|
61
|
+
{
|
62
|
+
int err = replace_s(z, 0, z->l, size, s, NULL);
|
63
|
+
z->c = 0;
|
64
|
+
return err;
|
65
|
+
}
|
66
|
+
|
@@ -0,0 +1,93 @@
|
|
1
|
+
|
2
|
+
#include <stdlib.h>
|
3
|
+
#include <string.h>
|
4
|
+
#include "libstemmer.h"
|
5
|
+
#include "api.h"
|
6
|
+
#include "modules.h"
|
7
|
+
|
8
|
+
struct sb_stemmer {
|
9
|
+
struct SN_env * (*create)(void);
|
10
|
+
void (*close)(struct SN_env *);
|
11
|
+
int (*stem)(struct SN_env *);
|
12
|
+
|
13
|
+
struct SN_env * env;
|
14
|
+
};
|
15
|
+
|
16
|
+
extern const char **
|
17
|
+
sb_stemmer_list(void)
|
18
|
+
{
|
19
|
+
return algorithm_names;
|
20
|
+
}
|
21
|
+
|
22
|
+
static stemmer_encoding_t
|
23
|
+
sb_getenc(const char * charenc)
|
24
|
+
{
|
25
|
+
struct stemmer_encoding * encoding;
|
26
|
+
if (charenc == NULL) return ENC_UTF_8;
|
27
|
+
for (encoding = encodings; encoding->name != 0; encoding++) {
|
28
|
+
if (strcmp(encoding->name, charenc) == 0) break;
|
29
|
+
}
|
30
|
+
if (encoding->name == NULL) return ENC_UNKNOWN;
|
31
|
+
return encoding->enc;
|
32
|
+
}
|
33
|
+
|
34
|
+
extern struct sb_stemmer *
|
35
|
+
sb_stemmer_new(const char * algorithm, const char * charenc)
|
36
|
+
{
|
37
|
+
stemmer_encoding_t enc;
|
38
|
+
struct stemmer_modules * module;
|
39
|
+
struct sb_stemmer * stemmer =
|
40
|
+
(struct sb_stemmer *) malloc(sizeof(struct sb_stemmer));
|
41
|
+
if (stemmer == NULL) return NULL;
|
42
|
+
enc = sb_getenc(charenc);
|
43
|
+
if (enc == ENC_UNKNOWN) return NULL;
|
44
|
+
|
45
|
+
for (module = modules; module->name != 0; module++) {
|
46
|
+
if (strcmp(module->name, algorithm) == 0 && module->enc == enc) break;
|
47
|
+
}
|
48
|
+
if (module->name == NULL) return NULL;
|
49
|
+
|
50
|
+
stemmer->create = module->create;
|
51
|
+
stemmer->close = module->close;
|
52
|
+
stemmer->stem = module->stem;
|
53
|
+
|
54
|
+
stemmer->env = stemmer->create();
|
55
|
+
if (stemmer->env == NULL)
|
56
|
+
{
|
57
|
+
sb_stemmer_delete(stemmer);
|
58
|
+
return NULL;
|
59
|
+
}
|
60
|
+
|
61
|
+
return stemmer;
|
62
|
+
}
|
63
|
+
|
64
|
+
void
|
65
|
+
sb_stemmer_delete(struct sb_stemmer * stemmer)
|
66
|
+
{
|
67
|
+
if (stemmer == 0) return;
|
68
|
+
if (stemmer->close == 0) return;
|
69
|
+
stemmer->close(stemmer->env);
|
70
|
+
stemmer->close = 0;
|
71
|
+
free(stemmer);
|
72
|
+
}
|
73
|
+
|
74
|
+
const sb_symbol *
|
75
|
+
sb_stemmer_stem(struct sb_stemmer * stemmer, const sb_symbol * word, int size)
|
76
|
+
{
|
77
|
+
int ret;
|
78
|
+
if (SN_set_current(stemmer->env, size, (const symbol *)(word)))
|
79
|
+
{
|
80
|
+
stemmer->env->l = 0;
|
81
|
+
return NULL;
|
82
|
+
}
|
83
|
+
ret = stemmer->stem(stemmer->env);
|
84
|
+
if (ret < 0) return NULL;
|
85
|
+
stemmer->env->p[stemmer->env->l] = 0;
|
86
|
+
return (const sb_symbol *)(stemmer->env->p);
|
87
|
+
}
|
88
|
+
|
89
|
+
int
|
90
|
+
sb_stemmer_length(struct sb_stemmer * stemmer)
|
91
|
+
{
|
92
|
+
return stemmer->env->l;
|
93
|
+
}
|