ferret 0.11.6 → 0.11.8.4
Sign up to get free protection for your applications and to get access to all the features.
- data/README +10 -22
- data/RELEASE_CHANGES +137 -0
- data/RELEASE_NOTES +60 -0
- data/Rakefile +379 -274
- data/TODO +100 -8
- data/bin/ferret-browser +0 -0
- data/ext/BZLIB_blocksort.c +1094 -0
- data/ext/BZLIB_bzlib.c +1578 -0
- data/ext/BZLIB_compress.c +672 -0
- data/ext/BZLIB_crctable.c +104 -0
- data/ext/BZLIB_decompress.c +626 -0
- data/ext/BZLIB_huffman.c +205 -0
- data/ext/BZLIB_randtable.c +84 -0
- data/ext/{api.c → STEMMER_api.c} +7 -10
- data/ext/{libstemmer.c → STEMMER_libstemmer.c} +3 -2
- data/ext/{stem_ISO_8859_1_danish.c → STEMMER_stem_ISO_8859_1_danish.c} +123 -124
- data/ext/{stem_ISO_8859_1_dutch.c → STEMMER_stem_ISO_8859_1_dutch.c} +177 -188
- data/ext/STEMMER_stem_ISO_8859_1_english.c +1117 -0
- data/ext/{stem_ISO_8859_1_finnish.c → STEMMER_stem_ISO_8859_1_finnish.c} +276 -306
- data/ext/STEMMER_stem_ISO_8859_1_french.c +1246 -0
- data/ext/{stem_ISO_8859_1_german.c → STEMMER_stem_ISO_8859_1_german.c} +161 -170
- data/ext/STEMMER_stem_ISO_8859_1_hungarian.c +1230 -0
- data/ext/STEMMER_stem_ISO_8859_1_italian.c +1065 -0
- data/ext/STEMMER_stem_ISO_8859_1_norwegian.c +297 -0
- data/ext/{stem_ISO_8859_1_porter.c → STEMMER_stem_ISO_8859_1_porter.c} +263 -290
- data/ext/{stem_ISO_8859_1_portuguese.c → STEMMER_stem_ISO_8859_1_portuguese.c} +362 -380
- data/ext/STEMMER_stem_ISO_8859_1_spanish.c +1093 -0
- data/ext/STEMMER_stem_ISO_8859_1_swedish.c +307 -0
- data/ext/STEMMER_stem_ISO_8859_2_romanian.c +998 -0
- data/ext/{stem_KOI8_R_russian.c → STEMMER_stem_KOI8_R_russian.c} +244 -245
- data/ext/STEMMER_stem_UTF_8_danish.c +339 -0
- data/ext/{stem_UTF_8_dutch.c → STEMMER_stem_UTF_8_dutch.c} +192 -211
- data/ext/STEMMER_stem_UTF_8_english.c +1125 -0
- data/ext/{stem_UTF_8_finnish.c → STEMMER_stem_UTF_8_finnish.c} +284 -324
- data/ext/STEMMER_stem_UTF_8_french.c +1256 -0
- data/ext/{stem_UTF_8_german.c → STEMMER_stem_UTF_8_german.c} +170 -187
- data/ext/STEMMER_stem_UTF_8_hungarian.c +1234 -0
- data/ext/STEMMER_stem_UTF_8_italian.c +1073 -0
- data/ext/STEMMER_stem_UTF_8_norwegian.c +299 -0
- data/ext/{stem_UTF_8_porter.c → STEMMER_stem_UTF_8_porter.c} +271 -310
- data/ext/STEMMER_stem_UTF_8_portuguese.c +1023 -0
- data/ext/STEMMER_stem_UTF_8_romanian.c +1004 -0
- data/ext/STEMMER_stem_UTF_8_russian.c +694 -0
- data/ext/STEMMER_stem_UTF_8_spanish.c +1097 -0
- data/ext/STEMMER_stem_UTF_8_swedish.c +309 -0
- data/ext/STEMMER_stem_UTF_8_turkish.c +2205 -0
- data/ext/{utilities.c → STEMMER_utilities.c} +100 -68
- data/ext/analysis.c +276 -121
- data/ext/analysis.h +190 -143
- data/ext/api.h +3 -4
- data/ext/array.c +5 -3
- data/ext/array.h +52 -43
- data/ext/bitvector.c +38 -482
- data/ext/bitvector.h +446 -124
- data/ext/bzlib.h +282 -0
- data/ext/bzlib_private.h +503 -0
- data/ext/compound_io.c +23 -22
- data/ext/config.h +21 -11
- data/ext/document.c +43 -40
- data/ext/document.h +31 -21
- data/ext/except.c +20 -38
- data/ext/except.h +89 -76
- data/ext/extconf.rb +3 -2
- data/ext/ferret.c +49 -35
- data/ext/ferret.h +14 -11
- data/ext/field_index.c +262 -0
- data/ext/field_index.h +52 -0
- data/ext/filter.c +11 -10
- data/ext/fs_store.c +65 -47
- data/ext/global.c +245 -165
- data/ext/global.h +252 -54
- data/ext/hash.c +200 -243
- data/ext/hash.h +205 -163
- data/ext/hashset.c +118 -96
- data/ext/hashset.h +110 -82
- data/ext/header.h +19 -19
- data/ext/helper.c +11 -10
- data/ext/helper.h +14 -6
- data/ext/index.c +745 -366
- data/ext/index.h +503 -529
- data/ext/internal.h +1020 -0
- data/ext/lang.c +10 -0
- data/ext/lang.h +35 -15
- data/ext/mempool.c +5 -4
- data/ext/mempool.h +30 -22
- data/ext/modules.h +35 -7
- data/ext/multimapper.c +43 -2
- data/ext/multimapper.h +32 -23
- data/ext/posh.c +0 -0
- data/ext/posh.h +4 -38
- data/ext/priorityqueue.c +10 -12
- data/ext/priorityqueue.h +33 -21
- data/ext/q_boolean.c +22 -9
- data/ext/q_const_score.c +3 -2
- data/ext/q_filtered_query.c +15 -12
- data/ext/q_fuzzy.c +147 -135
- data/ext/q_match_all.c +3 -2
- data/ext/q_multi_term.c +28 -32
- data/ext/q_parser.c +451 -173
- data/ext/q_phrase.c +158 -79
- data/ext/q_prefix.c +16 -18
- data/ext/q_range.c +363 -31
- data/ext/q_span.c +130 -141
- data/ext/q_term.c +21 -21
- data/ext/q_wildcard.c +19 -23
- data/ext/r_analysis.c +369 -242
- data/ext/r_index.c +421 -434
- data/ext/r_qparser.c +142 -92
- data/ext/r_search.c +790 -407
- data/ext/r_store.c +44 -44
- data/ext/r_utils.c +264 -96
- data/ext/ram_store.c +29 -23
- data/ext/scanner.c +895 -0
- data/ext/scanner.h +36 -0
- data/ext/scanner_mb.c +6701 -0
- data/ext/scanner_utf8.c +4415 -0
- data/ext/search.c +210 -87
- data/ext/search.h +556 -488
- data/ext/similarity.c +17 -16
- data/ext/similarity.h +51 -44
- data/ext/sort.c +157 -354
- data/ext/stem_ISO_8859_1_hungarian.h +16 -0
- data/ext/stem_ISO_8859_2_romanian.h +16 -0
- data/ext/stem_UTF_8_hungarian.h +16 -0
- data/ext/stem_UTF_8_romanian.h +16 -0
- data/ext/stem_UTF_8_turkish.h +16 -0
- data/ext/stopwords.c +287 -278
- data/ext/store.c +57 -51
- data/ext/store.h +308 -286
- data/ext/symbol.c +10 -0
- data/ext/symbol.h +23 -0
- data/ext/term_vectors.c +14 -293
- data/ext/threading.h +22 -22
- data/ext/win32.h +12 -4
- data/lib/ferret.rb +2 -1
- data/lib/ferret/browser.rb +1 -1
- data/lib/ferret/field_symbol.rb +94 -0
- data/lib/ferret/index.rb +221 -34
- data/lib/ferret/number_tools.rb +6 -6
- data/lib/ferret/version.rb +3 -0
- data/test/{unit → long_running}/largefile/tc_largefile.rb +1 -1
- data/test/test_helper.rb +7 -2
- data/test/test_installed.rb +1 -0
- data/test/threading/thread_safety_index_test.rb +10 -1
- data/test/threading/thread_safety_read_write_test.rb +4 -7
- data/test/threading/thread_safety_test.rb +0 -0
- data/test/unit/analysis/tc_analyzer.rb +29 -27
- data/test/unit/analysis/tc_token_stream.rb +23 -16
- data/test/unit/index/tc_index.rb +116 -11
- data/test/unit/index/tc_index_reader.rb +27 -27
- data/test/unit/index/tc_index_writer.rb +10 -0
- data/test/unit/index/th_doc.rb +38 -21
- data/test/unit/search/tc_filter.rb +31 -10
- data/test/unit/search/tc_index_searcher.rb +6 -0
- data/test/unit/search/tm_searcher.rb +53 -1
- data/test/unit/store/tc_fs_store.rb +40 -2
- data/test/unit/store/tc_ram_store.rb +0 -0
- data/test/unit/store/tm_store.rb +0 -0
- data/test/unit/store/tm_store_lock.rb +7 -6
- data/test/unit/tc_field_symbol.rb +26 -0
- data/test/unit/ts_analysis.rb +0 -0
- data/test/unit/ts_index.rb +0 -0
- data/test/unit/ts_store.rb +0 -0
- data/test/unit/ts_utils.rb +0 -0
- data/test/unit/utils/tc_number_tools.rb +0 -0
- data/test/utils/content_generator.rb +226 -0
- metadata +262 -221
- data/ext/inc/lang.h +0 -48
- data/ext/inc/threading.h +0 -31
- data/ext/stem_ISO_8859_1_english.c +0 -1156
- data/ext/stem_ISO_8859_1_french.c +0 -1276
- data/ext/stem_ISO_8859_1_italian.c +0 -1091
- data/ext/stem_ISO_8859_1_norwegian.c +0 -296
- data/ext/stem_ISO_8859_1_spanish.c +0 -1119
- data/ext/stem_ISO_8859_1_swedish.c +0 -307
- data/ext/stem_UTF_8_danish.c +0 -344
- data/ext/stem_UTF_8_english.c +0 -1176
- data/ext/stem_UTF_8_french.c +0 -1296
- data/ext/stem_UTF_8_italian.c +0 -1113
- data/ext/stem_UTF_8_norwegian.c +0 -302
- data/ext/stem_UTF_8_portuguese.c +0 -1055
- data/ext/stem_UTF_8_russian.c +0 -709
- data/ext/stem_UTF_8_spanish.c +0 -1137
- data/ext/stem_UTF_8_swedish.c +0 -313
- data/lib/ferret_version.rb +0 -3
data/ext/ferret.h
CHANGED
@@ -4,6 +4,8 @@
|
|
4
4
|
#include "global.h"
|
5
5
|
#include "hashset.h"
|
6
6
|
#include "document.h"
|
7
|
+
#include "symbol.h"
|
8
|
+
#include "internal.h"
|
7
9
|
|
8
10
|
/* IDs */
|
9
11
|
extern ID id_new;
|
@@ -56,18 +58,19 @@ extern void object_set2(void *key, VALUE obj, const char *file, int line);
|
|
56
58
|
//extern void object_del(void *key);
|
57
59
|
#define object_del(key) object_del2(key, __FILE__, __LINE__)
|
58
60
|
extern void object_del2(void *key, const char *file, int line);
|
59
|
-
extern void
|
61
|
+
extern void frb_gc_mark(void *key);
|
60
62
|
extern VALUE object_get(void *key);
|
61
|
-
extern VALUE
|
62
|
-
extern void
|
63
|
-
extern void
|
64
|
-
extern VALUE
|
65
|
-
extern void *
|
66
|
-
extern
|
67
|
-
extern VALUE
|
63
|
+
extern VALUE frb_data_alloc(VALUE klass);
|
64
|
+
extern void frb_deref_free(void *p);
|
65
|
+
extern void frb_create_dir(VALUE rpath);
|
66
|
+
extern VALUE frb_hs_to_rb_ary(HashSet *hs);
|
67
|
+
extern void *frb_rb_data_ptr(VALUE val);
|
68
|
+
extern Symbol frb_field(VALUE rfield);
|
69
|
+
extern VALUE frb_get_term(Symbol field, const char *term);
|
68
70
|
extern char *json_concat_string(char *s, char *field);
|
69
71
|
extern char *rs2s(VALUE rstr);
|
70
|
-
extern char *
|
72
|
+
extern char *rstrdup(VALUE rstr);
|
73
|
+
extern Symbol rintern(VALUE rstr);
|
71
74
|
#define Frt_Make_Struct(klass)\
|
72
75
|
rb_data_object_alloc(klass,NULL,(RUBY_DATA_FUNC)NULL,(RUBY_DATA_FUNC)NULL)
|
73
76
|
|
@@ -87,5 +90,5 @@ extern char *nstrdup(VALUE rstr);
|
|
87
90
|
|
88
91
|
#endif
|
89
92
|
|
90
|
-
#define
|
91
|
-
#define
|
93
|
+
#define frb_mark_cclass(klass) rb_ivar_set(klass, id_cclass, Qtrue)
|
94
|
+
#define frb_is_cclass(obj) (rb_ivar_get(CLASS_OF(obj), id_cclass) == Qtrue)
|
data/ext/field_index.c
ADDED
@@ -0,0 +1,262 @@
|
|
1
|
+
#include <string.h>
|
2
|
+
#include "field_index.h"
|
3
|
+
#include "internal.h"
|
4
|
+
|
5
|
+
/***************************************************************************
|
6
|
+
*
|
7
|
+
* FieldIndex
|
8
|
+
*
|
9
|
+
***************************************************************************/
|
10
|
+
|
11
|
+
static unsigned long field_index_hash(const void *p)
|
12
|
+
{
|
13
|
+
FieldIndex *self = (FieldIndex *)p;
|
14
|
+
return sym_hash(self->field) ^ (unsigned long)(self->klass);
|
15
|
+
}
|
16
|
+
|
17
|
+
static int field_index_eq(const void *p1, const void *p2)
|
18
|
+
{
|
19
|
+
FieldIndex *fi1 = (FieldIndex *)p1;
|
20
|
+
FieldIndex *fi2 = (FieldIndex *)p2;
|
21
|
+
return (fi1->field == fi2->field) &&
|
22
|
+
(fi1->klass->type == fi2->klass->type);
|
23
|
+
}
|
24
|
+
|
25
|
+
static void field_index_destroy(void *p)
|
26
|
+
{
|
27
|
+
FieldIndex *self = (FieldIndex *)p;
|
28
|
+
if (self->index) {
|
29
|
+
self->klass->destroy_index(self->index);
|
30
|
+
}
|
31
|
+
free(self);
|
32
|
+
}
|
33
|
+
|
34
|
+
FieldIndex *field_index_get(IndexReader *ir, Symbol field,
|
35
|
+
const FieldIndexClass *klass)
|
36
|
+
{
|
37
|
+
int length = 0;
|
38
|
+
TermEnum *volatile te = NULL;
|
39
|
+
TermDocEnum *volatile tde = NULL;
|
40
|
+
FieldInfo *fi = fis_get_field(ir->fis, field);
|
41
|
+
const volatile int field_num = fi ? fi->number : -1;
|
42
|
+
FieldIndex *volatile self = NULL;
|
43
|
+
FieldIndex key;
|
44
|
+
|
45
|
+
if (field_num < 0) {
|
46
|
+
RAISE(ARG_ERROR,
|
47
|
+
"Cannot sort by field \"%s\". It doesn't exist in the index.",
|
48
|
+
S(field));
|
49
|
+
}
|
50
|
+
|
51
|
+
if (!ir->field_index_cache) {
|
52
|
+
ir->field_index_cache = h_new(&field_index_hash, &field_index_eq,
|
53
|
+
NULL, &field_index_destroy);
|
54
|
+
}
|
55
|
+
|
56
|
+
key.field = field;
|
57
|
+
key.klass = klass;
|
58
|
+
self = (FieldIndex *)h_get(ir->field_index_cache, &key);
|
59
|
+
|
60
|
+
if (self == NULL) {
|
61
|
+
self = ALLOC(FieldIndex);
|
62
|
+
self->klass = klass;
|
63
|
+
/* FieldIndex only lives as long as the IndexReader lives so we can
|
64
|
+
* just use the field_infos field string */
|
65
|
+
self->field = fi->name;
|
66
|
+
|
67
|
+
length = ir->max_doc(ir);
|
68
|
+
if (length > 0) {
|
69
|
+
TRY
|
70
|
+
{
|
71
|
+
void *index;
|
72
|
+
tde = ir->term_docs(ir);
|
73
|
+
te = ir->terms(ir, field_num);
|
74
|
+
index = self->index = klass->create_index(length);
|
75
|
+
while (te->next(te)) {
|
76
|
+
tde->seek_te(tde, te);
|
77
|
+
klass->handle_term(index, tde, te->curr_term);
|
78
|
+
}
|
79
|
+
}
|
80
|
+
XFINALLY
|
81
|
+
tde->close(tde);
|
82
|
+
te->close(te);
|
83
|
+
XENDTRY
|
84
|
+
}
|
85
|
+
h_set(ir->field_index_cache, self, self);
|
86
|
+
}
|
87
|
+
|
88
|
+
return self;
|
89
|
+
}
|
90
|
+
|
91
|
+
/******************************************************************************
|
92
|
+
* ByteFieldIndex < FieldIndex
|
93
|
+
*
|
94
|
+
* The ByteFieldIndex holds an array of integers for each document in the
|
95
|
+
* index where the integer represents the sort value for the document. This
|
96
|
+
* index should only be used for sorting and not as a field cache of the
|
97
|
+
* column's value.
|
98
|
+
******************************************************************************/
|
99
|
+
static void byte_handle_term(void *index_ptr,
|
100
|
+
TermDocEnum *tde,
|
101
|
+
const char *text)
|
102
|
+
{
|
103
|
+
long *index = (long *)index_ptr;
|
104
|
+
long val = index[-1]++;
|
105
|
+
(void)text;
|
106
|
+
while (tde->next(tde)) {
|
107
|
+
index[tde->doc_num(tde)] = val;
|
108
|
+
}
|
109
|
+
}
|
110
|
+
|
111
|
+
static void *byte_create_index(int size)
|
112
|
+
{
|
113
|
+
long *index = ALLOC_AND_ZERO_N(long, size + 1);
|
114
|
+
index[0] = 1;
|
115
|
+
return &index[1];
|
116
|
+
}
|
117
|
+
|
118
|
+
static void byte_destroy_index(void *p)
|
119
|
+
{
|
120
|
+
long *index = (long *)p;
|
121
|
+
free(&index[-1]);
|
122
|
+
}
|
123
|
+
|
124
|
+
const FieldIndexClass BYTE_FIELD_INDEX_CLASS = {
|
125
|
+
"byte",
|
126
|
+
&byte_create_index,
|
127
|
+
&byte_destroy_index,
|
128
|
+
&byte_handle_term
|
129
|
+
};
|
130
|
+
|
131
|
+
/******************************************************************************
|
132
|
+
* IntegerFieldIndex < FieldIndex
|
133
|
+
******************************************************************************/
|
134
|
+
static void *integer_create_index(int size)
|
135
|
+
{
|
136
|
+
return ALLOC_AND_ZERO_N(long, size);
|
137
|
+
}
|
138
|
+
|
139
|
+
static void integer_handle_term(void *index_ptr,
|
140
|
+
TermDocEnum *tde,
|
141
|
+
const char *text)
|
142
|
+
{
|
143
|
+
long *index = (long *)index_ptr;
|
144
|
+
long val;
|
145
|
+
sscanf(text, "%ld", &val);
|
146
|
+
while (tde->next(tde)) {
|
147
|
+
index[tde->doc_num(tde)] = val;
|
148
|
+
}
|
149
|
+
}
|
150
|
+
|
151
|
+
const FieldIndexClass INTEGER_FIELD_INDEX_CLASS = {
|
152
|
+
"integer",
|
153
|
+
&integer_create_index,
|
154
|
+
&free,
|
155
|
+
&integer_handle_term
|
156
|
+
};
|
157
|
+
|
158
|
+
long get_integer_value(FieldIndex *field_index, long doc_num)
|
159
|
+
{
|
160
|
+
if (field_index->klass == &INTEGER_FIELD_INDEX_CLASS && doc_num >= 0) {
|
161
|
+
return ((long *)field_index->index)[doc_num];
|
162
|
+
}
|
163
|
+
return 0l;
|
164
|
+
}
|
165
|
+
|
166
|
+
|
167
|
+
/******************************************************************************
|
168
|
+
* FloatFieldIndex < FieldIndex
|
169
|
+
******************************************************************************/
|
170
|
+
#define VALUES_ARRAY_START_SIZE 8
|
171
|
+
static void *float_create_index(int size)
|
172
|
+
{
|
173
|
+
return ALLOC_AND_ZERO_N(float, size);
|
174
|
+
}
|
175
|
+
|
176
|
+
static void float_handle_term(void *index_ptr,
|
177
|
+
TermDocEnum *tde,
|
178
|
+
const char *text)
|
179
|
+
{
|
180
|
+
float *index = (float *)index_ptr;
|
181
|
+
float val;
|
182
|
+
sscanf(text, "%g", &val);
|
183
|
+
while (tde->next(tde)) {
|
184
|
+
index[tde->doc_num(tde)] = val;
|
185
|
+
}
|
186
|
+
}
|
187
|
+
|
188
|
+
const FieldIndexClass FLOAT_FIELD_INDEX_CLASS = {
|
189
|
+
"float",
|
190
|
+
&float_create_index,
|
191
|
+
&free,
|
192
|
+
&float_handle_term
|
193
|
+
};
|
194
|
+
|
195
|
+
float get_float_value(FieldIndex *field_index, long doc_num)
|
196
|
+
{
|
197
|
+
if (field_index->klass == &FLOAT_FIELD_INDEX_CLASS && doc_num >= 0) {
|
198
|
+
return ((float *)field_index->index)[doc_num];
|
199
|
+
}
|
200
|
+
return 0.0f;
|
201
|
+
}
|
202
|
+
|
203
|
+
/******************************************************************************
|
204
|
+
* StringFieldIndex < FieldIndex
|
205
|
+
******************************************************************************/
|
206
|
+
|
207
|
+
static void *string_create_index(int size)
|
208
|
+
{
|
209
|
+
StringIndex *self = ALLOC_AND_ZERO(StringIndex);
|
210
|
+
self->size = size;
|
211
|
+
self->index = ALLOC_AND_ZERO_N(long, size);
|
212
|
+
self->v_capa = VALUES_ARRAY_START_SIZE;
|
213
|
+
self->v_size = 1; /* leave the first value as NULL */
|
214
|
+
self->values = ALLOC_AND_ZERO_N(char *, VALUES_ARRAY_START_SIZE);
|
215
|
+
return self;
|
216
|
+
}
|
217
|
+
|
218
|
+
static void string_destroy_index(void *p)
|
219
|
+
{
|
220
|
+
StringIndex *self = (StringIndex *)p;
|
221
|
+
int i;
|
222
|
+
free(self->index);
|
223
|
+
for (i = 0; i < self->v_size; i++) {
|
224
|
+
free(self->values[i]);
|
225
|
+
}
|
226
|
+
free(self->values);
|
227
|
+
free(self);
|
228
|
+
}
|
229
|
+
|
230
|
+
static void string_handle_term(void *index_ptr,
|
231
|
+
TermDocEnum *tde,
|
232
|
+
const char *text)
|
233
|
+
{
|
234
|
+
StringIndex *index = (StringIndex *)index_ptr;
|
235
|
+
if (index->v_size >= index->v_capa) {
|
236
|
+
index->v_capa *= 2;
|
237
|
+
index->values = REALLOC_N(index->values, char *, index->v_capa);
|
238
|
+
}
|
239
|
+
index->values[index->v_size] = estrdup(text);
|
240
|
+
while (tde->next(tde)) {
|
241
|
+
index->index[tde->doc_num(tde)] = index->v_size;
|
242
|
+
}
|
243
|
+
index->v_size++;
|
244
|
+
}
|
245
|
+
|
246
|
+
const FieldIndexClass STRING_FIELD_INDEX_CLASS = {
|
247
|
+
"string",
|
248
|
+
&string_create_index,
|
249
|
+
&string_destroy_index,
|
250
|
+
&string_handle_term
|
251
|
+
};
|
252
|
+
|
253
|
+
const char *get_string_value(FieldIndex *field_index, long doc_num)
|
254
|
+
{
|
255
|
+
if (field_index->klass == &STRING_FIELD_INDEX_CLASS) {
|
256
|
+
StringIndex *string_index = (StringIndex *)field_index->index;
|
257
|
+
if (doc_num >= 0 && doc_num < string_index->size) {
|
258
|
+
return string_index->values[string_index->index[doc_num]];
|
259
|
+
}
|
260
|
+
}
|
261
|
+
return NULL;
|
262
|
+
}
|
data/ext/field_index.h
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
#ifndef FRT_FIELD_INDEX_H
|
2
|
+
#define FRT_FIELD_INDEX_H
|
3
|
+
|
4
|
+
#ifdef __cplusplus
|
5
|
+
extern "C" {
|
6
|
+
#endif
|
7
|
+
|
8
|
+
#include "index.h"
|
9
|
+
|
10
|
+
/***************************************************************************
|
11
|
+
*
|
12
|
+
* FrtFieldIndex
|
13
|
+
*
|
14
|
+
***************************************************************************/
|
15
|
+
|
16
|
+
typedef struct FrtStringIndex {
|
17
|
+
int size;
|
18
|
+
long *index;
|
19
|
+
char **values;
|
20
|
+
int v_size;
|
21
|
+
int v_capa;
|
22
|
+
} FrtStringIndex;
|
23
|
+
|
24
|
+
typedef struct FrtFieldIndexClass {
|
25
|
+
const char *type;
|
26
|
+
void *(*create_index)(int size);
|
27
|
+
void (*destroy_index)(void *p);
|
28
|
+
void (*handle_term)(void *index, FrtTermDocEnum *tde, const char *text);
|
29
|
+
} FrtFieldIndexClass;
|
30
|
+
|
31
|
+
typedef struct FrtFieldIndex {
|
32
|
+
FrtSymbol field;
|
33
|
+
const FrtFieldIndexClass *klass;
|
34
|
+
void *index;
|
35
|
+
} FrtFieldIndex;
|
36
|
+
|
37
|
+
extern const FrtFieldIndexClass FRT_INTEGER_FIELD_INDEX_CLASS;
|
38
|
+
extern const FrtFieldIndexClass FRT_FLOAT_FIELD_INDEX_CLASS;
|
39
|
+
extern const FrtFieldIndexClass FRT_STRING_FIELD_INDEX_CLASS;
|
40
|
+
extern const FrtFieldIndexClass FRT_BYTE_FIELD_INDEX_CLASS;
|
41
|
+
|
42
|
+
extern FrtFieldIndex *frt_field_index_get(FrtIndexReader *ir, FrtSymbol field,
|
43
|
+
const FrtFieldIndexClass *klass);
|
44
|
+
extern const char *frt_get_string_value(FrtFieldIndex *field_index, long doc_num);
|
45
|
+
extern float frt_get_float_value(FrtFieldIndex *field_index, long doc_num);
|
46
|
+
extern long frt_get_integer_value(FrtFieldIndex *field_index, long doc_num);
|
47
|
+
|
48
|
+
#ifdef __cplusplus
|
49
|
+
} // extern "C"
|
50
|
+
#endif
|
51
|
+
|
52
|
+
#endif
|
data/ext/filter.c
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
#include "search.h"
|
2
|
+
#include "symbol.h"
|
2
3
|
#include <string.h>
|
4
|
+
#include "internal.h"
|
3
5
|
|
4
6
|
/***************************************************************************
|
5
7
|
*
|
@@ -10,7 +12,6 @@
|
|
10
12
|
void filt_destroy_i(Filter *filt)
|
11
13
|
{
|
12
14
|
h_destroy(filt->cache);
|
13
|
-
free(filt->name);
|
14
15
|
free(filt);
|
15
16
|
}
|
16
17
|
void filt_deref(Filter *filt)
|
@@ -22,7 +23,7 @@ void filt_deref(Filter *filt)
|
|
22
23
|
|
23
24
|
BitVector *filt_get_bv(Filter *filt, IndexReader *ir)
|
24
25
|
{
|
25
|
-
CacheObject *co = h_get(filt->cache, ir);
|
26
|
+
CacheObject *co = (CacheObject *)h_get(filt->cache, ir);
|
26
27
|
|
27
28
|
if (!co) {
|
28
29
|
BitVector *bv;
|
@@ -38,26 +39,26 @@ BitVector *filt_get_bv(Filter *filt, IndexReader *ir)
|
|
38
39
|
|
39
40
|
static char *filt_to_s_i(Filter *filt)
|
40
41
|
{
|
41
|
-
return estrdup(filt->name);
|
42
|
+
return estrdup(S(filt->name));
|
42
43
|
}
|
43
44
|
|
44
|
-
unsigned long filt_hash_default(Filter *filt)
|
45
|
+
static unsigned long filt_hash_default(Filter *filt)
|
45
46
|
{
|
46
47
|
(void)filt;
|
47
48
|
return 0;
|
48
49
|
}
|
49
50
|
|
50
|
-
int filt_eq_default(Filter *filt, Filter *o)
|
51
|
+
static int filt_eq_default(Filter *filt, Filter *o)
|
51
52
|
{
|
52
53
|
(void)filt; (void)o;
|
53
54
|
return false;
|
54
55
|
}
|
55
56
|
|
56
|
-
Filter *filt_create(size_t size,
|
57
|
+
Filter *filt_create(size_t size, Symbol name)
|
57
58
|
{
|
58
59
|
Filter *filt = (Filter *)emalloc(size);
|
59
60
|
filt->cache = co_hash_create();
|
60
|
-
filt->name =
|
61
|
+
filt->name = name;
|
61
62
|
filt->to_s = &filt_to_s_i;
|
62
63
|
filt->hash = &filt_hash_default;
|
63
64
|
filt->eq = &filt_eq_default;
|
@@ -68,13 +69,13 @@ Filter *filt_create(size_t size, const char *name)
|
|
68
69
|
|
69
70
|
unsigned long filt_hash(Filter *filt)
|
70
71
|
{
|
71
|
-
return
|
72
|
+
return sym_hash(filt->name) ^ filt->hash(filt);
|
72
73
|
}
|
73
74
|
|
74
75
|
int filt_eq(Filter *filt, Filter *o)
|
75
76
|
{
|
76
77
|
return ((filt == o)
|
77
|
-
|| ((
|
78
|
+
|| ((filt->name == o->name)
|
78
79
|
&& (filt->eq == o->eq)
|
79
80
|
&& (filt->eq(filt, o))));
|
80
81
|
}
|
@@ -95,7 +96,7 @@ typedef struct QueryFilter
|
|
95
96
|
static char *qfilt_to_s(Filter *filt)
|
96
97
|
{
|
97
98
|
Query *query = QF(filt)->query;
|
98
|
-
char *query_str = query->to_s(query,
|
99
|
+
char *query_str = query->to_s(query, NULL);
|
99
100
|
char *filter_str = strfmt("QueryFilter< %s >", query_str);
|
100
101
|
free(query_str);
|
101
102
|
return filter_str;
|
data/ext/fs_store.c
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
#include "store.h"
|
2
|
+
#include "lang.h"
|
2
3
|
#include <time.h>
|
3
4
|
#include <sys/types.h>
|
4
5
|
#include <fcntl.h>
|
@@ -14,6 +15,7 @@
|
|
14
15
|
# endif
|
15
16
|
# ifndef DIR_SEPARATOR
|
16
17
|
# define DIR_SEPARATOR "\\"
|
18
|
+
# define DIR_SEPARATOR_CHAR '\\'
|
17
19
|
# endif
|
18
20
|
# ifndef S_IRUSR
|
19
21
|
# define S_IRUSR _S_IREAD
|
@@ -23,18 +25,14 @@
|
|
23
25
|
# endif
|
24
26
|
#else
|
25
27
|
# define DIR_SEPARATOR "/"
|
28
|
+
# define DIR_SEPARATOR_CHAR '/'
|
26
29
|
# include <unistd.h>
|
27
30
|
# include <dirent.h>
|
28
31
|
#endif
|
29
32
|
#ifndef O_BINARY
|
30
33
|
# define O_BINARY 0
|
31
34
|
#endif
|
32
|
-
|
33
|
-
extern Store *store_new();
|
34
|
-
extern void store_destroy(Store *store);
|
35
|
-
extern OutStream *os_new();
|
36
|
-
extern InStream *is_new();
|
37
|
-
extern int file_is_lock(char *filename);
|
35
|
+
#include "internal.h"
|
38
36
|
|
39
37
|
/**
|
40
38
|
* Create a filepath for a file in the store using the operating systems
|
@@ -46,7 +44,7 @@ static char *join_path(char *buf, const char *base, const char *filename)
|
|
46
44
|
return buf;
|
47
45
|
}
|
48
46
|
|
49
|
-
static void fs_touch(Store *store, char *filename)
|
47
|
+
static void fs_touch(Store *store, const char *filename)
|
50
48
|
{
|
51
49
|
int f;
|
52
50
|
char path[MAX_FILE_PATH];
|
@@ -58,7 +56,7 @@ static void fs_touch(Store *store, char *filename)
|
|
58
56
|
close(f);
|
59
57
|
}
|
60
58
|
|
61
|
-
static int fs_exists(Store *store, char *filename)
|
59
|
+
static int fs_exists(Store *store, const char *filename)
|
62
60
|
{
|
63
61
|
int fd;
|
64
62
|
char path[MAX_FILE_PATH];
|
@@ -75,13 +73,13 @@ static int fs_exists(Store *store, char *filename)
|
|
75
73
|
return true;
|
76
74
|
}
|
77
75
|
|
78
|
-
static int fs_remove(Store *store, char *filename)
|
76
|
+
static int fs_remove(Store *store, const char *filename)
|
79
77
|
{
|
80
78
|
char path[MAX_FILE_PATH];
|
81
79
|
return remove(join_path(path, store->dir.path, filename));
|
82
80
|
}
|
83
81
|
|
84
|
-
static void fs_rename(Store *store, char *from, char *to)
|
82
|
+
static void fs_rename(Store *store, const char *from, const char *to)
|
85
83
|
{
|
86
84
|
char path1[MAX_FILE_PATH], path2[MAX_FILE_PATH];
|
87
85
|
|
@@ -117,7 +115,7 @@ static int fs_count(Store *store)
|
|
117
115
|
return cnt;
|
118
116
|
}
|
119
117
|
|
120
|
-
static void fs_each(Store *store, void (*func)(char *fname, void *arg), void *arg)
|
118
|
+
static void fs_each(Store *store, void (*func)(const char *fname, void *arg), void *arg)
|
121
119
|
{
|
122
120
|
struct dirent *de;
|
123
121
|
DIR *d = opendir(store->dir.path);
|
@@ -155,6 +153,20 @@ static void fs_clear_locks(Store *store)
|
|
155
153
|
closedir(d);
|
156
154
|
}
|
157
155
|
|
156
|
+
static void remove_if_index_file(const char *base_path, const char *file_name)
|
157
|
+
{
|
158
|
+
char path[MAX_FILE_PATH];
|
159
|
+
char *basename;
|
160
|
+
join_path(path, base_path, file_name);
|
161
|
+
/* get basename of path */
|
162
|
+
basename = strrchr(path, DIR_SEPARATOR_CHAR);
|
163
|
+
basename = (basename ? basename + 1 : path);
|
164
|
+
/* we don't want to delete non-index files here */
|
165
|
+
if (file_name_filter_is_index_file(basename, true)) {
|
166
|
+
remove(path);
|
167
|
+
}
|
168
|
+
}
|
169
|
+
|
158
170
|
static void fs_clear(Store *store)
|
159
171
|
{
|
160
172
|
struct dirent *de;
|
@@ -168,13 +180,16 @@ static void fs_clear(Store *store)
|
|
168
180
|
while ((de = readdir(d)) != NULL) {
|
169
181
|
if (de->d_name[0] > '/' /* skip ., .., / and '\0'*/
|
170
182
|
&& !file_is_lock(de->d_name)) {
|
171
|
-
|
172
|
-
remove(join_path(path, store->dir.path, de->d_name));
|
183
|
+
remove_if_index_file(store->dir.path, de->d_name);
|
173
184
|
}
|
174
185
|
}
|
175
186
|
closedir(d);
|
176
187
|
}
|
177
188
|
|
189
|
+
/**
|
190
|
+
* Clear all files which belong to the index. Use fs_clear to clear the
|
191
|
+
* directory regardless of the files origin.
|
192
|
+
*/
|
178
193
|
static void fs_clear_all(Store *store)
|
179
194
|
{
|
180
195
|
struct dirent *de;
|
@@ -187,8 +202,7 @@ static void fs_clear_all(Store *store)
|
|
187
202
|
|
188
203
|
while ((de = readdir(d)) != NULL) {
|
189
204
|
if (de->d_name[0] > '/') { /* skip ., .., / and '\0'*/
|
190
|
-
|
191
|
-
remove(join_path(path, store->dir.path, de->d_name));
|
205
|
+
remove_if_index_file(store->dir.path, de->d_name);
|
192
206
|
}
|
193
207
|
}
|
194
208
|
closedir(d);
|
@@ -211,7 +225,7 @@ static void fs_destroy(Store *store)
|
|
211
225
|
store_destroy(store);
|
212
226
|
}
|
213
227
|
|
214
|
-
static off_t fs_length(Store *store, char *filename)
|
228
|
+
static off_t fs_length(Store *store, const char *filename)
|
215
229
|
{
|
216
230
|
char path[MAX_FILE_PATH];
|
217
231
|
struct stat stt;
|
@@ -224,7 +238,7 @@ static off_t fs_length(Store *store, char *filename)
|
|
224
238
|
return stt.st_size;
|
225
239
|
}
|
226
240
|
|
227
|
-
static void fso_flush_i(OutStream *os, uchar *src, int len)
|
241
|
+
static void fso_flush_i(OutStream *os, const uchar *src, int len)
|
228
242
|
{
|
229
243
|
if (len != write(os->file.fd, src, len)) {
|
230
244
|
RAISE(IO_ERROR, "flushing src of length %d, <%s>", len,
|
@@ -235,7 +249,7 @@ static void fso_flush_i(OutStream *os, uchar *src, int len)
|
|
235
249
|
static void fso_seek_i(OutStream *os, off_t pos)
|
236
250
|
{
|
237
251
|
if (lseek(os->file.fd, pos, SEEK_SET) < 0) {
|
238
|
-
RAISE(IO_ERROR, "seeking position %"
|
252
|
+
RAISE(IO_ERROR, "seeking position %"OFF_T_PFX"d: <%s>",
|
239
253
|
pos, strerror(errno));
|
240
254
|
}
|
241
255
|
}
|
@@ -247,7 +261,7 @@ static void fso_close_i(OutStream *os)
|
|
247
261
|
}
|
248
262
|
}
|
249
263
|
|
250
|
-
const struct OutStreamMethods FS_OUT_STREAM_METHODS = {
|
264
|
+
static const struct OutStreamMethods FS_OUT_STREAM_METHODS = {
|
251
265
|
fso_flush_i,
|
252
266
|
fso_seek_i,
|
253
267
|
fso_close_i
|
@@ -289,7 +303,7 @@ static void fsi_read_i(InStream *is, uchar *path, int len)
|
|
289
303
|
static void fsi_seek_i(InStream *is, off_t pos)
|
290
304
|
{
|
291
305
|
if (lseek(is->file.fd, pos, SEEK_SET) < 0) {
|
292
|
-
RAISE(IO_ERROR, "seeking pos %"
|
306
|
+
RAISE(IO_ERROR, "seeking pos %"OFF_T_PFX"d: <%s>",
|
293
307
|
pos, strerror(errno));
|
294
308
|
}
|
295
309
|
}
|
@@ -297,7 +311,7 @@ static void fsi_seek_i(InStream *is, off_t pos)
|
|
297
311
|
static void fsi_close_i(InStream *is)
|
298
312
|
{
|
299
313
|
if (close(is->file.fd)) {
|
300
|
-
RAISE(IO_ERROR, strerror(errno));
|
314
|
+
RAISE(IO_ERROR, "%s", strerror(errno));
|
301
315
|
}
|
302
316
|
free(is->d.path);
|
303
317
|
}
|
@@ -337,10 +351,6 @@ static InStream *fs_open_input(Store *store, const char *filename)
|
|
337
351
|
|
338
352
|
#define LOCK_OBTAIN_TIMEOUT 10
|
339
353
|
|
340
|
-
#ifdef RUBY_BINDINGS
|
341
|
-
struct timeval rb_time_interval _((VALUE));
|
342
|
-
#endif
|
343
|
-
|
344
354
|
static int fs_lock_obtain(Lock *lock)
|
345
355
|
{
|
346
356
|
int f;
|
@@ -377,12 +387,12 @@ static int fs_lock_is_locked(Lock *lock)
|
|
377
387
|
}
|
378
388
|
}
|
379
389
|
|
380
|
-
void fs_lock_release(Lock *lock)
|
390
|
+
static void fs_lock_release(Lock *lock)
|
381
391
|
{
|
382
392
|
remove(lock->name);
|
383
393
|
}
|
384
394
|
|
385
|
-
static Lock *fs_open_lock_i(Store *store, char *lockname)
|
395
|
+
static Lock *fs_open_lock_i(Store *store, const char *lockname)
|
386
396
|
{
|
387
397
|
Lock *lock = ALLOC(Lock);
|
388
398
|
char lname[100];
|
@@ -403,19 +413,7 @@ static void fs_close_lock_i(Lock *lock)
|
|
403
413
|
free(lock);
|
404
414
|
}
|
405
415
|
|
406
|
-
static
|
407
|
-
/* fill */ 0,
|
408
|
-
/* used */ 0,
|
409
|
-
/* mask */ HASH_MINSIZE - 1,
|
410
|
-
/* ref_cnt */ 1,
|
411
|
-
/* table */ stores.smalltable,
|
412
|
-
/* smalltable */ {{0, NULL, NULL}},
|
413
|
-
/* lookup */ (h_lookup_ft)&h_lookup_str,
|
414
|
-
/* hash */ NULL,
|
415
|
-
/* eq */ NULL,
|
416
|
-
/* free_key */ (free_ft)&dummy_free,
|
417
|
-
/* free_value */ (free_ft)&fs_destroy
|
418
|
-
};
|
416
|
+
static Hash *stores = NULL;
|
419
417
|
|
420
418
|
#ifndef UNTHREADED
|
421
419
|
static mutex_t stores_mutex = MUTEX_INITIALIZER;
|
@@ -424,7 +422,7 @@ static mutex_t stores_mutex = MUTEX_INITIALIZER;
|
|
424
422
|
static void fs_close_i(Store *store)
|
425
423
|
{
|
426
424
|
mutex_lock(&stores_mutex);
|
427
|
-
h_del(
|
425
|
+
h_del(stores, store->dir.path);
|
428
426
|
mutex_unlock(&stores_mutex);
|
429
427
|
}
|
430
428
|
|
@@ -435,11 +433,26 @@ static Store *fs_store_new(const char *pathname)
|
|
435
433
|
|
436
434
|
new_store->file_mode = S_IRUSR | S_IWUSR;
|
437
435
|
#ifndef POSH_OS_WIN32
|
438
|
-
if (!stat(pathname, &stt)
|
439
|
-
|
440
|
-
|
436
|
+
if (!stat(pathname, &stt)) {
|
437
|
+
gid_t st_gid = stt.st_gid;
|
438
|
+
bool is_grp = (st_gid == getgid());
|
439
|
+
|
440
|
+
if (!is_grp) {
|
441
|
+
int size = getgroups(0, NULL);
|
442
|
+
gid_t list[size];
|
443
|
+
|
444
|
+
if (getgroups(size, list) != -1) {
|
445
|
+
int i = 0;
|
446
|
+
while (i < size && !(is_grp = (st_gid == list[i]))) i++;
|
447
|
+
}
|
448
|
+
}
|
449
|
+
|
450
|
+
if (is_grp) {
|
451
|
+
if (stt.st_mode & S_IWGRP) {
|
452
|
+
umask(S_IWOTH);
|
453
|
+
}
|
454
|
+
new_store->file_mode |= stt.st_mode & (S_IRGRP | S_IWGRP);
|
441
455
|
}
|
442
|
-
new_store->file_mode |= stt.st_mode & (S_IRGRP | S_IWGRP);
|
443
456
|
}
|
444
457
|
#endif
|
445
458
|
|
@@ -466,8 +479,13 @@ Store *open_fs_store(const char *pathname)
|
|
466
479
|
{
|
467
480
|
Store *store = NULL;
|
468
481
|
|
482
|
+
if (!stores) {
|
483
|
+
stores = h_new_str(NULL, (free_ft)fs_destroy);
|
484
|
+
register_for_cleanup(stores, (free_ft)h_destroy);
|
485
|
+
}
|
486
|
+
|
469
487
|
mutex_lock(&stores_mutex);
|
470
|
-
store = h_get(
|
488
|
+
store = (Store *)h_get(stores, pathname);
|
471
489
|
if (store) {
|
472
490
|
mutex_lock(&store->mutex);
|
473
491
|
store->ref_cnt++;
|
@@ -475,7 +493,7 @@ Store *open_fs_store(const char *pathname)
|
|
475
493
|
}
|
476
494
|
else {
|
477
495
|
store = fs_store_new(pathname);
|
478
|
-
h_set(
|
496
|
+
h_set(stores, store->dir.path, store);
|
479
497
|
}
|
480
498
|
mutex_unlock(&stores_mutex);
|
481
499
|
|