ferret 0.11.6 → 0.11.8.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (185) hide show
  1. data/README +10 -22
  2. data/RELEASE_CHANGES +137 -0
  3. data/RELEASE_NOTES +60 -0
  4. data/Rakefile +379 -274
  5. data/TODO +100 -8
  6. data/bin/ferret-browser +0 -0
  7. data/ext/BZLIB_blocksort.c +1094 -0
  8. data/ext/BZLIB_bzlib.c +1578 -0
  9. data/ext/BZLIB_compress.c +672 -0
  10. data/ext/BZLIB_crctable.c +104 -0
  11. data/ext/BZLIB_decompress.c +626 -0
  12. data/ext/BZLIB_huffman.c +205 -0
  13. data/ext/BZLIB_randtable.c +84 -0
  14. data/ext/{api.c → STEMMER_api.c} +7 -10
  15. data/ext/{libstemmer.c → STEMMER_libstemmer.c} +3 -2
  16. data/ext/{stem_ISO_8859_1_danish.c → STEMMER_stem_ISO_8859_1_danish.c} +123 -124
  17. data/ext/{stem_ISO_8859_1_dutch.c → STEMMER_stem_ISO_8859_1_dutch.c} +177 -188
  18. data/ext/STEMMER_stem_ISO_8859_1_english.c +1117 -0
  19. data/ext/{stem_ISO_8859_1_finnish.c → STEMMER_stem_ISO_8859_1_finnish.c} +276 -306
  20. data/ext/STEMMER_stem_ISO_8859_1_french.c +1246 -0
  21. data/ext/{stem_ISO_8859_1_german.c → STEMMER_stem_ISO_8859_1_german.c} +161 -170
  22. data/ext/STEMMER_stem_ISO_8859_1_hungarian.c +1230 -0
  23. data/ext/STEMMER_stem_ISO_8859_1_italian.c +1065 -0
  24. data/ext/STEMMER_stem_ISO_8859_1_norwegian.c +297 -0
  25. data/ext/{stem_ISO_8859_1_porter.c → STEMMER_stem_ISO_8859_1_porter.c} +263 -290
  26. data/ext/{stem_ISO_8859_1_portuguese.c → STEMMER_stem_ISO_8859_1_portuguese.c} +362 -380
  27. data/ext/STEMMER_stem_ISO_8859_1_spanish.c +1093 -0
  28. data/ext/STEMMER_stem_ISO_8859_1_swedish.c +307 -0
  29. data/ext/STEMMER_stem_ISO_8859_2_romanian.c +998 -0
  30. data/ext/{stem_KOI8_R_russian.c → STEMMER_stem_KOI8_R_russian.c} +244 -245
  31. data/ext/STEMMER_stem_UTF_8_danish.c +339 -0
  32. data/ext/{stem_UTF_8_dutch.c → STEMMER_stem_UTF_8_dutch.c} +192 -211
  33. data/ext/STEMMER_stem_UTF_8_english.c +1125 -0
  34. data/ext/{stem_UTF_8_finnish.c → STEMMER_stem_UTF_8_finnish.c} +284 -324
  35. data/ext/STEMMER_stem_UTF_8_french.c +1256 -0
  36. data/ext/{stem_UTF_8_german.c → STEMMER_stem_UTF_8_german.c} +170 -187
  37. data/ext/STEMMER_stem_UTF_8_hungarian.c +1234 -0
  38. data/ext/STEMMER_stem_UTF_8_italian.c +1073 -0
  39. data/ext/STEMMER_stem_UTF_8_norwegian.c +299 -0
  40. data/ext/{stem_UTF_8_porter.c → STEMMER_stem_UTF_8_porter.c} +271 -310
  41. data/ext/STEMMER_stem_UTF_8_portuguese.c +1023 -0
  42. data/ext/STEMMER_stem_UTF_8_romanian.c +1004 -0
  43. data/ext/STEMMER_stem_UTF_8_russian.c +694 -0
  44. data/ext/STEMMER_stem_UTF_8_spanish.c +1097 -0
  45. data/ext/STEMMER_stem_UTF_8_swedish.c +309 -0
  46. data/ext/STEMMER_stem_UTF_8_turkish.c +2205 -0
  47. data/ext/{utilities.c → STEMMER_utilities.c} +100 -68
  48. data/ext/analysis.c +276 -121
  49. data/ext/analysis.h +190 -143
  50. data/ext/api.h +3 -4
  51. data/ext/array.c +5 -3
  52. data/ext/array.h +52 -43
  53. data/ext/bitvector.c +38 -482
  54. data/ext/bitvector.h +446 -124
  55. data/ext/bzlib.h +282 -0
  56. data/ext/bzlib_private.h +503 -0
  57. data/ext/compound_io.c +23 -22
  58. data/ext/config.h +21 -11
  59. data/ext/document.c +43 -40
  60. data/ext/document.h +31 -21
  61. data/ext/except.c +20 -38
  62. data/ext/except.h +89 -76
  63. data/ext/extconf.rb +3 -2
  64. data/ext/ferret.c +49 -35
  65. data/ext/ferret.h +14 -11
  66. data/ext/field_index.c +262 -0
  67. data/ext/field_index.h +52 -0
  68. data/ext/filter.c +11 -10
  69. data/ext/fs_store.c +65 -47
  70. data/ext/global.c +245 -165
  71. data/ext/global.h +252 -54
  72. data/ext/hash.c +200 -243
  73. data/ext/hash.h +205 -163
  74. data/ext/hashset.c +118 -96
  75. data/ext/hashset.h +110 -82
  76. data/ext/header.h +19 -19
  77. data/ext/helper.c +11 -10
  78. data/ext/helper.h +14 -6
  79. data/ext/index.c +745 -366
  80. data/ext/index.h +503 -529
  81. data/ext/internal.h +1020 -0
  82. data/ext/lang.c +10 -0
  83. data/ext/lang.h +35 -15
  84. data/ext/mempool.c +5 -4
  85. data/ext/mempool.h +30 -22
  86. data/ext/modules.h +35 -7
  87. data/ext/multimapper.c +43 -2
  88. data/ext/multimapper.h +32 -23
  89. data/ext/posh.c +0 -0
  90. data/ext/posh.h +4 -38
  91. data/ext/priorityqueue.c +10 -12
  92. data/ext/priorityqueue.h +33 -21
  93. data/ext/q_boolean.c +22 -9
  94. data/ext/q_const_score.c +3 -2
  95. data/ext/q_filtered_query.c +15 -12
  96. data/ext/q_fuzzy.c +147 -135
  97. data/ext/q_match_all.c +3 -2
  98. data/ext/q_multi_term.c +28 -32
  99. data/ext/q_parser.c +451 -173
  100. data/ext/q_phrase.c +158 -79
  101. data/ext/q_prefix.c +16 -18
  102. data/ext/q_range.c +363 -31
  103. data/ext/q_span.c +130 -141
  104. data/ext/q_term.c +21 -21
  105. data/ext/q_wildcard.c +19 -23
  106. data/ext/r_analysis.c +369 -242
  107. data/ext/r_index.c +421 -434
  108. data/ext/r_qparser.c +142 -92
  109. data/ext/r_search.c +790 -407
  110. data/ext/r_store.c +44 -44
  111. data/ext/r_utils.c +264 -96
  112. data/ext/ram_store.c +29 -23
  113. data/ext/scanner.c +895 -0
  114. data/ext/scanner.h +36 -0
  115. data/ext/scanner_mb.c +6701 -0
  116. data/ext/scanner_utf8.c +4415 -0
  117. data/ext/search.c +210 -87
  118. data/ext/search.h +556 -488
  119. data/ext/similarity.c +17 -16
  120. data/ext/similarity.h +51 -44
  121. data/ext/sort.c +157 -354
  122. data/ext/stem_ISO_8859_1_hungarian.h +16 -0
  123. data/ext/stem_ISO_8859_2_romanian.h +16 -0
  124. data/ext/stem_UTF_8_hungarian.h +16 -0
  125. data/ext/stem_UTF_8_romanian.h +16 -0
  126. data/ext/stem_UTF_8_turkish.h +16 -0
  127. data/ext/stopwords.c +287 -278
  128. data/ext/store.c +57 -51
  129. data/ext/store.h +308 -286
  130. data/ext/symbol.c +10 -0
  131. data/ext/symbol.h +23 -0
  132. data/ext/term_vectors.c +14 -293
  133. data/ext/threading.h +22 -22
  134. data/ext/win32.h +12 -4
  135. data/lib/ferret.rb +2 -1
  136. data/lib/ferret/browser.rb +1 -1
  137. data/lib/ferret/field_symbol.rb +94 -0
  138. data/lib/ferret/index.rb +221 -34
  139. data/lib/ferret/number_tools.rb +6 -6
  140. data/lib/ferret/version.rb +3 -0
  141. data/test/{unit → long_running}/largefile/tc_largefile.rb +1 -1
  142. data/test/test_helper.rb +7 -2
  143. data/test/test_installed.rb +1 -0
  144. data/test/threading/thread_safety_index_test.rb +10 -1
  145. data/test/threading/thread_safety_read_write_test.rb +4 -7
  146. data/test/threading/thread_safety_test.rb +0 -0
  147. data/test/unit/analysis/tc_analyzer.rb +29 -27
  148. data/test/unit/analysis/tc_token_stream.rb +23 -16
  149. data/test/unit/index/tc_index.rb +116 -11
  150. data/test/unit/index/tc_index_reader.rb +27 -27
  151. data/test/unit/index/tc_index_writer.rb +10 -0
  152. data/test/unit/index/th_doc.rb +38 -21
  153. data/test/unit/search/tc_filter.rb +31 -10
  154. data/test/unit/search/tc_index_searcher.rb +6 -0
  155. data/test/unit/search/tm_searcher.rb +53 -1
  156. data/test/unit/store/tc_fs_store.rb +40 -2
  157. data/test/unit/store/tc_ram_store.rb +0 -0
  158. data/test/unit/store/tm_store.rb +0 -0
  159. data/test/unit/store/tm_store_lock.rb +7 -6
  160. data/test/unit/tc_field_symbol.rb +26 -0
  161. data/test/unit/ts_analysis.rb +0 -0
  162. data/test/unit/ts_index.rb +0 -0
  163. data/test/unit/ts_store.rb +0 -0
  164. data/test/unit/ts_utils.rb +0 -0
  165. data/test/unit/utils/tc_number_tools.rb +0 -0
  166. data/test/utils/content_generator.rb +226 -0
  167. metadata +262 -221
  168. data/ext/inc/lang.h +0 -48
  169. data/ext/inc/threading.h +0 -31
  170. data/ext/stem_ISO_8859_1_english.c +0 -1156
  171. data/ext/stem_ISO_8859_1_french.c +0 -1276
  172. data/ext/stem_ISO_8859_1_italian.c +0 -1091
  173. data/ext/stem_ISO_8859_1_norwegian.c +0 -296
  174. data/ext/stem_ISO_8859_1_spanish.c +0 -1119
  175. data/ext/stem_ISO_8859_1_swedish.c +0 -307
  176. data/ext/stem_UTF_8_danish.c +0 -344
  177. data/ext/stem_UTF_8_english.c +0 -1176
  178. data/ext/stem_UTF_8_french.c +0 -1296
  179. data/ext/stem_UTF_8_italian.c +0 -1113
  180. data/ext/stem_UTF_8_norwegian.c +0 -302
  181. data/ext/stem_UTF_8_portuguese.c +0 -1055
  182. data/ext/stem_UTF_8_russian.c +0 -709
  183. data/ext/stem_UTF_8_spanish.c +0 -1137
  184. data/ext/stem_UTF_8_swedish.c +0 -313
  185. data/lib/ferret_version.rb +0 -3
@@ -4,6 +4,8 @@
4
4
  #include "global.h"
5
5
  #include "hashset.h"
6
6
  #include "document.h"
7
+ #include "symbol.h"
8
+ #include "internal.h"
7
9
 
8
10
  /* IDs */
9
11
  extern ID id_new;
@@ -56,18 +58,19 @@ extern void object_set2(void *key, VALUE obj, const char *file, int line);
56
58
  //extern void object_del(void *key);
57
59
  #define object_del(key) object_del2(key, __FILE__, __LINE__)
58
60
  extern void object_del2(void *key, const char *file, int line);
59
- extern void frt_gc_mark(void *key);
61
+ extern void frb_gc_mark(void *key);
60
62
  extern VALUE object_get(void *key);
61
- extern VALUE frt_data_alloc(VALUE klass);
62
- extern void frt_deref_free(void *p);
63
- extern void frt_create_dir(VALUE rpath);
64
- extern VALUE frt_hs_to_rb_ary(HashSet *hs);
65
- extern void *frt_rb_data_ptr(VALUE val);
66
- extern char * frt_field(VALUE rfield);
67
- extern VALUE frt_get_term(const char *field, const char *term);
63
+ extern VALUE frb_data_alloc(VALUE klass);
64
+ extern void frb_deref_free(void *p);
65
+ extern void frb_create_dir(VALUE rpath);
66
+ extern VALUE frb_hs_to_rb_ary(HashSet *hs);
67
+ extern void *frb_rb_data_ptr(VALUE val);
68
+ extern Symbol frb_field(VALUE rfield);
69
+ extern VALUE frb_get_term(Symbol field, const char *term);
68
70
  extern char *json_concat_string(char *s, char *field);
69
71
  extern char *rs2s(VALUE rstr);
70
- extern char *nstrdup(VALUE rstr);
72
+ extern char *rstrdup(VALUE rstr);
73
+ extern Symbol rintern(VALUE rstr);
71
74
  #define Frt_Make_Struct(klass)\
72
75
  rb_data_object_alloc(klass,NULL,(RUBY_DATA_FUNC)NULL,(RUBY_DATA_FUNC)NULL)
73
76
 
@@ -87,5 +90,5 @@ extern char *nstrdup(VALUE rstr);
87
90
 
88
91
  #endif
89
92
 
90
- #define frt_mark_cclass(klass) rb_ivar_set(klass, id_cclass, Qtrue)
91
- #define frt_is_cclass(obj) (rb_ivar_get(CLASS_OF(obj), id_cclass) == Qtrue)
93
+ #define frb_mark_cclass(klass) rb_ivar_set(klass, id_cclass, Qtrue)
94
+ #define frb_is_cclass(obj) (rb_ivar_get(CLASS_OF(obj), id_cclass) == Qtrue)
@@ -0,0 +1,262 @@
1
+ #include <string.h>
2
+ #include "field_index.h"
3
+ #include "internal.h"
4
+
5
+ /***************************************************************************
6
+ *
7
+ * FieldIndex
8
+ *
9
+ ***************************************************************************/
10
+
11
+ static unsigned long field_index_hash(const void *p)
12
+ {
13
+ FieldIndex *self = (FieldIndex *)p;
14
+ return sym_hash(self->field) ^ (unsigned long)(self->klass);
15
+ }
16
+
17
+ static int field_index_eq(const void *p1, const void *p2)
18
+ {
19
+ FieldIndex *fi1 = (FieldIndex *)p1;
20
+ FieldIndex *fi2 = (FieldIndex *)p2;
21
+ return (fi1->field == fi2->field) &&
22
+ (fi1->klass->type == fi2->klass->type);
23
+ }
24
+
25
+ static void field_index_destroy(void *p)
26
+ {
27
+ FieldIndex *self = (FieldIndex *)p;
28
+ if (self->index) {
29
+ self->klass->destroy_index(self->index);
30
+ }
31
+ free(self);
32
+ }
33
+
34
+ FieldIndex *field_index_get(IndexReader *ir, Symbol field,
35
+ const FieldIndexClass *klass)
36
+ {
37
+ int length = 0;
38
+ TermEnum *volatile te = NULL;
39
+ TermDocEnum *volatile tde = NULL;
40
+ FieldInfo *fi = fis_get_field(ir->fis, field);
41
+ const volatile int field_num = fi ? fi->number : -1;
42
+ FieldIndex *volatile self = NULL;
43
+ FieldIndex key;
44
+
45
+ if (field_num < 0) {
46
+ RAISE(ARG_ERROR,
47
+ "Cannot sort by field \"%s\". It doesn't exist in the index.",
48
+ S(field));
49
+ }
50
+
51
+ if (!ir->field_index_cache) {
52
+ ir->field_index_cache = h_new(&field_index_hash, &field_index_eq,
53
+ NULL, &field_index_destroy);
54
+ }
55
+
56
+ key.field = field;
57
+ key.klass = klass;
58
+ self = (FieldIndex *)h_get(ir->field_index_cache, &key);
59
+
60
+ if (self == NULL) {
61
+ self = ALLOC(FieldIndex);
62
+ self->klass = klass;
63
+ /* FieldIndex only lives as long as the IndexReader lives so we can
64
+ * just use the field_infos field string */
65
+ self->field = fi->name;
66
+
67
+ length = ir->max_doc(ir);
68
+ if (length > 0) {
69
+ TRY
70
+ {
71
+ void *index;
72
+ tde = ir->term_docs(ir);
73
+ te = ir->terms(ir, field_num);
74
+ index = self->index = klass->create_index(length);
75
+ while (te->next(te)) {
76
+ tde->seek_te(tde, te);
77
+ klass->handle_term(index, tde, te->curr_term);
78
+ }
79
+ }
80
+ XFINALLY
81
+ tde->close(tde);
82
+ te->close(te);
83
+ XENDTRY
84
+ }
85
+ h_set(ir->field_index_cache, self, self);
86
+ }
87
+
88
+ return self;
89
+ }
90
+
91
+ /******************************************************************************
92
+ * ByteFieldIndex < FieldIndex
93
+ *
94
+ * The ByteFieldIndex holds an array of integers for each document in the
95
+ * index where the integer represents the sort value for the document. This
96
+ * index should only be used for sorting and not as a field cache of the
97
+ * column's value.
98
+ ******************************************************************************/
99
+ static void byte_handle_term(void *index_ptr,
100
+ TermDocEnum *tde,
101
+ const char *text)
102
+ {
103
+ long *index = (long *)index_ptr;
104
+ long val = index[-1]++;
105
+ (void)text;
106
+ while (tde->next(tde)) {
107
+ index[tde->doc_num(tde)] = val;
108
+ }
109
+ }
110
+
111
+ static void *byte_create_index(int size)
112
+ {
113
+ long *index = ALLOC_AND_ZERO_N(long, size + 1);
114
+ index[0] = 1;
115
+ return &index[1];
116
+ }
117
+
118
+ static void byte_destroy_index(void *p)
119
+ {
120
+ long *index = (long *)p;
121
+ free(&index[-1]);
122
+ }
123
+
124
+ const FieldIndexClass BYTE_FIELD_INDEX_CLASS = {
125
+ "byte",
126
+ &byte_create_index,
127
+ &byte_destroy_index,
128
+ &byte_handle_term
129
+ };
130
+
131
+ /******************************************************************************
132
+ * IntegerFieldIndex < FieldIndex
133
+ ******************************************************************************/
134
+ static void *integer_create_index(int size)
135
+ {
136
+ return ALLOC_AND_ZERO_N(long, size);
137
+ }
138
+
139
+ static void integer_handle_term(void *index_ptr,
140
+ TermDocEnum *tde,
141
+ const char *text)
142
+ {
143
+ long *index = (long *)index_ptr;
144
+ long val;
145
+ sscanf(text, "%ld", &val);
146
+ while (tde->next(tde)) {
147
+ index[tde->doc_num(tde)] = val;
148
+ }
149
+ }
150
+
151
+ const FieldIndexClass INTEGER_FIELD_INDEX_CLASS = {
152
+ "integer",
153
+ &integer_create_index,
154
+ &free,
155
+ &integer_handle_term
156
+ };
157
+
158
+ long get_integer_value(FieldIndex *field_index, long doc_num)
159
+ {
160
+ if (field_index->klass == &INTEGER_FIELD_INDEX_CLASS && doc_num >= 0) {
161
+ return ((long *)field_index->index)[doc_num];
162
+ }
163
+ return 0l;
164
+ }
165
+
166
+
167
+ /******************************************************************************
168
+ * FloatFieldIndex < FieldIndex
169
+ ******************************************************************************/
170
+ #define VALUES_ARRAY_START_SIZE 8
171
+ static void *float_create_index(int size)
172
+ {
173
+ return ALLOC_AND_ZERO_N(float, size);
174
+ }
175
+
176
+ static void float_handle_term(void *index_ptr,
177
+ TermDocEnum *tde,
178
+ const char *text)
179
+ {
180
+ float *index = (float *)index_ptr;
181
+ float val;
182
+ sscanf(text, "%g", &val);
183
+ while (tde->next(tde)) {
184
+ index[tde->doc_num(tde)] = val;
185
+ }
186
+ }
187
+
188
+ const FieldIndexClass FLOAT_FIELD_INDEX_CLASS = {
189
+ "float",
190
+ &float_create_index,
191
+ &free,
192
+ &float_handle_term
193
+ };
194
+
195
+ float get_float_value(FieldIndex *field_index, long doc_num)
196
+ {
197
+ if (field_index->klass == &FLOAT_FIELD_INDEX_CLASS && doc_num >= 0) {
198
+ return ((float *)field_index->index)[doc_num];
199
+ }
200
+ return 0.0f;
201
+ }
202
+
203
+ /******************************************************************************
204
+ * StringFieldIndex < FieldIndex
205
+ ******************************************************************************/
206
+
207
+ static void *string_create_index(int size)
208
+ {
209
+ StringIndex *self = ALLOC_AND_ZERO(StringIndex);
210
+ self->size = size;
211
+ self->index = ALLOC_AND_ZERO_N(long, size);
212
+ self->v_capa = VALUES_ARRAY_START_SIZE;
213
+ self->v_size = 1; /* leave the first value as NULL */
214
+ self->values = ALLOC_AND_ZERO_N(char *, VALUES_ARRAY_START_SIZE);
215
+ return self;
216
+ }
217
+
218
+ static void string_destroy_index(void *p)
219
+ {
220
+ StringIndex *self = (StringIndex *)p;
221
+ int i;
222
+ free(self->index);
223
+ for (i = 0; i < self->v_size; i++) {
224
+ free(self->values[i]);
225
+ }
226
+ free(self->values);
227
+ free(self);
228
+ }
229
+
230
+ static void string_handle_term(void *index_ptr,
231
+ TermDocEnum *tde,
232
+ const char *text)
233
+ {
234
+ StringIndex *index = (StringIndex *)index_ptr;
235
+ if (index->v_size >= index->v_capa) {
236
+ index->v_capa *= 2;
237
+ index->values = REALLOC_N(index->values, char *, index->v_capa);
238
+ }
239
+ index->values[index->v_size] = estrdup(text);
240
+ while (tde->next(tde)) {
241
+ index->index[tde->doc_num(tde)] = index->v_size;
242
+ }
243
+ index->v_size++;
244
+ }
245
+
246
+ const FieldIndexClass STRING_FIELD_INDEX_CLASS = {
247
+ "string",
248
+ &string_create_index,
249
+ &string_destroy_index,
250
+ &string_handle_term
251
+ };
252
+
253
+ const char *get_string_value(FieldIndex *field_index, long doc_num)
254
+ {
255
+ if (field_index->klass == &STRING_FIELD_INDEX_CLASS) {
256
+ StringIndex *string_index = (StringIndex *)field_index->index;
257
+ if (doc_num >= 0 && doc_num < string_index->size) {
258
+ return string_index->values[string_index->index[doc_num]];
259
+ }
260
+ }
261
+ return NULL;
262
+ }
@@ -0,0 +1,52 @@
1
+ #ifndef FRT_FIELD_INDEX_H
2
+ #define FRT_FIELD_INDEX_H
3
+
4
+ #ifdef __cplusplus
5
+ extern "C" {
6
+ #endif
7
+
8
+ #include "index.h"
9
+
10
+ /***************************************************************************
11
+ *
12
+ * FrtFieldIndex
13
+ *
14
+ ***************************************************************************/
15
+
16
+ typedef struct FrtStringIndex {
17
+ int size;
18
+ long *index;
19
+ char **values;
20
+ int v_size;
21
+ int v_capa;
22
+ } FrtStringIndex;
23
+
24
+ typedef struct FrtFieldIndexClass {
25
+ const char *type;
26
+ void *(*create_index)(int size);
27
+ void (*destroy_index)(void *p);
28
+ void (*handle_term)(void *index, FrtTermDocEnum *tde, const char *text);
29
+ } FrtFieldIndexClass;
30
+
31
+ typedef struct FrtFieldIndex {
32
+ FrtSymbol field;
33
+ const FrtFieldIndexClass *klass;
34
+ void *index;
35
+ } FrtFieldIndex;
36
+
37
+ extern const FrtFieldIndexClass FRT_INTEGER_FIELD_INDEX_CLASS;
38
+ extern const FrtFieldIndexClass FRT_FLOAT_FIELD_INDEX_CLASS;
39
+ extern const FrtFieldIndexClass FRT_STRING_FIELD_INDEX_CLASS;
40
+ extern const FrtFieldIndexClass FRT_BYTE_FIELD_INDEX_CLASS;
41
+
42
+ extern FrtFieldIndex *frt_field_index_get(FrtIndexReader *ir, FrtSymbol field,
43
+ const FrtFieldIndexClass *klass);
44
+ extern const char *frt_get_string_value(FrtFieldIndex *field_index, long doc_num);
45
+ extern float frt_get_float_value(FrtFieldIndex *field_index, long doc_num);
46
+ extern long frt_get_integer_value(FrtFieldIndex *field_index, long doc_num);
47
+
48
+ #ifdef __cplusplus
49
+ } // extern "C"
50
+ #endif
51
+
52
+ #endif
@@ -1,5 +1,7 @@
1
1
  #include "search.h"
2
+ #include "symbol.h"
2
3
  #include <string.h>
4
+ #include "internal.h"
3
5
 
4
6
  /***************************************************************************
5
7
  *
@@ -10,7 +12,6 @@
10
12
  void filt_destroy_i(Filter *filt)
11
13
  {
12
14
  h_destroy(filt->cache);
13
- free(filt->name);
14
15
  free(filt);
15
16
  }
16
17
  void filt_deref(Filter *filt)
@@ -22,7 +23,7 @@ void filt_deref(Filter *filt)
22
23
 
23
24
  BitVector *filt_get_bv(Filter *filt, IndexReader *ir)
24
25
  {
25
- CacheObject *co = h_get(filt->cache, ir);
26
+ CacheObject *co = (CacheObject *)h_get(filt->cache, ir);
26
27
 
27
28
  if (!co) {
28
29
  BitVector *bv;
@@ -38,26 +39,26 @@ BitVector *filt_get_bv(Filter *filt, IndexReader *ir)
38
39
 
39
40
  static char *filt_to_s_i(Filter *filt)
40
41
  {
41
- return estrdup(filt->name);
42
+ return estrdup(S(filt->name));
42
43
  }
43
44
 
44
- unsigned long filt_hash_default(Filter *filt)
45
+ static unsigned long filt_hash_default(Filter *filt)
45
46
  {
46
47
  (void)filt;
47
48
  return 0;
48
49
  }
49
50
 
50
- int filt_eq_default(Filter *filt, Filter *o)
51
+ static int filt_eq_default(Filter *filt, Filter *o)
51
52
  {
52
53
  (void)filt; (void)o;
53
54
  return false;
54
55
  }
55
56
 
56
- Filter *filt_create(size_t size, const char *name)
57
+ Filter *filt_create(size_t size, Symbol name)
57
58
  {
58
59
  Filter *filt = (Filter *)emalloc(size);
59
60
  filt->cache = co_hash_create();
60
- filt->name = estrdup(name);
61
+ filt->name = name;
61
62
  filt->to_s = &filt_to_s_i;
62
63
  filt->hash = &filt_hash_default;
63
64
  filt->eq = &filt_eq_default;
@@ -68,13 +69,13 @@ Filter *filt_create(size_t size, const char *name)
68
69
 
69
70
  unsigned long filt_hash(Filter *filt)
70
71
  {
71
- return str_hash(filt->name) ^ filt->hash(filt);
72
+ return sym_hash(filt->name) ^ filt->hash(filt);
72
73
  }
73
74
 
74
75
  int filt_eq(Filter *filt, Filter *o)
75
76
  {
76
77
  return ((filt == o)
77
- || ((strcmp(filt->name, o->name) == 0)
78
+ || ((filt->name == o->name)
78
79
  && (filt->eq == o->eq)
79
80
  && (filt->eq(filt, o))));
80
81
  }
@@ -95,7 +96,7 @@ typedef struct QueryFilter
95
96
  static char *qfilt_to_s(Filter *filt)
96
97
  {
97
98
  Query *query = QF(filt)->query;
98
- char *query_str = query->to_s(query, "");
99
+ char *query_str = query->to_s(query, NULL);
99
100
  char *filter_str = strfmt("QueryFilter< %s >", query_str);
100
101
  free(query_str);
101
102
  return filter_str;
@@ -1,4 +1,5 @@
1
1
  #include "store.h"
2
+ #include "lang.h"
2
3
  #include <time.h>
3
4
  #include <sys/types.h>
4
5
  #include <fcntl.h>
@@ -14,6 +15,7 @@
14
15
  # endif
15
16
  # ifndef DIR_SEPARATOR
16
17
  # define DIR_SEPARATOR "\\"
18
+ # define DIR_SEPARATOR_CHAR '\\'
17
19
  # endif
18
20
  # ifndef S_IRUSR
19
21
  # define S_IRUSR _S_IREAD
@@ -23,18 +25,14 @@
23
25
  # endif
24
26
  #else
25
27
  # define DIR_SEPARATOR "/"
28
+ # define DIR_SEPARATOR_CHAR '/'
26
29
  # include <unistd.h>
27
30
  # include <dirent.h>
28
31
  #endif
29
32
  #ifndef O_BINARY
30
33
  # define O_BINARY 0
31
34
  #endif
32
-
33
- extern Store *store_new();
34
- extern void store_destroy(Store *store);
35
- extern OutStream *os_new();
36
- extern InStream *is_new();
37
- extern int file_is_lock(char *filename);
35
+ #include "internal.h"
38
36
 
39
37
  /**
40
38
  * Create a filepath for a file in the store using the operating systems
@@ -46,7 +44,7 @@ static char *join_path(char *buf, const char *base, const char *filename)
46
44
  return buf;
47
45
  }
48
46
 
49
- static void fs_touch(Store *store, char *filename)
47
+ static void fs_touch(Store *store, const char *filename)
50
48
  {
51
49
  int f;
52
50
  char path[MAX_FILE_PATH];
@@ -58,7 +56,7 @@ static void fs_touch(Store *store, char *filename)
58
56
  close(f);
59
57
  }
60
58
 
61
- static int fs_exists(Store *store, char *filename)
59
+ static int fs_exists(Store *store, const char *filename)
62
60
  {
63
61
  int fd;
64
62
  char path[MAX_FILE_PATH];
@@ -75,13 +73,13 @@ static int fs_exists(Store *store, char *filename)
75
73
  return true;
76
74
  }
77
75
 
78
- static int fs_remove(Store *store, char *filename)
76
+ static int fs_remove(Store *store, const char *filename)
79
77
  {
80
78
  char path[MAX_FILE_PATH];
81
79
  return remove(join_path(path, store->dir.path, filename));
82
80
  }
83
81
 
84
- static void fs_rename(Store *store, char *from, char *to)
82
+ static void fs_rename(Store *store, const char *from, const char *to)
85
83
  {
86
84
  char path1[MAX_FILE_PATH], path2[MAX_FILE_PATH];
87
85
 
@@ -117,7 +115,7 @@ static int fs_count(Store *store)
117
115
  return cnt;
118
116
  }
119
117
 
120
- static void fs_each(Store *store, void (*func)(char *fname, void *arg), void *arg)
118
+ static void fs_each(Store *store, void (*func)(const char *fname, void *arg), void *arg)
121
119
  {
122
120
  struct dirent *de;
123
121
  DIR *d = opendir(store->dir.path);
@@ -155,6 +153,20 @@ static void fs_clear_locks(Store *store)
155
153
  closedir(d);
156
154
  }
157
155
 
156
+ static void remove_if_index_file(const char *base_path, const char *file_name)
157
+ {
158
+ char path[MAX_FILE_PATH];
159
+ char *basename;
160
+ join_path(path, base_path, file_name);
161
+ /* get basename of path */
162
+ basename = strrchr(path, DIR_SEPARATOR_CHAR);
163
+ basename = (basename ? basename + 1 : path);
164
+ /* we don't want to delete non-index files here */
165
+ if (file_name_filter_is_index_file(basename, true)) {
166
+ remove(path);
167
+ }
168
+ }
169
+
158
170
  static void fs_clear(Store *store)
159
171
  {
160
172
  struct dirent *de;
@@ -168,13 +180,16 @@ static void fs_clear(Store *store)
168
180
  while ((de = readdir(d)) != NULL) {
169
181
  if (de->d_name[0] > '/' /* skip ., .., / and '\0'*/
170
182
  && !file_is_lock(de->d_name)) {
171
- char path[MAX_FILE_PATH];
172
- remove(join_path(path, store->dir.path, de->d_name));
183
+ remove_if_index_file(store->dir.path, de->d_name);
173
184
  }
174
185
  }
175
186
  closedir(d);
176
187
  }
177
188
 
189
+ /**
190
+ * Clear all files which belong to the index. Use fs_clear to clear the
191
+ * directory regardless of the files origin.
192
+ */
178
193
  static void fs_clear_all(Store *store)
179
194
  {
180
195
  struct dirent *de;
@@ -187,8 +202,7 @@ static void fs_clear_all(Store *store)
187
202
 
188
203
  while ((de = readdir(d)) != NULL) {
189
204
  if (de->d_name[0] > '/') { /* skip ., .., / and '\0'*/
190
- char path[MAX_FILE_PATH];
191
- remove(join_path(path, store->dir.path, de->d_name));
205
+ remove_if_index_file(store->dir.path, de->d_name);
192
206
  }
193
207
  }
194
208
  closedir(d);
@@ -211,7 +225,7 @@ static void fs_destroy(Store *store)
211
225
  store_destroy(store);
212
226
  }
213
227
 
214
- static off_t fs_length(Store *store, char *filename)
228
+ static off_t fs_length(Store *store, const char *filename)
215
229
  {
216
230
  char path[MAX_FILE_PATH];
217
231
  struct stat stt;
@@ -224,7 +238,7 @@ static off_t fs_length(Store *store, char *filename)
224
238
  return stt.st_size;
225
239
  }
226
240
 
227
- static void fso_flush_i(OutStream *os, uchar *src, int len)
241
+ static void fso_flush_i(OutStream *os, const uchar *src, int len)
228
242
  {
229
243
  if (len != write(os->file.fd, src, len)) {
230
244
  RAISE(IO_ERROR, "flushing src of length %d, <%s>", len,
@@ -235,7 +249,7 @@ static void fso_flush_i(OutStream *os, uchar *src, int len)
235
249
  static void fso_seek_i(OutStream *os, off_t pos)
236
250
  {
237
251
  if (lseek(os->file.fd, pos, SEEK_SET) < 0) {
238
- RAISE(IO_ERROR, "seeking position %"F_OFF_T_PFX"d: <%s>",
252
+ RAISE(IO_ERROR, "seeking position %"OFF_T_PFX"d: <%s>",
239
253
  pos, strerror(errno));
240
254
  }
241
255
  }
@@ -247,7 +261,7 @@ static void fso_close_i(OutStream *os)
247
261
  }
248
262
  }
249
263
 
250
- const struct OutStreamMethods FS_OUT_STREAM_METHODS = {
264
+ static const struct OutStreamMethods FS_OUT_STREAM_METHODS = {
251
265
  fso_flush_i,
252
266
  fso_seek_i,
253
267
  fso_close_i
@@ -289,7 +303,7 @@ static void fsi_read_i(InStream *is, uchar *path, int len)
289
303
  static void fsi_seek_i(InStream *is, off_t pos)
290
304
  {
291
305
  if (lseek(is->file.fd, pos, SEEK_SET) < 0) {
292
- RAISE(IO_ERROR, "seeking pos %"F_OFF_T_PFX"d: <%s>",
306
+ RAISE(IO_ERROR, "seeking pos %"OFF_T_PFX"d: <%s>",
293
307
  pos, strerror(errno));
294
308
  }
295
309
  }
@@ -297,7 +311,7 @@ static void fsi_seek_i(InStream *is, off_t pos)
297
311
  static void fsi_close_i(InStream *is)
298
312
  {
299
313
  if (close(is->file.fd)) {
300
- RAISE(IO_ERROR, strerror(errno));
314
+ RAISE(IO_ERROR, "%s", strerror(errno));
301
315
  }
302
316
  free(is->d.path);
303
317
  }
@@ -337,10 +351,6 @@ static InStream *fs_open_input(Store *store, const char *filename)
337
351
 
338
352
  #define LOCK_OBTAIN_TIMEOUT 10
339
353
 
340
- #ifdef RUBY_BINDINGS
341
- struct timeval rb_time_interval _((VALUE));
342
- #endif
343
-
344
354
  static int fs_lock_obtain(Lock *lock)
345
355
  {
346
356
  int f;
@@ -377,12 +387,12 @@ static int fs_lock_is_locked(Lock *lock)
377
387
  }
378
388
  }
379
389
 
380
- void fs_lock_release(Lock *lock)
390
+ static void fs_lock_release(Lock *lock)
381
391
  {
382
392
  remove(lock->name);
383
393
  }
384
394
 
385
- static Lock *fs_open_lock_i(Store *store, char *lockname)
395
+ static Lock *fs_open_lock_i(Store *store, const char *lockname)
386
396
  {
387
397
  Lock *lock = ALLOC(Lock);
388
398
  char lname[100];
@@ -403,19 +413,7 @@ static void fs_close_lock_i(Lock *lock)
403
413
  free(lock);
404
414
  }
405
415
 
406
- static HashTable stores = {
407
- /* fill */ 0,
408
- /* used */ 0,
409
- /* mask */ HASH_MINSIZE - 1,
410
- /* ref_cnt */ 1,
411
- /* table */ stores.smalltable,
412
- /* smalltable */ {{0, NULL, NULL}},
413
- /* lookup */ (h_lookup_ft)&h_lookup_str,
414
- /* hash */ NULL,
415
- /* eq */ NULL,
416
- /* free_key */ (free_ft)&dummy_free,
417
- /* free_value */ (free_ft)&fs_destroy
418
- };
416
+ static Hash *stores = NULL;
419
417
 
420
418
  #ifndef UNTHREADED
421
419
  static mutex_t stores_mutex = MUTEX_INITIALIZER;
@@ -424,7 +422,7 @@ static mutex_t stores_mutex = MUTEX_INITIALIZER;
424
422
  static void fs_close_i(Store *store)
425
423
  {
426
424
  mutex_lock(&stores_mutex);
427
- h_del(&stores, store->dir.path);
425
+ h_del(stores, store->dir.path);
428
426
  mutex_unlock(&stores_mutex);
429
427
  }
430
428
 
@@ -435,11 +433,26 @@ static Store *fs_store_new(const char *pathname)
435
433
 
436
434
  new_store->file_mode = S_IRUSR | S_IWUSR;
437
435
  #ifndef POSH_OS_WIN32
438
- if (!stat(pathname, &stt) && stt.st_gid == getgid()) {
439
- if (stt.st_mode & S_IWGRP) {
440
- umask(S_IWOTH);
436
+ if (!stat(pathname, &stt)) {
437
+ gid_t st_gid = stt.st_gid;
438
+ bool is_grp = (st_gid == getgid());
439
+
440
+ if (!is_grp) {
441
+ int size = getgroups(0, NULL);
442
+ gid_t list[size];
443
+
444
+ if (getgroups(size, list) != -1) {
445
+ int i = 0;
446
+ while (i < size && !(is_grp = (st_gid == list[i]))) i++;
447
+ }
448
+ }
449
+
450
+ if (is_grp) {
451
+ if (stt.st_mode & S_IWGRP) {
452
+ umask(S_IWOTH);
453
+ }
454
+ new_store->file_mode |= stt.st_mode & (S_IRGRP | S_IWGRP);
441
455
  }
442
- new_store->file_mode |= stt.st_mode & (S_IRGRP | S_IWGRP);
443
456
  }
444
457
  #endif
445
458
 
@@ -466,8 +479,13 @@ Store *open_fs_store(const char *pathname)
466
479
  {
467
480
  Store *store = NULL;
468
481
 
482
+ if (!stores) {
483
+ stores = h_new_str(NULL, (free_ft)fs_destroy);
484
+ register_for_cleanup(stores, (free_ft)h_destroy);
485
+ }
486
+
469
487
  mutex_lock(&stores_mutex);
470
- store = h_get(&stores, pathname);
488
+ store = (Store *)h_get(stores, pathname);
471
489
  if (store) {
472
490
  mutex_lock(&store->mutex);
473
491
  store->ref_cnt++;
@@ -475,7 +493,7 @@ Store *open_fs_store(const char *pathname)
475
493
  }
476
494
  else {
477
495
  store = fs_store_new(pathname);
478
- h_set(&stores, store->dir.path, store);
496
+ h_set(stores, store->dir.path, store);
479
497
  }
480
498
  mutex_unlock(&stores_mutex);
481
499