isomorfeus-ferret 0.13.7 → 0.13.10
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +10 -4
- data/ext/isomorfeus_ferret_ext/bm_store.c +3 -3
- data/ext/isomorfeus_ferret_ext/frb_analysis.c +7 -4
- data/ext/isomorfeus_ferret_ext/frb_index.c +18 -24
- data/ext/isomorfeus_ferret_ext/frb_qparser.c +2 -1
- data/ext/isomorfeus_ferret_ext/frb_search.c +23 -19
- data/ext/isomorfeus_ferret_ext/frb_store.c +34 -36
- data/ext/isomorfeus_ferret_ext/frt_analysis.c +4 -4
- data/ext/isomorfeus_ferret_ext/frt_analysis.h +9 -9
- data/ext/isomorfeus_ferret_ext/frt_bitvector.c +1 -1
- data/ext/isomorfeus_ferret_ext/frt_bitvector.h +1 -1
- data/ext/isomorfeus_ferret_ext/frt_compound_io.c +41 -66
- data/ext/isomorfeus_ferret_ext/frt_config.h +8 -0
- data/ext/isomorfeus_ferret_ext/frt_except.c +1 -0
- data/ext/isomorfeus_ferret_ext/frt_field_index.c +1 -1
- data/ext/isomorfeus_ferret_ext/frt_filter.c +2 -4
- data/ext/isomorfeus_ferret_ext/frt_fs_store.c +24 -19
- data/ext/isomorfeus_ferret_ext/frt_global.c +6 -4
- data/ext/isomorfeus_ferret_ext/frt_global.h +1 -1
- data/ext/isomorfeus_ferret_ext/frt_hash.c +40 -48
- data/ext/isomorfeus_ferret_ext/frt_hash.h +14 -16
- data/ext/isomorfeus_ferret_ext/frt_ind.c +3 -4
- data/ext/isomorfeus_ferret_ext/frt_index.c +152 -222
- data/ext/isomorfeus_ferret_ext/frt_index.h +31 -31
- data/ext/isomorfeus_ferret_ext/frt_lang.c +1 -4
- data/ext/isomorfeus_ferret_ext/frt_multimapper.c +8 -9
- data/ext/isomorfeus_ferret_ext/frt_multimapper.h +1 -1
- data/ext/isomorfeus_ferret_ext/frt_q_boolean.c +7 -7
- data/ext/isomorfeus_ferret_ext/frt_q_fuzzy.c +1 -1
- data/ext/isomorfeus_ferret_ext/frt_q_match_all.c +8 -3
- data/ext/isomorfeus_ferret_ext/frt_q_multi_term.c +2 -2
- data/ext/isomorfeus_ferret_ext/frt_q_parser.c +1742 -1742
- data/ext/isomorfeus_ferret_ext/frt_q_phrase.c +2 -3
- data/ext/isomorfeus_ferret_ext/frt_q_prefix.c +1 -1
- data/ext/isomorfeus_ferret_ext/frt_q_range.c +1 -1
- data/ext/isomorfeus_ferret_ext/frt_q_span.c +12 -11
- data/ext/isomorfeus_ferret_ext/frt_q_term.c +2 -2
- data/ext/isomorfeus_ferret_ext/frt_q_wildcard.c +1 -1
- data/ext/isomorfeus_ferret_ext/frt_ram_store.c +24 -40
- data/ext/isomorfeus_ferret_ext/frt_search.c +30 -29
- data/ext/isomorfeus_ferret_ext/frt_search.h +18 -19
- data/ext/isomorfeus_ferret_ext/frt_sort.c +1 -1
- data/ext/isomorfeus_ferret_ext/frt_store.c +47 -40
- data/ext/isomorfeus_ferret_ext/frt_store.h +45 -47
- data/ext/isomorfeus_ferret_ext/frt_threading.h +12 -5
- data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.c +4 -3
- data/ext/isomorfeus_ferret_ext/test_1710.c +1 -2
- data/ext/isomorfeus_ferret_ext/test_compound_io.c +8 -8
- data/ext/isomorfeus_ferret_ext/test_fields.c +7 -7
- data/ext/isomorfeus_ferret_ext/test_file_deleter.c +1 -1
- data/ext/isomorfeus_ferret_ext/test_filter.c +5 -4
- data/ext/isomorfeus_ferret_ext/test_fs_store.c +1 -1
- data/ext/isomorfeus_ferret_ext/test_highlighter.c +4 -2
- data/ext/isomorfeus_ferret_ext/test_index.c +63 -63
- data/ext/isomorfeus_ferret_ext/test_q_const_score.c +3 -2
- data/ext/isomorfeus_ferret_ext/test_q_filtered.c +4 -3
- data/ext/isomorfeus_ferret_ext/test_q_fuzzy.c +4 -2
- data/ext/isomorfeus_ferret_ext/test_q_span.c +9 -2
- data/ext/isomorfeus_ferret_ext/test_ram_store.c +4 -4
- data/ext/isomorfeus_ferret_ext/test_search.c +10 -5
- data/ext/isomorfeus_ferret_ext/test_segments.c +4 -3
- data/ext/isomorfeus_ferret_ext/test_sort.c +18 -10
- data/ext/isomorfeus_ferret_ext/test_store.c +1 -1
- data/ext/isomorfeus_ferret_ext/test_term.c +5 -3
- data/ext/isomorfeus_ferret_ext/test_term_vectors.c +2 -2
- data/ext/isomorfeus_ferret_ext/test_threading.c +5 -4
- data/lib/isomorfeus/ferret/index/index.rb +8 -3
- data/lib/isomorfeus/ferret/version.rb +1 -1
- metadata +16 -2
@@ -100,7 +100,7 @@ typedef struct FrtFieldInfo {
|
|
100
100
|
float boost;
|
101
101
|
unsigned int bits;
|
102
102
|
int number;
|
103
|
-
int
|
103
|
+
_Atomic unsigned int ref_cnt;
|
104
104
|
VALUE rfi;
|
105
105
|
} FrtFieldInfo;
|
106
106
|
|
@@ -133,7 +133,7 @@ extern void frt_fi_deref(FrtFieldInfo *fi);
|
|
133
133
|
#define FIELD_INFOS_INIT_CAPA 4
|
134
134
|
/* carry changes over to dummy_fis in test/test_segments.c */
|
135
135
|
typedef struct FrtFieldInfos {
|
136
|
-
FrtStoreValue
|
136
|
+
FrtStoreValue store_val;
|
137
137
|
FrtCompressionType compression;
|
138
138
|
FrtIndexValue index;
|
139
139
|
FrtTermVectorValue term_vector;
|
@@ -141,7 +141,7 @@ typedef struct FrtFieldInfos {
|
|
141
141
|
int capa;
|
142
142
|
FrtFieldInfo **fields;
|
143
143
|
FrtHash *field_dict;
|
144
|
-
int
|
144
|
+
_Atomic unsigned int ref_cnt;
|
145
145
|
VALUE rfis;
|
146
146
|
} FrtFieldInfos;
|
147
147
|
|
@@ -167,18 +167,18 @@ extern void frt_fis_deref(FrtFieldInfos *fis);
|
|
167
167
|
#define FRT_SEGMENTS_FILE_NAME "segments"
|
168
168
|
|
169
169
|
typedef struct FrtSegmentInfo {
|
170
|
-
int ref_cnt;
|
171
|
-
char
|
172
|
-
FrtStore
|
173
|
-
int
|
174
|
-
int
|
175
|
-
int
|
176
|
-
int
|
177
|
-
bool
|
170
|
+
_Atomic unsigned int ref_cnt;
|
171
|
+
char *name;
|
172
|
+
FrtStore *store;
|
173
|
+
int doc_cnt;
|
174
|
+
int del_gen;
|
175
|
+
int *norm_gens;
|
176
|
+
int norm_gens_size;
|
177
|
+
bool use_compound_file;
|
178
178
|
} FrtSegmentInfo;
|
179
179
|
|
180
180
|
extern FrtSegmentInfo *frt_si_new(char *name, int doc_cnt, FrtStore *store);
|
181
|
-
extern void
|
181
|
+
extern void frt_si_close(FrtSegmentInfo *si);
|
182
182
|
extern bool frt_si_has_deletions(FrtSegmentInfo *si);
|
183
183
|
extern bool frt_si_has_separate_norms(FrtSegmentInfo *si);
|
184
184
|
extern void frt_si_advance_norm_gen(FrtSegmentInfo *si, int field_num);
|
@@ -224,9 +224,9 @@ extern void frt_sis_put(FrtSegmentInfos *sis, FILE *stream);
|
|
224
224
|
|
225
225
|
typedef struct FrtTermInfo {
|
226
226
|
int doc_freq;
|
227
|
-
|
228
|
-
|
229
|
-
|
227
|
+
frt_off_t frq_ptr;
|
228
|
+
frt_off_t prx_ptr;
|
229
|
+
frt_off_t skip_offset;
|
230
230
|
} FrtTermInfo;
|
231
231
|
|
232
232
|
#define frt_ti_set(ti, mdf, mfp, mpp, mso) do {\
|
@@ -265,14 +265,14 @@ FrtTermInfo *frt_te_get_ti(struct FrtTermEnum *te);
|
|
265
265
|
/* FrtSegmentTermIndex */
|
266
266
|
|
267
267
|
typedef struct FrtSegmentTermIndex {
|
268
|
-
|
269
|
-
|
268
|
+
frt_off_t index_ptr;
|
269
|
+
frt_off_t ptr;
|
270
270
|
int index_cnt;
|
271
271
|
int size;
|
272
272
|
char **index_terms;
|
273
273
|
int *index_term_lens;
|
274
274
|
FrtTermInfo *index_term_infos;
|
275
|
-
|
275
|
+
frt_off_t *index_ptrs;
|
276
276
|
} FrtSegmentTermIndex;
|
277
277
|
|
278
278
|
/* FrtSegmentFieldIndex */
|
@@ -281,7 +281,7 @@ typedef struct FrtSegmentFieldIndex {
|
|
281
281
|
frt_mutex_t mutex;
|
282
282
|
int skip_interval;
|
283
283
|
int index_interval;
|
284
|
-
|
284
|
+
frt_off_t index_ptr;
|
285
285
|
FrtTermEnum *index_te;
|
286
286
|
FrtHash *field_dict;
|
287
287
|
} FrtSegmentFieldIndex;
|
@@ -349,7 +349,7 @@ typedef struct FrtTermInfosWriter {
|
|
349
349
|
int field_count;
|
350
350
|
int index_interval;
|
351
351
|
int skip_interval;
|
352
|
-
|
352
|
+
frt_off_t last_index_ptr;
|
353
353
|
FrtOutStream *tfx_out;
|
354
354
|
FrtTermWriter *tix_writer;
|
355
355
|
FrtTermWriter *tis_writer;
|
@@ -385,7 +385,7 @@ struct FrtTermDocEnum {
|
|
385
385
|
typedef struct FrtSegmentTermDocEnum FrtSegmentTermDocEnum;
|
386
386
|
struct FrtSegmentTermDocEnum {
|
387
387
|
FrtTermDocEnum tde;
|
388
|
-
void (*seek_prox)(FrtSegmentTermDocEnum *stde,
|
388
|
+
void (*seek_prox)(FrtSegmentTermDocEnum *stde, frt_off_t prx_ptr);
|
389
389
|
void (*skip_prox)(FrtSegmentTermDocEnum *stde);
|
390
390
|
FrtTermInfosReader *tir;
|
391
391
|
FrtInStream *frq_in;
|
@@ -402,9 +402,9 @@ struct FrtSegmentTermDocEnum {
|
|
402
402
|
int skip_doc;
|
403
403
|
int prx_cnt;
|
404
404
|
int position;
|
405
|
-
|
406
|
-
|
407
|
-
|
405
|
+
frt_off_t frq_ptr;
|
406
|
+
frt_off_t prx_ptr;
|
407
|
+
frt_off_t skip_ptr;
|
408
408
|
bool have_skipped : 1;
|
409
409
|
};
|
410
410
|
|
@@ -429,8 +429,8 @@ extern FrtTermDocEnum *frt_mtdpe_new(FrtIndexReader *ir, int field_num, char **t
|
|
429
429
|
****************************************************************************/
|
430
430
|
|
431
431
|
typedef struct FrtOffset {
|
432
|
-
|
433
|
-
|
432
|
+
frt_off_t start;
|
433
|
+
frt_off_t end;
|
434
434
|
} FrtOffset;
|
435
435
|
|
436
436
|
/****************************************************************************
|
@@ -529,7 +529,7 @@ extern FrtTVTerm *frt_tv_get_tv_term(FrtTermVector *tv, const char *term);
|
|
529
529
|
|
530
530
|
/* * * FrtLazyDocField * * */
|
531
531
|
typedef struct FrtLazyDocFieldData {
|
532
|
-
|
532
|
+
frt_off_t start;
|
533
533
|
int length;
|
534
534
|
rb_encoding *encoding;
|
535
535
|
FrtCompressionType compression; /* as stored */
|
@@ -595,7 +595,7 @@ typedef struct FrtFieldsWriter {
|
|
595
595
|
FrtOutStream *fdx_out;
|
596
596
|
FrtOutStream *buffer;
|
597
597
|
FrtTVField *tv_fields;
|
598
|
-
|
598
|
+
frt_off_t start_ptr;
|
599
599
|
} FrtFieldsWriter;
|
600
600
|
|
601
601
|
extern FrtFieldsWriter *frt_fw_open(FrtStore *store, const char *segment, FrtFieldInfos *fis);
|
@@ -671,7 +671,7 @@ struct FrtIndexReader {
|
|
671
671
|
bool (*is_latest_i)(FrtIndexReader *ir);
|
672
672
|
void (*commit_i)(FrtIndexReader *ir);
|
673
673
|
void (*close_i)(FrtIndexReader *ir);
|
674
|
-
int
|
674
|
+
_Atomic unsigned int ref_cnt;
|
675
675
|
FrtDeleter *deleter;
|
676
676
|
FrtStore *store;
|
677
677
|
FrtLock *write_lock;
|
@@ -846,8 +846,8 @@ extern void frt_iw_add_readers(FrtIndexWriter *iw, FrtIndexReader **readers, con
|
|
846
846
|
#define FRT_CW_INIT_CAPA 16
|
847
847
|
typedef struct FrtCWFileEntry {
|
848
848
|
char *name;
|
849
|
-
|
850
|
-
|
849
|
+
frt_off_t dir_offset;
|
850
|
+
frt_off_t data_offset;
|
851
851
|
} FrtCWFileEntry;
|
852
852
|
|
853
853
|
typedef struct FrtCompoundWriter {
|
@@ -43,10 +43,7 @@ void *frt_erealloc(void *ptr, size_t size)
|
|
43
43
|
return p;
|
44
44
|
}
|
45
45
|
|
46
|
-
|
47
|
-
struct timeval rb_time_interval _((VALUE));
|
48
|
-
|
49
|
-
extern void frt_micro_sleep(const int micro_seconds) {
|
46
|
+
void frt_micro_sleep(const int micro_seconds) {
|
50
47
|
rb_thread_wait_for(rb_time_interval(rb_float_new((double)micro_seconds/1000000.0)));
|
51
48
|
}
|
52
49
|
|
@@ -328,19 +328,18 @@ char *frt_mulmap_dynamic_map(FrtMultiMapper *self, char *from)
|
|
328
328
|
return to;
|
329
329
|
}
|
330
330
|
|
331
|
-
void frt_mulmap_destroy(FrtMultiMapper *
|
332
|
-
{
|
333
|
-
if (--(self->ref_cnt) <= 0) {
|
331
|
+
void frt_mulmap_destroy(FrtMultiMapper *mulm) {
|
332
|
+
if (FRT_DEREF(mulm) <= 0) {
|
334
333
|
int i;
|
335
|
-
mulmap_free_dstates(
|
336
|
-
for (i =
|
337
|
-
FrtMapping *mapping =
|
334
|
+
mulmap_free_dstates(mulm);
|
335
|
+
for (i = mulm->size - 1; i >= 0; i--) {
|
336
|
+
FrtMapping *mapping = mulm->mappings[i];
|
338
337
|
free(mapping->pattern);
|
339
338
|
free(mapping->replacement);
|
340
339
|
free(mapping);
|
341
340
|
}
|
342
|
-
free(
|
343
|
-
free(
|
344
|
-
free(
|
341
|
+
free(mulm->mappings);
|
342
|
+
free(mulm->dstates);
|
343
|
+
free(mulm);
|
345
344
|
}
|
346
345
|
}
|
@@ -1264,10 +1264,10 @@ void frt_bc_set_occur(FrtBooleanClause *self, FrtBCType occur) {
|
|
1264
1264
|
}
|
1265
1265
|
}
|
1266
1266
|
|
1267
|
-
void frt_bc_deref(FrtBooleanClause *
|
1268
|
-
if (
|
1269
|
-
frt_q_deref(
|
1270
|
-
free(
|
1267
|
+
void frt_bc_deref(FrtBooleanClause *bc) {
|
1268
|
+
if (FRT_DEREF(bc) == 0) {
|
1269
|
+
frt_q_deref(bc->query);
|
1270
|
+
free(bc);
|
1271
1271
|
}
|
1272
1272
|
}
|
1273
1273
|
|
@@ -1280,7 +1280,7 @@ static int bc_eq(FrtBooleanClause *self, FrtBooleanClause *o) {
|
|
1280
1280
|
}
|
1281
1281
|
|
1282
1282
|
FrtBooleanClause *frt_bc_alloc(void) {
|
1283
|
-
return
|
1283
|
+
return FRT_ALLOC_AND_ZERO(FrtBooleanClause);
|
1284
1284
|
}
|
1285
1285
|
|
1286
1286
|
FrtBooleanClause *frt_bc_init(FrtBooleanClause *self, FrtQuery *query, FrtBCType occur) {
|
@@ -1345,7 +1345,7 @@ static FrtQuery *bq_rewrite(FrtQuery *self, FrtIndexReader *ir) {
|
|
1345
1345
|
}
|
1346
1346
|
}
|
1347
1347
|
|
1348
|
-
self
|
1348
|
+
FRT_REF(self);
|
1349
1349
|
/* replace each clause's query with its rewritten query */
|
1350
1350
|
for (i = 0; i < clause_cnt; i++) {
|
1351
1351
|
FrtBooleanClause *clause = BQ(self)->clauses[i];
|
@@ -1364,7 +1364,7 @@ static FrtQuery *bq_rewrite(FrtQuery *self, FrtIndexReader *ir) {
|
|
1364
1364
|
for (j = 0; j < clause_cnt; j++) {
|
1365
1365
|
FRT_REF(BQ(self)->clauses[j]);
|
1366
1366
|
}
|
1367
|
-
self
|
1367
|
+
FRT_DEREF(self);
|
1368
1368
|
self = new_self;
|
1369
1369
|
self->ref_cnt = 1;
|
1370
1370
|
rewritten = true;
|
@@ -45,9 +45,15 @@ static FrtExplanation *masc_explain(FrtScorer *self, int doc_num)
|
|
45
45
|
return frt_expl_new(1.0, "MatchAllScorer");
|
46
46
|
}
|
47
47
|
|
48
|
+
void masc_destroy_i(FrtScorer *self) {
|
49
|
+
frt_ir_close(MASc(self)->ir);
|
50
|
+
frt_scorer_destroy_i(self);
|
51
|
+
}
|
52
|
+
|
48
53
|
static FrtScorer *masc_new(FrtWeight *weight, FrtIndexReader *ir) {
|
49
|
-
FrtScorer *self
|
54
|
+
FrtScorer *self = frt_scorer_new(MatchAllScorer, weight->similarity);
|
50
55
|
MASc(self)->ir = ir;
|
56
|
+
FRT_REF(ir);
|
51
57
|
MASc(self)->max_doc = ir->max_doc(ir);
|
52
58
|
MASc(self)->score = weight->value;
|
53
59
|
self->doc = -1;
|
@@ -55,7 +61,7 @@ static FrtScorer *masc_new(FrtWeight *weight, FrtIndexReader *ir) {
|
|
55
61
|
self->next = &masc_next;
|
56
62
|
self->skip_to = &masc_skip_to;
|
57
63
|
self->explain = &masc_explain;
|
58
|
-
self->destroy = &
|
64
|
+
self->destroy = &masc_destroy_i;
|
59
65
|
|
60
66
|
return self;
|
61
67
|
}
|
@@ -142,4 +148,3 @@ FrtQuery *frt_maq_new(void) {
|
|
142
148
|
FrtQuery *self = frt_maq_alloc();
|
143
149
|
return frt_maq_init(self);
|
144
150
|
}
|
145
|
-
|