isomorfeus-ferret 0.17.2 → 0.17.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ext/isomorfeus_ferret_ext/benchmark.c +9 -20
- data/ext/isomorfeus_ferret_ext/benchmarks_all.h +1 -2
- data/ext/isomorfeus_ferret_ext/bm_hash.c +1 -2
- data/ext/isomorfeus_ferret_ext/brotli_dec_decode.c +4 -2
- data/ext/isomorfeus_ferret_ext/brotli_enc_encode.c +3 -2
- data/ext/isomorfeus_ferret_ext/frb_analysis.c +4 -5
- data/ext/isomorfeus_ferret_ext/frb_field_info.c +3 -4
- data/ext/isomorfeus_ferret_ext/frb_index.c +118 -125
- data/ext/isomorfeus_ferret_ext/frb_lazy_doc.c +14 -16
- data/ext/isomorfeus_ferret_ext/frb_search.c +31 -23
- data/ext/isomorfeus_ferret_ext/frb_store.c +27 -13
- data/ext/isomorfeus_ferret_ext/frb_utils.c +3 -6
- data/ext/isomorfeus_ferret_ext/frt_analysis.c +39 -46
- data/ext/isomorfeus_ferret_ext/frt_analysis.h +9 -9
- data/ext/isomorfeus_ferret_ext/frt_array.c +11 -22
- data/ext/isomorfeus_ferret_ext/frt_bitvector.h +3 -6
- data/ext/isomorfeus_ferret_ext/frt_doc_field.c +87 -0
- data/ext/isomorfeus_ferret_ext/frt_doc_field.h +26 -0
- data/ext/isomorfeus_ferret_ext/frt_document.c +4 -97
- data/ext/isomorfeus_ferret_ext/frt_document.h +2 -27
- data/ext/isomorfeus_ferret_ext/frt_except.c +8 -6
- data/ext/isomorfeus_ferret_ext/frt_except.h +1 -2
- data/ext/isomorfeus_ferret_ext/frt_field_index.c +13 -32
- data/ext/isomorfeus_ferret_ext/frt_field_index.h +0 -6
- data/ext/isomorfeus_ferret_ext/frt_field_info.c +69 -0
- data/ext/isomorfeus_ferret_ext/frt_field_info.h +49 -0
- data/ext/isomorfeus_ferret_ext/frt_field_infos.c +196 -0
- data/ext/isomorfeus_ferret_ext/frt_field_infos.h +35 -0
- data/ext/isomorfeus_ferret_ext/frt_global.c +10 -4
- data/ext/isomorfeus_ferret_ext/frt_global.h +11 -15
- data/ext/isomorfeus_ferret_ext/frt_hash.c +8 -8
- data/ext/isomorfeus_ferret_ext/frt_hash.h +1 -2
- data/ext/isomorfeus_ferret_ext/frt_hashset.c +20 -40
- data/ext/isomorfeus_ferret_ext/frt_hashset.h +1 -2
- data/ext/isomorfeus_ferret_ext/frt_helper.c +7 -15
- data/ext/isomorfeus_ferret_ext/frt_in_stream.c +35 -45
- data/ext/isomorfeus_ferret_ext/frt_in_stream.h +3 -2
- data/ext/isomorfeus_ferret_ext/frt_ind.c +20 -38
- data/ext/isomorfeus_ferret_ext/frt_index.c +292 -790
- data/ext/isomorfeus_ferret_ext/frt_index.h +1 -102
- data/ext/isomorfeus_ferret_ext/frt_lang.c +5 -10
- data/ext/isomorfeus_ferret_ext/frt_lazy_doc.c +3 -3
- data/ext/isomorfeus_ferret_ext/frt_lazy_doc.h +1 -1
- data/ext/isomorfeus_ferret_ext/frt_lazy_doc_field.c +18 -25
- data/ext/isomorfeus_ferret_ext/frt_lazy_doc_field.h +5 -5
- data/ext/isomorfeus_ferret_ext/frt_mdbx_store.c +102 -70
- data/ext/isomorfeus_ferret_ext/frt_mempool.c +8 -16
- data/ext/isomorfeus_ferret_ext/frt_multimapper.c +23 -46
- data/ext/isomorfeus_ferret_ext/frt_multimapper.h +4 -8
- data/ext/isomorfeus_ferret_ext/frt_out_stream.c +31 -43
- data/ext/isomorfeus_ferret_ext/frt_out_stream.h +2 -2
- data/ext/isomorfeus_ferret_ext/frt_posh.c +6 -819
- data/ext/isomorfeus_ferret_ext/frt_posh.h +0 -57
- data/ext/isomorfeus_ferret_ext/frt_priorityqueue.c +11 -22
- data/ext/isomorfeus_ferret_ext/frt_priorityqueue.h +1 -2
- data/ext/isomorfeus_ferret_ext/frt_q_boolean.c +85 -171
- data/ext/isomorfeus_ferret_ext/frt_q_match_all.c +8 -16
- data/ext/isomorfeus_ferret_ext/frt_q_multi_term.c +1 -2
- data/ext/isomorfeus_ferret_ext/frt_q_parser.c +49 -98
- data/ext/isomorfeus_ferret_ext/frt_q_phrase.c +52 -104
- data/ext/isomorfeus_ferret_ext/frt_q_range.c +6 -12
- data/ext/isomorfeus_ferret_ext/frt_q_span.c +113 -226
- data/ext/isomorfeus_ferret_ext/frt_q_wildcard.c +1 -2
- data/ext/isomorfeus_ferret_ext/frt_ram_store.c +134 -85
- data/ext/isomorfeus_ferret_ext/frt_search.c +82 -164
- data/ext/isomorfeus_ferret_ext/frt_similarity.c +11 -22
- data/ext/isomorfeus_ferret_ext/frt_similarity.h +1 -2
- data/ext/isomorfeus_ferret_ext/frt_store.c +13 -25
- data/ext/isomorfeus_ferret_ext/frt_store.h +86 -52
- data/ext/isomorfeus_ferret_ext/frt_term_vectors.c +8 -16
- data/ext/isomorfeus_ferret_ext/frt_win32.h +5 -10
- data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.c +12 -11
- data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.h +11 -13
- data/ext/isomorfeus_ferret_ext/lz4.c +422 -195
- data/ext/isomorfeus_ferret_ext/lz4.h +114 -46
- data/ext/isomorfeus_ferret_ext/lz4frame.c +421 -242
- data/ext/isomorfeus_ferret_ext/lz4frame.h +122 -53
- data/ext/isomorfeus_ferret_ext/lz4hc.c +127 -111
- data/ext/isomorfeus_ferret_ext/lz4hc.h +14 -14
- data/ext/isomorfeus_ferret_ext/lz4xxhash.h +1 -1
- data/ext/isomorfeus_ferret_ext/mdbx.c +3762 -2526
- data/ext/isomorfeus_ferret_ext/mdbx.h +115 -70
- data/ext/isomorfeus_ferret_ext/test.c +40 -87
- data/ext/isomorfeus_ferret_ext/test.h +3 -6
- data/ext/isomorfeus_ferret_ext/test_1710.c +11 -13
- data/ext/isomorfeus_ferret_ext/test_analysis.c +32 -64
- data/ext/isomorfeus_ferret_ext/test_array.c +6 -12
- data/ext/isomorfeus_ferret_ext/test_bitvector.c +12 -24
- data/ext/isomorfeus_ferret_ext/test_document.c +23 -33
- data/ext/isomorfeus_ferret_ext/test_except.c +10 -21
- data/ext/isomorfeus_ferret_ext/test_fields.c +62 -68
- data/ext/isomorfeus_ferret_ext/test_file_deleter.c +15 -23
- data/ext/isomorfeus_ferret_ext/test_filter.c +17 -27
- data/ext/isomorfeus_ferret_ext/test_global.c +14 -29
- data/ext/isomorfeus_ferret_ext/test_hash.c +19 -38
- data/ext/isomorfeus_ferret_ext/test_hashset.c +8 -16
- data/ext/isomorfeus_ferret_ext/test_helper.c +4 -8
- data/ext/isomorfeus_ferret_ext/test_highlighter.c +16 -28
- data/ext/isomorfeus_ferret_ext/test_index.c +277 -487
- data/ext/isomorfeus_ferret_ext/test_lang.c +7 -14
- data/ext/isomorfeus_ferret_ext/test_mdbx_store.c +2 -5
- data/ext/isomorfeus_ferret_ext/test_mempool.c +5 -10
- data/ext/isomorfeus_ferret_ext/test_multimapper.c +3 -6
- data/ext/isomorfeus_ferret_ext/test_priorityqueue.c +9 -18
- data/ext/isomorfeus_ferret_ext/test_q_const_score.c +4 -6
- data/ext/isomorfeus_ferret_ext/test_q_filtered.c +3 -4
- data/ext/isomorfeus_ferret_ext/test_q_fuzzy.c +9 -15
- data/ext/isomorfeus_ferret_ext/test_q_parser.c +8 -16
- data/ext/isomorfeus_ferret_ext/test_q_span.c +19 -35
- data/ext/isomorfeus_ferret_ext/test_ram_store.c +14 -13
- data/ext/isomorfeus_ferret_ext/test_search.c +60 -109
- data/ext/isomorfeus_ferret_ext/test_segments.c +8 -13
- data/ext/isomorfeus_ferret_ext/test_similarity.c +2 -4
- data/ext/isomorfeus_ferret_ext/test_sort.c +14 -24
- data/ext/isomorfeus_ferret_ext/test_store.c +96 -115
- data/ext/isomorfeus_ferret_ext/test_term.c +9 -15
- data/ext/isomorfeus_ferret_ext/test_term_vectors.c +9 -14
- data/ext/isomorfeus_ferret_ext/test_test.c +4 -8
- data/ext/isomorfeus_ferret_ext/test_threading.c +14 -20
- data/ext/isomorfeus_ferret_ext/testhelper.c +11 -21
- data/ext/isomorfeus_ferret_ext/testhelper.h +1 -1
- data/ext/isomorfeus_ferret_ext/tests_all.h +1 -2
- data/lib/isomorfeus/ferret/index/index.rb +1 -1
- data/lib/isomorfeus/ferret/version.rb +1 -1
- metadata +24 -4
@@ -7,8 +7,7 @@
|
|
7
7
|
* over all alements in the HashSet. It also uses a Hash to keep track of
|
8
8
|
* which elements have been added and their index in the +elems+ array.
|
9
9
|
*/
|
10
|
-
static FrtHashSet *hs_alloc(frt_free_ft free_func)
|
11
|
-
{
|
10
|
+
static FrtHashSet *hs_alloc(frt_free_ft free_func) {
|
12
11
|
FrtHashSet *hs = FRT_ALLOC(FrtHashSet);
|
13
12
|
hs->size = 0;
|
14
13
|
hs->first = hs->last = NULL;
|
@@ -16,29 +15,25 @@ static FrtHashSet *hs_alloc(frt_free_ft free_func)
|
|
16
15
|
return hs;
|
17
16
|
}
|
18
17
|
|
19
|
-
FrtHashSet *frt_hs_new(frt_hash_ft hash_func, frt_eq_ft eq_func, frt_free_ft free_func)
|
20
|
-
{
|
18
|
+
FrtHashSet *frt_hs_new(frt_hash_ft hash_func, frt_eq_ft eq_func, frt_free_ft free_func) {
|
21
19
|
FrtHashSet *hs = hs_alloc(free_func);
|
22
20
|
hs->ht = frt_h_new(hash_func, eq_func, NULL, NULL);
|
23
21
|
return hs;
|
24
22
|
}
|
25
23
|
|
26
|
-
FrtHashSet *frt_hs_new_str(frt_free_ft free_func)
|
27
|
-
{
|
24
|
+
FrtHashSet *frt_hs_new_str(frt_free_ft free_func) {
|
28
25
|
FrtHashSet *hs = hs_alloc(free_func);
|
29
26
|
hs->ht = frt_h_new_str((frt_free_ft) NULL, NULL);
|
30
27
|
return hs;
|
31
28
|
}
|
32
29
|
|
33
|
-
FrtHashSet *frt_hs_new_ptr(frt_free_ft free_func)
|
34
|
-
{
|
30
|
+
FrtHashSet *frt_hs_new_ptr(frt_free_ft free_func) {
|
35
31
|
FrtHashSet *hs = hs_alloc(free_func);
|
36
32
|
hs->ht = frt_h_new_ptr(NULL);
|
37
33
|
return hs;
|
38
34
|
}
|
39
35
|
|
40
|
-
static void hs_clear(FrtHashSet *hs, bool destroy)
|
41
|
-
{
|
36
|
+
static void hs_clear(FrtHashSet *hs, bool destroy) {
|
42
37
|
FrtHashSetEntry *curr, *next = hs->first;
|
43
38
|
frt_free_ft do_free = destroy ? hs->free_elem_i : &frt_dummy_free;
|
44
39
|
while (NULL != (curr = next)) {
|
@@ -50,36 +45,31 @@ static void hs_clear(FrtHashSet *hs, bool destroy)
|
|
50
45
|
hs->size = 0;
|
51
46
|
}
|
52
47
|
|
53
|
-
void frt_hs_clear(FrtHashSet *hs)
|
54
|
-
{
|
48
|
+
void frt_hs_clear(FrtHashSet *hs) {
|
55
49
|
hs_clear(hs, true);
|
56
50
|
frt_h_clear(hs->ht);
|
57
51
|
}
|
58
52
|
|
59
|
-
void frt_hs_free(FrtHashSet *hs)
|
60
|
-
{
|
53
|
+
void frt_hs_free(FrtHashSet *hs) {
|
61
54
|
hs_clear(hs, false);
|
62
55
|
frt_h_destroy(hs->ht);
|
63
56
|
free(hs);
|
64
57
|
}
|
65
58
|
|
66
|
-
void frt_hs_destroy(FrtHashSet *hs)
|
67
|
-
{
|
59
|
+
void frt_hs_destroy(FrtHashSet *hs) {
|
68
60
|
hs_clear(hs, true);
|
69
61
|
frt_h_destroy(hs->ht);
|
70
62
|
free(hs);
|
71
63
|
}
|
72
64
|
|
73
|
-
static void append(FrtHashSet *hs, void *elem)
|
74
|
-
{
|
65
|
+
static void append(FrtHashSet *hs, void *elem) {
|
75
66
|
FrtHashSetEntry *entry = FRT_ALLOC(FrtHashSetEntry);
|
76
67
|
entry->elem = elem;
|
77
68
|
entry->prev = hs->last;
|
78
69
|
entry->next = NULL;
|
79
70
|
if (!hs->first) {
|
80
71
|
hs->first = hs->last = entry;
|
81
|
-
}
|
82
|
-
else {
|
72
|
+
} else {
|
83
73
|
hs->last->next = entry;
|
84
74
|
hs->last = entry;
|
85
75
|
}
|
@@ -87,8 +77,7 @@ static void append(FrtHashSet *hs, void *elem)
|
|
87
77
|
hs->size++;
|
88
78
|
}
|
89
79
|
|
90
|
-
FrtHashKeyStatus frt_hs_add(FrtHashSet *hs, void *elem)
|
91
|
-
{
|
80
|
+
FrtHashKeyStatus frt_hs_add(FrtHashSet *hs, void *elem) {
|
92
81
|
FrtHashKeyStatus has_elem = frt_h_has_key(hs->ht, elem);
|
93
82
|
switch (has_elem)
|
94
83
|
{
|
@@ -111,8 +100,7 @@ FrtHashKeyStatus frt_hs_add(FrtHashSet *hs, void *elem)
|
|
111
100
|
return has_elem;
|
112
101
|
}
|
113
102
|
|
114
|
-
int frt_hs_add_safe(FrtHashSet *hs, void *elem)
|
115
|
-
{
|
103
|
+
int frt_hs_add_safe(FrtHashSet *hs, void *elem) {
|
116
104
|
switch(frt_h_has_key(hs->ht, elem))
|
117
105
|
{
|
118
106
|
/* element can't be added */
|
@@ -128,24 +116,20 @@ int frt_hs_add_safe(FrtHashSet *hs, void *elem)
|
|
128
116
|
return true;
|
129
117
|
}
|
130
118
|
|
131
|
-
void *frt_hs_rem(FrtHashSet *hs, const void *elem)
|
132
|
-
{
|
119
|
+
void *frt_hs_rem(FrtHashSet *hs, const void *elem) {
|
133
120
|
void *return_elem;
|
134
121
|
FrtHashSetEntry *entry = (FrtHashSetEntry *)frt_h_get(hs->ht, elem);
|
135
122
|
if (entry == NULL) return NULL;
|
136
123
|
|
137
124
|
if (hs->first == hs->last) {
|
138
125
|
hs->first = hs->last = NULL;
|
139
|
-
}
|
140
|
-
else if (hs->first == entry) {
|
126
|
+
} else if (hs->first == entry) {
|
141
127
|
hs->first = entry->next;
|
142
128
|
hs->first->prev = NULL;
|
143
|
-
}
|
144
|
-
else if (hs->last == entry) {
|
129
|
+
} else if (hs->last == entry) {
|
145
130
|
hs->last = entry->prev;
|
146
131
|
hs->last->next = NULL;
|
147
|
-
}
|
148
|
-
else {
|
132
|
+
} else {
|
149
133
|
entry->prev->next = entry->next;
|
150
134
|
entry->next->prev = entry->prev;
|
151
135
|
}
|
@@ -156,8 +140,7 @@ void *frt_hs_rem(FrtHashSet *hs, const void *elem)
|
|
156
140
|
return return_elem;
|
157
141
|
}
|
158
142
|
|
159
|
-
int frt_hs_del(FrtHashSet *hs, const void *elem)
|
160
|
-
{
|
143
|
+
int frt_hs_del(FrtHashSet *hs, const void *elem) {
|
161
144
|
void *tmp_elem = frt_hs_rem(hs, elem);
|
162
145
|
if (tmp_elem != NULL) {
|
163
146
|
hs->free_elem_i(tmp_elem);
|
@@ -166,13 +149,11 @@ int frt_hs_del(FrtHashSet *hs, const void *elem)
|
|
166
149
|
return 0;
|
167
150
|
}
|
168
151
|
|
169
|
-
FrtHashKeyStatus frt_hs_exists(FrtHashSet *hs, const void *elem)
|
170
|
-
{
|
152
|
+
FrtHashKeyStatus frt_hs_exists(FrtHashSet *hs, const void *elem) {
|
171
153
|
return frt_h_has_key(hs->ht, elem);
|
172
154
|
}
|
173
155
|
|
174
|
-
FrtHashSet *frt_hs_merge(FrtHashSet *hs, FrtHashSet * other)
|
175
|
-
{
|
156
|
+
FrtHashSet *frt_hs_merge(FrtHashSet *hs, FrtHashSet * other) {
|
176
157
|
FrtHashSetEntry *entry = other->first;
|
177
158
|
for (; entry != NULL; entry = entry->next) {
|
178
159
|
frt_hs_add(hs, entry->elem);
|
@@ -184,8 +165,7 @@ FrtHashSet *frt_hs_merge(FrtHashSet *hs, FrtHashSet * other)
|
|
184
165
|
return hs;
|
185
166
|
}
|
186
167
|
|
187
|
-
void *frt_hs_orig(FrtHashSet *hs, const void *elem)
|
188
|
-
{
|
168
|
+
void *frt_hs_orig(FrtHashSet *hs, const void *elem) {
|
189
169
|
FrtHashSetEntry *entry = (FrtHashSetEntry *)frt_h_get(hs->ht, elem);
|
190
170
|
return entry ? entry->elem : NULL;
|
191
171
|
}
|
@@ -1,8 +1,6 @@
|
|
1
1
|
#include "frt_helper.h"
|
2
2
|
|
3
|
-
int frt_hlp_string_diff(register const char *const s1,
|
4
|
-
register const char *const s2)
|
5
|
-
{
|
3
|
+
int frt_hlp_string_diff(register const char *const s1, register const char *const s2) {
|
6
4
|
register int i = 0;
|
7
5
|
while (s1[i] && (s1[i] == s2[i])) {
|
8
6
|
i++;
|
@@ -10,26 +8,22 @@ int frt_hlp_string_diff(register const char *const s1,
|
|
10
8
|
return i;
|
11
9
|
}
|
12
10
|
|
13
|
-
frt_i32 frt_float2int(float f)
|
14
|
-
{
|
11
|
+
frt_i32 frt_float2int(float f) {
|
15
12
|
union { frt_i32 i; float f; } tmp;
|
16
13
|
tmp.f = f;
|
17
14
|
return tmp.i;
|
18
15
|
}
|
19
16
|
|
20
|
-
float frt_int2float(frt_i32 v)
|
21
|
-
{
|
17
|
+
float frt_int2float(frt_i32 v) {
|
22
18
|
union { frt_i32 i; float f; } tmp;
|
23
19
|
tmp.i = v;
|
24
20
|
return tmp.f;
|
25
21
|
}
|
26
22
|
|
27
|
-
float frt_byte2float(unsigned char b)
|
28
|
-
{
|
23
|
+
float frt_byte2float(unsigned char b) {
|
29
24
|
if (b == 0) {
|
30
25
|
return 0.0;
|
31
|
-
}
|
32
|
-
else {
|
26
|
+
} else {
|
33
27
|
frt_u32 mantissa = b & 0x07;
|
34
28
|
frt_u32 exponent = (b >> 3) & 0x1f;
|
35
29
|
|
@@ -37,12 +31,10 @@ float frt_byte2float(unsigned char b)
|
|
37
31
|
}
|
38
32
|
}
|
39
33
|
|
40
|
-
unsigned char frt_float2byte(float f)
|
41
|
-
{
|
34
|
+
unsigned char frt_float2byte(float f) {
|
42
35
|
if (f <= 0.0) {
|
43
36
|
return 0;
|
44
|
-
}
|
45
|
-
else {
|
37
|
+
} else {
|
46
38
|
/* correctly order the bytes for encoding */
|
47
39
|
frt_u32 i = frt_float2int(f);
|
48
40
|
int mantissa = (i & 0xEf0000) >> 21;
|
@@ -1,3 +1,8 @@
|
|
1
|
+
/* prevent warning: #warning Please include winsock2.h before windows.h [-Wcpp] */
|
2
|
+
#ifdef _WIN32
|
3
|
+
# include <winsock2.h>
|
4
|
+
#endif
|
5
|
+
|
1
6
|
#include "brotli_decode.h"
|
2
7
|
#include "brotli_encode.h"
|
3
8
|
#include "bzlib.h"
|
@@ -12,6 +17,7 @@
|
|
12
17
|
*/
|
13
18
|
FrtInStream *frt_is_new(void) {
|
14
19
|
FrtInStream *is = FRT_ALLOC(FrtInStream);
|
20
|
+
is->folder_name = NULL;
|
15
21
|
is->f = FRT_ALLOC_AND_ZERO(FrtInStreamFile);
|
16
22
|
is->f->ref_cnt = 1;
|
17
23
|
is->buf.start = 0;
|
@@ -28,8 +34,7 @@ FrtInStream *frt_is_new(void) {
|
|
28
34
|
* @raise FRT_IO_ERROR if there is a error reading from the filesystem
|
29
35
|
* @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
|
30
36
|
*/
|
31
|
-
static void frt_is_refill(FrtInStream *is)
|
32
|
-
{
|
37
|
+
static void frt_is_refill(FrtInStream *is) {
|
33
38
|
frt_off_t start = is->buf.start + is->buf.pos;
|
34
39
|
frt_off_t last = start + FRT_BUFFER_SIZE;
|
35
40
|
frt_off_t flen = is->m->length_i(is);
|
@@ -65,22 +70,18 @@ static void frt_is_refill(FrtInStream *is)
|
|
65
70
|
* @raise FRT_IO_ERROR if there is a error reading from the filesystem
|
66
71
|
* @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
|
67
72
|
*/
|
68
|
-
frt_uchar frt_is_read_byte(FrtInStream *is)
|
69
|
-
{
|
73
|
+
frt_uchar frt_is_read_byte(FrtInStream *is) {
|
70
74
|
if (is->buf.pos >= is->buf.len) {
|
71
75
|
frt_is_refill(is);
|
72
76
|
}
|
73
|
-
|
74
77
|
return read_byte(is);
|
75
78
|
}
|
76
79
|
|
77
|
-
off_t frt_is_pos(FrtInStream *is)
|
78
|
-
{
|
80
|
+
off_t frt_is_pos(FrtInStream *is) {
|
79
81
|
return is->buf.start + is->buf.pos;
|
80
82
|
}
|
81
83
|
|
82
|
-
frt_uchar *frt_is_read_bytes(FrtInStream *is, frt_uchar *buf, int len)
|
83
|
-
{
|
84
|
+
frt_uchar *frt_is_read_bytes(FrtInStream *is, frt_uchar *buf, int len) {
|
84
85
|
int i;
|
85
86
|
frt_off_t start;
|
86
87
|
|
@@ -88,8 +89,7 @@ frt_uchar *frt_is_read_bytes(FrtInStream *is, frt_uchar *buf, int len)
|
|
88
89
|
for (i = 0; i < len; i++) {
|
89
90
|
buf[i] = read_byte(is);
|
90
91
|
}
|
91
|
-
}
|
92
|
-
else { /* read all-at-once */
|
92
|
+
} else { /* read all-at-once */
|
93
93
|
start = frt_is_pos(is);
|
94
94
|
is->m->seek_i(is, start);
|
95
95
|
is->m->read_i(is, buf, len);
|
@@ -122,23 +122,27 @@ void frt_is_close(FrtInStream *is) {
|
|
122
122
|
is->m->close_i(is);
|
123
123
|
free(is->f);
|
124
124
|
}
|
125
|
+
if (is->folder_name) {
|
126
|
+
free((void *)is->folder_name);
|
127
|
+
}
|
125
128
|
free(is);
|
126
129
|
}
|
127
130
|
}
|
128
131
|
|
129
|
-
FrtInStream *frt_is_clone(FrtInStream *is)
|
130
|
-
{
|
132
|
+
FrtInStream *frt_is_clone(FrtInStream *is) {
|
131
133
|
if (!(is->f))
|
132
134
|
return NULL;
|
133
135
|
FrtInStream *new_is = FRT_ALLOC(FrtInStream);
|
134
136
|
memcpy(new_is, is, sizeof(FrtInStream));
|
137
|
+
if (is->folder_name) {
|
138
|
+
new_is->folder_name = frt_estrdup(is->folder_name);
|
139
|
+
}
|
135
140
|
new_is->ref_cnt = 1;
|
136
141
|
FRT_REF(new_is->f);
|
137
142
|
return new_is;
|
138
143
|
}
|
139
144
|
|
140
|
-
frt_i32 frt_is_read_i32(FrtInStream *is)
|
141
|
-
{
|
145
|
+
frt_i32 frt_is_read_i32(FrtInStream *is) {
|
142
146
|
return ((frt_i32)frt_is_read_byte(is) << 24) |
|
143
147
|
((frt_i32)frt_is_read_byte(is) << 16) |
|
144
148
|
((frt_i32)frt_is_read_byte(is) << 8) |
|
@@ -146,8 +150,7 @@ frt_i32 frt_is_read_i32(FrtInStream *is)
|
|
146
150
|
}
|
147
151
|
|
148
152
|
|
149
|
-
frt_i64 frt_is_read_i64(FrtInStream *is)
|
150
|
-
{
|
153
|
+
frt_i64 frt_is_read_i64(FrtInStream *is) {
|
151
154
|
return ((frt_i64)frt_is_read_byte(is) << 56) |
|
152
155
|
((frt_i64)frt_is_read_byte(is) << 48) |
|
153
156
|
((frt_i64)frt_is_read_byte(is) << 40) |
|
@@ -159,16 +162,14 @@ frt_i64 frt_is_read_i64(FrtInStream *is)
|
|
159
162
|
}
|
160
163
|
|
161
164
|
|
162
|
-
frt_u32 frt_is_read_u32(FrtInStream *is)
|
163
|
-
{
|
165
|
+
frt_u32 frt_is_read_u32(FrtInStream *is) {
|
164
166
|
return ((frt_u32)frt_is_read_byte(is) << 24) |
|
165
167
|
((frt_u32)frt_is_read_byte(is) << 16) |
|
166
168
|
((frt_u32)frt_is_read_byte(is) << 8) |
|
167
169
|
((frt_u32)frt_is_read_byte(is));
|
168
170
|
}
|
169
171
|
|
170
|
-
frt_u64 frt_is_read_u64(FrtInStream *is)
|
171
|
-
{
|
172
|
+
frt_u64 frt_is_read_u64(FrtInStream *is) {
|
172
173
|
return ((frt_u64)frt_is_read_byte(is) << 56) |
|
173
174
|
((frt_u64)frt_is_read_byte(is) << 48) |
|
174
175
|
((frt_u64)frt_is_read_byte(is) << 40) |
|
@@ -181,8 +182,7 @@ frt_u64 frt_is_read_u64(FrtInStream *is)
|
|
181
182
|
|
182
183
|
|
183
184
|
/* optimized to use unchecked read_byte if there is definitely space */
|
184
|
-
unsigned int frt_is_read_vint(FrtInStream *is)
|
185
|
-
{
|
185
|
+
unsigned int frt_is_read_vint(FrtInStream *is) {
|
186
186
|
register unsigned int res, b;
|
187
187
|
register int shift = 7;
|
188
188
|
|
@@ -195,8 +195,7 @@ unsigned int frt_is_read_vint(FrtInStream *is)
|
|
195
195
|
res |= (b & 0x7F) << shift;
|
196
196
|
shift += 7;
|
197
197
|
}
|
198
|
-
}
|
199
|
-
else { /* unchecked optimization */
|
198
|
+
} else { /* unchecked optimization */
|
200
199
|
b = read_byte(is);
|
201
200
|
res = b & 0x7F; /* 0x7F = 0b01111111 */
|
202
201
|
|
@@ -212,8 +211,7 @@ unsigned int frt_is_read_vint(FrtInStream *is)
|
|
212
211
|
|
213
212
|
|
214
213
|
/* optimized to use unchecked read_byte if there is definitely space */
|
215
|
-
off_t frt_is_read_voff_t(FrtInStream *is)
|
216
|
-
{
|
214
|
+
off_t frt_is_read_voff_t(FrtInStream *is) {
|
217
215
|
register frt_off_t res, b;
|
218
216
|
register int shift = 7;
|
219
217
|
|
@@ -226,8 +224,7 @@ off_t frt_is_read_voff_t(FrtInStream *is)
|
|
226
224
|
res |= (b & 0x7F) << shift;
|
227
225
|
shift += 7;
|
228
226
|
}
|
229
|
-
}
|
230
|
-
else { /* unchecked optimization */
|
227
|
+
} else { /* unchecked optimization */
|
231
228
|
b = read_byte(is);
|
232
229
|
res = b & 0x7F; /* 0x7F = 0b01111111 */
|
233
230
|
|
@@ -242,8 +239,7 @@ off_t frt_is_read_voff_t(FrtInStream *is)
|
|
242
239
|
}
|
243
240
|
|
244
241
|
/* optimized to use unchecked read_byte if there is definitely space */
|
245
|
-
frt_u64 frt_is_read_vll(FrtInStream *is)
|
246
|
-
{
|
242
|
+
frt_u64 frt_is_read_vll(FrtInStream *is) {
|
247
243
|
register frt_u64 res, b;
|
248
244
|
register int shift = 7;
|
249
245
|
|
@@ -256,8 +252,7 @@ frt_u64 frt_is_read_vll(FrtInStream *is)
|
|
256
252
|
res |= (b & 0x7F) << shift;
|
257
253
|
shift += 7;
|
258
254
|
}
|
259
|
-
}
|
260
|
-
else { /* unchecked optimization */
|
255
|
+
} else { /* unchecked optimization */
|
261
256
|
b = read_byte(is);
|
262
257
|
res = b & 0x7F; /* 0x7F = 0b01111111 */
|
263
258
|
|
@@ -271,16 +266,14 @@ frt_u64 frt_is_read_vll(FrtInStream *is)
|
|
271
266
|
return res;
|
272
267
|
}
|
273
268
|
|
274
|
-
void frt_is_skip_vints(FrtInStream *is, register int cnt)
|
275
|
-
{
|
269
|
+
void frt_is_skip_vints(FrtInStream *is, register int cnt) {
|
276
270
|
for (; cnt > 0; cnt--) {
|
277
271
|
while ((frt_is_read_byte(is) & 0x80) != 0) {
|
278
272
|
}
|
279
273
|
}
|
280
274
|
}
|
281
275
|
|
282
|
-
char *frt_is_read_string(FrtInStream *is)
|
283
|
-
{
|
276
|
+
char *frt_is_read_string(FrtInStream *is) {
|
284
277
|
register int length = (int) frt_is_read_vint(is);
|
285
278
|
char *str = FRT_ALLOC_N(char, length + 1);
|
286
279
|
str[length] = '\0';
|
@@ -290,8 +283,7 @@ char *frt_is_read_string(FrtInStream *is)
|
|
290
283
|
for (i = 0; i < length; i++) {
|
291
284
|
str[i] = frt_is_read_byte(is);
|
292
285
|
}
|
293
|
-
}
|
294
|
-
else { /* unchecked optimization */
|
286
|
+
} else { /* unchecked optimization */
|
295
287
|
memcpy(str, is->buf.buf + is->buf.pos, length);
|
296
288
|
is->buf.pos += length;
|
297
289
|
}
|
@@ -299,8 +291,7 @@ char *frt_is_read_string(FrtInStream *is)
|
|
299
291
|
return str;
|
300
292
|
}
|
301
293
|
|
302
|
-
char *frt_is_read_string_safe(FrtInStream *is)
|
303
|
-
{
|
294
|
+
char *frt_is_read_string_safe(FrtInStream *is) {
|
304
295
|
register int length = (int) frt_is_read_vint(is);
|
305
296
|
char *str = FRT_ALLOC_N(char, length + 1);
|
306
297
|
str[length] = '\0';
|
@@ -311,8 +302,7 @@ char *frt_is_read_string_safe(FrtInStream *is)
|
|
311
302
|
for (i = 0; i < length; i++) {
|
312
303
|
str[i] = frt_is_read_byte(is);
|
313
304
|
}
|
314
|
-
}
|
315
|
-
else { /* unchecked optimization */
|
305
|
+
} else { /* unchecked optimization */
|
316
306
|
memcpy(str, is->buf.buf + is->buf.pos, length);
|
317
307
|
is->buf.pos += length;
|
318
308
|
}
|
@@ -478,8 +468,8 @@ static char *frt_is_read_lz4_compressed_bytes(FrtInStream *is, int compressed_le
|
|
478
468
|
return buf_out;
|
479
469
|
}
|
480
470
|
|
481
|
-
char *frt_is_read_compressed_bytes(FrtInStream *is, int compressed_len, int *len, FrtCompressionType
|
482
|
-
switch (
|
471
|
+
char *frt_is_read_compressed_bytes(FrtInStream *is, int compressed_len, int *len, FrtCompressionType compression_type) {
|
472
|
+
switch (compression_type) {
|
483
473
|
case FRT_COMPRESSION_BROTLI:
|
484
474
|
return frt_is_read_brotli_compressed_bytes(is, compressed_len, len);
|
485
475
|
case FRT_COMPRESSION_BZ2:
|
@@ -56,6 +56,7 @@ typedef struct FrtInStreamFile {
|
|
56
56
|
} FrtInStreamFile;
|
57
57
|
|
58
58
|
struct FrtInStream {
|
59
|
+
const char *folder_name;
|
59
60
|
FrtBuffer buf;
|
60
61
|
struct FrtInStreamFile *f;
|
61
62
|
union {
|
@@ -233,8 +234,8 @@ extern char *frt_is_read_string(FrtInStream *is);
|
|
233
234
|
*/
|
234
235
|
extern char *frt_is_read_string_safe(FrtInStream *is);
|
235
236
|
|
236
|
-
extern char *frt_is_read_compressed_bytes(FrtInStream *is, int compressed_len, int *len, FrtCompressionType
|
237
|
+
extern char *frt_is_read_compressed_bytes(FrtInStream *is, int compressed_len, int *len, FrtCompressionType compression_type);
|
237
238
|
|
238
239
|
#define frt_is_length(mis) mis->m->length_i(mis)
|
239
240
|
|
240
|
-
#endif
|
241
|
+
#endif
|
@@ -44,6 +44,7 @@ FrtIndex *frt_index_new(FrtStore *store, FrtAnalyzer *analyzer, FrtHashSet *def_
|
|
44
44
|
FRT_REF(store);
|
45
45
|
} else {
|
46
46
|
self->store = frt_open_ram_store(NULL);
|
47
|
+
self->store->create_folder(self->store, segm_idx_name);
|
47
48
|
create = true;
|
48
49
|
}
|
49
50
|
if (analyzer) {
|
@@ -79,8 +80,7 @@ FrtIndex *frt_index_new(FrtStore *store, FrtAnalyzer *analyzer, FrtHashSet *def_
|
|
79
80
|
return self;
|
80
81
|
}
|
81
82
|
|
82
|
-
void frt_index_destroy(FrtIndex *self)
|
83
|
-
{
|
83
|
+
void frt_index_destroy(FrtIndex *self) {
|
84
84
|
pthread_mutex_destroy(&self->mutex);
|
85
85
|
INDEX_CLOSE_READER(self);
|
86
86
|
if (self->iw) frt_iw_close(self->iw);
|
@@ -91,8 +91,7 @@ void frt_index_destroy(FrtIndex *self)
|
|
91
91
|
free(self);
|
92
92
|
}
|
93
93
|
|
94
|
-
void frt_ensure_writer_open(FrtIndex *self)
|
95
|
-
{
|
94
|
+
void frt_ensure_writer_open(FrtIndex *self) {
|
96
95
|
if (!self->iw) {
|
97
96
|
INDEX_CLOSE_READER(self);
|
98
97
|
|
@@ -102,8 +101,7 @@ void frt_ensure_writer_open(FrtIndex *self)
|
|
102
101
|
}
|
103
102
|
}
|
104
103
|
|
105
|
-
void frt_ensure_reader_open(FrtIndex *self)
|
106
|
-
{
|
104
|
+
void frt_ensure_reader_open(FrtIndex *self) {
|
107
105
|
if (self->ir) {
|
108
106
|
if (self->check_latest && !frt_ir_is_latest(self->ir)) {
|
109
107
|
INDEX_CLOSE_READER(self);
|
@@ -118,16 +116,14 @@ void frt_ensure_reader_open(FrtIndex *self)
|
|
118
116
|
self->ir = frt_ir_open(NULL, self->store);
|
119
117
|
}
|
120
118
|
|
121
|
-
void frt_ensure_searcher_open(FrtIndex *self)
|
122
|
-
{
|
119
|
+
void frt_ensure_searcher_open(FrtIndex *self) {
|
123
120
|
frt_ensure_reader_open(self);
|
124
121
|
if (!self->sea) {
|
125
122
|
self->sea = frt_isea_new(self->ir);
|
126
123
|
}
|
127
124
|
}
|
128
125
|
|
129
|
-
int frt_index_size(FrtIndex *self)
|
130
|
-
{
|
126
|
+
int frt_index_size(FrtIndex *self) {
|
131
127
|
int size;
|
132
128
|
pthread_mutex_lock(&self->mutex);
|
133
129
|
{
|
@@ -138,8 +134,7 @@ int frt_index_size(FrtIndex *self)
|
|
138
134
|
return size;
|
139
135
|
}
|
140
136
|
|
141
|
-
bool frt_index_is_deleted(FrtIndex *self, int doc_num)
|
142
|
-
{
|
137
|
+
bool frt_index_is_deleted(FrtIndex *self, int doc_num) {
|
143
138
|
bool is_del;
|
144
139
|
pthread_mutex_lock(&self->mutex);
|
145
140
|
{
|
@@ -151,8 +146,7 @@ bool frt_index_is_deleted(FrtIndex *self, int doc_num)
|
|
151
146
|
}
|
152
147
|
|
153
148
|
static void index_del_doc_with_key_i(FrtIndex *self, FrtDocument *doc,
|
154
|
-
FrtHashSet *key)
|
155
|
-
{
|
149
|
+
FrtHashSet *key) {
|
156
150
|
FrtQuery *q;
|
157
151
|
FrtTopDocs *td;
|
158
152
|
FrtDocField *df;
|
@@ -188,8 +182,7 @@ static void index_del_doc_with_key_i(FrtIndex *self, FrtDocument *doc,
|
|
188
182
|
frt_td_destroy(td);
|
189
183
|
}
|
190
184
|
|
191
|
-
static void index_add_doc_i(FrtIndex *self, FrtDocument *doc)
|
192
|
-
{
|
185
|
+
static void index_add_doc_i(FrtIndex *self, FrtDocument *doc) {
|
193
186
|
if (self->key) {
|
194
187
|
index_del_doc_with_key_i(self, doc, self->key);
|
195
188
|
}
|
@@ -198,8 +191,7 @@ static void index_add_doc_i(FrtIndex *self, FrtDocument *doc)
|
|
198
191
|
AUTOFLUSH_IW(self);
|
199
192
|
}
|
200
193
|
|
201
|
-
void frt_index_add_doc(FrtIndex *self, FrtDocument *doc)
|
202
|
-
{
|
194
|
+
void frt_index_add_doc(FrtIndex *self, FrtDocument *doc) {
|
203
195
|
pthread_mutex_lock(&self->mutex);
|
204
196
|
{
|
205
197
|
index_add_doc_i(self, doc);
|
@@ -207,8 +199,7 @@ void frt_index_add_doc(FrtIndex *self, FrtDocument *doc)
|
|
207
199
|
pthread_mutex_unlock(&self->mutex);
|
208
200
|
}
|
209
201
|
|
210
|
-
FrtQuery *frt_index_get_query(FrtIndex *self, char *qstr, rb_encoding *encoding)
|
211
|
-
{
|
202
|
+
FrtQuery *frt_index_get_query(FrtIndex *self, char *qstr, rb_encoding *encoding) {
|
212
203
|
int i;
|
213
204
|
FrtFieldInfos *fis;
|
214
205
|
frt_ensure_searcher_open(self);
|
@@ -221,8 +212,7 @@ FrtQuery *frt_index_get_query(FrtIndex *self, char *qstr, rb_encoding *encoding)
|
|
221
212
|
|
222
213
|
FrtTopDocs *frt_index_search_str(FrtIndex *self, char *qstr, int first_doc,
|
223
214
|
int num_docs, FrtFilter *filter, FrtSort *sort,
|
224
|
-
FrtPostFilter *post_filter, rb_encoding *encoding)
|
225
|
-
{
|
215
|
+
FrtPostFilter *post_filter, rb_encoding *encoding) {
|
226
216
|
FrtQuery *query;
|
227
217
|
FrtTopDocs *td;
|
228
218
|
query = frt_index_get_query(self, qstr, encoding); /* will ensure_searcher is open */
|
@@ -232,16 +222,14 @@ FrtTopDocs *frt_index_search_str(FrtIndex *self, char *qstr, int first_doc,
|
|
232
222
|
return td;
|
233
223
|
}
|
234
224
|
|
235
|
-
FrtDocument *frt_index_get_doc(FrtIndex *self, int doc_num)
|
236
|
-
{
|
225
|
+
FrtDocument *frt_index_get_doc(FrtIndex *self, int doc_num) {
|
237
226
|
FrtDocument *doc;
|
238
227
|
frt_ensure_reader_open(self);
|
239
228
|
doc = self->ir->get_doc(self->ir, doc_num);
|
240
229
|
return doc;
|
241
230
|
}
|
242
231
|
|
243
|
-
FrtDocument *frt_index_get_doc_ts(FrtIndex *self, int doc_num)
|
244
|
-
{
|
232
|
+
FrtDocument *frt_index_get_doc_ts(FrtIndex *self, int doc_num) {
|
245
233
|
FrtDocument *doc;
|
246
234
|
pthread_mutex_lock(&self->mutex);
|
247
235
|
{
|
@@ -267,13 +255,11 @@ FrtDocument *frt_index_get_doc_term(FrtIndex *self, ID field, const char *term)
|
|
267
255
|
return doc;
|
268
256
|
}
|
269
257
|
|
270
|
-
FrtDocument *frt_index_get_doc_id(FrtIndex *self, const char *id)
|
271
|
-
{
|
258
|
+
FrtDocument *frt_index_get_doc_id(FrtIndex *self, const char *id) {
|
272
259
|
return frt_index_get_doc_term(self, self->id_field, id);
|
273
260
|
}
|
274
261
|
|
275
|
-
void frt_index_delete(FrtIndex *self, int doc_num)
|
276
|
-
{
|
262
|
+
void frt_index_delete(FrtIndex *self, int doc_num) {
|
277
263
|
pthread_mutex_lock(&self->mutex);
|
278
264
|
{
|
279
265
|
frt_ensure_reader_open(self);
|
@@ -305,20 +291,17 @@ void frt_index_delete_term(FrtIndex *self, ID field, const char *term) {
|
|
305
291
|
pthread_mutex_unlock(&self->mutex);
|
306
292
|
}
|
307
293
|
|
308
|
-
void frt_index_delete_id(FrtIndex *self, const char *id)
|
309
|
-
{
|
294
|
+
void frt_index_delete_id(FrtIndex *self, const char *id) {
|
310
295
|
frt_index_delete_term(self, self->id_field, id);
|
311
296
|
}
|
312
297
|
|
313
|
-
static void index_qdel_i(FrtSearcher *sea, int doc_num, float score, void *arg)
|
314
|
-
{
|
298
|
+
static void index_qdel_i(FrtSearcher *sea, int doc_num, float score, void *arg) {
|
315
299
|
(void)score; (void)arg;
|
316
300
|
frt_ir_delete_doc(((FrtIndexSearcher *)sea)->ir, doc_num);
|
317
301
|
}
|
318
302
|
|
319
303
|
void frt_index_delete_query(FrtIndex *self, FrtQuery *q, FrtFilter *f,
|
320
|
-
FrtPostFilter *post_filter)
|
321
|
-
{
|
304
|
+
FrtPostFilter *post_filter) {
|
322
305
|
pthread_mutex_lock(&self->mutex);
|
323
306
|
{
|
324
307
|
frt_ensure_searcher_open(self);
|
@@ -329,8 +312,7 @@ void frt_index_delete_query(FrtIndex *self, FrtQuery *q, FrtFilter *f,
|
|
329
312
|
}
|
330
313
|
|
331
314
|
void frt_index_delete_query_str(FrtIndex *self, char *qstr, FrtFilter *f,
|
332
|
-
FrtPostFilter *post_filter, rb_encoding *encoding)
|
333
|
-
{
|
315
|
+
FrtPostFilter *post_filter, rb_encoding *encoding) {
|
334
316
|
FrtQuery *q = frt_index_get_query(self, qstr, encoding);
|
335
317
|
frt_index_delete_query(self, q, f, post_filter);
|
336
318
|
frt_q_deref(q);
|