isomorfeus-ferret 0.14.4 → 0.16.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/LICENSE +56 -6
- data/ext/isomorfeus_ferret_ext/extconf.rb +19 -1
- data/ext/isomorfeus_ferret_ext/frb_analysis.c +0 -4
- data/ext/isomorfeus_ferret_ext/frb_index.c +170 -52
- data/ext/isomorfeus_ferret_ext/frb_lazy_doc.c +4 -0
- data/ext/isomorfeus_ferret_ext/frb_qparser.c +1 -6
- data/ext/isomorfeus_ferret_ext/frb_search.c +1 -6
- data/ext/isomorfeus_ferret_ext/frb_store.c +246 -61
- data/ext/isomorfeus_ferret_ext/frb_utils.c +0 -4
- data/ext/isomorfeus_ferret_ext/frt_compound_io.c +5 -5
- data/ext/isomorfeus_ferret_ext/frt_except.c +11 -11
- data/ext/isomorfeus_ferret_ext/frt_hash.c +32 -70
- data/ext/isomorfeus_ferret_ext/frt_ind.c +20 -20
- data/ext/isomorfeus_ferret_ext/frt_ind.h +1 -1
- data/ext/isomorfeus_ferret_ext/frt_index.c +84 -90
- data/ext/isomorfeus_ferret_ext/frt_index.h +11 -11
- data/ext/isomorfeus_ferret_ext/frt_mdbx_store.c +749 -0
- data/ext/isomorfeus_ferret_ext/frt_q_parser.c +4 -4
- data/ext/isomorfeus_ferret_ext/frt_ram_store.c +9 -9
- data/ext/isomorfeus_ferret_ext/frt_search.c +0 -3
- data/ext/isomorfeus_ferret_ext/frt_search.h +1 -1
- data/ext/isomorfeus_ferret_ext/frt_sort.c +2 -2
- data/ext/isomorfeus_ferret_ext/frt_store.c +6 -8
- data/ext/isomorfeus_ferret_ext/frt_store.h +37 -24
- data/ext/isomorfeus_ferret_ext/frt_threading.h +0 -16
- data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.c +2 -11
- data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.h +1 -1
- data/ext/isomorfeus_ferret_ext/mdbx.c +33675 -0
- data/ext/isomorfeus_ferret_ext/mdbx.h +5495 -0
- data/ext/isomorfeus_ferret_ext/test.c +9 -9
- data/ext/isomorfeus_ferret_ext/test_index.c +3 -3
- data/ext/isomorfeus_ferret_ext/{test_fs_store.c → test_mdbx_store.c} +4 -10
- data/ext/isomorfeus_ferret_ext/test_ram_store.c +1 -1
- data/ext/isomorfeus_ferret_ext/test_segments.c +1 -1
- data/ext/isomorfeus_ferret_ext/test_sort.c +2 -2
- data/ext/isomorfeus_ferret_ext/test_store.c +4 -8
- data/ext/isomorfeus_ferret_ext/test_threading.c +4 -4
- data/ext/isomorfeus_ferret_ext/tests_all.h +2 -3
- data/lib/isomorfeus/ferret/index/index.rb +8 -9
- data/lib/isomorfeus/ferret/version.rb +1 -1
- metadata +9 -7
- data/ext/isomorfeus_ferret_ext/frt_fs_store.c +0 -505
@@ -2115,7 +2115,7 @@ static int yyerror(FrtQParser *qp, rb_encoding *encoding, char const *msg)
|
|
2115
2115
|
if (qp->clean_str) {
|
2116
2116
|
free(qp->qstr);
|
2117
2117
|
}
|
2118
|
-
|
2118
|
+
pthread_mutex_unlock(&qp->mutex);
|
2119
2119
|
snprintf(frt_xmsg_buffer, FRT_XMSG_BUFFER_SIZE,
|
2120
2120
|
"couldn't parse query ``%s''. Error message "
|
2121
2121
|
" was %s", buf, (char *)msg);
|
@@ -2806,7 +2806,7 @@ FrtQParser *frt_qp_init(FrtQParser *self, FrtAnalyzer *analyzer) {
|
|
2806
2806
|
self->buf_index = 0;
|
2807
2807
|
self->dynbuf = NULL;
|
2808
2808
|
self->non_tokenizer = frt_non_tokenizer_new();
|
2809
|
-
|
2809
|
+
pthread_mutex_init(&self->mutex, NULL);
|
2810
2810
|
return self;
|
2811
2811
|
}
|
2812
2812
|
|
@@ -2974,7 +2974,7 @@ FrtQuery *qp_parse(FrtQParser *self, char *query_string, rb_encoding *encoding)
|
|
2974
2974
|
char *qstr;
|
2975
2975
|
unsigned char *dp_start = NULL;
|
2976
2976
|
|
2977
|
-
|
2977
|
+
pthread_mutex_lock(&self->mutex);
|
2978
2978
|
/* if qp->fields_top->next is not NULL we have a left over field-stack
|
2979
2979
|
* object that was not popped during the last query parse */
|
2980
2980
|
assert(NULL == self->fields_top->next);
|
@@ -3025,6 +3025,6 @@ FrtQuery *qp_parse(FrtQParser *self, char *query_string, rb_encoding *encoding)
|
|
3025
3025
|
if (dp_start)
|
3026
3026
|
free(dp_start);
|
3027
3027
|
|
3028
|
-
|
3028
|
+
pthread_mutex_unlock(&self->mutex);
|
3029
3029
|
return result;
|
3030
3030
|
}
|
@@ -1,7 +1,7 @@
|
|
1
1
|
#include "frt_store.h"
|
2
2
|
#include <string.h>
|
3
3
|
|
4
|
-
|
4
|
+
FrtRAMFile *rf_new(const char *name) {
|
5
5
|
FrtRAMFile *rf = FRT_ALLOC(FrtRAMFile);
|
6
6
|
rf->buffers = FRT_ALLOC(frt_uchar *);
|
7
7
|
rf->buffers[0] = FRT_ALLOC_N(frt_uchar, FRT_BUFFER_SIZE);
|
@@ -12,14 +12,14 @@ static FrtRAMFile *rf_new(const char *name) {
|
|
12
12
|
return rf;
|
13
13
|
}
|
14
14
|
|
15
|
-
|
15
|
+
void rf_extend_if_necessary(FrtRAMFile *rf, int buf_num) {
|
16
16
|
while (rf->bufcnt <= buf_num) {
|
17
17
|
FRT_REALLOC_N(rf->buffers, frt_uchar *, (rf->bufcnt + 1));
|
18
18
|
rf->buffers[rf->bufcnt++] = FRT_ALLOC_N(frt_uchar, FRT_BUFFER_SIZE);
|
19
19
|
}
|
20
20
|
}
|
21
21
|
|
22
|
-
|
22
|
+
void rf_close(void *p) {
|
23
23
|
FrtRAMFile *rf = (FrtRAMFile *)p;
|
24
24
|
if (FRT_DEREF(rf) == 0) {
|
25
25
|
int i;
|
@@ -145,7 +145,7 @@ static frt_off_t ram_length(FrtStore *store, const char *filename) {
|
|
145
145
|
}
|
146
146
|
}
|
147
147
|
|
148
|
-
|
148
|
+
void ramo_flush_i(FrtOutStream *os, const frt_uchar *src, int len) {
|
149
149
|
if (len == 0) { return; }
|
150
150
|
frt_uchar *buffer;
|
151
151
|
FrtRAMFile *rf = os->file.rf;
|
@@ -179,7 +179,7 @@ static void ramo_flush_i(FrtOutStream *os, const frt_uchar *src, int len) {
|
|
179
179
|
}
|
180
180
|
}
|
181
181
|
|
182
|
-
|
182
|
+
void ramo_seek_i(FrtOutStream *os, frt_off_t pos) {
|
183
183
|
os->pointer = pos;
|
184
184
|
}
|
185
185
|
|
@@ -242,7 +242,7 @@ static FrtOutStream *ram_new_output(FrtStore *store, const char *filename) {
|
|
242
242
|
return os;
|
243
243
|
}
|
244
244
|
|
245
|
-
|
245
|
+
void rami_read_i(FrtInStream *is, frt_uchar *b, int len) {
|
246
246
|
FrtRAMFile *rf = is->f->file.rf;
|
247
247
|
|
248
248
|
int offset = 0;
|
@@ -271,15 +271,15 @@ static void rami_read_i(FrtInStream *is, frt_uchar *b, int len) {
|
|
271
271
|
is->d.pointer += len;
|
272
272
|
}
|
273
273
|
|
274
|
-
|
274
|
+
frt_off_t rami_length_i(FrtInStream *is) {
|
275
275
|
return is->f->file.rf->len;
|
276
276
|
}
|
277
277
|
|
278
|
-
|
278
|
+
void rami_seek_i(FrtInStream *is, frt_off_t pos) {
|
279
279
|
is->d.pointer = pos;
|
280
280
|
}
|
281
281
|
|
282
|
-
|
282
|
+
void rami_close_i(FrtInStream *is) {
|
283
283
|
FrtRAMFile *rf = is->f->file.rf;
|
284
284
|
rf_close(rf);
|
285
285
|
}
|
@@ -480,8 +480,6 @@ static int match_range_cmp(const void *p1, const void *p2)
|
|
480
480
|
}
|
481
481
|
}
|
482
482
|
|
483
|
-
|
484
|
-
|
485
483
|
/* ** FrtMatchVector ** */
|
486
484
|
FrtMatchVector *frt_matchv_new(void) {
|
487
485
|
FrtMatchVector *matchv = FRT_ALLOC(FrtMatchVector);
|
@@ -559,7 +557,6 @@ FrtMatchVector *frt_matchv_compact_with_breaks(FrtMatchVector *self)
|
|
559
557
|
return self;
|
560
558
|
}
|
561
559
|
|
562
|
-
|
563
560
|
static FrtMatchVector *matchv_set_offsets(FrtMatchVector *mv, FrtOffset *offsets)
|
564
561
|
{
|
565
562
|
int i;
|
@@ -372,9 +372,9 @@ static Comparator *sorter_get_comparator(FrtSortField *sf, FrtIndexReader *ir) {
|
|
372
372
|
te->close(te);
|
373
373
|
}
|
374
374
|
}
|
375
|
-
|
375
|
+
pthread_mutex_lock(&ir->field_index_mutex);
|
376
376
|
field_index = frt_field_index_get(ir, sf->field, sf->field_index_class);
|
377
|
-
|
377
|
+
pthread_mutex_unlock(&ir->field_index_mutex);
|
378
378
|
index = field_index->index;
|
379
379
|
}
|
380
380
|
return comparator_new(index, sf->reverse, sf->compare);
|
@@ -54,8 +54,8 @@ FrtStore *frt_store_alloc(void) {
|
|
54
54
|
|
55
55
|
FrtStore *frt_store_init(FrtStore *store) {
|
56
56
|
store->ref_cnt = 1;
|
57
|
-
|
58
|
-
|
57
|
+
pthread_mutex_init(&store->mutex_i, NULL);
|
58
|
+
pthread_mutex_init(&store->mutex, NULL);
|
59
59
|
store->locks = frt_hs_new_ptr((frt_free_ft)&frt_close_lock_i);
|
60
60
|
store->rstore = Qnil;
|
61
61
|
return store;
|
@@ -73,17 +73,16 @@ FrtStore *frt_store_new(void) {
|
|
73
73
|
*/
|
74
74
|
void frt_store_close(FrtStore *store) {
|
75
75
|
if (store->ref_cnt == 0) {
|
76
|
-
fprintf(stderr, "store ref_cnt to low\n");
|
77
76
|
FRT_RAISE(FRT_STATE_ERROR, "store ref_cnt to low\n");
|
78
77
|
}
|
79
78
|
|
80
79
|
if (FRT_DEREF(store) == 0) {
|
81
|
-
|
80
|
+
pthread_mutex_lock(&store->mutex_i);
|
82
81
|
store->close_i(store);
|
83
82
|
frt_hs_destroy(store->locks);
|
84
|
-
|
85
|
-
|
86
|
-
|
83
|
+
pthread_mutex_destroy(&store->mutex);
|
84
|
+
pthread_mutex_unlock(&store->mutex_i);
|
85
|
+
pthread_mutex_destroy(&store->mutex_i);
|
87
86
|
free(store);
|
88
87
|
}
|
89
88
|
}
|
@@ -286,7 +285,6 @@ void frt_is_seek(FrtInStream *is, frt_off_t pos) {
|
|
286
285
|
|
287
286
|
void frt_is_close(FrtInStream *is) {
|
288
287
|
if (is->ref_cnt == 0) {
|
289
|
-
fprintf(stderr, "is ref_cnt to low\n");
|
290
288
|
FRT_RAISE(FRT_STATE_ERROR, "is ref_cnt to low\n");
|
291
289
|
}
|
292
290
|
|
@@ -6,18 +6,22 @@
|
|
6
6
|
#include "frt_hash.h"
|
7
7
|
#include "frt_hashset.h"
|
8
8
|
#include "frt_threading.h"
|
9
|
+
#if defined POSH_OS_WIN32 || defined POSH_OS_WIN64
|
10
|
+
#define HAVE_STRUCT_IOVEC
|
11
|
+
#endif
|
12
|
+
#include "mdbx.h"
|
9
13
|
|
10
14
|
#define FRT_LOCK_PREFIX "ferret-"
|
11
15
|
#define FRT_LOCK_EXT ".lck"
|
12
16
|
|
13
|
-
typedef struct FrtBuffer
|
14
|
-
{
|
17
|
+
typedef struct FrtBuffer {
|
15
18
|
frt_uchar buf[FRT_BUFFER_SIZE];
|
16
19
|
frt_off_t start;
|
17
20
|
frt_off_t pos;
|
18
21
|
frt_off_t len;
|
19
22
|
} FrtBuffer;
|
20
23
|
|
24
|
+
typedef struct FrtStore FrtStore;
|
21
25
|
typedef struct FrtOutStream FrtOutStream;
|
22
26
|
struct FrtOutStreamMethods {
|
23
27
|
/* internal functions for the FrtInStream */
|
@@ -49,24 +53,22 @@ struct FrtOutStreamMethods {
|
|
49
53
|
void (*close_i)(struct FrtOutStream *os);
|
50
54
|
};
|
51
55
|
|
52
|
-
typedef struct FrtRAMFile
|
53
|
-
{
|
56
|
+
typedef struct FrtRAMFile {
|
54
57
|
char *name;
|
55
58
|
frt_uchar **buffers;
|
56
59
|
int bufcnt;
|
57
|
-
frt_off_t
|
60
|
+
frt_off_t len;
|
58
61
|
_Atomic unsigned int ref_cnt;
|
59
62
|
} FrtRAMFile;
|
60
63
|
|
61
|
-
struct FrtOutStream
|
62
|
-
{
|
64
|
+
struct FrtOutStream {
|
63
65
|
FrtBuffer buf;
|
64
|
-
|
65
|
-
|
66
|
+
frt_off_t pointer; /* only used by RAMOut */
|
67
|
+
FrtStore *store;
|
68
|
+
union {
|
66
69
|
int fd;
|
67
70
|
FrtRAMFile *rf;
|
68
71
|
} file;
|
69
|
-
frt_off_t pointer; /* only used by RAMOut */
|
70
72
|
const struct FrtOutStreamMethods *m;
|
71
73
|
};
|
72
74
|
|
@@ -74,8 +76,7 @@ typedef struct FrtCompoundInStream FrtCompoundInStream;
|
|
74
76
|
|
75
77
|
typedef struct FrtInStream FrtInStream;
|
76
78
|
|
77
|
-
struct FrtInStreamMethods
|
78
|
-
{
|
79
|
+
struct FrtInStreamMethods {
|
79
80
|
/**
|
80
81
|
* Read +len+ characters from the input stream into the +offset+ position in
|
81
82
|
* +buf+, an array of unsigned characters.
|
@@ -134,8 +135,7 @@ struct FrtInStream {
|
|
134
135
|
const struct FrtInStreamMethods *m;
|
135
136
|
};
|
136
137
|
|
137
|
-
struct FrtCompoundInStream
|
138
|
-
{
|
138
|
+
struct FrtCompoundInStream {
|
139
139
|
FrtInStream *sub;
|
140
140
|
frt_off_t offset;
|
141
141
|
frt_off_t length;
|
@@ -143,10 +143,8 @@ struct FrtCompoundInStream
|
|
143
143
|
|
144
144
|
#define frt_is_length(mis) mis->m->length_i(mis)
|
145
145
|
|
146
|
-
typedef struct FrtStore FrtStore;
|
147
146
|
typedef struct FrtLock FrtLock;
|
148
|
-
struct FrtLock
|
149
|
-
{
|
147
|
+
struct FrtLock {
|
150
148
|
char *name;
|
151
149
|
FrtStore *store;
|
152
150
|
int (*obtain)(FrtLock *lock);
|
@@ -155,22 +153,27 @@ struct FrtLock
|
|
155
153
|
VALUE rlock;
|
156
154
|
};
|
157
155
|
|
158
|
-
typedef struct FrtCompoundStore
|
159
|
-
{
|
156
|
+
typedef struct FrtCompoundStore {
|
160
157
|
FrtStore *store;
|
161
158
|
const char *name;
|
162
159
|
FrtHash *entries;
|
163
160
|
FrtInStream *stream;
|
164
161
|
} FrtCompoundStore;
|
165
162
|
|
163
|
+
typedef struct MDBXInfo {
|
164
|
+
MDBX_env *env;
|
165
|
+
char *path;
|
166
|
+
} MDBXInfo;
|
167
|
+
|
166
168
|
struct FrtStore {
|
167
169
|
_Atomic unsigned int ref_cnt;
|
168
|
-
|
169
|
-
|
170
|
+
pthread_mutex_t mutex_i; /* for internal use only */
|
171
|
+
pthread_mutex_t mutex; /* external mutex for use outside */
|
170
172
|
union {
|
171
173
|
char *path; /* for fs_store only */
|
172
174
|
FrtHash *ht; /* for ram_store only */
|
173
175
|
FrtCompoundStore *cmpd; /* for compound_store only */
|
176
|
+
MDBXInfo *mdbx; /* for mdbx store only */
|
174
177
|
} dir;
|
175
178
|
#if defined POSH_OS_WIN32 || defined POSH_OS_WIN64
|
176
179
|
int file_mode;
|
@@ -328,13 +331,13 @@ struct FrtStore {
|
|
328
331
|
};
|
329
332
|
|
330
333
|
/**
|
331
|
-
* Create a newly allocated
|
334
|
+
* Create a newly allocated MDBX FrtStore at the pathname designated. The
|
332
335
|
* pathname must be the name of an existing directory.
|
333
336
|
*
|
334
337
|
* @param pathname the pathname of the directory to be used by the index
|
335
|
-
* @return a newly allocated
|
338
|
+
* @return a newly allocated MDBX FrtStore.
|
336
339
|
*/
|
337
|
-
extern FrtStore *
|
340
|
+
extern FrtStore *frt_open_mdbx_store(const char *pathname);
|
338
341
|
|
339
342
|
/**
|
340
343
|
* Create a newly allocated in-memory or RAM FrtStore.
|
@@ -792,4 +795,14 @@ extern FrtInStream *frt_is_new();
|
|
792
795
|
extern int frt_file_is_lock(const char *filename);
|
793
796
|
extern bool frt_file_name_filter_is_index_file(const char *file_name, bool include_locks);
|
794
797
|
|
798
|
+
extern void ramo_flush_i(FrtOutStream *os, const frt_uchar *src, int len);
|
799
|
+
extern void ramo_seek_i(FrtOutStream *os, frt_off_t pos);
|
800
|
+
extern void rami_read_i(FrtInStream *is, frt_uchar *b, int len);
|
801
|
+
extern void rami_seek_i(FrtInStream *is, frt_off_t pos);
|
802
|
+
extern void rami_close_i(FrtInStream *is);
|
803
|
+
extern frt_off_t rami_length_i(FrtInStream *is);
|
804
|
+
extern void rf_close(void *p);
|
805
|
+
extern FrtRAMFile *rf_new(const char *name);
|
806
|
+
extern void rf_extend_if_necessary(FrtRAMFile *rf, int buf_num);
|
807
|
+
|
795
808
|
#endif
|
@@ -3,27 +3,11 @@
|
|
3
3
|
|
4
4
|
#include <pthread.h>
|
5
5
|
|
6
|
-
typedef pthread_mutex_t frt_mutex_t;
|
7
6
|
typedef struct FrtHash *frt_thread_key_t;
|
8
|
-
typedef pthread_once_t frt_thread_once_t;
|
9
7
|
|
10
|
-
// extern void frb_thread_once(int *once_control, void (*init_routine) (void));
|
11
8
|
extern int frb_thread_key_create(frt_thread_key_t *key, void (*destr_function)(void *));
|
12
9
|
extern void frb_thread_key_delete(frt_thread_key_t key);
|
13
10
|
extern void frb_thread_setspecific(frt_thread_key_t key, const void *pointer);
|
14
11
|
extern void *frb_thread_getspecific(frt_thread_key_t key);
|
15
12
|
|
16
|
-
#define FRT_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
|
17
|
-
#define FRT_THREAD_ONCE_INIT PTHREAD_ONCE_INIT
|
18
|
-
#define frt_mutex_init(a, b) pthread_mutex_init(a, b)
|
19
|
-
#define frt_mutex_lock(a) pthread_mutex_lock(a)
|
20
|
-
#define frt_mutex_trylock(a) pthread_mutex_trylock(a)
|
21
|
-
#define frt_mutex_unlock(a) pthread_mutex_unlock(a)
|
22
|
-
#define frt_mutex_destroy(a) pthread_mutex_destroy(a)
|
23
|
-
#define frt_thread_key_create(a, b) frb_thread_key_create(a, b)
|
24
|
-
#define frt_thread_key_delete(a) frb_thread_key_delete(a)
|
25
|
-
#define frt_thread_setspecific(a, b) frb_thread_setspecific(a, b)
|
26
|
-
#define frt_thread_getspecific(a) frb_thread_getspecific(a)
|
27
|
-
#define frt_thread_once(a, b) pthread_once(a, b)
|
28
|
-
|
29
13
|
#endif
|
@@ -1,11 +1,9 @@
|
|
1
1
|
#include <errno.h>
|
2
2
|
#include "isomorfeus_ferret.h"
|
3
|
-
#include "frt_global.h"
|
4
3
|
#include "frt_except.h"
|
5
4
|
#include "frt_hash.h"
|
6
5
|
#include "frt_hashset.h"
|
7
6
|
#include "frt_threading.h"
|
8
|
-
#include "ruby.h"
|
9
7
|
|
10
8
|
/* IDs */
|
11
9
|
ID id_new;
|
@@ -15,7 +13,7 @@ ID id_hash;
|
|
15
13
|
ID id_capacity;
|
16
14
|
ID id_less_than;
|
17
15
|
ID id_lt;
|
18
|
-
ID
|
16
|
+
ID id_exist;
|
19
17
|
ID id_close;
|
20
18
|
ID id_cclass;
|
21
19
|
ID id_data;
|
@@ -55,13 +53,6 @@ int value_eq(const void *key1, const void *key2) {
|
|
55
53
|
return key1 == key2;
|
56
54
|
}
|
57
55
|
|
58
|
-
void frb_thread_once(int *once_control, void (*init_routine) (void)) {
|
59
|
-
if (*once_control) {
|
60
|
-
init_routine();
|
61
|
-
*once_control = 0;
|
62
|
-
}
|
63
|
-
}
|
64
|
-
|
65
56
|
int frb_thread_key_create(frt_thread_key_t *key, void (*destr_function)(void *)) {
|
66
57
|
*key = frt_h_new(&value_hash, &value_eq, NULL, destr_function);
|
67
58
|
return 0;
|
@@ -265,7 +256,7 @@ void Init_isomorfeus_ferret_ext(void) {
|
|
265
256
|
id_lt = rb_intern("<");
|
266
257
|
|
267
258
|
id_mkdir_p = rb_intern("mkdir_p");
|
268
|
-
|
259
|
+
id_exist = rb_intern("exist?");
|
269
260
|
id_close = rb_intern("close");
|
270
261
|
|
271
262
|
id_cclass = rb_intern("cclass");
|