isomorfeus-ferret 0.17.0 → 0.17.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/ext/isomorfeus_ferret_ext/bm_store.c +2 -0
- data/ext/isomorfeus_ferret_ext/frb_index.c +0 -35
- data/ext/isomorfeus_ferret_ext/frt_except.c +42 -0
- data/ext/isomorfeus_ferret_ext/frt_except.h +2 -0
- data/ext/isomorfeus_ferret_ext/frt_in_stream.c +492 -0
- data/ext/isomorfeus_ferret_ext/frt_in_stream.h +240 -0
- data/ext/isomorfeus_ferret_ext/frt_ind.c +0 -11
- data/ext/isomorfeus_ferret_ext/frt_ind.h +0 -1
- data/ext/isomorfeus_ferret_ext/frt_index.c +6 -1069
- data/ext/isomorfeus_ferret_ext/frt_index.h +1 -43
- data/ext/isomorfeus_ferret_ext/frt_lazy_doc.c +29 -0
- data/ext/isomorfeus_ferret_ext/frt_lazy_doc.h +19 -0
- data/ext/isomorfeus_ferret_ext/frt_lazy_doc_field.c +100 -0
- data/ext/isomorfeus_ferret_ext/frt_lazy_doc_field.h +33 -0
- data/ext/isomorfeus_ferret_ext/frt_out_stream.c +346 -0
- data/ext/isomorfeus_ferret_ext/frt_out_stream.h +198 -0
- data/ext/isomorfeus_ferret_ext/frt_ram_store.h +12 -0
- data/ext/isomorfeus_ferret_ext/frt_store.c +2 -513
- data/ext/isomorfeus_ferret_ext/frt_store.h +4 -443
- data/ext/isomorfeus_ferret_ext/frt_stream.h +18 -0
- data/ext/isomorfeus_ferret_ext/test_file_deleter.c +0 -1
- data/ext/isomorfeus_ferret_ext/test_index.c +0 -8
- data/ext/isomorfeus_ferret_ext/test_threading.c +1 -10
- data/lib/isomorfeus/ferret/index/index.rb +0 -11
- data/lib/isomorfeus/ferret/version.rb +1 -1
- metadata +22 -3
@@ -0,0 +1,240 @@
|
|
1
|
+
#ifndef FRT_IN_STREAM_H
|
2
|
+
#define FRT_IN_STREAM_H
|
3
|
+
|
4
|
+
#include "frt_config.h"
|
5
|
+
#include "frt_global.h"
|
6
|
+
#include "frt_stream.h"
|
7
|
+
#include "frt_ram_store.h"
|
8
|
+
|
9
|
+
typedef struct FrtInStream FrtInStream;
|
10
|
+
|
11
|
+
struct FrtInStreamMethods {
|
12
|
+
/**
|
13
|
+
* Read +len+ characters from the input stream into the +offset+ position in
|
14
|
+
* +buf+, an array of unsigned characters.
|
15
|
+
*
|
16
|
+
* @param is self
|
17
|
+
* @param buf an array of characters which must be allocated with at least
|
18
|
+
* +offset+ + +len+ bytes
|
19
|
+
* @param len the number of bytes to read
|
20
|
+
* @raise FRT_IO_ERROR if there is an error reading from the input stream
|
21
|
+
*/
|
22
|
+
void (*read_i)(struct FrtInStream *is, frt_uchar *buf, int len);
|
23
|
+
|
24
|
+
/**
|
25
|
+
* Seek position +pos+ in input stream +is+
|
26
|
+
*
|
27
|
+
* @param is self
|
28
|
+
* @param pos the position to seek
|
29
|
+
* @raise FRT_IO_ERROR if the seek fails
|
30
|
+
*/
|
31
|
+
void (*seek_i)(struct FrtInStream *is, frt_off_t pos);
|
32
|
+
|
33
|
+
/**
|
34
|
+
* Returns the length of the input stream +is+
|
35
|
+
*
|
36
|
+
* @param is self
|
37
|
+
* @raise FRT_IO_ERROR if there is an error getting the file length
|
38
|
+
*/
|
39
|
+
frt_off_t (*length_i)(struct FrtInStream *is);
|
40
|
+
|
41
|
+
/**
|
42
|
+
* Close the resources allocated to the inputstream +is+
|
43
|
+
*
|
44
|
+
* @param is self
|
45
|
+
* @raise FRT_IO_ERROR if the close fails
|
46
|
+
*/
|
47
|
+
void (*close_i)(struct FrtInStream *is);
|
48
|
+
};
|
49
|
+
|
50
|
+
typedef struct FrtInStreamFile {
|
51
|
+
_Atomic unsigned int ref_cnt;
|
52
|
+
union {
|
53
|
+
int fd;
|
54
|
+
FrtRAMFile *rf;
|
55
|
+
} file;
|
56
|
+
} FrtInStreamFile;
|
57
|
+
|
58
|
+
struct FrtInStream {
|
59
|
+
FrtBuffer buf;
|
60
|
+
struct FrtInStreamFile *f;
|
61
|
+
union {
|
62
|
+
frt_off_t pointer; /* only used by RAMIn */
|
63
|
+
char *path; /* only used by FSIn */
|
64
|
+
} d;
|
65
|
+
_Atomic unsigned int ref_cnt;
|
66
|
+
const struct FrtInStreamMethods *m;
|
67
|
+
};
|
68
|
+
|
69
|
+
/**
|
70
|
+
* Get the current position within an FrtInStream.
|
71
|
+
*
|
72
|
+
* @param is the FrtInStream to get the current position from
|
73
|
+
* @return the current position within the FrtInStream +is+
|
74
|
+
*/
|
75
|
+
extern frt_off_t frt_is_pos(FrtInStream *is);
|
76
|
+
|
77
|
+
/**
|
78
|
+
* Set the current position in FrtInStream +is+ to +pos+.
|
79
|
+
*
|
80
|
+
* @param is the FrtInStream to set the current position in
|
81
|
+
* @param pos the position in FrtInStream to seek
|
82
|
+
* @raise FRT_IO_ERROR if there is a error seeking from the file-system
|
83
|
+
* @raise FRT_EOF_ERROR if there is an attempt to seek past the end of the file
|
84
|
+
*/
|
85
|
+
extern void frt_is_seek(FrtInStream *is, frt_off_t pos);
|
86
|
+
|
87
|
+
/**
|
88
|
+
* Close the FrtInStream freeing all allocated resources.
|
89
|
+
*
|
90
|
+
* @param is the FrtInStream to close
|
91
|
+
* @raise FRT_IO_ERROR if there is an error closing the associated file
|
92
|
+
*/
|
93
|
+
extern void frt_is_close(FrtInStream *is);
|
94
|
+
|
95
|
+
/**
|
96
|
+
* Clone the FrtInStream allocating a new FrtInStream structure
|
97
|
+
*
|
98
|
+
* @param is the FrtInStream to clone
|
99
|
+
* @return a newly allocated FrtInStream which is a clone of +is+
|
100
|
+
*/
|
101
|
+
extern FrtInStream *frt_is_clone(FrtInStream *is);
|
102
|
+
|
103
|
+
/**
|
104
|
+
* Read a singly byte (unsigned char) from the FrtInStream +is+.
|
105
|
+
*
|
106
|
+
* @param is the Instream to read from
|
107
|
+
* @return a single unsigned char read from the FrtInStream +is+
|
108
|
+
* @raise FRT_IO_ERROR if there is a error reading from the file-system
|
109
|
+
* @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
|
110
|
+
*/
|
111
|
+
extern frt_uchar frt_is_read_byte(FrtInStream *is);
|
112
|
+
|
113
|
+
/**
|
114
|
+
* Read +len+ bytes from FrtInStream +is+ and write them to buffer +buf+
|
115
|
+
*
|
116
|
+
* @param is the FrtInStream to read from
|
117
|
+
* @param buf the buffer to read into, that is copy the bytes read to
|
118
|
+
* @param len the number of bytes to read
|
119
|
+
* @return the resultant buffer +buf+
|
120
|
+
* @raise FRT_IO_ERROR if there is a error reading from the file-system
|
121
|
+
* @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
|
122
|
+
*/
|
123
|
+
extern frt_uchar *frt_is_read_bytes(FrtInStream *is, frt_uchar *buf, int len);
|
124
|
+
|
125
|
+
/**
|
126
|
+
* Read a 32-bit unsigned integer from the FrtInStream.
|
127
|
+
*
|
128
|
+
* @param is the FrtInStream to read from
|
129
|
+
* @return a 32-bit unsigned integer
|
130
|
+
* @raise FRT_IO_ERROR if there is a error reading from the file-system
|
131
|
+
* @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
|
132
|
+
*/
|
133
|
+
extern frt_i32 frt_is_read_i32(FrtInStream *is);
|
134
|
+
|
135
|
+
/**
|
136
|
+
* Read a 64-bit unsigned integer from the FrtInStream.
|
137
|
+
*
|
138
|
+
* @param is the FrtInStream to read from
|
139
|
+
* @return a 64-bit unsigned integer
|
140
|
+
* @raise FRT_IO_ERROR if there is a error reading from the file-system
|
141
|
+
* @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
|
142
|
+
*/
|
143
|
+
extern frt_i64 frt_is_read_i64(FrtInStream *is);
|
144
|
+
|
145
|
+
/**
|
146
|
+
* Read a 32-bit signed integer from the FrtInStream.
|
147
|
+
*
|
148
|
+
* @param is the FrtInStream to read from
|
149
|
+
* @return a 32-bit signed integer
|
150
|
+
* @raise FRT_IO_ERROR if there is a error reading from the file-system
|
151
|
+
* @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
|
152
|
+
*/
|
153
|
+
extern frt_u32 frt_is_read_u32(FrtInStream *is);
|
154
|
+
|
155
|
+
/**
|
156
|
+
* Read a 64-bit signed integer from the FrtInStream.
|
157
|
+
*
|
158
|
+
* @param is the FrtInStream to read from
|
159
|
+
* @return a 64-bit signed integer
|
160
|
+
* @raise FRT_IO_ERROR if there is a error reading from the file-system
|
161
|
+
* @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
|
162
|
+
*/
|
163
|
+
extern frt_u64 frt_is_read_u64(FrtInStream *is);
|
164
|
+
|
165
|
+
/**
|
166
|
+
* Read a compressed (VINT) unsigned integer from the FrtInStream.
|
167
|
+
* TODO: describe VINT format
|
168
|
+
*
|
169
|
+
* @param is the FrtInStream to read from
|
170
|
+
* @return an int
|
171
|
+
* @raise FRT_IO_ERROR if there is a error reading from the file-system
|
172
|
+
* @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
|
173
|
+
*/
|
174
|
+
extern unsigned int frt_is_read_vint(FrtInStream *is);
|
175
|
+
|
176
|
+
/**
|
177
|
+
* Skip _cnt_ vints. This is a convenience method used for performance reasons
|
178
|
+
* to skip large numbers of vints. It is mostly used by TermDocEnums. When
|
179
|
+
* skipping positions os the proximity index file.
|
180
|
+
*
|
181
|
+
* @param is the FrtInStream to read from
|
182
|
+
* @param cnt the number of vints to skip
|
183
|
+
* @raise FRT_IO_ERROR if there is a error reading from the file-system
|
184
|
+
* @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
|
185
|
+
*/
|
186
|
+
extern void frt_is_skip_vints(FrtInStream *is, register int cnt);
|
187
|
+
|
188
|
+
/**
|
189
|
+
* Read a compressed (VINT) unsigned frt_off_t from the FrtInStream.
|
190
|
+
* TODO: describe VINT format
|
191
|
+
*
|
192
|
+
* @param is the FrtInStream to read from
|
193
|
+
* @return a frt_off_t
|
194
|
+
* @raise FRT_IO_ERROR if there is a error reading from the file-system
|
195
|
+
* @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
|
196
|
+
*/
|
197
|
+
extern frt_off_t frt_is_read_voff_t(FrtInStream *is);
|
198
|
+
|
199
|
+
/**
|
200
|
+
* Read a compressed (VINT) unsigned 64bit int from the FrtInStream.
|
201
|
+
* TODO: describe VINT format
|
202
|
+
*
|
203
|
+
* @param is the FrtInStream to read from
|
204
|
+
* @return a 64bit int
|
205
|
+
* @raise FRT_IO_ERROR if there is a error reading from the file-system
|
206
|
+
* @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
|
207
|
+
*/
|
208
|
+
extern frt_u64 frt_is_read_vll(FrtInStream *is);
|
209
|
+
|
210
|
+
/**
|
211
|
+
* Read a string from the FrtInStream. A string is an integer +length+ in vint
|
212
|
+
* format (see frt_is_read_vint) followed by +length+ bytes. This is the format
|
213
|
+
* used by frt_os_write_string.
|
214
|
+
*
|
215
|
+
* @param is the FrtInStream to read from
|
216
|
+
* @return a null byte delimited string
|
217
|
+
* @raise FRT_IO_ERROR if there is a error reading from the file-system
|
218
|
+
* @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
|
219
|
+
*/
|
220
|
+
extern char *frt_is_read_string(FrtInStream *is);
|
221
|
+
|
222
|
+
/**
|
223
|
+
* Read a string from the FrtInStream. A string is an integer +length+ in vint
|
224
|
+
* format (see frt_is_read_vint) followed by +length+ bytes. This is the format
|
225
|
+
* used by frt_os_write_string. This method is similar to +frt_is_read_string+ except
|
226
|
+
* that it will safely free all memory if there is an error reading the
|
227
|
+
* string.
|
228
|
+
*
|
229
|
+
* @param is the FrtInStream to read from
|
230
|
+
* @return a null byte delimited string
|
231
|
+
* @raise FRT_IO_ERROR if there is a error reading from the file-system
|
232
|
+
* @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
|
233
|
+
*/
|
234
|
+
extern char *frt_is_read_string_safe(FrtInStream *is);
|
235
|
+
|
236
|
+
extern char *frt_is_read_compressed_bytes(FrtInStream *is, int compressed_len, int *len, FrtCompressionType compression);
|
237
|
+
|
238
|
+
#define frt_is_length(mis) mis->m->length_i(mis)
|
239
|
+
|
240
|
+
#endif
|
@@ -138,17 +138,6 @@ int frt_index_size(FrtIndex *self)
|
|
138
138
|
return size;
|
139
139
|
}
|
140
140
|
|
141
|
-
void frt_index_optimize(FrtIndex *self)
|
142
|
-
{
|
143
|
-
pthread_mutex_lock(&self->mutex);
|
144
|
-
{
|
145
|
-
frt_ensure_writer_open(self);
|
146
|
-
frt_iw_optimize(self->iw);
|
147
|
-
AUTOFLUSH_IW(self);
|
148
|
-
}
|
149
|
-
pthread_mutex_unlock(&self->mutex);
|
150
|
-
}
|
151
|
-
|
152
141
|
bool frt_index_is_deleted(FrtIndex *self, int doc_num)
|
153
142
|
{
|
154
143
|
bool is_del;
|
@@ -32,7 +32,6 @@ typedef struct FrtIndex {
|
|
32
32
|
extern FrtIndex *frt_index_new(FrtStore *store, FrtAnalyzer *analyzer, FrtHashSet *def_fields, bool create);
|
33
33
|
extern void frt_index_destroy(FrtIndex *self);
|
34
34
|
extern int frt_index_size(FrtIndex *self);
|
35
|
-
extern void frt_index_optimize(FrtIndex *self);
|
36
35
|
extern bool frt_index_is_deleted(FrtIndex *self, int doc_num);
|
37
36
|
extern void frt_index_add_doc(FrtIndex *self, FrtDocument *doc);
|
38
37
|
extern FrtTopDocs *frt_index_search_str(FrtIndex *self, char *query, int first_doc, int num_docs, FrtFilter *filter, FrtSort *sort, FrtPostFilter *post_filter, rb_encoding *encoding);
|