isomorfeus-ferret 0.13.9 → 0.13.12
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/ext/isomorfeus_ferret_ext/bm_store.c +3 -3
- data/ext/isomorfeus_ferret_ext/bzlib.h +83 -82
- data/ext/isomorfeus_ferret_ext/frb_index.c +2 -2
- data/ext/isomorfeus_ferret_ext/frb_store.c +1 -1
- data/ext/isomorfeus_ferret_ext/frt_analysis.c +2 -2
- data/ext/isomorfeus_ferret_ext/frt_analysis.h +4 -4
- data/ext/isomorfeus_ferret_ext/frt_compound_io.c +11 -11
- data/ext/isomorfeus_ferret_ext/frt_config.h +8 -0
- data/ext/isomorfeus_ferret_ext/frt_fs_store.c +13 -7
- data/ext/isomorfeus_ferret_ext/frt_hashset.c +5 -5
- data/ext/isomorfeus_ferret_ext/frt_index.c +28 -26
- data/ext/isomorfeus_ferret_ext/frt_index.h +18 -18
- data/ext/isomorfeus_ferret_ext/frt_q_parser.c +1 -0
- data/ext/isomorfeus_ferret_ext/frt_ram_store.c +6 -6
- data/ext/isomorfeus_ferret_ext/frt_store.c +8 -8
- data/ext/isomorfeus_ferret_ext/frt_store.h +23 -23
- data/ext/isomorfeus_ferret_ext/test_index.c +2 -2
- data/lib/isomorfeus/ferret/version.rb +1 -1
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 60ce50bdaf067d95199abf381bd96e0b92fcce70ac7f64ea7e2f73ac1564ff64
|
4
|
+
data.tar.gz: 9c6eba0bc2630a6d95b55ea5887466a041e50425f1472c9008ef82a841194cf6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 878e93d18fe9a2847599be12478baea42fd010d4aa52f639977a7fb4bc8901771035908c9ceebfd34508ccd02f5d909569d06f88e55df36d8345e5bf35942d3e
|
7
|
+
data.tar.gz: c00f2909a8ef647a81deff946bedbfe5e4bfd86c175537a114866ca278a388a36c885a612f3cb1c49b582aaea9c9751f86b439d111e1b72205f46f653d7913cc
|
@@ -6,7 +6,7 @@
|
|
6
6
|
#define N 10
|
7
7
|
#define write_byte(os, b) os->buf.buf[os->buf.pos++] = (frt_uchar)b
|
8
8
|
|
9
|
-
void my_os_write_voff_t(FrtOutStream *os, register
|
9
|
+
void my_os_write_voff_t(FrtOutStream *os, register frt_off_t num) {
|
10
10
|
if (!(num&0x7f)) {
|
11
11
|
if (os->buf.pos >= FRT_BUFFER_SIZE) {
|
12
12
|
frt_os_write_byte(os, (frt_uchar)num);
|
@@ -48,7 +48,7 @@ void my_os_write_voff_t(FrtOutStream *os, register off_t num) {
|
|
48
48
|
|
49
49
|
static void vint_out(void) {
|
50
50
|
int n;
|
51
|
-
|
51
|
+
frt_off_t i;
|
52
52
|
FrtOutStream *os;
|
53
53
|
|
54
54
|
for (n = 0; n < N; n++) {
|
@@ -63,7 +63,7 @@ static void vint_out(void) {
|
|
63
63
|
|
64
64
|
static void unrolled_vint_out(void) {
|
65
65
|
int n;
|
66
|
-
|
66
|
+
frt_off_t i;
|
67
67
|
FrtOutStream *os;
|
68
68
|
|
69
69
|
for (n = 0; n < N; n++) {
|
@@ -11,7 +11,7 @@
|
|
11
11
|
bzip2/libbzip2 version 1.0.8 of 13 July 2019
|
12
12
|
Copyright (C) 1996-2019 Julian Seward <jseward@acm.org>
|
13
13
|
|
14
|
-
Please read the WARNING, DISCLAIMER and PATENTS sections in the
|
14
|
+
Please read the WARNING, DISCLAIMER and PATENTS sections in the
|
15
15
|
README file.
|
16
16
|
|
17
17
|
This program is released under the terms of the license contained
|
@@ -25,6 +25,7 @@
|
|
25
25
|
#ifdef __cplusplus
|
26
26
|
extern "C" {
|
27
27
|
#endif
|
28
|
+
#define BZ_DEBUG 0
|
28
29
|
|
29
30
|
#define BZ_RUN 0
|
30
31
|
#define BZ_FLUSH 1
|
@@ -45,7 +46,7 @@ extern "C" {
|
|
45
46
|
#define BZ_OUTBUFF_FULL (-8)
|
46
47
|
#define BZ_CONFIG_ERROR (-9)
|
47
48
|
|
48
|
-
typedef
|
49
|
+
typedef
|
49
50
|
struct {
|
50
51
|
char *next_in;
|
51
52
|
unsigned int avail_in;
|
@@ -62,7 +63,7 @@ typedef
|
|
62
63
|
void *(*bzalloc)(void *,int,int);
|
63
64
|
void (*bzfree)(void *,void *);
|
64
65
|
void *opaque;
|
65
|
-
}
|
66
|
+
}
|
66
67
|
bz_stream;
|
67
68
|
|
68
69
|
|
@@ -97,34 +98,34 @@ typedef
|
|
97
98
|
|
98
99
|
/*-- Core (low-level) library functions --*/
|
99
100
|
|
100
|
-
BZ_EXTERN int BZ_API(BZ2_bzCompressInit) (
|
101
|
-
bz_stream* strm,
|
102
|
-
int blockSize100k,
|
103
|
-
int verbosity,
|
104
|
-
int workFactor
|
101
|
+
BZ_EXTERN int BZ_API(BZ2_bzCompressInit) (
|
102
|
+
bz_stream* strm,
|
103
|
+
int blockSize100k,
|
104
|
+
int verbosity,
|
105
|
+
int workFactor
|
105
106
|
);
|
106
107
|
|
107
|
-
BZ_EXTERN int BZ_API(BZ2_bzCompress) (
|
108
|
-
bz_stream* strm,
|
109
|
-
int action
|
108
|
+
BZ_EXTERN int BZ_API(BZ2_bzCompress) (
|
109
|
+
bz_stream* strm,
|
110
|
+
int action
|
110
111
|
);
|
111
112
|
|
112
|
-
BZ_EXTERN int BZ_API(BZ2_bzCompressEnd) (
|
113
|
-
bz_stream* strm
|
113
|
+
BZ_EXTERN int BZ_API(BZ2_bzCompressEnd) (
|
114
|
+
bz_stream* strm
|
114
115
|
);
|
115
116
|
|
116
|
-
BZ_EXTERN int BZ_API(BZ2_bzDecompressInit) (
|
117
|
-
bz_stream *strm,
|
118
|
-
int verbosity,
|
117
|
+
BZ_EXTERN int BZ_API(BZ2_bzDecompressInit) (
|
118
|
+
bz_stream *strm,
|
119
|
+
int verbosity,
|
119
120
|
int small
|
120
121
|
);
|
121
122
|
|
122
|
-
BZ_EXTERN int BZ_API(BZ2_bzDecompress) (
|
123
|
-
bz_stream* strm
|
123
|
+
BZ_EXTERN int BZ_API(BZ2_bzDecompress) (
|
124
|
+
bz_stream* strm
|
124
125
|
);
|
125
126
|
|
126
|
-
BZ_EXTERN int BZ_API(BZ2_bzDecompressEnd) (
|
127
|
-
bz_stream *strm
|
127
|
+
BZ_EXTERN int BZ_API(BZ2_bzDecompressEnd) (
|
128
|
+
bz_stream *strm
|
128
129
|
);
|
129
130
|
|
130
131
|
|
@@ -136,64 +137,64 @@ BZ_EXTERN int BZ_API(BZ2_bzDecompressEnd) (
|
|
136
137
|
|
137
138
|
typedef void BZFILE;
|
138
139
|
|
139
|
-
BZ_EXTERN BZFILE* BZ_API(BZ2_bzReadOpen) (
|
140
|
-
int* bzerror,
|
141
|
-
FILE* f,
|
142
|
-
int verbosity,
|
140
|
+
BZ_EXTERN BZFILE* BZ_API(BZ2_bzReadOpen) (
|
141
|
+
int* bzerror,
|
142
|
+
FILE* f,
|
143
|
+
int verbosity,
|
143
144
|
int small,
|
144
|
-
void* unused,
|
145
|
-
int nUnused
|
145
|
+
void* unused,
|
146
|
+
int nUnused
|
146
147
|
);
|
147
148
|
|
148
|
-
BZ_EXTERN void BZ_API(BZ2_bzReadClose) (
|
149
|
-
int* bzerror,
|
150
|
-
BZFILE* b
|
149
|
+
BZ_EXTERN void BZ_API(BZ2_bzReadClose) (
|
150
|
+
int* bzerror,
|
151
|
+
BZFILE* b
|
151
152
|
);
|
152
153
|
|
153
|
-
BZ_EXTERN void BZ_API(BZ2_bzReadGetUnused) (
|
154
|
-
int* bzerror,
|
155
|
-
BZFILE* b,
|
156
|
-
void** unused,
|
157
|
-
int* nUnused
|
154
|
+
BZ_EXTERN void BZ_API(BZ2_bzReadGetUnused) (
|
155
|
+
int* bzerror,
|
156
|
+
BZFILE* b,
|
157
|
+
void** unused,
|
158
|
+
int* nUnused
|
158
159
|
);
|
159
160
|
|
160
|
-
BZ_EXTERN int BZ_API(BZ2_bzRead) (
|
161
|
-
int* bzerror,
|
162
|
-
BZFILE* b,
|
163
|
-
void* buf,
|
164
|
-
int len
|
161
|
+
BZ_EXTERN int BZ_API(BZ2_bzRead) (
|
162
|
+
int* bzerror,
|
163
|
+
BZFILE* b,
|
164
|
+
void* buf,
|
165
|
+
int len
|
165
166
|
);
|
166
167
|
|
167
|
-
BZ_EXTERN BZFILE* BZ_API(BZ2_bzWriteOpen) (
|
168
|
-
int* bzerror,
|
169
|
-
FILE* f,
|
170
|
-
int blockSize100k,
|
171
|
-
int verbosity,
|
172
|
-
int workFactor
|
168
|
+
BZ_EXTERN BZFILE* BZ_API(BZ2_bzWriteOpen) (
|
169
|
+
int* bzerror,
|
170
|
+
FILE* f,
|
171
|
+
int blockSize100k,
|
172
|
+
int verbosity,
|
173
|
+
int workFactor
|
173
174
|
);
|
174
175
|
|
175
|
-
BZ_EXTERN void BZ_API(BZ2_bzWrite) (
|
176
|
-
int* bzerror,
|
177
|
-
BZFILE* b,
|
178
|
-
void* buf,
|
179
|
-
int len
|
176
|
+
BZ_EXTERN void BZ_API(BZ2_bzWrite) (
|
177
|
+
int* bzerror,
|
178
|
+
BZFILE* b,
|
179
|
+
void* buf,
|
180
|
+
int len
|
180
181
|
);
|
181
182
|
|
182
|
-
BZ_EXTERN void BZ_API(BZ2_bzWriteClose) (
|
183
|
-
int* bzerror,
|
184
|
-
BZFILE* b,
|
185
|
-
int abandon,
|
186
|
-
unsigned int* nbytes_in,
|
187
|
-
unsigned int* nbytes_out
|
183
|
+
BZ_EXTERN void BZ_API(BZ2_bzWriteClose) (
|
184
|
+
int* bzerror,
|
185
|
+
BZFILE* b,
|
186
|
+
int abandon,
|
187
|
+
unsigned int* nbytes_in,
|
188
|
+
unsigned int* nbytes_out
|
188
189
|
);
|
189
190
|
|
190
|
-
BZ_EXTERN void BZ_API(BZ2_bzWriteClose64) (
|
191
|
-
int* bzerror,
|
192
|
-
BZFILE* b,
|
193
|
-
int abandon,
|
194
|
-
unsigned int* nbytes_in_lo32,
|
195
|
-
unsigned int* nbytes_in_hi32,
|
196
|
-
unsigned int* nbytes_out_lo32,
|
191
|
+
BZ_EXTERN void BZ_API(BZ2_bzWriteClose64) (
|
192
|
+
int* bzerror,
|
193
|
+
BZFILE* b,
|
194
|
+
int abandon,
|
195
|
+
unsigned int* nbytes_in_lo32,
|
196
|
+
unsigned int* nbytes_in_hi32,
|
197
|
+
unsigned int* nbytes_out_lo32,
|
197
198
|
unsigned int* nbytes_out_hi32
|
198
199
|
);
|
199
200
|
#endif
|
@@ -201,23 +202,23 @@ BZ_EXTERN void BZ_API(BZ2_bzWriteClose64) (
|
|
201
202
|
|
202
203
|
/*-- Utility functions --*/
|
203
204
|
|
204
|
-
BZ_EXTERN int BZ_API(BZ2_bzBuffToBuffCompress) (
|
205
|
-
char* dest,
|
205
|
+
BZ_EXTERN int BZ_API(BZ2_bzBuffToBuffCompress) (
|
206
|
+
char* dest,
|
206
207
|
unsigned int* destLen,
|
207
|
-
char* source,
|
208
|
+
char* source,
|
208
209
|
unsigned int sourceLen,
|
209
|
-
int blockSize100k,
|
210
|
-
int verbosity,
|
211
|
-
int workFactor
|
210
|
+
int blockSize100k,
|
211
|
+
int verbosity,
|
212
|
+
int workFactor
|
212
213
|
);
|
213
214
|
|
214
|
-
BZ_EXTERN int BZ_API(BZ2_bzBuffToBuffDecompress) (
|
215
|
-
char* dest,
|
215
|
+
BZ_EXTERN int BZ_API(BZ2_bzBuffToBuffDecompress) (
|
216
|
+
char* dest,
|
216
217
|
unsigned int* destLen,
|
217
|
-
char* source,
|
218
|
+
char* source,
|
218
219
|
unsigned int sourceLen,
|
219
|
-
int small,
|
220
|
-
int verbosity
|
220
|
+
int small,
|
221
|
+
int verbosity
|
221
222
|
);
|
222
223
|
|
223
224
|
|
@@ -244,17 +245,17 @@ BZ_EXTERN BZFILE * BZ_API(BZ2_bzdopen) (
|
|
244
245
|
int fd,
|
245
246
|
const char *mode
|
246
247
|
);
|
247
|
-
|
248
|
+
|
248
249
|
BZ_EXTERN int BZ_API(BZ2_bzread) (
|
249
|
-
BZFILE* b,
|
250
|
-
void* buf,
|
251
|
-
int len
|
250
|
+
BZFILE* b,
|
251
|
+
void* buf,
|
252
|
+
int len
|
252
253
|
);
|
253
254
|
|
254
255
|
BZ_EXTERN int BZ_API(BZ2_bzwrite) (
|
255
|
-
BZFILE* b,
|
256
|
-
void* buf,
|
257
|
-
int len
|
256
|
+
BZFILE* b,
|
257
|
+
void* buf,
|
258
|
+
int len
|
258
259
|
);
|
259
260
|
|
260
261
|
BZ_EXTERN int BZ_API(BZ2_bzflush) (
|
@@ -266,7 +267,7 @@ BZ_EXTERN void BZ_API(BZ2_bzclose) (
|
|
266
267
|
);
|
267
268
|
|
268
269
|
BZ_EXTERN const char * BZ_API(BZ2_bzerror) (
|
269
|
-
BZFILE *b,
|
270
|
+
BZFILE *b,
|
270
271
|
int *errnum
|
271
272
|
);
|
272
273
|
#endif
|
@@ -229,7 +229,7 @@ static VALUE frb_fi_init(int argc, VALUE *argv, VALUE self) {
|
|
229
229
|
*/
|
230
230
|
static VALUE frb_fi_name(VALUE self) {
|
231
231
|
FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
|
232
|
-
return
|
232
|
+
return ID2SYM(fi->name);
|
233
233
|
}
|
234
234
|
|
235
235
|
/*
|
@@ -659,7 +659,7 @@ frb_fis_get_tk_fields(VALUE self)
|
|
659
659
|
int i;
|
660
660
|
for (i = 0; i < fis->size; i++) {
|
661
661
|
if (!fi_is_tokenized(fis->fields[i])) continue;
|
662
|
-
rb_ary_push(rfield_names,
|
662
|
+
rb_ary_push(rfield_names, ID2SYM(fis->fields[i]->name));
|
663
663
|
}
|
664
664
|
return rfield_names;
|
665
665
|
}
|
@@ -402,7 +402,7 @@ static VALUE frb_fsdir_new(int argc, VALUE *argv, VALUE klass) {
|
|
402
402
|
frb_create_dir(rpath);
|
403
403
|
}
|
404
404
|
if (!rb_funcall(rb_cFile, id_is_directory, 1, rpath)) {
|
405
|
-
rb_raise(
|
405
|
+
rb_raise(cFileNotFoundError, "No directory <%s> found. Use :create => true to create one.", rs2s(rpath));
|
406
406
|
}
|
407
407
|
store = frt_open_fs_store(rs2s(rpath));
|
408
408
|
if (create) store->clear_all(store);
|
@@ -64,7 +64,7 @@ static inline int get_cp(char *start, char *end, int *cp_len, rb_encoding *enc)
|
|
64
64
|
/*** FrtToken ****************************************************************/
|
65
65
|
/*****************************************************************************/
|
66
66
|
|
67
|
-
FrtToken *frt_tk_set(FrtToken *tk, char *text, int tlen,
|
67
|
+
FrtToken *frt_tk_set(FrtToken *tk, char *text, int tlen, frt_off_t start, frt_off_t end, int pos_inc, rb_encoding *encoding) {
|
68
68
|
if (tlen >= FRT_MAX_WORD_SIZE) {
|
69
69
|
tlen = FRT_MAX_WORD_SIZE - 1; // TODO: this may invalidate mbc's
|
70
70
|
}
|
@@ -92,7 +92,7 @@ static FrtToken *frt_tk_set_ts(FrtToken *tk, char *start, char *end, char *text,
|
|
92
92
|
return frt_tk_set(tk, start, (int)(end - start), (off_t)(start - text), (off_t)(end - text), pos_inc, encoding);
|
93
93
|
}
|
94
94
|
|
95
|
-
FrtToken *frt_tk_set_no_len(FrtToken *tk, char *text,
|
95
|
+
FrtToken *frt_tk_set_no_len(FrtToken *tk, char *text, frt_off_t start, frt_off_t end, int pos_inc, rb_encoding *encoding) {
|
96
96
|
return frt_tk_set(tk, text, (int)strlen(text), start, end, pos_inc, encoding);
|
97
97
|
}
|
98
98
|
|
@@ -13,15 +13,15 @@
|
|
13
13
|
typedef struct FrtToken {
|
14
14
|
char text[FRT_MAX_WORD_SIZE];
|
15
15
|
int len;
|
16
|
-
|
17
|
-
|
16
|
+
frt_off_t start;
|
17
|
+
frt_off_t end;
|
18
18
|
int pos_inc;
|
19
19
|
} FrtToken;
|
20
20
|
|
21
21
|
extern FrtToken *frt_tk_new();
|
22
22
|
extern void frt_tk_destroy(void *p);
|
23
|
-
extern FrtToken *frt_tk_set(FrtToken *tk, char *text, int tlen,
|
24
|
-
extern FrtToken *frt_tk_set_no_len(FrtToken *tk, char *text,
|
23
|
+
extern FrtToken *frt_tk_set(FrtToken *tk, char *text, int tlen, frt_off_t start, frt_off_t end, int pos_inc, rb_encoding *encoding);
|
24
|
+
extern FrtToken *frt_tk_set_no_len(FrtToken *tk, char *text, frt_off_t start, frt_off_t end, int pos_inc, rb_encoding *encoding);
|
25
25
|
extern int frt_tk_eq(FrtToken *tk1, FrtToken *tk2);
|
26
26
|
extern int frt_tk_cmp(FrtToken *tk1, FrtToken *tk2);
|
27
27
|
|
@@ -12,8 +12,8 @@ extern FrtStore *frt_store_new();
|
|
12
12
|
****************************************************************************/
|
13
13
|
|
14
14
|
typedef struct FileEntry {
|
15
|
-
|
16
|
-
|
15
|
+
frt_off_t offset;
|
16
|
+
frt_off_t length;
|
17
17
|
} FileEntry;
|
18
18
|
|
19
19
|
static void cmpd_touch(FrtStore *store, const char *file_name) {
|
@@ -76,7 +76,7 @@ static void cmpd_close_i(FrtStore *store) {
|
|
76
76
|
free(store->dir.cmpd);
|
77
77
|
}
|
78
78
|
|
79
|
-
static
|
79
|
+
static frt_off_t cmpd_length(FrtStore *store, const char *file_name) {
|
80
80
|
FileEntry *fe = (FileEntry *)frt_h_get(store->dir.cmpd->entries, file_name);
|
81
81
|
if (fe != NULL) {
|
82
82
|
return fe->length;
|
@@ -85,7 +85,7 @@ static off_t cmpd_length(FrtStore *store, const char *file_name) {
|
|
85
85
|
}
|
86
86
|
}
|
87
87
|
|
88
|
-
static void cmpdi_seek_i(FrtInStream *is,
|
88
|
+
static void cmpdi_seek_i(FrtInStream *is, frt_off_t pos) {
|
89
89
|
(void)is;
|
90
90
|
(void)pos;
|
91
91
|
}
|
@@ -95,7 +95,7 @@ static void cmpdi_close_i(FrtInStream *is) {
|
|
95
95
|
free(is->d.cis);
|
96
96
|
}
|
97
97
|
|
98
|
-
static
|
98
|
+
static frt_off_t cmpdi_length_i(FrtInStream *is) {
|
99
99
|
return (is->d.cis->length);
|
100
100
|
}
|
101
101
|
|
@@ -104,7 +104,7 @@ static off_t cmpdi_length_i(FrtInStream *is) {
|
|
104
104
|
*/
|
105
105
|
static void cmpdi_read_i(FrtInStream *is, frt_uchar *b, int len) {
|
106
106
|
FrtCompoundInStream *cis = is->d.cis;
|
107
|
-
|
107
|
+
frt_off_t start = frt_is_pos(is);
|
108
108
|
|
109
109
|
if ((start + len) > cis->length) {
|
110
110
|
FRT_RAISE(FRT_EOF_ERROR, "Tried to read past end of file. File length is "
|
@@ -123,7 +123,7 @@ static const struct FrtInStreamMethods CMPD_IN_STREAM_METHODS = {
|
|
123
123
|
cmpdi_close_i
|
124
124
|
};
|
125
125
|
|
126
|
-
static FrtInStream *cmpd_create_input(FrtInStream *sub_is,
|
126
|
+
static FrtInStream *cmpd_create_input(FrtInStream *sub_is, frt_off_t offset, frt_off_t length) {
|
127
127
|
FrtInStream *is = frt_is_new();
|
128
128
|
FrtCompoundInStream *cis = FRT_ALLOC(FrtCompoundInStream);
|
129
129
|
|
@@ -182,7 +182,7 @@ static void cmpd_close_lock_i(FrtLock *lock) {
|
|
182
182
|
|
183
183
|
FrtStore *frt_open_cmpd_store(FrtStore *store, const char *name) {
|
184
184
|
int count, i;
|
185
|
-
|
185
|
+
frt_off_t offset;
|
186
186
|
char *fname;
|
187
187
|
FileEntry *volatile entry = NULL;
|
188
188
|
FrtStore *new_store = NULL;
|
@@ -273,9 +273,9 @@ void frt_cw_add_file(FrtCompoundWriter *cw, char *id) {
|
|
273
273
|
}
|
274
274
|
|
275
275
|
static void cw_copy_file(FrtCompoundWriter *cw, FrtCWFileEntry *src, FrtOutStream *os) {
|
276
|
-
|
277
|
-
|
278
|
-
|
276
|
+
frt_off_t start_ptr = frt_os_pos(os);
|
277
|
+
frt_off_t end_ptr;
|
278
|
+
frt_off_t remainder, length, len;
|
279
279
|
frt_uchar buffer[FRT_BUFFER_SIZE];
|
280
280
|
|
281
281
|
FrtInStream *is = cw->store->open_input(cw->store, src->name);
|
@@ -1,6 +1,8 @@
|
|
1
1
|
#ifndef FRT_DEFINES_H
|
2
2
|
#define FRT_DEFINES_H
|
3
3
|
|
4
|
+
#define _FILE_OFFSET_BITS 64
|
5
|
+
|
4
6
|
#include <sys/types.h>
|
5
7
|
#include <limits.h>
|
6
8
|
#include "frt_posh.h"
|
@@ -24,6 +26,12 @@ typedef posh_i32_t frt_i32;
|
|
24
26
|
typedef posh_u64_t frt_u64;
|
25
27
|
typedef posh_i64_t frt_i64;
|
26
28
|
|
29
|
+
#if (defined POSH_OS_WIN32 || defined POSH_OS_WIN64)
|
30
|
+
typedef off64_t frt_off_t;
|
31
|
+
#else
|
32
|
+
typedef off_t frt_off_t;
|
33
|
+
#endif
|
34
|
+
|
27
35
|
#if ( LONG_MAX == 2147483647 ) && defined(_FILE_OFFSET_BITS) && (_FILE_OFFSET_BITS == 64)
|
28
36
|
#define FRT_OFF_T_PFX "ll"
|
29
37
|
#else
|
@@ -225,7 +225,7 @@ static void fs_destroy(FrtStore *store)
|
|
225
225
|
free(store->dir.path);
|
226
226
|
}
|
227
227
|
|
228
|
-
static
|
228
|
+
static frt_off_t fs_length(FrtStore *store, const char *filename)
|
229
229
|
{
|
230
230
|
char path[FRT_MAX_FILE_PATH];
|
231
231
|
struct stat stt;
|
@@ -247,9 +247,12 @@ static void fso_flush_i(FrtOutStream *os, const frt_uchar *src, int len)
|
|
247
247
|
}
|
248
248
|
}
|
249
249
|
|
250
|
-
static void fso_seek_i(FrtOutStream *os,
|
251
|
-
|
250
|
+
static void fso_seek_i(FrtOutStream *os, frt_off_t pos) {
|
251
|
+
#if (defined POSH_OS_WIN32 || defined POSH_OS_WIN64)
|
252
|
+
if (_lseeki64(os->file.fd, pos, SEEK_SET) < 0) {
|
253
|
+
#else
|
252
254
|
if (lseek(os->file.fd, pos, SEEK_SET) < 0) {
|
255
|
+
#endif
|
253
256
|
FRT_RAISE(FRT_IO_ERROR, "seeking position %"FRT_OFF_T_PFX"d: <%s>",
|
254
257
|
pos, strerror(errno));
|
255
258
|
}
|
@@ -286,7 +289,7 @@ static FrtOutStream *fs_new_output(FrtStore *store, const char *filename)
|
|
286
289
|
static void fsi_read_i(FrtInStream *is, frt_uchar *path, int len)
|
287
290
|
{
|
288
291
|
int fd = is->f->file.fd;
|
289
|
-
|
292
|
+
frt_off_t pos = frt_is_pos(is);
|
290
293
|
if (pos != lseek(fd, 0, SEEK_CUR)) {
|
291
294
|
lseek(fd, pos, SEEK_SET);
|
292
295
|
}
|
@@ -299,9 +302,12 @@ static void fsi_read_i(FrtInStream *is, frt_uchar *path, int len)
|
|
299
302
|
}
|
300
303
|
}
|
301
304
|
|
302
|
-
static void fsi_seek_i(FrtInStream *is,
|
303
|
-
|
305
|
+
static void fsi_seek_i(FrtInStream *is, frt_off_t pos) {
|
306
|
+
#if (defined POSH_OS_WIN32 || defined POSH_OS_WIN64)
|
307
|
+
if (_lseeki64(is->f->file.fd, pos, SEEK_SET) < 0) {
|
308
|
+
#else
|
304
309
|
if (lseek(is->f->file.fd, pos, SEEK_SET) < 0) {
|
310
|
+
#endif
|
305
311
|
FRT_RAISE(FRT_IO_ERROR, "seeking pos %"FRT_OFF_T_PFX"d: <%s>",
|
306
312
|
pos, strerror(errno));
|
307
313
|
}
|
@@ -315,7 +321,7 @@ static void fsi_close_i(FrtInStream *is)
|
|
315
321
|
if (is->d.path) free(is->d.path);
|
316
322
|
}
|
317
323
|
|
318
|
-
static
|
324
|
+
static frt_off_t fsi_length_i(FrtInStream *is)
|
319
325
|
{
|
320
326
|
struct stat stt;
|
321
327
|
if (fstat(is->f->file.fd, &stt)) {
|
@@ -3,7 +3,7 @@
|
|
3
3
|
|
4
4
|
/*
|
5
5
|
* The HashSet contains an array +elems+ of the elements that have been added.
|
6
|
-
* It always has +size+ elements so +size+
|
6
|
+
* It always has +size+ elements so +size+ and +elems+ can be used to iterate
|
7
7
|
* over all alements in the HashSet. It also uses a Hash to keep track of
|
8
8
|
* which elements have been added and their index in the +elems+ array.
|
9
9
|
*/
|
@@ -37,7 +37,7 @@ FrtHashSet *frt_hs_new_ptr(frt_free_ft free_func)
|
|
37
37
|
return hs;
|
38
38
|
}
|
39
39
|
|
40
|
-
static void
|
40
|
+
static void hs_clear(FrtHashSet *hs, bool destroy)
|
41
41
|
{
|
42
42
|
FrtHashSetEntry *curr, *next = hs->first;
|
43
43
|
frt_free_ft do_free = destroy ? hs->free_elem_i : &frt_dummy_free;
|
@@ -52,20 +52,20 @@ static void clear(FrtHashSet *hs, bool destroy)
|
|
52
52
|
|
53
53
|
void frt_hs_clear(FrtHashSet *hs)
|
54
54
|
{
|
55
|
-
|
55
|
+
hs_clear(hs, true);
|
56
56
|
frt_h_clear(hs->ht);
|
57
57
|
}
|
58
58
|
|
59
59
|
void frt_hs_free(FrtHashSet *hs)
|
60
60
|
{
|
61
|
-
|
61
|
+
hs_clear(hs, false);
|
62
62
|
frt_h_destroy(hs->ht);
|
63
63
|
free(hs);
|
64
64
|
}
|
65
65
|
|
66
66
|
void frt_hs_destroy(FrtHashSet *hs)
|
67
67
|
{
|
68
|
-
|
68
|
+
hs_clear(hs, true);
|
69
69
|
frt_h_destroy(hs->ht);
|
70
70
|
free(hs);
|
71
71
|
}
|
@@ -116,7 +116,7 @@ char *frt_fn_for_generation(char *buf, const char *base, const char *ext, frt_i6
|
|
116
116
|
} else {
|
117
117
|
char b[FRT_SEGMENT_NAME_MAX_LENGTH];
|
118
118
|
char *u = u64_to_str36(b, FRT_SEGMENT_NAME_MAX_LENGTH, (frt_u64)gen);
|
119
|
-
if (ext == NULL) {
|
119
|
+
if (ext == NULL || (strcmp(ext, "") == 0)) {
|
120
120
|
sprintf(buf, "%s_%s", base, u);
|
121
121
|
} else {
|
122
122
|
sprintf(buf, "%s_%s.%s", base, u, ext);
|
@@ -303,6 +303,7 @@ FrtFieldInfo *frt_fi_init(FrtFieldInfo *fi, ID name, FrtStoreValue store, FrtCom
|
|
303
303
|
fi_set_compression(fi, compression);
|
304
304
|
fi_set_index(fi, index);
|
305
305
|
fi_set_term_vector(fi, term_vector);
|
306
|
+
fi->number = 0;
|
306
307
|
fi->ref_cnt = 1;
|
307
308
|
fi->rfi = Qnil;
|
308
309
|
return fi;
|
@@ -394,6 +395,7 @@ FrtFieldInfo *frt_fis_add_field(FrtFieldInfos *fis, FrtFieldInfo *fi) {
|
|
394
395
|
if (!frt_h_set_safe(fis->field_dict, (void *)fi->name, fi)) {
|
395
396
|
FRT_RAISE(FRT_ARG_ERROR, "Field :%s already exists", rb_id2name(fi->name));
|
396
397
|
}
|
398
|
+
FRT_REF(fi);
|
397
399
|
fi->number = fis->size;
|
398
400
|
fis->fields[fis->size] = fi;
|
399
401
|
fis->size++;
|
@@ -1565,7 +1567,7 @@ static void frt_fr_read_compressed_fields(FrtFieldsReader *fr, FrtDocField *df,
|
|
1565
1567
|
FrtDocument *frt_fr_get_doc(FrtFieldsReader *fr, int doc_num)
|
1566
1568
|
{
|
1567
1569
|
int i, j;
|
1568
|
-
|
1570
|
+
frt_off_t pos;
|
1569
1571
|
int stored_cnt;
|
1570
1572
|
FrtDocument *doc = frt_doc_new();
|
1571
1573
|
FrtInStream *fdx_in = fr->fdx_in;
|
@@ -1612,7 +1614,7 @@ FrtLazyDoc *frt_fr_get_lazy_doc(FrtFieldsReader *fr, int doc_num)
|
|
1612
1614
|
{
|
1613
1615
|
int start = 0;
|
1614
1616
|
int i, j;
|
1615
|
-
|
1617
|
+
frt_off_t pos;
|
1616
1618
|
int stored_cnt;
|
1617
1619
|
FrtLazyDoc *lazy_doc;
|
1618
1620
|
FrtInStream *fdx_in = fr->fdx_in;
|
@@ -1642,7 +1644,7 @@ FrtLazyDoc *frt_fr_get_lazy_doc(FrtFieldsReader *fr, int doc_num)
|
|
1642
1644
|
lazy_doc_add_field(lazy_doc, lazy_df, i);
|
1643
1645
|
}
|
1644
1646
|
/* correct the starts to their correct absolute positions */
|
1645
|
-
const
|
1647
|
+
const frt_off_t abs_start = frt_is_pos(fdt_in);
|
1646
1648
|
for (i = 0; i < stored_cnt; i++) {
|
1647
1649
|
FrtLazyDocField *lazy_df = lazy_doc->fields[i];
|
1648
1650
|
const int df_size = lazy_df->size;
|
@@ -1719,7 +1721,7 @@ FrtHash *frt_fr_get_tv(FrtFieldsReader *fr, int doc_num)
|
|
1719
1721
|
int i;
|
1720
1722
|
FrtInStream *fdx_in = fr->fdx_in;
|
1721
1723
|
FrtInStream *fdt_in = fr->fdt_in;
|
1722
|
-
|
1724
|
+
frt_off_t data_ptr, field_index_ptr;
|
1723
1725
|
int field_cnt;
|
1724
1726
|
int *field_nums;
|
1725
1727
|
|
@@ -1757,7 +1759,7 @@ FrtTermVector *frt_fr_get_field_tv(FrtFieldsReader *fr, int doc_num, int field_n
|
|
1757
1759
|
|
1758
1760
|
if (doc_num >= 0 && doc_num < fr->size) {
|
1759
1761
|
int i, fnum = -1;
|
1760
|
-
|
1762
|
+
frt_off_t field_index_ptr;
|
1761
1763
|
int field_cnt;
|
1762
1764
|
int offset = 0;
|
1763
1765
|
FrtInStream *fdx_in = fr->fdx_in;
|
@@ -2038,7 +2040,7 @@ void frt_fw_add_postings(FrtFieldsWriter *fw,
|
|
2038
2040
|
int i, delta_start, delta_length;
|
2039
2041
|
const char *last_term = FRT_EMPTY_STRING;
|
2040
2042
|
FrtOutStream *fdt_out = fw->fdt_out;
|
2041
|
-
|
2043
|
+
frt_off_t fdt_start_pos = frt_os_pos(fdt_out);
|
2042
2044
|
FrtPostingList *plist;
|
2043
2045
|
FrtPosting *posting;
|
2044
2046
|
FrtOccurence *occ;
|
@@ -2152,7 +2154,7 @@ static void sti_ensure_index_is_read(FrtSegmentTermIndex *sti, FrtTermEnum *inde
|
|
2152
2154
|
if (NULL == sti->index_terms) {
|
2153
2155
|
int i;
|
2154
2156
|
int index_cnt = sti->index_cnt;
|
2155
|
-
|
2157
|
+
frt_off_t index_ptr = 0;
|
2156
2158
|
ste_reset(index_te);
|
2157
2159
|
frt_is_seek(STE(index_te)->is, sti->index_ptr);
|
2158
2160
|
STE(index_te)->size = sti->index_cnt;
|
@@ -2789,7 +2791,7 @@ static void tw_add(FrtTermWriter *tw, const char *term, int term_len, FrtTermInf
|
|
2789
2791
|
}
|
2790
2792
|
|
2791
2793
|
void frt_tiw_add(FrtTermInfosWriter *tiw, const char *term, int term_len, FrtTermInfo *ti) {
|
2792
|
-
|
2794
|
+
frt_off_t tis_pos;
|
2793
2795
|
|
2794
2796
|
if (0 == (tiw->tis_writer->counter % tiw->index_interval)) {
|
2795
2797
|
/* add an index term */
|
@@ -2962,8 +2964,8 @@ static bool stde_skip_to(FrtTermDocEnum *tde, int target_doc_num) {
|
|
2962
2964
|
if (stde->doc_freq >= stde->skip_interval
|
2963
2965
|
&& target_doc_num > stde->doc_num) { /* optimized case */
|
2964
2966
|
int last_skip_doc;
|
2965
|
-
|
2966
|
-
|
2967
|
+
frt_off_t last_frq_ptr;
|
2968
|
+
frt_off_t last_prx_ptr;
|
2967
2969
|
int num_skipped;
|
2968
2970
|
|
2969
2971
|
if (NULL == stde->skip_in) {
|
@@ -3034,7 +3036,7 @@ static void stde_skip_prox(FrtSegmentTermDocEnum *stde) {
|
|
3034
3036
|
(void)stde;
|
3035
3037
|
}
|
3036
3038
|
|
3037
|
-
static void stde_seek_prox(FrtSegmentTermDocEnum *stde,
|
3039
|
+
static void stde_seek_prox(FrtSegmentTermDocEnum *stde, frt_off_t prx_ptr) {
|
3038
3040
|
(void)stde;
|
3039
3041
|
(void)prx_ptr;
|
3040
3042
|
}
|
@@ -3130,7 +3132,7 @@ static void stpe_skip_prox(FrtSegmentTermDocEnum *stde)
|
|
3130
3132
|
frt_is_skip_vints(stde->prx_in, stde->freq);
|
3131
3133
|
}
|
3132
3134
|
|
3133
|
-
static void stpe_seek_prox(FrtSegmentTermDocEnum *stde,
|
3135
|
+
static void stpe_seek_prox(FrtSegmentTermDocEnum *stde, frt_off_t prx_ptr)
|
3134
3136
|
{
|
3135
3137
|
frt_is_seek(stde->prx_in, prx_ptr);
|
3136
3138
|
stde->prx_cnt = 0;
|
@@ -5108,8 +5110,8 @@ typedef struct SkipBuffer
|
|
5108
5110
|
FrtOutStream *frq_out;
|
5109
5111
|
FrtOutStream *prx_out;
|
5110
5112
|
int last_doc;
|
5111
|
-
|
5112
|
-
|
5113
|
+
frt_off_t last_frq_ptr;
|
5114
|
+
frt_off_t last_prx_ptr;
|
5113
5115
|
} SkipBuffer;
|
5114
5116
|
|
5115
5117
|
static void skip_buf_reset(SkipBuffer *skip_buf)
|
@@ -5131,8 +5133,8 @@ static SkipBuffer *skip_buf_new(FrtOutStream *frq_out, FrtOutStream *prx_out)
|
|
5131
5133
|
|
5132
5134
|
static void skip_buf_add(SkipBuffer *skip_buf, int doc)
|
5133
5135
|
{
|
5134
|
-
|
5135
|
-
|
5136
|
+
frt_off_t frq_ptr = frt_os_pos(skip_buf->frq_out);
|
5137
|
+
frt_off_t prx_ptr = frt_os_pos(skip_buf->prx_out);
|
5136
5138
|
|
5137
5139
|
frt_os_write_vint(skip_buf->buf, doc - skip_buf->last_doc);
|
5138
5140
|
frt_os_write_vint(skip_buf->buf, frq_ptr - skip_buf->last_frq_ptr);
|
@@ -5143,9 +5145,9 @@ static void skip_buf_add(SkipBuffer *skip_buf, int doc)
|
|
5143
5145
|
skip_buf->last_prx_ptr = prx_ptr;
|
5144
5146
|
}
|
5145
5147
|
|
5146
|
-
static
|
5148
|
+
static frt_off_t skip_buf_write(SkipBuffer *skip_buf)
|
5147
5149
|
{
|
5148
|
-
|
5150
|
+
frt_off_t skip_ptr = frt_os_pos(skip_buf->frq_out);
|
5149
5151
|
frt_ramo_write_to(skip_buf->buf, skip_buf->frq_out);
|
5150
5152
|
return skip_ptr;
|
5151
5153
|
}
|
@@ -5378,7 +5380,7 @@ static void dw_add_posting(FrtMemoryPool *mp,
|
|
5378
5380
|
}
|
5379
5381
|
}
|
5380
5382
|
|
5381
|
-
static void dw_add_offsets(FrtDocWriter *dw, int pos,
|
5383
|
+
static void dw_add_offsets(FrtDocWriter *dw, int pos, frt_off_t start, frt_off_t end)
|
5382
5384
|
{
|
5383
5385
|
if (pos >= dw->offsets_capa) {
|
5384
5386
|
int old_capa = dw->offsets_capa;
|
@@ -5402,7 +5404,7 @@ FrtHash *frt_dw_invert_field(FrtDocWriter *dw, FrtFieldInverter *fld_inv, FrtDoc
|
|
5402
5404
|
int doc_num = dw->doc_num;
|
5403
5405
|
int i;
|
5404
5406
|
const int df_size = df->size;
|
5405
|
-
|
5407
|
+
frt_off_t start_offset = 0;
|
5406
5408
|
|
5407
5409
|
if (fld_inv->is_tokenized) {
|
5408
5410
|
FrtToken *tk;
|
@@ -5701,7 +5703,7 @@ static void sm_destroy(SegmentMerger *sm)
|
|
5701
5703
|
static void sm_merge_fields(SegmentMerger *sm)
|
5702
5704
|
{
|
5703
5705
|
int i, j;
|
5704
|
-
|
5706
|
+
frt_off_t start, end = 0;
|
5705
5707
|
char file_name[FRT_SEGMENT_NAME_MAX_LENGTH];
|
5706
5708
|
FrtOutStream *fdt_out, *fdx_out;
|
5707
5709
|
FrtStore *store = sm->store;
|
@@ -5818,12 +5820,12 @@ static char *sm_cache_term(SegmentMerger *sm, char *term, int term_len)
|
|
5818
5820
|
static void sm_merge_term_info(SegmentMerger *sm, SegmentMergeInfo **matches,
|
5819
5821
|
int match_size)
|
5820
5822
|
{
|
5821
|
-
|
5822
|
-
|
5823
|
+
frt_off_t frq_ptr = frt_os_pos(sm->frq_out);
|
5824
|
+
frt_off_t prx_ptr = frt_os_pos(sm->prx_out);
|
5823
5825
|
|
5824
5826
|
int df = sm_append_postings(sm, matches, match_size); /* append posting data */
|
5825
5827
|
|
5826
|
-
|
5828
|
+
frt_off_t skip_ptr = skip_buf_write(sm->skip_buf);
|
5827
5829
|
|
5828
5830
|
if (df > 0) {
|
5829
5831
|
/* add an entry to the dictionary with ptrs to prox and freq files */
|
@@ -6370,7 +6372,7 @@ static void iw_cp_fields(FrtIndexWriter *iw, FrtSegmentReader *sr, const char *s
|
|
6370
6372
|
int j, data_len = 0;
|
6371
6373
|
const int field_cnt = frt_is_read_vint(fdt_in);
|
6372
6374
|
int tv_cnt;
|
6373
|
-
|
6375
|
+
frt_off_t doc_start_ptr = frt_os_pos(fdt_out);
|
6374
6376
|
|
6375
6377
|
frt_os_write_u64(fdx_out, doc_start_ptr);
|
6376
6378
|
frt_os_write_vint(fdt_out, field_cnt);
|
@@ -224,9 +224,9 @@ extern void frt_sis_put(FrtSegmentInfos *sis, FILE *stream);
|
|
224
224
|
|
225
225
|
typedef struct FrtTermInfo {
|
226
226
|
int doc_freq;
|
227
|
-
|
228
|
-
|
229
|
-
|
227
|
+
frt_off_t frq_ptr;
|
228
|
+
frt_off_t prx_ptr;
|
229
|
+
frt_off_t skip_offset;
|
230
230
|
} FrtTermInfo;
|
231
231
|
|
232
232
|
#define frt_ti_set(ti, mdf, mfp, mpp, mso) do {\
|
@@ -265,14 +265,14 @@ FrtTermInfo *frt_te_get_ti(struct FrtTermEnum *te);
|
|
265
265
|
/* FrtSegmentTermIndex */
|
266
266
|
|
267
267
|
typedef struct FrtSegmentTermIndex {
|
268
|
-
|
269
|
-
|
268
|
+
frt_off_t index_ptr;
|
269
|
+
frt_off_t ptr;
|
270
270
|
int index_cnt;
|
271
271
|
int size;
|
272
272
|
char **index_terms;
|
273
273
|
int *index_term_lens;
|
274
274
|
FrtTermInfo *index_term_infos;
|
275
|
-
|
275
|
+
frt_off_t *index_ptrs;
|
276
276
|
} FrtSegmentTermIndex;
|
277
277
|
|
278
278
|
/* FrtSegmentFieldIndex */
|
@@ -281,7 +281,7 @@ typedef struct FrtSegmentFieldIndex {
|
|
281
281
|
frt_mutex_t mutex;
|
282
282
|
int skip_interval;
|
283
283
|
int index_interval;
|
284
|
-
|
284
|
+
frt_off_t index_ptr;
|
285
285
|
FrtTermEnum *index_te;
|
286
286
|
FrtHash *field_dict;
|
287
287
|
} FrtSegmentFieldIndex;
|
@@ -349,7 +349,7 @@ typedef struct FrtTermInfosWriter {
|
|
349
349
|
int field_count;
|
350
350
|
int index_interval;
|
351
351
|
int skip_interval;
|
352
|
-
|
352
|
+
frt_off_t last_index_ptr;
|
353
353
|
FrtOutStream *tfx_out;
|
354
354
|
FrtTermWriter *tix_writer;
|
355
355
|
FrtTermWriter *tis_writer;
|
@@ -385,7 +385,7 @@ struct FrtTermDocEnum {
|
|
385
385
|
typedef struct FrtSegmentTermDocEnum FrtSegmentTermDocEnum;
|
386
386
|
struct FrtSegmentTermDocEnum {
|
387
387
|
FrtTermDocEnum tde;
|
388
|
-
void (*seek_prox)(FrtSegmentTermDocEnum *stde,
|
388
|
+
void (*seek_prox)(FrtSegmentTermDocEnum *stde, frt_off_t prx_ptr);
|
389
389
|
void (*skip_prox)(FrtSegmentTermDocEnum *stde);
|
390
390
|
FrtTermInfosReader *tir;
|
391
391
|
FrtInStream *frq_in;
|
@@ -402,9 +402,9 @@ struct FrtSegmentTermDocEnum {
|
|
402
402
|
int skip_doc;
|
403
403
|
int prx_cnt;
|
404
404
|
int position;
|
405
|
-
|
406
|
-
|
407
|
-
|
405
|
+
frt_off_t frq_ptr;
|
406
|
+
frt_off_t prx_ptr;
|
407
|
+
frt_off_t skip_ptr;
|
408
408
|
bool have_skipped : 1;
|
409
409
|
};
|
410
410
|
|
@@ -429,8 +429,8 @@ extern FrtTermDocEnum *frt_mtdpe_new(FrtIndexReader *ir, int field_num, char **t
|
|
429
429
|
****************************************************************************/
|
430
430
|
|
431
431
|
typedef struct FrtOffset {
|
432
|
-
|
433
|
-
|
432
|
+
frt_off_t start;
|
433
|
+
frt_off_t end;
|
434
434
|
} FrtOffset;
|
435
435
|
|
436
436
|
/****************************************************************************
|
@@ -529,7 +529,7 @@ extern FrtTVTerm *frt_tv_get_tv_term(FrtTermVector *tv, const char *term);
|
|
529
529
|
|
530
530
|
/* * * FrtLazyDocField * * */
|
531
531
|
typedef struct FrtLazyDocFieldData {
|
532
|
-
|
532
|
+
frt_off_t start;
|
533
533
|
int length;
|
534
534
|
rb_encoding *encoding;
|
535
535
|
FrtCompressionType compression; /* as stored */
|
@@ -595,7 +595,7 @@ typedef struct FrtFieldsWriter {
|
|
595
595
|
FrtOutStream *fdx_out;
|
596
596
|
FrtOutStream *buffer;
|
597
597
|
FrtTVField *tv_fields;
|
598
|
-
|
598
|
+
frt_off_t start_ptr;
|
599
599
|
} FrtFieldsWriter;
|
600
600
|
|
601
601
|
extern FrtFieldsWriter *frt_fw_open(FrtStore *store, const char *segment, FrtFieldInfos *fis);
|
@@ -846,8 +846,8 @@ extern void frt_iw_add_readers(FrtIndexWriter *iw, FrtIndexReader **readers, con
|
|
846
846
|
#define FRT_CW_INIT_CAPA 16
|
847
847
|
typedef struct FrtCWFileEntry {
|
848
848
|
char *name;
|
849
|
-
|
850
|
-
|
849
|
+
frt_off_t dir_offset;
|
850
|
+
frt_off_t data_offset;
|
851
851
|
} FrtCWFileEntry;
|
852
852
|
|
853
853
|
typedef struct FrtCompoundWriter {
|
@@ -2801,6 +2801,7 @@ FrtQParser *frt_qp_init(FrtQParser *self, FrtAnalyzer *analyzer) {
|
|
2801
2801
|
qp_push_fields(self, self->def_fields, false);
|
2802
2802
|
|
2803
2803
|
self->analyzer = analyzer;
|
2804
|
+
if (analyzer) FRT_REF(analyzer);
|
2804
2805
|
self->ts_cache = frt_h_new_ptr((frt_free_ft)&frt_ts_deref);
|
2805
2806
|
self->buf_index = 0;
|
2806
2807
|
self->dynbuf = NULL;
|
@@ -136,7 +136,7 @@ static void ram_clear_all(FrtStore *store) {
|
|
136
136
|
}
|
137
137
|
}
|
138
138
|
|
139
|
-
static
|
139
|
+
static frt_off_t ram_length(FrtStore *store, const char *filename) {
|
140
140
|
FrtRAMFile *rf = (FrtRAMFile *)frt_h_get(store->dir.ht, filename);
|
141
141
|
if (rf != NULL) {
|
142
142
|
return rf->len;
|
@@ -151,7 +151,7 @@ static void ramo_flush_i(FrtOutStream *os, const frt_uchar *src, int len) {
|
|
151
151
|
FrtRAMFile *rf = os->file.rf;
|
152
152
|
int buffer_number, buffer_offset, bytes_in_buffer, bytes_to_copy;
|
153
153
|
int src_offset;
|
154
|
-
|
154
|
+
frt_off_t pointer = os->pointer;
|
155
155
|
|
156
156
|
buffer_number = (int)(pointer / FRT_BUFFER_SIZE);
|
157
157
|
buffer_offset = pointer % FRT_BUFFER_SIZE;
|
@@ -179,7 +179,7 @@ static void ramo_flush_i(FrtOutStream *os, const frt_uchar *src, int len) {
|
|
179
179
|
}
|
180
180
|
}
|
181
181
|
|
182
|
-
static void ramo_seek_i(FrtOutStream *os,
|
182
|
+
static void ramo_seek_i(FrtOutStream *os, frt_off_t pos) {
|
183
183
|
os->pointer = pos;
|
184
184
|
}
|
185
185
|
|
@@ -248,7 +248,7 @@ static void rami_read_i(FrtInStream *is, frt_uchar *b, int len) {
|
|
248
248
|
int offset = 0;
|
249
249
|
int buffer_number, buffer_offset, bytes_in_buffer, bytes_to_copy;
|
250
250
|
int remainder = len;
|
251
|
-
|
251
|
+
frt_off_t start = is->d.pointer;
|
252
252
|
frt_uchar *buffer;
|
253
253
|
|
254
254
|
while (remainder > 0) {
|
@@ -271,11 +271,11 @@ static void rami_read_i(FrtInStream *is, frt_uchar *b, int len) {
|
|
271
271
|
is->d.pointer += len;
|
272
272
|
}
|
273
273
|
|
274
|
-
static
|
274
|
+
static frt_off_t rami_length_i(FrtInStream *is) {
|
275
275
|
return is->f->file.rf->len;
|
276
276
|
}
|
277
277
|
|
278
|
-
static void rami_seek_i(FrtInStream *is,
|
278
|
+
static void rami_seek_i(FrtInStream *is, frt_off_t pos) {
|
279
279
|
is->d.pointer = pos;
|
280
280
|
}
|
281
281
|
|
@@ -125,7 +125,7 @@ off_t frt_os_pos(FrtOutStream *os)
|
|
125
125
|
return os->buf.start + os->buf.pos;
|
126
126
|
}
|
127
127
|
|
128
|
-
void frt_os_seek(FrtOutStream *os,
|
128
|
+
void frt_os_seek(FrtOutStream *os, frt_off_t new_pos)
|
129
129
|
{
|
130
130
|
frt_os_flush(os);
|
131
131
|
os->buf.start = new_pos;
|
@@ -202,9 +202,9 @@ FrtInStream *frt_is_new(void) {
|
|
202
202
|
*/
|
203
203
|
static void is_refill(FrtInStream *is)
|
204
204
|
{
|
205
|
-
|
206
|
-
|
207
|
-
|
205
|
+
frt_off_t start = is->buf.start + is->buf.pos;
|
206
|
+
frt_off_t last = start + FRT_BUFFER_SIZE;
|
207
|
+
frt_off_t flen = is->m->length_i(is);
|
208
208
|
|
209
209
|
if (last > flen) { /* don't read past EOF */
|
210
210
|
last = flen;
|
@@ -254,7 +254,7 @@ off_t frt_is_pos(FrtInStream *is)
|
|
254
254
|
frt_uchar *frt_is_read_bytes(FrtInStream *is, frt_uchar *buf, int len)
|
255
255
|
{
|
256
256
|
int i;
|
257
|
-
|
257
|
+
frt_off_t start;
|
258
258
|
|
259
259
|
if ((is->buf.pos + len) < is->buf.len) {
|
260
260
|
for (i = 0; i < len; i++) {
|
@@ -273,7 +273,7 @@ frt_uchar *frt_is_read_bytes(FrtInStream *is, frt_uchar *buf, int len)
|
|
273
273
|
return buf;
|
274
274
|
}
|
275
275
|
|
276
|
-
void frt_is_seek(FrtInStream *is,
|
276
|
+
void frt_is_seek(FrtInStream *is, frt_off_t pos) {
|
277
277
|
if (pos >= is->buf.start && pos < (is->buf.start + is->buf.len)) {
|
278
278
|
is->buf.pos = pos - is->buf.start; /* seek within buffer */
|
279
279
|
} else {
|
@@ -384,7 +384,7 @@ unsigned int frt_is_read_vint(FrtInStream *is)
|
|
384
384
|
/* optimized to use unchecked read_byte if there is definitely space */
|
385
385
|
off_t frt_is_read_voff_t(FrtInStream *is)
|
386
386
|
{
|
387
|
-
register
|
387
|
+
register frt_off_t res, b;
|
388
388
|
register int shift = 7;
|
389
389
|
|
390
390
|
if (is->buf.pos > (is->buf.len - VINT_MAX_LEN)) {
|
@@ -553,7 +553,7 @@ void frt_os_write_vint(FrtOutStream *os, register unsigned int num)
|
|
553
553
|
}
|
554
554
|
|
555
555
|
/* optimized to use an unchecked write if there is space */
|
556
|
-
void frt_os_write_voff_t(FrtOutStream *os, register
|
556
|
+
void frt_os_write_voff_t(FrtOutStream *os, register frt_off_t num)
|
557
557
|
{
|
558
558
|
if (os->buf.pos > VINT_END) {
|
559
559
|
while (num > 127) {
|
@@ -13,9 +13,9 @@
|
|
13
13
|
typedef struct FrtBuffer
|
14
14
|
{
|
15
15
|
frt_uchar buf[FRT_BUFFER_SIZE];
|
16
|
-
|
17
|
-
|
18
|
-
|
16
|
+
frt_off_t start;
|
17
|
+
frt_off_t pos;
|
18
|
+
frt_off_t len;
|
19
19
|
} FrtBuffer;
|
20
20
|
|
21
21
|
typedef struct FrtOutStream FrtOutStream;
|
@@ -38,13 +38,13 @@ struct FrtOutStreamMethods {
|
|
38
38
|
* @param pos the position to seek in the stream
|
39
39
|
* @raise FRT_IO_ERROR if there is an error seeking in the output stream
|
40
40
|
*/
|
41
|
-
void (*seek_i)(struct FrtOutStream *os,
|
41
|
+
void (*seek_i)(struct FrtOutStream *os, frt_off_t pos);
|
42
42
|
|
43
43
|
/**
|
44
44
|
* Close any resources used by the output stream +os+
|
45
45
|
*
|
46
46
|
* @param os self
|
47
|
-
* @raise FRT_IO_ERROR if there is an closing the file
|
47
|
+
* @raise FRT_IO_ERROR if there is an error closing the file
|
48
48
|
*/
|
49
49
|
void (*close_i)(struct FrtOutStream *os);
|
50
50
|
};
|
@@ -54,7 +54,7 @@ typedef struct FrtRAMFile
|
|
54
54
|
char *name;
|
55
55
|
frt_uchar **buffers;
|
56
56
|
int bufcnt;
|
57
|
-
|
57
|
+
frt_off_t len;
|
58
58
|
_Atomic unsigned int ref_cnt;
|
59
59
|
} FrtRAMFile;
|
60
60
|
|
@@ -66,7 +66,7 @@ struct FrtOutStream
|
|
66
66
|
int fd;
|
67
67
|
FrtRAMFile *rf;
|
68
68
|
} file;
|
69
|
-
|
69
|
+
frt_off_t pointer; /* only used by RAMOut */
|
70
70
|
const struct FrtOutStreamMethods *m;
|
71
71
|
};
|
72
72
|
|
@@ -95,7 +95,7 @@ struct FrtInStreamMethods
|
|
95
95
|
* @param pos the position to seek
|
96
96
|
* @raise FRT_IO_ERROR if the seek fails
|
97
97
|
*/
|
98
|
-
void (*seek_i)(struct FrtInStream *is,
|
98
|
+
void (*seek_i)(struct FrtInStream *is, frt_off_t pos);
|
99
99
|
|
100
100
|
/**
|
101
101
|
* Returns the length of the input stream +is+
|
@@ -103,7 +103,7 @@ struct FrtInStreamMethods
|
|
103
103
|
* @param is self
|
104
104
|
* @raise FRT_IO_ERROR if there is an error getting the file length
|
105
105
|
*/
|
106
|
-
|
106
|
+
frt_off_t (*length_i)(struct FrtInStream *is);
|
107
107
|
|
108
108
|
/**
|
109
109
|
* Close the resources allocated to the inputstream +is+
|
@@ -126,7 +126,7 @@ struct FrtInStream {
|
|
126
126
|
FrtBuffer buf;
|
127
127
|
struct FrtInStreamFile *f;
|
128
128
|
union {
|
129
|
-
|
129
|
+
frt_off_t pointer; /* only used by RAMIn */
|
130
130
|
char *path; /* only used by FSIn */
|
131
131
|
FrtCompoundInStream *cis;
|
132
132
|
} d;
|
@@ -137,8 +137,8 @@ struct FrtInStream {
|
|
137
137
|
struct FrtCompoundInStream
|
138
138
|
{
|
139
139
|
FrtInStream *sub;
|
140
|
-
|
141
|
-
|
140
|
+
frt_off_t offset;
|
141
|
+
frt_off_t length;
|
142
142
|
};
|
143
143
|
|
144
144
|
#define frt_is_length(mis) mis->m->length_i(mis)
|
@@ -274,7 +274,7 @@ struct FrtStore {
|
|
274
274
|
* @return the length of the file in bytes
|
275
275
|
* @raise FRT_IO_ERROR if there is an error checking the file length
|
276
276
|
*/
|
277
|
-
|
277
|
+
frt_off_t (*length)(FrtStore *store, const char *filename);
|
278
278
|
|
279
279
|
/**
|
280
280
|
* Allocate the resources needed for the output stream in the +store+ with
|
@@ -465,7 +465,7 @@ extern void frt_os_close(FrtOutStream *os);
|
|
465
465
|
* @param os the FrtOutStream to get the position from
|
466
466
|
* @return the current position in FrtOutStream +os+
|
467
467
|
*/
|
468
|
-
extern
|
468
|
+
extern frt_off_t frt_os_pos(FrtOutStream *os);
|
469
469
|
|
470
470
|
/**
|
471
471
|
* Set the current position in FrtOutStream +os+.
|
@@ -474,7 +474,7 @@ extern off_t frt_os_pos(FrtOutStream *os);
|
|
474
474
|
* @param pos the new position in the FrtOutStream
|
475
475
|
* @raise FRT_IO_ERROR if there is a file-system IO error seeking the file
|
476
476
|
*/
|
477
|
-
extern void frt_os_seek(FrtOutStream *os,
|
477
|
+
extern void frt_os_seek(FrtOutStream *os, frt_off_t new_pos);
|
478
478
|
|
479
479
|
/**
|
480
480
|
* Write a single byte +b+ to the FrtOutStream +os+
|
@@ -541,14 +541,14 @@ extern void frt_os_write_u64(FrtOutStream *os, frt_u64 num);
|
|
541
541
|
extern void frt_os_write_vint(FrtOutStream *os, register unsigned int num);
|
542
542
|
|
543
543
|
/**
|
544
|
-
* Write an unsigned
|
544
|
+
* Write an unsigned frt_off_t to FrtOutStream in compressed VINT format.
|
545
545
|
* TODO: describe VINT format
|
546
546
|
*
|
547
547
|
* @param os FrtOutStream to write to
|
548
|
-
* @param num the
|
548
|
+
* @param num the frt_off_t to write
|
549
549
|
* @raise FRT_IO_ERROR if there is an error writing to the file-system
|
550
550
|
*/
|
551
|
-
extern void frt_os_write_voff_t(FrtOutStream *os, register
|
551
|
+
extern void frt_os_write_voff_t(FrtOutStream *os, register frt_off_t num);
|
552
552
|
|
553
553
|
/**
|
554
554
|
* Write an unsigned 64bit int to FrtOutStream in compressed VINT format.
|
@@ -591,7 +591,7 @@ extern void frt_os_write_string(FrtOutStream *os, const char *str);
|
|
591
591
|
* @param is the FrtInStream to get the current position from
|
592
592
|
* @return the current position within the FrtInStream +is+
|
593
593
|
*/
|
594
|
-
extern
|
594
|
+
extern frt_off_t frt_is_pos(FrtInStream *is);
|
595
595
|
|
596
596
|
/**
|
597
597
|
* Set the current position in FrtInStream +is+ to +pos+.
|
@@ -601,7 +601,7 @@ extern off_t frt_is_pos(FrtInStream *is);
|
|
601
601
|
* @raise FRT_IO_ERROR if there is a error seeking from the file-system
|
602
602
|
* @raise FRT_EOF_ERROR if there is an attempt to seek past the end of the file
|
603
603
|
*/
|
604
|
-
extern void frt_is_seek(FrtInStream *is,
|
604
|
+
extern void frt_is_seek(FrtInStream *is, frt_off_t pos);
|
605
605
|
|
606
606
|
/**
|
607
607
|
* Close the FrtInStream freeing all allocated resources.
|
@@ -705,15 +705,15 @@ extern unsigned int frt_is_read_vint(FrtInStream *is);
|
|
705
705
|
extern void frt_is_skip_vints(FrtInStream *is, register int cnt);
|
706
706
|
|
707
707
|
/**
|
708
|
-
* Read a compressed (VINT) unsigned
|
708
|
+
* Read a compressed (VINT) unsigned frt_off_t from the FrtInStream.
|
709
709
|
* TODO: describe VINT format
|
710
710
|
*
|
711
711
|
* @param is the FrtInStream to read from
|
712
|
-
* @return a
|
712
|
+
* @return a frt_off_t
|
713
713
|
* @raise FRT_IO_ERROR if there is a error reading from the file-system
|
714
714
|
* @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
|
715
715
|
*/
|
716
|
-
extern
|
716
|
+
extern frt_off_t frt_is_read_voff_t(FrtInStream *is);
|
717
717
|
|
718
718
|
/**
|
719
719
|
* Read a compressed (VINT) unsigned 64bit int from the FrtInStream.
|
@@ -998,7 +998,7 @@ static void test_simulated_crashed_writer(TestCase *tc, void *data)
|
|
998
998
|
{
|
999
999
|
int i;
|
1000
1000
|
long gen;
|
1001
|
-
|
1001
|
+
frt_off_t length;
|
1002
1002
|
FrtStore *store = (FrtStore *)data;
|
1003
1003
|
FrtIndexWriter *iw;
|
1004
1004
|
FrtIndexReader *ir;
|
@@ -1059,7 +1059,7 @@ static void test_simulated_corrupt_index1(TestCase *tc, void *data)
|
|
1059
1059
|
{
|
1060
1060
|
int i;
|
1061
1061
|
long gen;
|
1062
|
-
|
1062
|
+
frt_off_t length;
|
1063
1063
|
FrtStore *store = (FrtStore *)data;
|
1064
1064
|
FrtIndexWriter *iw;
|
1065
1065
|
FrtIndexReader *ir;
|
metadata
CHANGED
@@ -1,15 +1,29 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: isomorfeus-ferret
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.13.
|
4
|
+
version: 0.13.12
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jan Biedermann
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-05-
|
11
|
+
date: 2022-05-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: oj
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
13
27
|
- !ruby/object:Gem::Dependency
|
14
28
|
name: rake
|
15
29
|
requirement: !ruby/object:Gem::Requirement
|