ferret 0.3.2 → 0.9.0
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG +9 -0
- data/Rakefile +51 -25
- data/ext/analysis.c +553 -0
- data/ext/analysis.h +76 -0
- data/ext/array.c +83 -0
- data/ext/array.h +19 -0
- data/ext/bitvector.c +164 -0
- data/ext/bitvector.h +29 -0
- data/ext/compound_io.c +335 -0
- data/ext/document.c +336 -0
- data/ext/document.h +87 -0
- data/ext/ferret.c +88 -47
- data/ext/ferret.h +43 -109
- data/ext/field.c +395 -0
- data/ext/filter.c +103 -0
- data/ext/fs_store.c +352 -0
- data/ext/global.c +219 -0
- data/ext/global.h +73 -0
- data/ext/hash.c +446 -0
- data/ext/hash.h +80 -0
- data/ext/hashset.c +141 -0
- data/ext/hashset.h +37 -0
- data/ext/helper.c +11 -0
- data/ext/helper.h +5 -0
- data/ext/inc/lang.h +41 -0
- data/ext/ind.c +389 -0
- data/ext/index.h +884 -0
- data/ext/index_io.c +269 -415
- data/ext/index_rw.c +2543 -0
- data/ext/lang.c +31 -0
- data/ext/lang.h +41 -0
- data/ext/priorityqueue.c +228 -0
- data/ext/priorityqueue.h +44 -0
- data/ext/q_boolean.c +1331 -0
- data/ext/q_const_score.c +154 -0
- data/ext/q_fuzzy.c +287 -0
- data/ext/q_match_all.c +142 -0
- data/ext/q_multi_phrase.c +343 -0
- data/ext/q_parser.c +2180 -0
- data/ext/q_phrase.c +657 -0
- data/ext/q_prefix.c +75 -0
- data/ext/q_range.c +247 -0
- data/ext/q_span.c +1566 -0
- data/ext/q_term.c +308 -0
- data/ext/q_wildcard.c +146 -0
- data/ext/r_analysis.c +255 -0
- data/ext/r_doc.c +578 -0
- data/ext/r_index_io.c +996 -0
- data/ext/r_qparser.c +158 -0
- data/ext/r_search.c +2321 -0
- data/ext/r_store.c +263 -0
- data/ext/r_term.c +219 -0
- data/ext/ram_store.c +447 -0
- data/ext/search.c +524 -0
- data/ext/search.h +1065 -0
- data/ext/similarity.c +143 -39
- data/ext/sort.c +661 -0
- data/ext/store.c +35 -0
- data/ext/store.h +152 -0
- data/ext/term.c +704 -143
- data/ext/termdocs.c +599 -0
- data/ext/vector.c +594 -0
- data/lib/ferret.rb +9 -10
- data/lib/ferret/analysis/analyzers.rb +2 -2
- data/lib/ferret/analysis/standard_tokenizer.rb +1 -1
- data/lib/ferret/analysis/token.rb +14 -14
- data/lib/ferret/analysis/token_filters.rb +3 -3
- data/lib/ferret/document/field.rb +16 -17
- data/lib/ferret/index/document_writer.rb +4 -4
- data/lib/ferret/index/index.rb +39 -23
- data/lib/ferret/index/index_writer.rb +2 -2
- data/lib/ferret/index/multiple_term_doc_pos_enum.rb +1 -8
- data/lib/ferret/index/segment_term_vector.rb +4 -4
- data/lib/ferret/index/term.rb +5 -1
- data/lib/ferret/index/term_vector_offset_info.rb +6 -6
- data/lib/ferret/index/term_vectors_io.rb +5 -5
- data/lib/ferret/query_parser/query_parser.tab.rb +81 -77
- data/lib/ferret/search.rb +1 -1
- data/lib/ferret/search/boolean_query.rb +2 -1
- data/lib/ferret/search/field_sorted_hit_queue.rb +3 -3
- data/lib/ferret/search/fuzzy_query.rb +2 -1
- data/lib/ferret/search/index_searcher.rb +3 -0
- data/lib/ferret/search/{match_all_docs_query.rb → match_all_query.rb} +7 -7
- data/lib/ferret/search/multi_phrase_query.rb +6 -5
- data/lib/ferret/search/phrase_query.rb +3 -6
- data/lib/ferret/search/prefix_query.rb +4 -4
- data/lib/ferret/search/sort.rb +3 -1
- data/lib/ferret/search/sort_field.rb +9 -9
- data/lib/ferret/search/spans/near_spans_enum.rb +1 -1
- data/lib/ferret/search/spans/span_near_query.rb +1 -1
- data/lib/ferret/search/spans/span_weight.rb +1 -1
- data/lib/ferret/search/spans/spans_enum.rb +7 -7
- data/lib/ferret/store/fs_store.rb +10 -6
- data/lib/ferret/store/ram_store.rb +3 -3
- data/lib/rferret.rb +36 -0
- data/test/functional/thread_safety_index_test.rb +2 -2
- data/test/test_helper.rb +16 -2
- data/test/unit/analysis/c_token.rb +25 -0
- data/test/unit/analysis/tc_per_field_analyzer_wrapper.rb +1 -1
- data/test/unit/analysis/tc_standard_analyzer.rb +1 -1
- data/test/unit/document/{tc_document.rb → c_document.rb} +0 -0
- data/test/unit/document/c_field.rb +98 -0
- data/test/unit/document/tc_field.rb +0 -66
- data/test/unit/index/{tc_index.rb → c_index.rb} +62 -6
- data/test/unit/index/{tc_index_reader.rb → c_index_reader.rb} +51 -10
- data/test/unit/index/{tc_index_writer.rb → c_index_writer.rb} +0 -4
- data/test/unit/index/{tc_term.rb → c_term.rb} +1 -3
- data/test/unit/index/{tc_term_vector_offset_info.rb → c_term_voi.rb} +5 -5
- data/test/unit/index/tc_segment_term_vector.rb +2 -2
- data/test/unit/index/tc_term_vectors_io.rb +4 -4
- data/test/unit/query_parser/c_query_parser.rb +138 -0
- data/test/unit/search/{tc_filter.rb → c_filter.rb} +24 -24
- data/test/unit/search/{tc_fuzzy_query.rb → c_fuzzy_query.rb} +0 -0
- data/test/unit/search/{tc_index_searcher.rb → c_index_searcher.rb} +9 -26
- data/test/unit/search/{tc_search_and_sort.rb → c_search_and_sort.rb} +15 -15
- data/test/unit/search/{tc_sort.rb → c_sort.rb} +2 -1
- data/test/unit/search/c_sort_field.rb +27 -0
- data/test/unit/search/{tc_spans.rb → c_spans.rb} +0 -0
- data/test/unit/search/tc_sort_field.rb +7 -20
- data/test/unit/store/c_fs_store.rb +76 -0
- data/test/unit/store/c_ram_store.rb +35 -0
- data/test/unit/store/m_store.rb +34 -0
- data/test/unit/store/m_store_lock.rb +68 -0
- data/test/unit/store/tc_fs_store.rb +0 -53
- data/test/unit/store/tc_ram_store.rb +0 -20
- data/test/unit/store/tm_store.rb +0 -30
- data/test/unit/store/tm_store_lock.rb +0 -66
- metadata +84 -31
- data/ext/Makefile +0 -140
- data/ext/ferret_ext.so +0 -0
- data/ext/priority_queue.c +0 -232
- data/ext/ram_directory.c +0 -321
- data/ext/segment_merge_queue.c +0 -37
- data/ext/segment_term_enum.c +0 -326
- data/ext/string_helper.c +0 -42
- data/ext/tags +0 -344
- data/ext/term_buffer.c +0 -230
- data/ext/term_infos_reader.c +0 -54
- data/ext/terminfo.c +0 -160
- data/ext/token.c +0 -93
- data/ext/util.c +0 -12
data/ext/index_io.c
CHANGED
@@ -1,523 +1,377 @@
|
|
1
|
-
#include
|
1
|
+
#include <store.h>
|
2
|
+
#include <string.h>
|
2
3
|
|
3
|
-
|
4
|
-
|
5
|
-
/****************************************************************************
|
6
|
-
*
|
7
|
-
* BufferIndexInput Methods
|
8
|
-
*
|
9
|
-
****************************************************************************/
|
10
|
-
|
11
|
-
void
|
12
|
-
frt_indexbuffer_free(void *p)
|
4
|
+
Buffer *buf_create()
|
13
5
|
{
|
14
|
-
|
15
|
-
|
16
|
-
|
6
|
+
Buffer *buf = ALLOC(Buffer);
|
7
|
+
buf->start = 0;
|
8
|
+
buf->pos = 0;
|
9
|
+
buf->len = 0;
|
10
|
+
return buf;
|
17
11
|
}
|
18
12
|
|
19
|
-
|
20
|
-
frt_indexbuffer_alloc(VALUE klass)
|
13
|
+
void buf_destroy(Buffer *buf)
|
21
14
|
{
|
22
|
-
|
23
|
-
|
24
|
-
my_buf = ALLOC(IndexBuffer);
|
25
|
-
MEMZERO(my_buf, IndexBuffer, 1);
|
26
|
-
my_buf->buffer = ALLOC_N(byte_t, BUFFER_SIZE);
|
27
|
-
|
28
|
-
return Data_Wrap_Struct(klass, NULL, frt_indexbuffer_free, my_buf);
|
15
|
+
free(buf);
|
29
16
|
}
|
30
17
|
|
31
|
-
|
32
|
-
static VALUE
|
33
|
-
frt_indexin_init_copy(VALUE self, VALUE orig)
|
18
|
+
OutStream *os_create()
|
34
19
|
{
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
Data_Get_Struct(orig, IndexBuffer, orig_buf);
|
42
|
-
|
43
|
-
len = orig_buf->len;
|
44
|
-
my_buf->len = len;
|
45
|
-
my_buf->pos = orig_buf->pos;
|
46
|
-
my_buf->start = orig_buf->start;
|
47
|
-
|
48
|
-
MEMCPY(my_buf->buffer, orig_buf->buffer, byte_t, len);
|
49
|
-
|
50
|
-
return self;
|
20
|
+
OutStream *os = ALLOC(OutStream);
|
21
|
+
os->buf.start = 0;
|
22
|
+
os->buf.pos = 0;
|
23
|
+
os->buf.len = 0;
|
24
|
+
return os;
|
51
25
|
}
|
52
26
|
|
53
|
-
|
54
|
-
frt_indexin_refill(VALUE self)
|
27
|
+
inline void os_flush(OutStream *os)
|
55
28
|
{
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
int input_len = FIX2INT(rb_funcall(self, frt_length, 0, NULL));
|
60
|
-
GET_MY_BUF;
|
61
|
-
|
62
|
-
start = my_buf->start + my_buf->pos;
|
63
|
-
stop = start + BUFFER_SIZE;
|
64
|
-
if (stop > input_len) {
|
65
|
-
stop = input_len;
|
66
|
-
}
|
67
|
-
|
68
|
-
len_to_read = stop - start;
|
69
|
-
if (len_to_read <= 0) {
|
70
|
-
rb_raise(rb_eEOFError, "IndexInput: Read past End of File");
|
71
|
-
}
|
72
|
-
|
73
|
-
rStr = rb_str_new((char *)my_buf->buffer, BUFFER_SIZE);
|
74
|
-
rb_funcall(self, frt_read_internal, 3,
|
75
|
-
rStr, INT2FIX(0), INT2FIX(len_to_read));
|
76
|
-
|
77
|
-
memcpy(my_buf->buffer, RSTRING(rStr)->ptr, BUFFER_SIZE);
|
78
|
-
/* my_buf->buffer = StringValuePtr(rStr); */
|
79
|
-
|
80
|
-
my_buf->len = len_to_read;
|
81
|
-
my_buf->start = start;
|
82
|
-
my_buf->pos = 0;
|
83
|
-
|
84
|
-
return Qnil;
|
29
|
+
os->flush_internal(os, os->buf.buf, os->buf.pos);
|
30
|
+
os->buf.start += os->buf.pos;
|
31
|
+
os->buf.pos = 0;
|
85
32
|
}
|
86
33
|
|
87
|
-
|
88
|
-
frt_read_byte(VALUE self, IndexBuffer *my_buf)
|
34
|
+
void os_close(OutStream *os)
|
89
35
|
{
|
90
|
-
|
91
|
-
|
92
|
-
|
36
|
+
os_flush(os);
|
37
|
+
os->close_internal(os);
|
38
|
+
free(os);
|
93
39
|
}
|
94
40
|
|
95
|
-
|
96
|
-
frt_indexin_read_byte(VALUE self)
|
97
|
-
{
|
98
|
-
GET_MY_BUF;
|
99
|
-
return INT2FIX(frt_read_byte(self, my_buf));
|
100
|
-
}
|
101
|
-
|
102
|
-
static VALUE
|
103
|
-
frt_indexin_pos(VALUE self)
|
41
|
+
int os_pos(OutStream *os)
|
104
42
|
{
|
105
|
-
|
106
|
-
Data_Get_Struct(self, IndexBuffer, my_buf);
|
107
|
-
return INT2FIX(my_buf->start + my_buf->pos);
|
43
|
+
return os->buf.start + os->buf.pos;
|
108
44
|
}
|
109
45
|
|
110
|
-
|
111
|
-
frt_read_bytes(VALUE self, VALUE rbuffer, int offset, int len)
|
46
|
+
void os_seek(OutStream *os, int new_pos)
|
112
47
|
{
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
GET_MY_BUF;
|
117
|
-
|
118
|
-
if (RSTRING(rbuf)->len < (offset + len)) {
|
119
|
-
rb_str_resize(rbuf, offset + len);
|
120
|
-
}
|
121
|
-
if ((len + offset) < BUFFER_SIZE) {
|
122
|
-
rb_str_modify(rbuf);
|
123
|
-
for (i = offset; i < offset + len; i++) {
|
124
|
-
RSTRING(rbuf)->ptr[i] = frt_read_byte(self, my_buf);
|
125
|
-
}
|
126
|
-
} else {
|
127
|
-
VALUE start = frt_indexin_pos(self);
|
128
|
-
rb_funcall(self, frt_seek_internal, 1, start);
|
129
|
-
rb_funcall(self, frt_read_internal, 3,
|
130
|
-
rbuf, INT2FIX(offset), INT2FIX(len));
|
131
|
-
|
132
|
-
my_buf->start = my_buf->start + len;
|
133
|
-
my_buf->pos = 0;
|
134
|
-
my_buf->len = 0; /* trigger refill() on read() */
|
135
|
-
}
|
136
|
-
|
137
|
-
return rbuf;
|
48
|
+
os_flush(os);
|
49
|
+
os->buf.start = new_pos;
|
50
|
+
os->seek_internal(os, new_pos);
|
138
51
|
}
|
139
52
|
|
140
|
-
|
141
|
-
frt_indexin_read_bytes(VALUE self, VALUE rbuf, VALUE roffset, VALUE rlen)
|
142
|
-
{
|
143
|
-
int len, offset;
|
53
|
+
#define write_byte(os, b) os->buf.buf[os->buf.pos++] = b
|
144
54
|
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
55
|
+
inline void os_write_byte(OutStream *os, uchar b)
|
56
|
+
{
|
57
|
+
if (os->buf.pos >= BUFFER_SIZE)
|
58
|
+
os_flush(os);
|
59
|
+
write_byte(os, b);
|
149
60
|
}
|
150
61
|
|
151
|
-
|
152
|
-
frt_indexin_seek(VALUE self, VALUE rpos)
|
62
|
+
void os_write_bytes(OutStream *os, uchar *b, int len)
|
153
63
|
{
|
154
|
-
|
155
|
-
|
156
|
-
GET_MY_BUF;
|
64
|
+
if (os->buf.pos > 0) // flush buffer
|
65
|
+
os_flush(os);
|
157
66
|
|
158
|
-
if (
|
159
|
-
|
67
|
+
if (len < BUFFER_SIZE) {
|
68
|
+
os->flush_internal(os, b, len);
|
69
|
+
os->buf.start += len;
|
160
70
|
} else {
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
71
|
+
int pos = 0;
|
72
|
+
int size;
|
73
|
+
while (pos < len) {
|
74
|
+
if (len - pos < BUFFER_SIZE) {
|
75
|
+
size = len - pos;
|
76
|
+
} else {
|
77
|
+
size = BUFFER_SIZE;
|
78
|
+
}
|
79
|
+
os->flush_internal(os, b + pos, size);
|
80
|
+
pos += size;
|
81
|
+
os->buf.start += size;
|
82
|
+
}
|
165
83
|
}
|
166
|
-
return Qnil;
|
167
84
|
}
|
168
85
|
|
169
|
-
|
170
|
-
frt_indexin_read_int(VALUE self)
|
86
|
+
InStream *is_create()
|
171
87
|
{
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
88
|
+
InStream *is = ALLOC(InStream);
|
89
|
+
is->buf.start = 0;
|
90
|
+
is->buf.pos = 0;
|
91
|
+
is->buf.len = 0;
|
92
|
+
return is;
|
177
93
|
}
|
178
94
|
|
179
|
-
|
180
|
-
frt_indexin_read_long(VALUE self)
|
95
|
+
void is_refill(InStream *is)
|
181
96
|
{
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
((long long)frt_read_byte(self, my_buf) << 24) |
|
188
|
-
((long long)frt_read_byte(self, my_buf) << 16) |
|
189
|
-
((long long)frt_read_byte(self, my_buf) << 8) |
|
190
|
-
(long long)frt_read_byte(self, my_buf));
|
191
|
-
}
|
97
|
+
int start = is->buf.start + is->buf.pos;
|
98
|
+
int last = start + BUFFER_SIZE;
|
99
|
+
int flen = is->length_internal(is);
|
100
|
+
if (last > flen) // don't read past EOF
|
101
|
+
last = flen;
|
192
102
|
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
return ULONG2NUM(((unsigned int)frt_read_byte(self, my_buf) << 24) |
|
198
|
-
((unsigned int)frt_read_byte(self, my_buf) << 16) |
|
199
|
-
((unsigned int)frt_read_byte(self, my_buf) << 8) |
|
200
|
-
(unsigned int)frt_read_byte(self, my_buf));
|
201
|
-
}
|
103
|
+
is->buf.len = last - start;
|
104
|
+
if (is->buf.len <= 0) {
|
105
|
+
eprintf(IO_ERROR, "EOF Error when trying to refill. flen was %d\n", flen);
|
106
|
+
}
|
202
107
|
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
return ULL2NUM(((unsigned long long)frt_read_byte(self, my_buf) << 56) |
|
208
|
-
((unsigned long long)frt_read_byte(self, my_buf) << 48) |
|
209
|
-
((unsigned long long)frt_read_byte(self, my_buf) << 40) |
|
210
|
-
((unsigned long long)frt_read_byte(self, my_buf) << 32) |
|
211
|
-
((unsigned long long)frt_read_byte(self, my_buf) << 24) |
|
212
|
-
((unsigned long long)frt_read_byte(self, my_buf) << 16) |
|
213
|
-
((unsigned long long)frt_read_byte(self, my_buf) << 8) |
|
214
|
-
(unsigned long long)frt_read_byte(self, my_buf));
|
108
|
+
is->read_internal(is, is->buf.buf, 0, is->buf.len);
|
109
|
+
|
110
|
+
is->buf.start = start;
|
111
|
+
is->buf.pos = 0;
|
215
112
|
}
|
216
113
|
|
217
|
-
|
218
|
-
|
114
|
+
#define read_byte(is) is->buf.buf[is->buf.pos++]
|
115
|
+
inline uchar is_read_byte(InStream *is)
|
219
116
|
{
|
220
|
-
|
221
|
-
|
117
|
+
if (is->buf.pos >= is->buf.len)
|
118
|
+
is_refill(is);
|
222
119
|
|
223
|
-
|
224
|
-
i = b & 0x7F; /* 0x7F = 0b01111111 */
|
225
|
-
|
226
|
-
while ((b & 0x80) != 0) {/* 0x80 = 0b10000000 */
|
227
|
-
b = frt_read_byte(self, my_buf);
|
228
|
-
i |= (b & 0x7F) << shift;
|
229
|
-
shift += 7;
|
230
|
-
}
|
231
|
-
|
232
|
-
return i;
|
120
|
+
return read_byte(is);
|
233
121
|
}
|
234
122
|
|
235
|
-
|
236
|
-
frt_indexin_read_vint(VALUE self)
|
123
|
+
int is_pos(InStream *is)
|
237
124
|
{
|
238
|
-
|
239
|
-
return ULL2NUM(frt_read_vint(self, my_buf));
|
125
|
+
return is->buf.start + is->buf.pos;
|
240
126
|
}
|
241
127
|
|
242
|
-
|
243
|
-
frt_read_chars(VALUE self, char* buffer, int off, int len)
|
128
|
+
uchar *is_read_bytes(InStream *is, uchar *b, int offset, int len)
|
244
129
|
{
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
130
|
+
int i, start;
|
131
|
+
if ((offset + len) < BUFFER_SIZE) {
|
132
|
+
for (i = offset; i < offset + len; i++) {
|
133
|
+
b[i] = is_read_byte(is);
|
134
|
+
}
|
135
|
+
} else { // read all-at-once
|
136
|
+
start = is_pos(is);
|
137
|
+
is->seek_internal(is, start);
|
138
|
+
is->read_internal(is, b, offset, len);
|
139
|
+
|
140
|
+
is->buf.start = start + len; // adjust stream variables
|
141
|
+
is->buf.pos = 0;
|
142
|
+
is->buf.len = 0; // trigger refill on read
|
255
143
|
}
|
144
|
+
return b;
|
256
145
|
}
|
257
146
|
|
258
|
-
|
259
|
-
frt_indexin_read_string(VALUE self)
|
147
|
+
void is_seek(InStream *is, int pos)
|
260
148
|
{
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
return rb_str_new(str, length);
|
149
|
+
if (pos >= is->buf.start && pos < (is->buf.start + is->buf.len)) {
|
150
|
+
is->buf.pos = pos - is->buf.start; // seek within buffer
|
151
|
+
} else {
|
152
|
+
is->buf.start = pos;
|
153
|
+
is->buf.pos = 0;
|
154
|
+
is->buf.len = 0; // trigger refill() on read()
|
155
|
+
is->seek_internal(is, pos);
|
156
|
+
}
|
270
157
|
}
|
271
158
|
|
272
|
-
|
273
|
-
*
|
274
|
-
* BufferIndexInput Methods
|
275
|
-
*
|
276
|
-
****************************************************************************/
|
277
|
-
|
278
|
-
static VALUE
|
279
|
-
frt_indexout_flush(VALUE self)
|
159
|
+
void is_close(InStream *is)
|
280
160
|
{
|
281
|
-
|
282
|
-
|
283
|
-
rb_funcall(self, frt_flush_buffer, 2,
|
284
|
-
rb_str_new((char *)my_buf->buffer, BUFFER_SIZE), INT2FIX(my_buf->pos));
|
285
|
-
|
286
|
-
my_buf->start += my_buf->pos;
|
287
|
-
my_buf->pos = 0;
|
288
|
-
|
289
|
-
return Qnil;
|
161
|
+
is->close_internal(is);
|
162
|
+
free(is);
|
290
163
|
}
|
291
164
|
|
292
|
-
|
293
|
-
frt_write_byte(VALUE self, byte_t b)
|
165
|
+
InStream *is_clone(InStream *is)
|
294
166
|
{
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
|
167
|
+
InStream *new_index_i = ALLOC(InStream);
|
168
|
+
memcpy(new_index_i, is, sizeof(InStream));
|
169
|
+
//new_index_i->buf.start = is->buf.start;
|
170
|
+
//new_index_i->buf.pos = is->buf.pos;
|
171
|
+
//new_index_i->buf.len = is->buf.len;
|
172
|
+
//new_index_i->file = is->file;
|
173
|
+
//new_index_i->d = is->d;
|
174
|
+
//new_index_i->read_internal = is->read_internal;
|
175
|
+
//new_index_i->seek_internal = is->seek_internal;
|
176
|
+
//new_index_i->length_internal = is->length_internal;
|
177
|
+
//new_index_i->clone_internal = is->clone_internal;
|
178
|
+
//new_index_i->close_internal = is->close_internal;
|
179
|
+
new_index_i->is_clone = true;
|
180
|
+
is->clone_internal(is, new_index_i);
|
181
|
+
return new_index_i;
|
302
182
|
}
|
303
183
|
|
304
|
-
|
305
|
-
|
184
|
+
int
|
185
|
+
is_read_int(InStream *is)
|
306
186
|
{
|
307
|
-
|
308
|
-
|
309
|
-
|
187
|
+
return ((int)is_read_byte(is) << 24) |
|
188
|
+
((int)is_read_byte(is) << 16) |
|
189
|
+
((int)is_read_byte(is) << 8) |
|
190
|
+
(int)is_read_byte(is);
|
310
191
|
}
|
311
192
|
|
312
|
-
|
313
|
-
|
193
|
+
long long
|
194
|
+
is_read_long(InStream *is)
|
314
195
|
{
|
315
|
-
|
316
|
-
|
317
|
-
|
196
|
+
return ((long long)is_read_byte(is) << 56) |
|
197
|
+
((long long)is_read_byte(is) << 48) |
|
198
|
+
((long long)is_read_byte(is) << 40) |
|
199
|
+
((long long)is_read_byte(is) << 32) |
|
200
|
+
((long long)is_read_byte(is) << 24) |
|
201
|
+
((long long)is_read_byte(is) << 16) |
|
202
|
+
((long long)is_read_byte(is) << 8) |
|
203
|
+
(long long)is_read_byte(is);
|
318
204
|
}
|
319
205
|
|
320
|
-
|
321
|
-
|
206
|
+
unsigned int
|
207
|
+
is_read_uint(InStream *is)
|
322
208
|
{
|
323
|
-
int
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
for (i = 0; i < len; i++)
|
328
|
-
frt_write_byte(self, RSTRING(rbuf)->ptr[i]);
|
329
|
-
|
330
|
-
return Qnil;
|
209
|
+
return ((unsigned int)is_read_byte(is) << 24) |
|
210
|
+
((unsigned int)is_read_byte(is) << 16) |
|
211
|
+
((unsigned int)is_read_byte(is) << 8) |
|
212
|
+
(unsigned int)is_read_byte(is);
|
331
213
|
}
|
332
214
|
|
333
|
-
|
334
|
-
|
215
|
+
unsigned long long
|
216
|
+
is_read_ulong(InStream *is)
|
335
217
|
{
|
336
|
-
|
337
|
-
|
218
|
+
return ((unsigned long long)is_read_byte(is) << 56) |
|
219
|
+
((unsigned long long)is_read_byte(is) << 48) |
|
220
|
+
((unsigned long long)is_read_byte(is) << 40) |
|
221
|
+
((unsigned long long)is_read_byte(is) << 32) |
|
222
|
+
((unsigned long long)is_read_byte(is) << 24) |
|
223
|
+
((unsigned long long)is_read_byte(is) << 16) |
|
224
|
+
((unsigned long long)is_read_byte(is) << 8) |
|
225
|
+
(unsigned long long)is_read_byte(is);
|
338
226
|
}
|
339
227
|
|
340
|
-
|
341
|
-
|
228
|
+
/* optimized to use unchecked read_byte if there is definitely space */
|
229
|
+
inline unsigned long long
|
230
|
+
is_read_vint(InStream *is)
|
342
231
|
{
|
343
|
-
|
344
|
-
|
345
|
-
frt_indexout_flush(self);
|
346
|
-
my_buf->start = FIX2INT(pos);
|
232
|
+
register unsigned long long res, b;
|
233
|
+
register int shift = 7;
|
347
234
|
|
348
|
-
|
235
|
+
if (is->buf.pos > (is->buf.len - VINT_MAX_LEN)) {
|
236
|
+
b = is_read_byte(is);
|
237
|
+
res = b & 0x7F; // 0x7F = 0b01111111
|
238
|
+
|
239
|
+
while ((b & 0x80) != 0) {// 0x80 = 0b10000000
|
240
|
+
b = is_read_byte(is);
|
241
|
+
res |= (b & 0x7F) << shift;
|
242
|
+
shift += 7;
|
243
|
+
}
|
244
|
+
} else { // unchecked
|
245
|
+
b = read_byte(is);
|
246
|
+
res = b & 0x7F; // 0x7F = 0b01111111
|
247
|
+
|
248
|
+
while ((b & 0x80) != 0) {// 0x80 = 0b10000000
|
249
|
+
b = read_byte(is);
|
250
|
+
res |= (b & 0x7F) << shift;
|
251
|
+
shift += 7;
|
252
|
+
}
|
253
|
+
}
|
254
|
+
|
255
|
+
return res;
|
349
256
|
}
|
350
257
|
|
351
|
-
|
352
|
-
|
258
|
+
inline void
|
259
|
+
is_read_chars(InStream *is, char* buffer, int off, int len)
|
353
260
|
{
|
354
|
-
|
355
|
-
frt_write_byte(self, (l >> 24) & 0xFF);
|
356
|
-
frt_write_byte(self, (l >> 16) & 0xFF);
|
357
|
-
frt_write_byte(self, (l >> 8) & 0xFF);
|
358
|
-
frt_write_byte(self, l & 0xFF);
|
359
|
-
|
360
|
-
return Qnil;
|
361
|
-
}
|
261
|
+
int end, i;
|
362
262
|
|
363
|
-
|
364
|
-
frt_indexout_write_long(VALUE self, VALUE rlong)
|
365
|
-
{
|
366
|
-
long long l = NUM2LL(rlong);
|
367
|
-
frt_write_byte(self, (l >> 56) & 0xFF);
|
368
|
-
frt_write_byte(self, (l >> 48) & 0xFF);
|
369
|
-
frt_write_byte(self, (l >> 40) & 0xFF);
|
370
|
-
frt_write_byte(self, (l >> 32) & 0xFF);
|
371
|
-
frt_write_byte(self, (l >> 24) & 0xFF);
|
372
|
-
frt_write_byte(self, (l >> 16) & 0xFF);
|
373
|
-
frt_write_byte(self, (l >> 8) & 0xFF);
|
374
|
-
frt_write_byte(self, l & 0xFF);
|
263
|
+
end = off + len;
|
375
264
|
|
376
|
-
|
265
|
+
for(i = off; i < end; i++) {
|
266
|
+
buffer[i] = is_read_byte(is);
|
267
|
+
}
|
377
268
|
}
|
378
269
|
|
379
|
-
|
380
|
-
|
270
|
+
char *
|
271
|
+
is_read_string(InStream *is)
|
381
272
|
{
|
382
|
-
|
383
|
-
|
384
|
-
|
385
|
-
frt_write_byte(self, (l >> 8) & 0xFF);
|
386
|
-
frt_write_byte(self, l & 0xFF);
|
273
|
+
register int length = (int)is_read_vint(is);
|
274
|
+
char *str = ALLOC_N(char, length + 1);
|
275
|
+
str[length] = '\0';
|
387
276
|
|
388
|
-
|
277
|
+
if (is->buf.pos > (is->buf.len - length)) {
|
278
|
+
register int i;
|
279
|
+
for(i = 0; i < length; i++) {
|
280
|
+
str[i] = is_read_byte(is);
|
281
|
+
}
|
282
|
+
} else { // unchecked
|
283
|
+
memcpy(str, is->buf.buf + is->buf.pos, length);
|
284
|
+
is->buf.pos += length;
|
285
|
+
}
|
286
|
+
//is_read_chars(is, str, 0, length);
|
287
|
+
|
288
|
+
return str;
|
389
289
|
}
|
390
290
|
|
391
|
-
|
392
|
-
|
291
|
+
void
|
292
|
+
os_write_int(OutStream *os, int l)
|
393
293
|
{
|
394
|
-
|
395
|
-
l
|
396
|
-
|
397
|
-
|
398
|
-
frt_write_byte(self, (l >> 40) & 0xFF);
|
399
|
-
frt_write_byte(self, (l >> 32) & 0xFF);
|
400
|
-
frt_write_byte(self, (l >> 24) & 0xFF);
|
401
|
-
frt_write_byte(self, (l >> 16) & 0xFF);
|
402
|
-
frt_write_byte(self, (l >> 8) & 0xFF);
|
403
|
-
frt_write_byte(self, l & 0xFF);
|
404
|
-
|
405
|
-
return Qnil;
|
294
|
+
os_write_byte(os, (l >> 24) & 0xFF);
|
295
|
+
os_write_byte(os, (l >> 16) & 0xFF);
|
296
|
+
os_write_byte(os, (l >> 8) & 0xFF);
|
297
|
+
os_write_byte(os, l & 0xFF);
|
406
298
|
}
|
407
299
|
|
408
|
-
|
409
|
-
|
300
|
+
void
|
301
|
+
os_write_long(OutStream *os, long long l)
|
410
302
|
{
|
411
|
-
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
303
|
+
os_write_byte(os, (l >> 56) & 0xFF);
|
304
|
+
os_write_byte(os, (l >> 48) & 0xFF);
|
305
|
+
os_write_byte(os, (l >> 40) & 0xFF);
|
306
|
+
os_write_byte(os, (l >> 32) & 0xFF);
|
307
|
+
os_write_byte(os, (l >> 24) & 0xFF);
|
308
|
+
os_write_byte(os, (l >> 16) & 0xFF);
|
309
|
+
os_write_byte(os, (l >> 8) & 0xFF);
|
310
|
+
os_write_byte(os, l & 0xFF);
|
418
311
|
}
|
419
312
|
|
420
|
-
|
421
|
-
|
313
|
+
void
|
314
|
+
os_write_uint(OutStream *os, unsigned int l)
|
422
315
|
{
|
423
|
-
|
316
|
+
os_write_byte(os, (l >> 24) & 0xFF);
|
317
|
+
os_write_byte(os, (l >> 16) & 0xFF);
|
318
|
+
os_write_byte(os, (l >> 8) & 0xFF);
|
319
|
+
os_write_byte(os, l & 0xFF);
|
320
|
+
}
|
424
321
|
|
425
|
-
|
426
|
-
|
427
|
-
|
322
|
+
void
|
323
|
+
os_write_ulong(OutStream *os, unsigned long long l)
|
324
|
+
{
|
325
|
+
os_write_byte(os, (l >> 56) & 0xFF);
|
326
|
+
os_write_byte(os, (l >> 48) & 0xFF);
|
327
|
+
os_write_byte(os, (l >> 40) & 0xFF);
|
328
|
+
os_write_byte(os, (l >> 32) & 0xFF);
|
329
|
+
os_write_byte(os, (l >> 24) & 0xFF);
|
330
|
+
os_write_byte(os, (l >> 16) & 0xFF);
|
331
|
+
os_write_byte(os, (l >> 8) & 0xFF);
|
332
|
+
os_write_byte(os, l & 0xFF);
|
333
|
+
}
|
334
|
+
|
335
|
+
/* optimized to use an unchecked write if there is space */
|
336
|
+
inline void
|
337
|
+
os_write_vint(OutStream *os, register unsigned long long i)
|
338
|
+
{
|
339
|
+
if (os->buf.pos > VINT_END) {
|
340
|
+
while (i > 127) {
|
341
|
+
os_write_byte(os, (i & 0x7f) | 0x80);
|
342
|
+
i >>= 7;
|
343
|
+
}
|
344
|
+
os_write_byte(os, i);
|
345
|
+
} else {
|
346
|
+
while (i > 127) {
|
347
|
+
write_byte(os, (i & 0x7f) | 0x80);
|
348
|
+
i >>= 7;
|
349
|
+
}
|
350
|
+
write_byte(os, i);
|
428
351
|
}
|
429
|
-
frt_write_byte(self, i);
|
430
|
-
|
431
|
-
return Qnil;
|
432
352
|
}
|
433
353
|
|
434
|
-
|
435
|
-
|
354
|
+
void
|
355
|
+
os_write_chars(OutStream *os, char *buf, int start, int length)
|
436
356
|
{
|
437
357
|
int i;
|
438
|
-
VALUE rstr = StringValue(rbuf);
|
439
358
|
|
440
359
|
for (i = start; i < start + length; i++) {
|
441
|
-
|
360
|
+
os_write_byte(os, buf[i]);
|
442
361
|
}
|
443
|
-
|
444
|
-
return Qnil;
|
445
362
|
}
|
446
363
|
|
447
|
-
|
448
|
-
|
364
|
+
void
|
365
|
+
os_write_string(OutStream *os, char *str)
|
449
366
|
{
|
450
|
-
int
|
451
|
-
|
367
|
+
int len = strlen(str);
|
368
|
+
os_write_vint(os, len);
|
452
369
|
|
453
|
-
|
370
|
+
os_write_chars(os, str, 0, len);
|
454
371
|
}
|
455
372
|
|
456
|
-
|
457
|
-
frt_indexout_write_string(VALUE self, VALUE rstr)
|
373
|
+
int file_is_lock(char *filename)
|
458
374
|
{
|
459
|
-
int
|
460
|
-
|
461
|
-
|
462
|
-
frt_write_chars(self, rstr, 0, len);
|
463
|
-
return Qnil;
|
464
|
-
}
|
465
|
-
|
466
|
-
/****************************************************************************
|
467
|
-
*
|
468
|
-
* Init Function
|
469
|
-
*
|
470
|
-
****************************************************************************/
|
471
|
-
|
472
|
-
void
|
473
|
-
Init_indexio(void)
|
474
|
-
{
|
475
|
-
/* IDs */
|
476
|
-
frt_length = rb_intern("length");
|
477
|
-
frt_flush_buffer = rb_intern("flush_buffer");
|
478
|
-
frt_read_internal = rb_intern("read_internal");
|
479
|
-
frt_seek_internal = rb_intern("seek_internal");
|
480
|
-
|
481
|
-
/* IndexInput */
|
482
|
-
cIndexIn = rb_define_class_under(mStore, "IndexInput", rb_cObject);
|
483
|
-
cBufferedIndexIn = rb_define_class_under(mStore, "BufferedIndexInput", cIndexIn);
|
484
|
-
rb_define_alloc_func(cBufferedIndexIn, frt_indexbuffer_alloc);
|
485
|
-
|
486
|
-
rb_define_method(cBufferedIndexIn, "initialize_copy", frt_indexin_init_copy, 1);
|
487
|
-
rb_define_method(cBufferedIndexIn, "refill", frt_indexin_refill, 0);
|
488
|
-
rb_define_method(cBufferedIndexIn, "read_byte", frt_indexin_read_byte, 0);
|
489
|
-
rb_define_method(cBufferedIndexIn, "read_bytes", frt_indexin_read_bytes, 3);
|
490
|
-
rb_define_method(cBufferedIndexIn, "pos", frt_indexin_pos, 0);
|
491
|
-
rb_define_method(cBufferedIndexIn, "seek", frt_indexin_seek, 1);
|
492
|
-
rb_define_method(cBufferedIndexIn, "read_int", frt_indexin_read_int, 0);
|
493
|
-
rb_define_method(cBufferedIndexIn, "read_long", frt_indexin_read_long, 0);
|
494
|
-
rb_define_method(cBufferedIndexIn, "read_uint", frt_indexin_read_uint, 0);
|
495
|
-
rb_define_method(cBufferedIndexIn, "read_ulong", frt_indexin_read_ulong, 0);
|
496
|
-
rb_define_method(cBufferedIndexIn, "read_vint", frt_indexin_read_vint, 0);
|
497
|
-
rb_define_method(cBufferedIndexIn, "read_vlong", frt_indexin_read_vint, 0);
|
498
|
-
rb_define_method(cBufferedIndexIn, "read_string", frt_indexin_read_string, 0);
|
499
|
-
rb_define_method(cBufferedIndexIn, "read_chars", frt_indexin_read_bytes, 3);
|
500
|
-
|
501
|
-
/* IndexOutput */
|
502
|
-
cIndexOut = rb_define_class_under(mStore, "IndexOutput", rb_cObject);
|
503
|
-
cBufferedIndexOut = rb_define_class_under(mStore, "BufferedIndexOutput", cIndexOut);
|
504
|
-
rb_define_alloc_func(cBufferedIndexOut, frt_indexbuffer_alloc);
|
505
|
-
|
506
|
-
rb_define_method(cBufferedIndexOut, "write_byte", frt_indexout_write_byte, 1);
|
507
|
-
rb_define_method(cBufferedIndexOut, "write_bytes", frt_indexout_write_bytes, 2);
|
508
|
-
rb_define_method(cBufferedIndexOut, "flush", frt_indexout_flush, 0);
|
509
|
-
rb_define_method(cBufferedIndexOut, "close", frt_indexout_flush, 0);
|
510
|
-
rb_define_method(cBufferedIndexOut, "pos", frt_indexout_pos, 0);
|
511
|
-
rb_define_method(cBufferedIndexOut, "seek", frt_indexout_seek, 1);
|
512
|
-
rb_define_method(cBufferedIndexOut, "write_int", frt_indexout_write_int, 1);
|
513
|
-
rb_define_method(cBufferedIndexOut, "write_long", frt_indexout_write_long, 1);
|
514
|
-
rb_define_method(cBufferedIndexOut, "write_uint", frt_indexout_write_uint, 1);
|
515
|
-
rb_define_method(cBufferedIndexOut, "write_ulong", frt_indexout_write_ulong, 1);
|
516
|
-
rb_define_method(cBufferedIndexOut, "write_vint", frt_indexout_write_vint, 1);
|
517
|
-
rb_define_method(cBufferedIndexOut, "write_vlong", frt_indexout_write_vint, 1);
|
518
|
-
rb_define_method(cBufferedIndexOut, "write_chars", frt_indexout_write_chars, 3);
|
519
|
-
rb_define_method(cBufferedIndexOut, "write_string", frt_indexout_write_string, 1);
|
520
|
-
|
521
|
-
/* FSIndexInput */
|
522
|
-
/*cFSIndexIn = rb_define_class_under(mStore, "FSIndexInput", cBufferedIndexIn); */
|
375
|
+
int start = strlen(filename) - 4;
|
376
|
+
return ((start > 0) && (strcmp(".lck", &filename[start]) == 0));
|
523
377
|
}
|