ferret 0.9.6 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/MIT-LICENSE +1 -1
- data/README +12 -24
- data/Rakefile +38 -54
- data/TODO +14 -17
- data/ext/analysis.c +982 -823
- data/ext/analysis.h +133 -76
- data/ext/array.c +96 -58
- data/ext/array.h +40 -13
- data/ext/bitvector.c +476 -118
- data/ext/bitvector.h +264 -22
- data/ext/compound_io.c +217 -229
- data/ext/defines.h +49 -0
- data/ext/document.c +107 -317
- data/ext/document.h +31 -65
- data/ext/except.c +81 -36
- data/ext/except.h +117 -55
- data/ext/extconf.rb +2 -9
- data/ext/ferret.c +211 -104
- data/ext/ferret.h +22 -11
- data/ext/filter.c +97 -82
- data/ext/fs_store.c +348 -367
- data/ext/global.c +226 -188
- data/ext/global.h +44 -26
- data/ext/hash.c +474 -391
- data/ext/hash.h +441 -68
- data/ext/hashset.c +124 -96
- data/ext/hashset.h +169 -20
- data/ext/helper.c +56 -5
- data/ext/helper.h +7 -0
- data/ext/inc/lang.h +29 -49
- data/ext/inc/threading.h +31 -0
- data/ext/ind.c +288 -278
- data/ext/ind.h +68 -0
- data/ext/index.c +5688 -0
- data/ext/index.h +663 -616
- data/ext/lang.h +29 -49
- data/ext/libstemmer.c +3 -3
- data/ext/mem_pool.c +84 -0
- data/ext/mem_pool.h +35 -0
- data/ext/posh.c +1006 -0
- data/ext/posh.h +1007 -0
- data/ext/priorityqueue.c +117 -194
- data/ext/priorityqueue.h +135 -39
- data/ext/q_boolean.c +1305 -1108
- data/ext/q_const_score.c +106 -93
- data/ext/q_filtered_query.c +138 -135
- data/ext/q_fuzzy.c +206 -242
- data/ext/q_match_all.c +94 -80
- data/ext/q_multi_term.c +663 -0
- data/ext/q_parser.c +667 -593
- data/ext/q_phrase.c +992 -555
- data/ext/q_prefix.c +72 -61
- data/ext/q_range.c +235 -210
- data/ext/q_span.c +1480 -1166
- data/ext/q_term.c +273 -246
- data/ext/q_wildcard.c +127 -114
- data/ext/r_analysis.c +1720 -711
- data/ext/r_index.c +3049 -0
- data/ext/r_qparser.c +433 -146
- data/ext/r_search.c +2934 -1993
- data/ext/r_store.c +372 -143
- data/ext/r_utils.c +941 -0
- data/ext/ram_store.c +330 -326
- data/ext/search.c +1291 -668
- data/ext/search.h +403 -702
- data/ext/similarity.c +91 -113
- data/ext/similarity.h +45 -30
- data/ext/sort.c +721 -484
- data/ext/stopwords.c +361 -273
- data/ext/store.c +556 -58
- data/ext/store.h +706 -126
- data/ext/tags +3578 -2780
- data/ext/term_vectors.c +352 -0
- data/ext/threading.h +31 -0
- data/ext/win32.h +54 -0
- data/lib/ferret.rb +5 -17
- data/lib/ferret/document.rb +130 -2
- data/lib/ferret/index.rb +577 -26
- data/lib/ferret/number_tools.rb +157 -0
- data/lib/ferret_version.rb +3 -0
- data/test/test_helper.rb +5 -13
- data/test/unit/analysis/tc_analyzer.rb +513 -1
- data/test/unit/analysis/{ctc_tokenstream.rb → tc_token_stream.rb} +23 -0
- data/test/unit/index/tc_index.rb +183 -240
- data/test/unit/index/tc_index_reader.rb +312 -479
- data/test/unit/index/tc_index_writer.rb +397 -13
- data/test/unit/index/th_doc.rb +269 -206
- data/test/unit/query_parser/tc_query_parser.rb +40 -33
- data/test/unit/search/tc_filter.rb +59 -71
- data/test/unit/search/tc_fuzzy_query.rb +24 -16
- data/test/unit/search/tc_index_searcher.rb +23 -201
- data/test/unit/search/tc_multi_searcher.rb +78 -226
- data/test/unit/search/tc_search_and_sort.rb +93 -81
- data/test/unit/search/tc_sort.rb +23 -23
- data/test/unit/search/tc_sort_field.rb +7 -7
- data/test/unit/search/tc_spans.rb +51 -47
- data/test/unit/search/tm_searcher.rb +339 -0
- data/test/unit/store/tc_fs_store.rb +1 -1
- data/test/unit/store/tm_store_lock.rb +3 -3
- data/test/unit/tc_document.rb +81 -0
- data/test/unit/ts_analysis.rb +1 -1
- data/test/unit/ts_utils.rb +1 -1
- data/test/unit/utils/tc_bit_vector.rb +288 -0
- data/test/unit/utils/tc_number_tools.rb +117 -0
- data/test/unit/utils/tc_priority_queue.rb +106 -0
- metadata +140 -301
- data/CHANGELOG +0 -9
- data/ext/dummy.exe +0 -0
- data/ext/field.c +0 -408
- data/ext/frtio.h +0 -13
- data/ext/inc/except.h +0 -90
- data/ext/index_io.c +0 -382
- data/ext/index_rw.c +0 -2658
- data/ext/lang.c +0 -41
- data/ext/nix_io.c +0 -134
- data/ext/q_multi_phrase.c +0 -380
- data/ext/r_doc.c +0 -582
- data/ext/r_index_io.c +0 -1021
- data/ext/r_term.c +0 -219
- data/ext/term.c +0 -820
- data/ext/termdocs.c +0 -611
- data/ext/vector.c +0 -637
- data/ext/w32_io.c +0 -150
- data/lib/ferret/analysis.rb +0 -11
- data/lib/ferret/analysis/analyzers.rb +0 -112
- data/lib/ferret/analysis/standard_tokenizer.rb +0 -71
- data/lib/ferret/analysis/token.rb +0 -100
- data/lib/ferret/analysis/token_filters.rb +0 -86
- data/lib/ferret/analysis/token_stream.rb +0 -26
- data/lib/ferret/analysis/tokenizers.rb +0 -112
- data/lib/ferret/analysis/word_list_loader.rb +0 -27
- data/lib/ferret/document/document.rb +0 -152
- data/lib/ferret/document/field.rb +0 -312
- data/lib/ferret/index/compound_file_io.rb +0 -338
- data/lib/ferret/index/document_writer.rb +0 -289
- data/lib/ferret/index/field_infos.rb +0 -279
- data/lib/ferret/index/fields_io.rb +0 -181
- data/lib/ferret/index/index.rb +0 -675
- data/lib/ferret/index/index_file_names.rb +0 -33
- data/lib/ferret/index/index_reader.rb +0 -503
- data/lib/ferret/index/index_writer.rb +0 -534
- data/lib/ferret/index/multi_reader.rb +0 -377
- data/lib/ferret/index/multiple_term_doc_pos_enum.rb +0 -98
- data/lib/ferret/index/segment_infos.rb +0 -130
- data/lib/ferret/index/segment_merge_info.rb +0 -49
- data/lib/ferret/index/segment_merge_queue.rb +0 -16
- data/lib/ferret/index/segment_merger.rb +0 -358
- data/lib/ferret/index/segment_reader.rb +0 -412
- data/lib/ferret/index/segment_term_enum.rb +0 -169
- data/lib/ferret/index/segment_term_vector.rb +0 -58
- data/lib/ferret/index/term.rb +0 -53
- data/lib/ferret/index/term_buffer.rb +0 -83
- data/lib/ferret/index/term_doc_enum.rb +0 -291
- data/lib/ferret/index/term_enum.rb +0 -52
- data/lib/ferret/index/term_info.rb +0 -37
- data/lib/ferret/index/term_infos_io.rb +0 -321
- data/lib/ferret/index/term_vector_offset_info.rb +0 -20
- data/lib/ferret/index/term_vectors_io.rb +0 -553
- data/lib/ferret/query_parser.rb +0 -312
- data/lib/ferret/query_parser/query_parser.tab.rb +0 -928
- data/lib/ferret/search.rb +0 -50
- data/lib/ferret/search/boolean_clause.rb +0 -100
- data/lib/ferret/search/boolean_query.rb +0 -299
- data/lib/ferret/search/boolean_scorer.rb +0 -294
- data/lib/ferret/search/caching_wrapper_filter.rb +0 -40
- data/lib/ferret/search/conjunction_scorer.rb +0 -99
- data/lib/ferret/search/disjunction_sum_scorer.rb +0 -205
- data/lib/ferret/search/exact_phrase_scorer.rb +0 -32
- data/lib/ferret/search/explanation.rb +0 -41
- data/lib/ferret/search/field_cache.rb +0 -215
- data/lib/ferret/search/field_doc.rb +0 -31
- data/lib/ferret/search/field_sorted_hit_queue.rb +0 -184
- data/lib/ferret/search/filter.rb +0 -11
- data/lib/ferret/search/filtered_query.rb +0 -130
- data/lib/ferret/search/filtered_term_enum.rb +0 -79
- data/lib/ferret/search/fuzzy_query.rb +0 -154
- data/lib/ferret/search/fuzzy_term_enum.rb +0 -247
- data/lib/ferret/search/hit_collector.rb +0 -34
- data/lib/ferret/search/hit_queue.rb +0 -11
- data/lib/ferret/search/index_searcher.rb +0 -200
- data/lib/ferret/search/match_all_query.rb +0 -104
- data/lib/ferret/search/multi_phrase_query.rb +0 -216
- data/lib/ferret/search/multi_searcher.rb +0 -261
- data/lib/ferret/search/multi_term_query.rb +0 -65
- data/lib/ferret/search/non_matching_scorer.rb +0 -22
- data/lib/ferret/search/phrase_positions.rb +0 -55
- data/lib/ferret/search/phrase_query.rb +0 -214
- data/lib/ferret/search/phrase_scorer.rb +0 -152
- data/lib/ferret/search/prefix_query.rb +0 -54
- data/lib/ferret/search/query.rb +0 -140
- data/lib/ferret/search/query_filter.rb +0 -51
- data/lib/ferret/search/range_filter.rb +0 -103
- data/lib/ferret/search/range_query.rb +0 -139
- data/lib/ferret/search/req_excl_scorer.rb +0 -125
- data/lib/ferret/search/req_opt_sum_scorer.rb +0 -70
- data/lib/ferret/search/score_doc.rb +0 -38
- data/lib/ferret/search/score_doc_comparator.rb +0 -114
- data/lib/ferret/search/scorer.rb +0 -91
- data/lib/ferret/search/similarity.rb +0 -278
- data/lib/ferret/search/sloppy_phrase_scorer.rb +0 -47
- data/lib/ferret/search/sort.rb +0 -112
- data/lib/ferret/search/sort_comparator.rb +0 -60
- data/lib/ferret/search/sort_field.rb +0 -91
- data/lib/ferret/search/spans.rb +0 -12
- data/lib/ferret/search/spans/near_spans_enum.rb +0 -304
- data/lib/ferret/search/spans/span_first_query.rb +0 -79
- data/lib/ferret/search/spans/span_near_query.rb +0 -108
- data/lib/ferret/search/spans/span_not_query.rb +0 -130
- data/lib/ferret/search/spans/span_or_query.rb +0 -176
- data/lib/ferret/search/spans/span_query.rb +0 -25
- data/lib/ferret/search/spans/span_scorer.rb +0 -74
- data/lib/ferret/search/spans/span_term_query.rb +0 -105
- data/lib/ferret/search/spans/span_weight.rb +0 -84
- data/lib/ferret/search/spans/spans_enum.rb +0 -44
- data/lib/ferret/search/term_query.rb +0 -128
- data/lib/ferret/search/term_scorer.rb +0 -183
- data/lib/ferret/search/top_docs.rb +0 -36
- data/lib/ferret/search/top_field_docs.rb +0 -17
- data/lib/ferret/search/weight.rb +0 -54
- data/lib/ferret/search/wildcard_query.rb +0 -26
- data/lib/ferret/search/wildcard_term_enum.rb +0 -61
- data/lib/ferret/stemmers.rb +0 -1
- data/lib/ferret/stemmers/porter_stemmer.rb +0 -218
- data/lib/ferret/store.rb +0 -5
- data/lib/ferret/store/buffered_index_io.rb +0 -190
- data/lib/ferret/store/directory.rb +0 -141
- data/lib/ferret/store/fs_store.rb +0 -381
- data/lib/ferret/store/index_io.rb +0 -245
- data/lib/ferret/store/ram_store.rb +0 -286
- data/lib/ferret/utils.rb +0 -8
- data/lib/ferret/utils/bit_vector.rb +0 -123
- data/lib/ferret/utils/date_tools.rb +0 -138
- data/lib/ferret/utils/number_tools.rb +0 -91
- data/lib/ferret/utils/parameter.rb +0 -41
- data/lib/ferret/utils/priority_queue.rb +0 -120
- data/lib/ferret/utils/string_helper.rb +0 -47
- data/lib/ferret/utils/thread_local.rb +0 -28
- data/lib/ferret/utils/weak_key_hash.rb +0 -60
- data/lib/rferret.rb +0 -37
- data/rake_utils/code_statistics.rb +0 -106
- data/test/benchmark/tb_ram_store.rb +0 -76
- data/test/benchmark/tb_rw_vint.rb +0 -26
- data/test/functional/thread_safety_index_test.rb +0 -81
- data/test/functional/thread_safety_test.rb +0 -137
- data/test/longrunning/tc_numbertools.rb +0 -60
- data/test/longrunning/tm_store.rb +0 -19
- data/test/unit/analysis/ctc_analyzer.rb +0 -532
- data/test/unit/analysis/data/wordfile +0 -6
- data/test/unit/analysis/rtc_letter_tokenizer.rb +0 -20
- data/test/unit/analysis/rtc_lower_case_filter.rb +0 -20
- data/test/unit/analysis/rtc_lower_case_tokenizer.rb +0 -27
- data/test/unit/analysis/rtc_per_field_analyzer_wrapper.rb +0 -39
- data/test/unit/analysis/rtc_porter_stem_filter.rb +0 -16
- data/test/unit/analysis/rtc_standard_analyzer.rb +0 -20
- data/test/unit/analysis/rtc_standard_tokenizer.rb +0 -20
- data/test/unit/analysis/rtc_stop_analyzer.rb +0 -20
- data/test/unit/analysis/rtc_stop_filter.rb +0 -14
- data/test/unit/analysis/rtc_white_space_analyzer.rb +0 -21
- data/test/unit/analysis/rtc_white_space_tokenizer.rb +0 -20
- data/test/unit/analysis/rtc_word_list_loader.rb +0 -32
- data/test/unit/analysis/tc_token.rb +0 -25
- data/test/unit/document/rtc_field.rb +0 -28
- data/test/unit/document/tc_document.rb +0 -47
- data/test/unit/document/tc_field.rb +0 -98
- data/test/unit/index/rtc_compound_file_io.rb +0 -107
- data/test/unit/index/rtc_field_infos.rb +0 -127
- data/test/unit/index/rtc_fields_io.rb +0 -167
- data/test/unit/index/rtc_multiple_term_doc_pos_enum.rb +0 -83
- data/test/unit/index/rtc_segment_infos.rb +0 -74
- data/test/unit/index/rtc_segment_term_docs.rb +0 -17
- data/test/unit/index/rtc_segment_term_enum.rb +0 -60
- data/test/unit/index/rtc_segment_term_vector.rb +0 -71
- data/test/unit/index/rtc_term_buffer.rb +0 -57
- data/test/unit/index/rtc_term_info.rb +0 -19
- data/test/unit/index/rtc_term_infos_io.rb +0 -192
- data/test/unit/index/rtc_term_vectors_io.rb +0 -108
- data/test/unit/index/tc_term.rb +0 -27
- data/test/unit/index/tc_term_voi.rb +0 -18
- data/test/unit/search/rtc_similarity.rb +0 -37
- data/test/unit/search/rtc_sort_field.rb +0 -14
- data/test/unit/search/tc_multi_searcher2.rb +0 -126
- data/test/unit/store/rtc_fs_store.rb +0 -62
- data/test/unit/store/rtc_ram_store.rb +0 -15
- data/test/unit/store/rtm_store.rb +0 -150
- data/test/unit/store/rtm_store_lock.rb +0 -2
- data/test/unit/ts_document.rb +0 -2
- data/test/unit/utils/rtc_bit_vector.rb +0 -73
- data/test/unit/utils/rtc_date_tools.rb +0 -50
- data/test/unit/utils/rtc_number_tools.rb +0 -59
- data/test/unit/utils/rtc_parameter.rb +0 -40
- data/test/unit/utils/rtc_priority_queue.rb +0 -62
- data/test/unit/utils/rtc_string_helper.rb +0 -21
- data/test/unit/utils/rtc_thread.rb +0 -61
- data/test/unit/utils/rtc_weak_key_hash.rb +0 -25
- data/test/utils/number_to_spoken.rb +0 -132
data/ext/ram_store.c
CHANGED
@@ -1,466 +1,470 @@
|
|
1
|
+
#include "store.h"
|
1
2
|
#include <string.h>
|
2
|
-
#include <store.h>
|
3
|
-
|
4
|
-
static char * const RENAME_ERROR_MSG = "tried to rename a file that doesn't exist";
|
5
|
-
static char * const MISSING_RAMFILE_ERROR_MSG ="Couldn't open the ram file to read";
|
6
3
|
|
4
|
+
extern Store *store_new();
|
7
5
|
extern void store_destroy(Store *store);
|
6
|
+
extern OutStream *os_new();
|
7
|
+
extern InStream *is_new();
|
8
|
+
extern int file_is_lock(char *filename);
|
8
9
|
|
9
|
-
|
10
|
-
char *name;
|
11
|
-
uchar **buffers;
|
12
|
-
int bufcnt;
|
13
|
-
int len;
|
14
|
-
int ref_cnt;
|
15
|
-
bool alive;
|
16
|
-
} RamFile;
|
17
|
-
|
18
|
-
RamFile *rf_create(const char *name)
|
10
|
+
static RAMFile *rf_new(const char *name)
|
19
11
|
{
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
return rf;
|
12
|
+
RAMFile *rf = ALLOC(RAMFile);
|
13
|
+
rf->buffers = ALLOC(uchar *);
|
14
|
+
rf->buffers[0] = ALLOC_N(uchar, BUFFER_SIZE);
|
15
|
+
rf->name = estrdup(name);
|
16
|
+
rf->len = 0;
|
17
|
+
rf->bufcnt = 1;
|
18
|
+
rf->ref_cnt = 1;
|
19
|
+
return rf;
|
29
20
|
}
|
30
21
|
|
31
|
-
void rf_extend_if_necessary(
|
22
|
+
static void rf_extend_if_necessary(RAMFile *rf, int buf_num)
|
32
23
|
{
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
24
|
+
while (rf->bufcnt <= buf_num) {
|
25
|
+
REALLOC_N(rf->buffers, uchar *, (rf->bufcnt + 1));
|
26
|
+
rf->buffers[rf->bufcnt++] = ALLOC_N(uchar, BUFFER_SIZE);
|
27
|
+
}
|
37
28
|
}
|
38
29
|
|
39
|
-
void rf_close(void *p)
|
30
|
+
static void rf_close(void *p)
|
40
31
|
{
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
free(rf->
|
47
|
-
|
48
|
-
|
49
|
-
|
32
|
+
int i;
|
33
|
+
RAMFile *rf = (RAMFile *)p;
|
34
|
+
if (rf->ref_cnt > 0) {
|
35
|
+
return;
|
36
|
+
}
|
37
|
+
free(rf->name);
|
38
|
+
for (i = 0; i < rf->bufcnt; i++) {
|
39
|
+
free(rf->buffers[i]);
|
40
|
+
}
|
41
|
+
free(rf->buffers);
|
42
|
+
free(rf);
|
50
43
|
}
|
51
44
|
|
52
|
-
void ram_touch(Store *store, char *filename)
|
45
|
+
static void ram_touch(Store *store, char *filename)
|
53
46
|
{
|
54
|
-
|
55
|
-
|
47
|
+
if (h_get(store->dir.ht, filename) == NULL) {
|
48
|
+
h_set(store->dir.ht, filename, rf_new(filename));
|
49
|
+
}
|
56
50
|
}
|
57
51
|
|
58
|
-
int ram_exists(Store *store, char *filename)
|
52
|
+
static int ram_exists(Store *store, char *filename)
|
59
53
|
{
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
54
|
+
if (h_get(store->dir.ht, filename) != NULL) {
|
55
|
+
return true;
|
56
|
+
}
|
57
|
+
else {
|
58
|
+
return false;
|
59
|
+
}
|
64
60
|
}
|
65
61
|
|
66
|
-
int ram_remove(Store *store, char *filename)
|
62
|
+
static int ram_remove(Store *store, char *filename)
|
67
63
|
{
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
64
|
+
RAMFile *rf = h_rem(store->dir.ht, filename, false);
|
65
|
+
if (rf != NULL) {
|
66
|
+
DEREF(rf);
|
67
|
+
rf_close(rf);
|
68
|
+
return true;
|
69
|
+
}
|
70
|
+
else {
|
71
|
+
return false;
|
72
|
+
}
|
76
73
|
}
|
77
74
|
|
78
|
-
|
75
|
+
static void ram_rename(Store *store, char *from, char *to)
|
79
76
|
{
|
80
|
-
|
81
|
-
|
77
|
+
RAMFile *rf = (RAMFile *)h_rem(store->dir.ht, from, false);
|
78
|
+
RAMFile *tmp;
|
82
79
|
|
83
|
-
|
84
|
-
|
85
|
-
|
80
|
+
if (rf == NULL) {
|
81
|
+
RAISE(IO_ERROR, "couldn't rename \"%s\" to \"%s\". \"%s\""
|
82
|
+
" doesn't exist", from, to, from);
|
83
|
+
}
|
86
84
|
|
87
|
-
|
85
|
+
free(rf->name);
|
88
86
|
|
89
|
-
|
87
|
+
rf->name = estrdup(to);
|
90
88
|
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
89
|
+
/* clean up the file we are overwriting */
|
90
|
+
tmp = (RAMFile *)h_get(store->dir.ht, to);
|
91
|
+
if (tmp != NULL) {
|
92
|
+
DEREF(tmp);
|
93
|
+
}
|
96
94
|
|
97
|
-
|
98
|
-
return true;
|
95
|
+
h_set(store->dir.ht, rf->name, rf);
|
99
96
|
}
|
100
97
|
|
101
|
-
int ram_count(Store *store)
|
98
|
+
static int ram_count(Store *store)
|
102
99
|
{
|
103
|
-
|
100
|
+
return store->dir.ht->size;
|
104
101
|
}
|
105
102
|
|
106
|
-
void ram_each(Store *store,
|
103
|
+
static void ram_each(Store *store,
|
104
|
+
void (*func)(char *fname, void *arg), void *arg)
|
107
105
|
{
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
106
|
+
HashTable *ht = store->dir.ht;
|
107
|
+
int i;
|
108
|
+
for (i = 0; i <= ht->mask; i++) {
|
109
|
+
RAMFile *rf = (RAMFile *)ht->table[i].value;
|
110
|
+
if (rf) {
|
111
|
+
if (strncmp(rf->name, LOCK_PREFIX, strlen(LOCK_PREFIX)) == 0) {
|
112
|
+
continue;
|
113
|
+
}
|
114
|
+
func(rf->name, arg);
|
115
|
+
}
|
117
116
|
}
|
118
|
-
}
|
119
117
|
}
|
120
118
|
|
121
|
-
void ram_close_i(Store *store)
|
119
|
+
static void ram_close_i(Store *store)
|
122
120
|
{
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
121
|
+
HashTable *ht = store->dir.ht;
|
122
|
+
int i;
|
123
|
+
for (i = 0; i <= ht->mask; i++) {
|
124
|
+
RAMFile *rf = (RAMFile *)ht->table[i].value;
|
125
|
+
if (rf) {
|
126
|
+
DEREF(rf);
|
127
|
+
}
|
128
|
+
}
|
129
|
+
h_destroy(store->dir.ht);
|
130
|
+
store_destroy(store);
|
132
131
|
}
|
133
132
|
|
134
133
|
/*
|
135
134
|
* Be sure to keep the locks
|
136
135
|
*/
|
137
|
-
void ram_clear(Store *store)
|
138
|
-
{
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
136
|
+
static void ram_clear(Store *store)
|
137
|
+
{
|
138
|
+
int i;
|
139
|
+
HashTable *ht = store->dir.ht;
|
140
|
+
for (i = 0; i <= ht->mask; i++) {
|
141
|
+
RAMFile *rf = (RAMFile *)ht->table[i].value;
|
142
|
+
if (rf && !file_is_lock(rf->name)) {
|
143
|
+
DEREF(rf);
|
144
|
+
h_del(ht, rf->name);
|
145
|
+
}
|
147
146
|
}
|
148
|
-
}
|
149
147
|
}
|
150
148
|
|
151
|
-
void ram_clear_locks(Store *store)
|
149
|
+
static void ram_clear_locks(Store *store)
|
152
150
|
{
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
151
|
+
int i;
|
152
|
+
HashTable *ht = store->dir.ht;
|
153
|
+
for (i = 0; i <= ht->mask; i++) {
|
154
|
+
RAMFile *rf = (RAMFile *)ht->table[i].value;
|
155
|
+
if (rf && file_is_lock(rf->name)) {
|
156
|
+
DEREF(rf);
|
157
|
+
h_del(ht, rf->name);
|
158
|
+
}
|
161
159
|
}
|
162
|
-
|
163
|
-
|
164
|
-
void ram_clear_all(Store *store)
|
165
|
-
{
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
160
|
+
}
|
161
|
+
|
162
|
+
static void ram_clear_all(Store *store)
|
163
|
+
{
|
164
|
+
int i;
|
165
|
+
HashTable *ht = store->dir.ht;
|
166
|
+
for (i = 0; i <= ht->mask; i++) {
|
167
|
+
RAMFile *rf = (RAMFile *)ht->table[i].value;
|
168
|
+
if (rf) {
|
169
|
+
DEREF(rf);
|
170
|
+
h_del(ht, rf->name);
|
171
|
+
}
|
174
172
|
}
|
175
|
-
}
|
176
173
|
}
|
177
174
|
|
178
|
-
|
175
|
+
static off_t ram_length(Store *store, char *filename)
|
179
176
|
{
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
177
|
+
RAMFile *rf = (RAMFile *)h_get(store->dir.ht, filename);
|
178
|
+
if (rf != NULL) {
|
179
|
+
return rf->len;
|
180
|
+
}
|
181
|
+
else {
|
182
|
+
return 0;
|
183
|
+
}
|
185
184
|
}
|
186
185
|
|
187
|
-
|
186
|
+
off_t ramo_length(OutStream *os)
|
188
187
|
{
|
189
|
-
|
188
|
+
return os->file.rf->len;
|
190
189
|
}
|
191
190
|
|
192
|
-
void
|
191
|
+
static void ramo_flush_i(OutStream *os, uchar *src, int len)
|
193
192
|
{
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
buffer_number = (int)(pointer / BUFFER_SIZE);
|
201
|
-
buffer_offset = pointer % BUFFER_SIZE;
|
202
|
-
bytes_in_buffer = BUFFER_SIZE - buffer_offset;
|
203
|
-
bytes_to_copy = bytes_in_buffer < len ? bytes_in_buffer : len;
|
193
|
+
uchar *buffer;
|
194
|
+
RAMFile *rf = os->file.rf;
|
195
|
+
int buffer_number, buffer_offset, bytes_in_buffer, bytes_to_copy;
|
196
|
+
int src_offset;
|
197
|
+
off_t pointer = os->pointer;
|
204
198
|
|
205
|
-
|
199
|
+
buffer_number = (int)(pointer / BUFFER_SIZE);
|
200
|
+
buffer_offset = pointer % BUFFER_SIZE;
|
201
|
+
bytes_in_buffer = BUFFER_SIZE - buffer_offset;
|
202
|
+
bytes_to_copy = bytes_in_buffer < len ? bytes_in_buffer : len;
|
206
203
|
|
207
|
-
buffer = rf->buffers[buffer_number];
|
208
|
-
memcpy(buffer + buffer_offset, src, bytes_to_copy);
|
209
|
-
|
210
|
-
if (bytes_to_copy < len) {
|
211
|
-
src_offset = bytes_to_copy;
|
212
|
-
bytes_to_copy = len - bytes_to_copy;
|
213
|
-
buffer_number += 1;
|
214
204
|
rf_extend_if_necessary(rf, buffer_number);
|
205
|
+
|
215
206
|
buffer = rf->buffers[buffer_number];
|
207
|
+
memcpy(buffer + buffer_offset, src, bytes_to_copy);
|
216
208
|
|
217
|
-
|
218
|
-
|
219
|
-
|
209
|
+
if (bytes_to_copy < len) {
|
210
|
+
src_offset = bytes_to_copy;
|
211
|
+
bytes_to_copy = len - bytes_to_copy;
|
212
|
+
buffer_number += 1;
|
213
|
+
rf_extend_if_necessary(rf, buffer_number);
|
214
|
+
buffer = rf->buffers[buffer_number];
|
220
215
|
|
221
|
-
|
222
|
-
|
216
|
+
memcpy(buffer, src + src_offset, bytes_to_copy);
|
217
|
+
}
|
218
|
+
os->pointer += len;
|
219
|
+
|
220
|
+
if (os->pointer > rf->len) {
|
221
|
+
rf->len = os->pointer;
|
222
|
+
}
|
223
223
|
}
|
224
224
|
|
225
|
-
void
|
225
|
+
static void ramo_seek_i(OutStream *os, off_t pos)
|
226
226
|
{
|
227
|
-
|
227
|
+
os->pointer = pos;
|
228
228
|
}
|
229
229
|
|
230
230
|
void ramo_reset(OutStream *os)
|
231
231
|
{
|
232
|
-
|
233
|
-
|
234
|
-
rf->len = 0;
|
232
|
+
os_seek(os, 0);
|
233
|
+
os->file.rf->len = 0;
|
235
234
|
}
|
236
235
|
|
237
|
-
void
|
236
|
+
static void ramo_close_i(OutStream *os)
|
238
237
|
{
|
239
|
-
|
240
|
-
|
241
|
-
|
238
|
+
RAMFile *rf = os->file.rf;
|
239
|
+
DEREF(rf);
|
240
|
+
rf_close(rf);
|
242
241
|
}
|
243
242
|
|
244
243
|
void ramo_write_to(OutStream *os, OutStream *other_o)
|
245
244
|
{
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
245
|
+
int i, len;
|
246
|
+
RAMFile *rf = os->file.rf;
|
247
|
+
int last_buffer_number;
|
248
|
+
int last_buffer_offset;
|
249
|
+
|
250
|
+
os_flush(os);
|
251
|
+
last_buffer_number = (int) (rf->len / BUFFER_SIZE);
|
252
|
+
last_buffer_offset = rf->len % BUFFER_SIZE;
|
253
|
+
for (i = 0; i <= last_buffer_number; i++) {
|
254
|
+
len = (i == last_buffer_number ? last_buffer_offset : BUFFER_SIZE);
|
255
|
+
os_write_bytes(other_o, rf->buffers[i], len);
|
256
|
+
}
|
258
257
|
}
|
259
258
|
|
260
|
-
|
259
|
+
const struct OutStreamMethods RAM_OUT_STREAM_METHODS = {
|
260
|
+
ramo_flush_i,
|
261
|
+
ramo_seek_i,
|
262
|
+
ramo_close_i
|
263
|
+
};
|
264
|
+
|
265
|
+
OutStream *ram_new_buffer()
|
261
266
|
{
|
262
|
-
|
263
|
-
|
267
|
+
RAMFile *rf = rf_new("");
|
268
|
+
OutStream *os = os_new();
|
264
269
|
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
os->close_internal = &ramo_close_internal;
|
271
|
-
return os;
|
270
|
+
DEREF(rf);
|
271
|
+
os->file.rf = rf;
|
272
|
+
os->pointer = 0;
|
273
|
+
os->m = &RAM_OUT_STREAM_METHODS;
|
274
|
+
return os;
|
272
275
|
}
|
273
276
|
|
274
277
|
void ram_destroy_buffer(OutStream *os)
|
275
278
|
{
|
276
|
-
|
277
|
-
|
279
|
+
rf_close(os->file.rf);
|
280
|
+
free(os);
|
278
281
|
}
|
279
282
|
|
280
|
-
OutStream *
|
283
|
+
static OutStream *ram_new_output(Store *store, const char *filename)
|
281
284
|
{
|
282
|
-
|
283
|
-
|
285
|
+
RAMFile *rf = (RAMFile *)h_get(store->dir.ht, filename);
|
286
|
+
OutStream *os = os_new();
|
284
287
|
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
288
|
+
if (rf == NULL) {
|
289
|
+
rf = rf_new(filename);
|
290
|
+
h_set(store->dir.ht, rf->name, rf);
|
291
|
+
}
|
292
|
+
REF(rf);
|
293
|
+
os->pointer = 0;
|
294
|
+
os->file.rf = rf;
|
295
|
+
os->m = &RAM_OUT_STREAM_METHODS;
|
296
|
+
return os;
|
297
|
+
}
|
298
|
+
|
299
|
+
static void rami_read_i(InStream *is, uchar *b, int len)
|
300
|
+
{
|
301
|
+
RAMFile *rf = is->file.rf;
|
302
|
+
|
303
|
+
int offset = 0;
|
304
|
+
int buffer_number, buffer_offset, bytes_in_buffer, bytes_to_copy;
|
305
|
+
int remainder = len;
|
306
|
+
off_t start = is->d.pointer;
|
307
|
+
uchar *buffer;
|
308
|
+
|
309
|
+
while (remainder > 0) {
|
310
|
+
buffer_number = (int) (start / BUFFER_SIZE);
|
311
|
+
buffer_offset = start % BUFFER_SIZE;
|
312
|
+
bytes_in_buffer = BUFFER_SIZE - buffer_offset;
|
313
|
+
|
314
|
+
if (bytes_in_buffer >= remainder) {
|
315
|
+
bytes_to_copy = remainder;
|
316
|
+
}
|
317
|
+
else {
|
318
|
+
bytes_to_copy = bytes_in_buffer;
|
319
|
+
}
|
320
|
+
buffer = rf->buffers[buffer_number];
|
321
|
+
memcpy(b + offset, buffer + buffer_offset, bytes_to_copy);
|
322
|
+
offset += bytes_to_copy;
|
323
|
+
start += bytes_to_copy;
|
324
|
+
remainder -= bytes_to_copy;
|
316
325
|
}
|
317
|
-
buffer = rf->buffers[buffer_number];
|
318
|
-
memcpy(b + offset, buffer + buffer_offset, bytes_to_copy);
|
319
|
-
offset += bytes_to_copy;
|
320
|
-
start += bytes_to_copy;
|
321
|
-
remainder -= bytes_to_copy;
|
322
|
-
}
|
323
|
-
|
324
|
-
is->d.pointer += len;
|
325
|
-
}
|
326
326
|
|
327
|
-
|
328
|
-
{
|
329
|
-
return ((RamFile *)is->file)->len;
|
327
|
+
is->d.pointer += len;
|
330
328
|
}
|
331
329
|
|
332
|
-
|
330
|
+
static off_t rami_length_i(InStream *is)
|
333
331
|
{
|
334
|
-
|
332
|
+
return is->file.rf->len;
|
335
333
|
}
|
336
334
|
|
337
|
-
void
|
335
|
+
static void rami_seek_i(InStream *is, off_t pos)
|
338
336
|
{
|
339
|
-
|
340
|
-
rf->ref_cnt--;
|
341
|
-
rf_close(rf);
|
337
|
+
is->d.pointer = pos;
|
342
338
|
}
|
343
339
|
|
344
|
-
void
|
340
|
+
static void rami_close_i(InStream *is)
|
345
341
|
{
|
346
|
-
|
342
|
+
RAMFile *rf = is->file.rf;
|
343
|
+
DEREF(rf);
|
344
|
+
rf_close(rf);
|
347
345
|
}
|
348
346
|
|
349
|
-
|
347
|
+
static const struct InStreamMethods RAM_IN_STREAM_METHODS = {
|
348
|
+
rami_read_i,
|
349
|
+
rami_seek_i,
|
350
|
+
rami_length_i,
|
351
|
+
rami_close_i
|
352
|
+
};
|
353
|
+
|
354
|
+
static InStream *ram_open_input(Store *store, const char *filename)
|
350
355
|
{
|
351
|
-
|
352
|
-
|
356
|
+
RAMFile *rf = (RAMFile *)h_get(store->dir.ht, filename);
|
357
|
+
InStream *is = is_new();
|
353
358
|
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
is->clone_internal = &rami_clone_internal;
|
365
|
-
is->length_internal = &rami_length;
|
366
|
-
return is;
|
359
|
+
if (rf == NULL) {
|
360
|
+
RAISE(IO_ERROR, "tried to open \"%s\" but it doesn't exist", filename);
|
361
|
+
}
|
362
|
+
REF(rf);
|
363
|
+
|
364
|
+
is->file.rf = rf;
|
365
|
+
is->d.pointer = 0;
|
366
|
+
is->m = &RAM_IN_STREAM_METHODS;
|
367
|
+
|
368
|
+
return is;
|
367
369
|
}
|
368
370
|
|
369
371
|
#define LOCK_OBTAIN_TIMEOUT 5
|
370
372
|
|
371
|
-
int ram_lock_obtain(Lock *lock)
|
373
|
+
static int ram_lock_obtain(Lock *lock)
|
372
374
|
{
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
375
|
+
int ret = true;
|
376
|
+
if (ram_exists(lock->store, lock->name))
|
377
|
+
ret = false;
|
378
|
+
ram_touch(lock->store, lock->name);
|
379
|
+
return ret;
|
378
380
|
}
|
379
381
|
|
380
|
-
int ram_lock_is_locked(Lock *lock)
|
382
|
+
static int ram_lock_is_locked(Lock *lock)
|
381
383
|
{
|
382
|
-
|
384
|
+
return ram_exists(lock->store, lock->name);
|
383
385
|
}
|
384
386
|
|
385
|
-
void ram_lock_release(Lock *lock)
|
387
|
+
static void ram_lock_release(Lock *lock)
|
386
388
|
{
|
387
|
-
|
389
|
+
ram_remove(lock->store, lock->name);
|
388
390
|
}
|
389
391
|
|
390
|
-
Lock *ram_open_lock(Store *store, char *lockname)
|
392
|
+
static Lock *ram_open_lock(Store *store, char *lockname)
|
391
393
|
{
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
398
|
-
|
399
|
-
|
400
|
-
|
394
|
+
Lock *lock = ALLOC(Lock);
|
395
|
+
char lname[100];
|
396
|
+
sprintf(lname, "%s%s.lck", LOCK_PREFIX, lockname);
|
397
|
+
lock->name = estrdup(lname);
|
398
|
+
lock->store = store;
|
399
|
+
lock->obtain = &ram_lock_obtain;
|
400
|
+
lock->release = &ram_lock_release;
|
401
|
+
lock->is_locked = &ram_lock_is_locked;
|
402
|
+
return lock;
|
401
403
|
}
|
402
404
|
|
403
|
-
void ram_close_lock(Lock *lock)
|
405
|
+
static void ram_close_lock(Lock *lock)
|
404
406
|
{
|
405
|
-
|
406
|
-
|
407
|
+
free(lock->name);
|
408
|
+
free(lock);
|
407
409
|
}
|
408
410
|
|
409
411
|
|
410
412
|
Store *open_ram_store()
|
411
413
|
{
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
|
420
|
-
|
421
|
-
|
422
|
-
|
423
|
-
|
424
|
-
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
}
|
432
|
-
|
433
|
-
struct CopyFileArg
|
434
|
-
|
414
|
+
Store *new_store = store_new();
|
415
|
+
|
416
|
+
new_store->dir.ht = h_new_str(NULL, rf_close);
|
417
|
+
new_store->touch = &ram_touch;
|
418
|
+
new_store->exists = &ram_exists;
|
419
|
+
new_store->remove = &ram_remove;
|
420
|
+
new_store->rename = &ram_rename;
|
421
|
+
new_store->count = &ram_count;
|
422
|
+
new_store->clear = &ram_clear;
|
423
|
+
new_store->clear_all = &ram_clear_all;
|
424
|
+
new_store->clear_locks = &ram_clear_locks;
|
425
|
+
new_store->length = &ram_length;
|
426
|
+
new_store->each = &ram_each;
|
427
|
+
new_store->new_output = &ram_new_output;
|
428
|
+
new_store->open_input = &ram_open_input;
|
429
|
+
new_store->open_lock = &ram_open_lock;
|
430
|
+
new_store->close_lock = &ram_close_lock;
|
431
|
+
new_store->close_i = &ram_close_i;
|
432
|
+
return new_store;
|
433
|
+
}
|
434
|
+
|
435
|
+
struct CopyFileArg
|
436
|
+
{
|
437
|
+
Store *to_store, *from_store;
|
435
438
|
};
|
436
439
|
|
437
440
|
static void copy_files(char *fname, void *arg)
|
438
441
|
{
|
439
|
-
|
440
|
-
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
is_read_bytes(is, buffer, 0, len);
|
446
|
-
os_write_bytes(os, buffer, len);
|
442
|
+
struct CopyFileArg *cfa = (struct CopyFileArg *)arg;
|
443
|
+
OutStream *os = cfa->to_store->new_output(cfa->to_store, fname);
|
444
|
+
InStream *is = cfa->from_store->open_input(cfa->from_store, fname);
|
445
|
+
int len = (int)is_length(is);
|
446
|
+
uchar *buffer = ALLOC_N(uchar, len + 1);
|
447
447
|
|
448
|
-
|
449
|
-
|
450
|
-
|
448
|
+
is_read_bytes(is, buffer, len);
|
449
|
+
os_write_bytes(os, buffer, len);
|
450
|
+
|
451
|
+
is_close(is);
|
452
|
+
os_close(os);
|
453
|
+
free(buffer);
|
451
454
|
}
|
452
455
|
|
453
456
|
Store *open_ram_store_and_copy(Store *from_store, bool close_dir)
|
454
457
|
{
|
455
|
-
|
456
|
-
|
457
|
-
|
458
|
-
|
458
|
+
Store *store = open_ram_store();
|
459
|
+
struct CopyFileArg cfa;
|
460
|
+
cfa.to_store = store;
|
461
|
+
cfa.from_store = from_store;
|
459
462
|
|
460
|
-
|
463
|
+
from_store->each(from_store, ©_files, &cfa);
|
461
464
|
|
462
|
-
|
463
|
-
|
465
|
+
if (close_dir) {
|
466
|
+
store_deref(from_store);
|
467
|
+
}
|
464
468
|
|
465
|
-
|
469
|
+
return store;
|
466
470
|
}
|