ferret 0.11.6 → 0.11.8.4
Sign up to get free protection for your applications and to get access to all the features.
- data/README +10 -22
- data/RELEASE_CHANGES +137 -0
- data/RELEASE_NOTES +60 -0
- data/Rakefile +379 -274
- data/TODO +100 -8
- data/bin/ferret-browser +0 -0
- data/ext/BZLIB_blocksort.c +1094 -0
- data/ext/BZLIB_bzlib.c +1578 -0
- data/ext/BZLIB_compress.c +672 -0
- data/ext/BZLIB_crctable.c +104 -0
- data/ext/BZLIB_decompress.c +626 -0
- data/ext/BZLIB_huffman.c +205 -0
- data/ext/BZLIB_randtable.c +84 -0
- data/ext/{api.c → STEMMER_api.c} +7 -10
- data/ext/{libstemmer.c → STEMMER_libstemmer.c} +3 -2
- data/ext/{stem_ISO_8859_1_danish.c → STEMMER_stem_ISO_8859_1_danish.c} +123 -124
- data/ext/{stem_ISO_8859_1_dutch.c → STEMMER_stem_ISO_8859_1_dutch.c} +177 -188
- data/ext/STEMMER_stem_ISO_8859_1_english.c +1117 -0
- data/ext/{stem_ISO_8859_1_finnish.c → STEMMER_stem_ISO_8859_1_finnish.c} +276 -306
- data/ext/STEMMER_stem_ISO_8859_1_french.c +1246 -0
- data/ext/{stem_ISO_8859_1_german.c → STEMMER_stem_ISO_8859_1_german.c} +161 -170
- data/ext/STEMMER_stem_ISO_8859_1_hungarian.c +1230 -0
- data/ext/STEMMER_stem_ISO_8859_1_italian.c +1065 -0
- data/ext/STEMMER_stem_ISO_8859_1_norwegian.c +297 -0
- data/ext/{stem_ISO_8859_1_porter.c → STEMMER_stem_ISO_8859_1_porter.c} +263 -290
- data/ext/{stem_ISO_8859_1_portuguese.c → STEMMER_stem_ISO_8859_1_portuguese.c} +362 -380
- data/ext/STEMMER_stem_ISO_8859_1_spanish.c +1093 -0
- data/ext/STEMMER_stem_ISO_8859_1_swedish.c +307 -0
- data/ext/STEMMER_stem_ISO_8859_2_romanian.c +998 -0
- data/ext/{stem_KOI8_R_russian.c → STEMMER_stem_KOI8_R_russian.c} +244 -245
- data/ext/STEMMER_stem_UTF_8_danish.c +339 -0
- data/ext/{stem_UTF_8_dutch.c → STEMMER_stem_UTF_8_dutch.c} +192 -211
- data/ext/STEMMER_stem_UTF_8_english.c +1125 -0
- data/ext/{stem_UTF_8_finnish.c → STEMMER_stem_UTF_8_finnish.c} +284 -324
- data/ext/STEMMER_stem_UTF_8_french.c +1256 -0
- data/ext/{stem_UTF_8_german.c → STEMMER_stem_UTF_8_german.c} +170 -187
- data/ext/STEMMER_stem_UTF_8_hungarian.c +1234 -0
- data/ext/STEMMER_stem_UTF_8_italian.c +1073 -0
- data/ext/STEMMER_stem_UTF_8_norwegian.c +299 -0
- data/ext/{stem_UTF_8_porter.c → STEMMER_stem_UTF_8_porter.c} +271 -310
- data/ext/STEMMER_stem_UTF_8_portuguese.c +1023 -0
- data/ext/STEMMER_stem_UTF_8_romanian.c +1004 -0
- data/ext/STEMMER_stem_UTF_8_russian.c +694 -0
- data/ext/STEMMER_stem_UTF_8_spanish.c +1097 -0
- data/ext/STEMMER_stem_UTF_8_swedish.c +309 -0
- data/ext/STEMMER_stem_UTF_8_turkish.c +2205 -0
- data/ext/{utilities.c → STEMMER_utilities.c} +100 -68
- data/ext/analysis.c +276 -121
- data/ext/analysis.h +190 -143
- data/ext/api.h +3 -4
- data/ext/array.c +5 -3
- data/ext/array.h +52 -43
- data/ext/bitvector.c +38 -482
- data/ext/bitvector.h +446 -124
- data/ext/bzlib.h +282 -0
- data/ext/bzlib_private.h +503 -0
- data/ext/compound_io.c +23 -22
- data/ext/config.h +21 -11
- data/ext/document.c +43 -40
- data/ext/document.h +31 -21
- data/ext/except.c +20 -38
- data/ext/except.h +89 -76
- data/ext/extconf.rb +3 -2
- data/ext/ferret.c +49 -35
- data/ext/ferret.h +14 -11
- data/ext/field_index.c +262 -0
- data/ext/field_index.h +52 -0
- data/ext/filter.c +11 -10
- data/ext/fs_store.c +65 -47
- data/ext/global.c +245 -165
- data/ext/global.h +252 -54
- data/ext/hash.c +200 -243
- data/ext/hash.h +205 -163
- data/ext/hashset.c +118 -96
- data/ext/hashset.h +110 -82
- data/ext/header.h +19 -19
- data/ext/helper.c +11 -10
- data/ext/helper.h +14 -6
- data/ext/index.c +745 -366
- data/ext/index.h +503 -529
- data/ext/internal.h +1020 -0
- data/ext/lang.c +10 -0
- data/ext/lang.h +35 -15
- data/ext/mempool.c +5 -4
- data/ext/mempool.h +30 -22
- data/ext/modules.h +35 -7
- data/ext/multimapper.c +43 -2
- data/ext/multimapper.h +32 -23
- data/ext/posh.c +0 -0
- data/ext/posh.h +4 -38
- data/ext/priorityqueue.c +10 -12
- data/ext/priorityqueue.h +33 -21
- data/ext/q_boolean.c +22 -9
- data/ext/q_const_score.c +3 -2
- data/ext/q_filtered_query.c +15 -12
- data/ext/q_fuzzy.c +147 -135
- data/ext/q_match_all.c +3 -2
- data/ext/q_multi_term.c +28 -32
- data/ext/q_parser.c +451 -173
- data/ext/q_phrase.c +158 -79
- data/ext/q_prefix.c +16 -18
- data/ext/q_range.c +363 -31
- data/ext/q_span.c +130 -141
- data/ext/q_term.c +21 -21
- data/ext/q_wildcard.c +19 -23
- data/ext/r_analysis.c +369 -242
- data/ext/r_index.c +421 -434
- data/ext/r_qparser.c +142 -92
- data/ext/r_search.c +790 -407
- data/ext/r_store.c +44 -44
- data/ext/r_utils.c +264 -96
- data/ext/ram_store.c +29 -23
- data/ext/scanner.c +895 -0
- data/ext/scanner.h +36 -0
- data/ext/scanner_mb.c +6701 -0
- data/ext/scanner_utf8.c +4415 -0
- data/ext/search.c +210 -87
- data/ext/search.h +556 -488
- data/ext/similarity.c +17 -16
- data/ext/similarity.h +51 -44
- data/ext/sort.c +157 -354
- data/ext/stem_ISO_8859_1_hungarian.h +16 -0
- data/ext/stem_ISO_8859_2_romanian.h +16 -0
- data/ext/stem_UTF_8_hungarian.h +16 -0
- data/ext/stem_UTF_8_romanian.h +16 -0
- data/ext/stem_UTF_8_turkish.h +16 -0
- data/ext/stopwords.c +287 -278
- data/ext/store.c +57 -51
- data/ext/store.h +308 -286
- data/ext/symbol.c +10 -0
- data/ext/symbol.h +23 -0
- data/ext/term_vectors.c +14 -293
- data/ext/threading.h +22 -22
- data/ext/win32.h +12 -4
- data/lib/ferret.rb +2 -1
- data/lib/ferret/browser.rb +1 -1
- data/lib/ferret/field_symbol.rb +94 -0
- data/lib/ferret/index.rb +221 -34
- data/lib/ferret/number_tools.rb +6 -6
- data/lib/ferret/version.rb +3 -0
- data/test/{unit → long_running}/largefile/tc_largefile.rb +1 -1
- data/test/test_helper.rb +7 -2
- data/test/test_installed.rb +1 -0
- data/test/threading/thread_safety_index_test.rb +10 -1
- data/test/threading/thread_safety_read_write_test.rb +4 -7
- data/test/threading/thread_safety_test.rb +0 -0
- data/test/unit/analysis/tc_analyzer.rb +29 -27
- data/test/unit/analysis/tc_token_stream.rb +23 -16
- data/test/unit/index/tc_index.rb +116 -11
- data/test/unit/index/tc_index_reader.rb +27 -27
- data/test/unit/index/tc_index_writer.rb +10 -0
- data/test/unit/index/th_doc.rb +38 -21
- data/test/unit/search/tc_filter.rb +31 -10
- data/test/unit/search/tc_index_searcher.rb +6 -0
- data/test/unit/search/tm_searcher.rb +53 -1
- data/test/unit/store/tc_fs_store.rb +40 -2
- data/test/unit/store/tc_ram_store.rb +0 -0
- data/test/unit/store/tm_store.rb +0 -0
- data/test/unit/store/tm_store_lock.rb +7 -6
- data/test/unit/tc_field_symbol.rb +26 -0
- data/test/unit/ts_analysis.rb +0 -0
- data/test/unit/ts_index.rb +0 -0
- data/test/unit/ts_store.rb +0 -0
- data/test/unit/ts_utils.rb +0 -0
- data/test/unit/utils/tc_number_tools.rb +0 -0
- data/test/utils/content_generator.rb +226 -0
- metadata +262 -221
- data/ext/inc/lang.h +0 -48
- data/ext/inc/threading.h +0 -31
- data/ext/stem_ISO_8859_1_english.c +0 -1156
- data/ext/stem_ISO_8859_1_french.c +0 -1276
- data/ext/stem_ISO_8859_1_italian.c +0 -1091
- data/ext/stem_ISO_8859_1_norwegian.c +0 -296
- data/ext/stem_ISO_8859_1_spanish.c +0 -1119
- data/ext/stem_ISO_8859_1_swedish.c +0 -307
- data/ext/stem_UTF_8_danish.c +0 -344
- data/ext/stem_UTF_8_english.c +0 -1176
- data/ext/stem_UTF_8_french.c +0 -1296
- data/ext/stem_UTF_8_italian.c +0 -1113
- data/ext/stem_UTF_8_norwegian.c +0 -302
- data/ext/stem_UTF_8_portuguese.c +0 -1055
- data/ext/stem_UTF_8_russian.c +0 -709
- data/ext/stem_UTF_8_spanish.c +0 -1137
- data/ext/stem_UTF_8_swedish.c +0 -313
- data/lib/ferret_version.rb +0 -3
data/ext/bitvector.h
CHANGED
@@ -1,13 +1,18 @@
|
|
1
1
|
#ifndef FRT_BIT_VECTOR_H
|
2
2
|
#define FRT_BIT_VECTOR_H
|
3
3
|
|
4
|
+
#ifdef __cplusplus
|
5
|
+
extern "C" {
|
6
|
+
#endif
|
7
|
+
|
4
8
|
#include "global.h"
|
5
9
|
|
6
|
-
#define
|
7
|
-
|
10
|
+
#define FRT_BV_INIT_CAPA 256
|
11
|
+
|
12
|
+
typedef struct FrtBitVector
|
8
13
|
{
|
9
14
|
/** The bits are held in an array of 32-bit integers */
|
10
|
-
|
15
|
+
frt_u32 *bits;
|
11
16
|
|
12
17
|
/** size is equal to 1 + the highest order bit set */
|
13
18
|
int size;
|
@@ -15,8 +20,9 @@ typedef struct BitVector
|
|
15
20
|
/** capa is the number of words (U32) allocated for the bits */
|
16
21
|
int capa;
|
17
22
|
|
18
|
-
/** count is the running count of bits set. This is kept up to
|
19
|
-
*
|
23
|
+
/** count is the running count of bits set. This is kept up to
|
24
|
+
* date by frt_bv_set and frt_bv_unset. You can reset this value
|
25
|
+
* by calling frt_bv_recount */
|
20
26
|
int count;
|
21
27
|
|
22
28
|
/** curr_bit is used by scan_next to record the previously scanned bit */
|
@@ -24,249 +30,565 @@ typedef struct BitVector
|
|
24
30
|
|
25
31
|
bool extends_as_ones : 1;
|
26
32
|
int ref_cnt;
|
27
|
-
}
|
33
|
+
} FrtBitVector;
|
28
34
|
|
29
35
|
/**
|
30
|
-
* Create a new
|
31
|
-
*
|
36
|
+
* Create a new FrtBitVector with a capacity of
|
37
|
+
* +FRT_BV_INIT_CAPA+. Note that the FrtBitVector is growable and will
|
38
|
+
* adjust it's capacity when you use frt_bv_set.
|
32
39
|
*
|
33
|
-
* @return
|
40
|
+
* @return FrtBitVector with a capacity of +FRT_BV_INIT_CAPA+.
|
34
41
|
*/
|
35
|
-
extern
|
42
|
+
extern FRT_ATTR_MALLOC
|
43
|
+
FrtBitVector *frt_bv_new();
|
36
44
|
|
37
45
|
/**
|
38
|
-
* Create a new
|
39
|
-
* is growable and will adjust it's capacity when you use
|
46
|
+
* Create a new FrtBitVector with a capacity of +capa+. Note that the
|
47
|
+
* FrtBitVector is growable and will adjust it's capacity when you use
|
48
|
+
* frt_bv_set.
|
40
49
|
*
|
41
|
-
* @param capa the initial capacity of the
|
42
|
-
* @return
|
50
|
+
* @param capa the initial capacity of the FrtBitVector
|
51
|
+
* @return FrtBitVector with a capacity of +capa+.
|
43
52
|
*/
|
44
|
-
extern
|
53
|
+
extern FRT_ATTR_MALLOC
|
54
|
+
FrtBitVector *frt_bv_new_capa(int capa);
|
45
55
|
|
46
56
|
/**
|
47
|
-
* Destroy a
|
57
|
+
* Destroy a FrtBitVector, freeing all memory allocated to that
|
58
|
+
* FrtBitVector
|
48
59
|
*
|
49
|
-
* @param bv
|
60
|
+
* @param bv FrtBitVector to destroy
|
50
61
|
*/
|
51
|
-
extern void
|
62
|
+
extern void frt_bv_destroy(FrtBitVector *bv);
|
52
63
|
|
53
64
|
/**
|
54
|
-
* Set the bit at position +index+. If +index+ is outside
|
55
|
-
*
|
56
|
-
* + 1. If it is greater than
|
57
|
-
* be expanded to
|
65
|
+
* Set the bit at position +index+ with +value+. If +index+ is outside
|
66
|
+
* of the range of the FrtBitVector, that is >= FrtBitVector.size,
|
67
|
+
* FrtBitVector.size will be set to +index+ + 1. If it is greater than
|
68
|
+
* the capacity of the FrtBitVector, the capacity will be expanded to
|
69
|
+
* accomodate.
|
58
70
|
*
|
59
|
-
* @param bv the
|
71
|
+
* @param bv the FrtBitVector to set the bit in
|
60
72
|
* @param index the index of the bit to set
|
73
|
+
* @param value the boolean value
|
61
74
|
*/
|
62
|
-
|
75
|
+
|
76
|
+
/*
|
77
|
+
* FIXME: if the top set bit is unset, size is not adjusted. This will not
|
78
|
+
* cause any bugs in this code but could cause problems if users are relying
|
79
|
+
* on the fact that size is accurate.
|
80
|
+
*/
|
81
|
+
static FRT_ATTR_ALWAYS_INLINE
|
82
|
+
void frt_bv_set_value(FrtBitVector *bv, int bit, bool value)
|
83
|
+
{
|
84
|
+
frt_u32 *word_p;
|
85
|
+
int word = bit >> 5;
|
86
|
+
frt_u32 bitmask = 1 << (bit & 31);
|
87
|
+
|
88
|
+
/* Check to see if we need to grow the BitVector */
|
89
|
+
if (unlikely(bit >= bv->size)) {
|
90
|
+
bv->size = bit + 1; /* size is max range of bits set */
|
91
|
+
if (word >= bv->capa) {
|
92
|
+
int capa = bv->capa << 1;
|
93
|
+
while (capa <= word) {
|
94
|
+
capa <<= 1;
|
95
|
+
}
|
96
|
+
FRT_REALLOC_N(bv->bits, frt_u32, capa);
|
97
|
+
memset(bv->bits + bv->capa, (bv->extends_as_ones ? 0xFF : 0),
|
98
|
+
sizeof(frt_u32) * (capa - bv->capa));
|
99
|
+
bv->capa = capa;
|
100
|
+
}
|
101
|
+
}
|
102
|
+
|
103
|
+
/* Set the required bit */
|
104
|
+
word_p = &(bv->bits[word]);
|
105
|
+
if ((!!(bitmask & *word_p)) != value) {
|
106
|
+
if (value) {
|
107
|
+
bv->count++;
|
108
|
+
*word_p |= bitmask;
|
109
|
+
}
|
110
|
+
else {
|
111
|
+
bv->count--;
|
112
|
+
*word_p &= ~bitmask;
|
113
|
+
}
|
114
|
+
}
|
115
|
+
}
|
116
|
+
|
117
|
+
/**
|
118
|
+
* Set the bit at position +index+. If +index+ is outside of the range
|
119
|
+
* of the FrtBitVector, that is >= FrtBitVector.size,
|
120
|
+
* FrtBitVector.size will be set to +index+ + 1. If it is greater than
|
121
|
+
* the capacity of the FrtBitVector, the capacity will be expanded to
|
122
|
+
* accomodate.
|
123
|
+
*
|
124
|
+
* @param bv the FrtBitVector to set the bit in
|
125
|
+
* @param index the index of the bit to set
|
126
|
+
*/
|
127
|
+
static FRT_ATTR_ALWAYS_INLINE
|
128
|
+
void frt_bv_set(FrtBitVector *bv, int bit)
|
129
|
+
{
|
130
|
+
frt_bv_set_value(bv, bit, 1);
|
131
|
+
}
|
63
132
|
|
64
133
|
/**
|
65
134
|
* Unsafely set the bit at position +index+. If you choose to use this
|
66
|
-
* function you must create the
|
67
|
-
* accomodate all of the
|
68
|
-
* order and only one time per bit. Otherwise,
|
135
|
+
* function you must create the FrtBitVector with a large enough
|
136
|
+
* capacity to accomodate all of the frt_bv_set_fast operations. You
|
137
|
+
* must also set bits in order and only one time per bit. Otherwise,
|
138
|
+
* use the safe frt_bv_set function.
|
69
139
|
*
|
70
140
|
* So this is ok;
|
71
141
|
* <pre>
|
72
|
-
*
|
73
|
-
*
|
74
|
-
*
|
75
|
-
*
|
142
|
+
* FrtBitVector *bv = frt_bv_new_capa(1000);
|
143
|
+
* frt_bv_set_fast(bv, 900);
|
144
|
+
* frt_bv_set_fast(bv, 920);
|
145
|
+
* frt_bv_set_fast(bv, 999);
|
76
146
|
* </pre>
|
77
147
|
*
|
78
148
|
* While these are not ok;
|
79
149
|
* <pre>
|
80
|
-
*
|
81
|
-
*
|
82
|
-
*
|
83
|
-
*
|
84
|
-
*
|
150
|
+
* FrtBitVector *bv = frt_bv_new_capa(90);
|
151
|
+
* frt_bv_set_fast(bv, 80);
|
152
|
+
* frt_bv_set_fast(bv, 79); // <= Bad: Out of Order
|
153
|
+
* frt_bv_set_fast(bv, 80); // <= Bad: Already set
|
154
|
+
* frt_bv_set_fast(bv, 90); // <= Bad: Out of Range. index must be < capa
|
85
155
|
* </pre>
|
86
156
|
*
|
87
|
-
* @param bv the
|
157
|
+
* @param bv the FrtBitVector to set the bit in
|
88
158
|
* @param index the index of the bit to set
|
89
159
|
*/
|
90
|
-
|
160
|
+
static FRT_ATTR_ALWAYS_INLINE
|
161
|
+
void frt_bv_set_fast(FrtBitVector *bv, int bit)
|
162
|
+
{
|
163
|
+
bv->count++;
|
164
|
+
bv->size = bit + 1;
|
165
|
+
bv->bits[bit >> 5] |= (1 << (bit & 31));
|
166
|
+
}
|
91
167
|
|
92
168
|
/**
|
93
|
-
* Return 1 if the bit at +index+ was set or 0 otherwise. If +index+
|
94
|
-
* range, that is greater then the BitVectors capacity, it
|
169
|
+
* Return 1 if the bit at +index+ was set or 0 otherwise. If +index+
|
170
|
+
* is out of range, that is greater then the BitVectors capacity, it
|
171
|
+
* will also return 0.
|
95
172
|
*
|
96
|
-
* @param bv the
|
173
|
+
* @param bv the FrtBitVector to check in
|
97
174
|
* @param index the index of the bit to check
|
98
175
|
* @return 1 if the bit was set, 0 otherwise
|
99
176
|
*/
|
100
|
-
|
177
|
+
static FRT_ATTR_ALWAYS_INLINE
|
178
|
+
int frt_bv_get(FrtBitVector *bv, int bit)
|
179
|
+
{
|
180
|
+
/* out of range so return 0 because it can't have been set */
|
181
|
+
if (unlikely(bit >= bv->size)) {
|
182
|
+
return bv->extends_as_ones;
|
183
|
+
}
|
184
|
+
return (bv->bits[bit >> 5] >> (bit & 31)) & 0x01;
|
185
|
+
}
|
101
186
|
|
102
187
|
/**
|
103
|
-
* Unset the bit at position +index+. If the +index+ was out of range,
|
104
|
-
* greater than the BitVectors capacity then do
|
105
|
-
* in this case anyway).
|
188
|
+
* Unset the bit at position +index+. If the +index+ was out of range,
|
189
|
+
* that is greater than the BitVectors capacity then do
|
190
|
+
* nothing. (frt_bv_get will return 0 in this case anyway).
|
106
191
|
*
|
107
|
-
* @param bv the
|
192
|
+
* @param bv the FrtBitVector to unset the bit in
|
108
193
|
* @param index the index of the bit to unset
|
109
194
|
*/
|
110
|
-
|
195
|
+
static FRT_ATTR_ALWAYS_INLINE
|
196
|
+
void frt_bv_unset(FrtBitVector *bv, int bit)
|
197
|
+
{
|
198
|
+
frt_bv_set_value(bv, bit, 0);
|
199
|
+
}
|
111
200
|
|
112
201
|
/**
|
113
202
|
* Clear all set bits. This function will set all set bits to 0.
|
114
203
|
*
|
115
|
-
* @param bv the
|
204
|
+
* @param bv the FrtBitVector to clear
|
116
205
|
*/
|
117
|
-
extern void
|
206
|
+
extern void frt_bv_clear(FrtBitVector *bv);
|
118
207
|
|
119
208
|
/**
|
120
|
-
* Resets the set bit count by running through the whole
|
121
|
-
* counting all set bits. A running count of the bits is kept by
|
122
|
-
*
|
123
|
-
*
|
124
|
-
*
|
209
|
+
* Resets the set bit count by running through the whole FrtBitVector
|
210
|
+
* and counting all set bits. A running count of the bits is kept by
|
211
|
+
* frt_bv_set, *frt_bv_get and frt_bv_set_fast so this function is
|
212
|
+
* only necessary if the count could have been corrupted somehow or if
|
213
|
+
* the FrtBitVector has been constructed in a different way (for
|
214
|
+
* example being read from the file_system).
|
125
215
|
*
|
126
|
-
* @param bv the
|
127
|
-
* @return the number of set bits in the
|
216
|
+
* @param bv the FrtBitVector to count the bits in
|
217
|
+
* @return the number of set bits in the FrtBitVector. FrtBitVector.count is also
|
128
218
|
* set
|
129
219
|
*/
|
130
|
-
|
220
|
+
static FRT_ATTR_ALWAYS_INLINE
|
221
|
+
int frt_bv_recount(FrtBitVector *bv)
|
222
|
+
{
|
223
|
+
unsigned int extra = ((bv->size & 31) >> 3) + 1;
|
224
|
+
unsigned int len = bv->size >> 5;
|
225
|
+
unsigned int idx, count = 0;
|
226
|
+
|
227
|
+
if (bv->extends_as_ones) {
|
228
|
+
for (idx = 0; idx < len; ++idx) {
|
229
|
+
count += frt_count_zeros(bv->bits[idx]);
|
230
|
+
}
|
231
|
+
switch (extra) {
|
232
|
+
case 4: count += frt_count_zeros(bv->bits[idx] | 0x00ffffff);
|
233
|
+
case 3: count += frt_count_zeros(bv->bits[idx] | 0xff00ffff);
|
234
|
+
case 2: count += frt_count_zeros(bv->bits[idx] | 0xffff00ff);
|
235
|
+
case 1: count += frt_count_zeros(bv->bits[idx] | 0xffffff00);
|
236
|
+
}
|
237
|
+
}
|
238
|
+
else {
|
239
|
+
for (idx = 0; idx < len; ++idx) {
|
240
|
+
count += frt_count_ones(bv->bits[idx]);
|
241
|
+
}
|
242
|
+
switch (extra) {
|
243
|
+
case 4: count += frt_count_ones(bv->bits[idx] & 0xff000000);
|
244
|
+
case 3: count += frt_count_ones(bv->bits[idx] & 0x00ff0000);
|
245
|
+
case 2: count += frt_count_ones(bv->bits[idx] & 0x0000ff00);
|
246
|
+
case 1: count += frt_count_ones(bv->bits[idx] & 0x000000ff);
|
247
|
+
}
|
248
|
+
}
|
249
|
+
return bv->count = count;
|
250
|
+
}
|
131
251
|
|
132
252
|
/**
|
133
|
-
* Reset the
|
134
|
-
* using
|
135
|
-
* not necessary when using
|
253
|
+
* Reset the FrtBitVector for scanning. This function should be called
|
254
|
+
* before using frt_bv_scan_next to scan through all set bits in the
|
255
|
+
* FrtBitVector. This is not necessary when using
|
256
|
+
* frt_bv_scan_next_from.
|
136
257
|
*
|
137
|
-
* @param bv the
|
258
|
+
* @param bv the FrtBitVector to reset for scanning
|
138
259
|
*/
|
139
|
-
extern void
|
260
|
+
extern void frt_bv_scan_reset(FrtBitVector *bv);
|
140
261
|
|
141
262
|
/**
|
142
|
-
* Scan the
|
143
|
-
*
|
144
|
-
*
|
145
|
-
* -1.
|
263
|
+
* Scan the FrtBitVector for the next set bit after +from+. If no more
|
264
|
+
* bits are set then return -1, otherwise return the index of teh next
|
265
|
+
* set bit.
|
146
266
|
*
|
147
|
-
* @param bv the
|
148
|
-
* @return the next set
|
267
|
+
* @param bv the FrtBitVector to scan
|
268
|
+
* @return the next set bit's index or -1 if no more bits are set
|
149
269
|
*/
|
150
|
-
|
270
|
+
static FRT_ATTR_ALWAYS_INLINE
|
271
|
+
int frt_bv_scan_next_from(FrtBitVector *bv, const int bit)
|
272
|
+
{
|
273
|
+
frt_u32 pos = bit >> 5;
|
274
|
+
frt_u32 word = bv->bits[pos];
|
275
|
+
|
276
|
+
if (bit >= bv->size)
|
277
|
+
return -1;
|
278
|
+
|
279
|
+
/* Keep only the bits above this position */
|
280
|
+
word &= ~0 << (bit & 31);
|
281
|
+
if (word) {
|
282
|
+
goto done;
|
283
|
+
}
|
284
|
+
else {
|
285
|
+
frt_u32 word_size = FRT_TO_WORD(bv->size);
|
286
|
+
for (pos++; pos < word_size; ++pos)
|
287
|
+
{
|
288
|
+
if ( (word = bv->bits[pos]) )
|
289
|
+
goto done;
|
290
|
+
}
|
291
|
+
}
|
292
|
+
return -1;
|
293
|
+
done:
|
294
|
+
return bv->curr_bit = (pos << 5) + frt_count_trailing_zeros(word);
|
295
|
+
}
|
151
296
|
|
152
297
|
/**
|
153
|
-
* Scan the
|
154
|
-
*
|
298
|
+
* Scan the FrtBitVector for the next set bit. Before using this
|
299
|
+
* function you should reset the FrtBitVector for scanning using
|
300
|
+
* +frt_bv_scan_reset+. You can the repeatedly call frt_bv_scan_next
|
301
|
+
* to get each set bit until it finally returns -1.
|
155
302
|
*
|
156
|
-
* @param bv the
|
157
|
-
* @return the next set
|
303
|
+
* @param bv the FrtBitVector to scan
|
304
|
+
* @return the next set bits index or -1 if no more bits are set
|
158
305
|
*/
|
306
|
+
static FRT_ATTR_ALWAYS_INLINE
|
307
|
+
int frt_bv_scan_next(FrtBitVector *bv)
|
308
|
+
{
|
309
|
+
return frt_bv_scan_next_from(bv, bv->curr_bit + 1);
|
310
|
+
}
|
159
311
|
|
160
|
-
extern int bv_scan_next_from(BitVector *bv, register const int from);
|
161
312
|
/**
|
162
|
-
* Scan the
|
163
|
-
*
|
164
|
-
*
|
165
|
-
* -1.
|
313
|
+
* Scan the FrtBitVector for the next unset bit after +from+. If no
|
314
|
+
* more bits are unset then return -1, otherwise return the index of
|
315
|
+
* teh next unset bit.
|
166
316
|
*
|
167
|
-
* @param bv the
|
168
|
-
* @return the next unset
|
317
|
+
* @param bv the FrtBitVector to scan
|
318
|
+
* @return the next unset bit's index or -1 if no more bits are unset
|
169
319
|
*/
|
170
|
-
|
320
|
+
static FRT_ATTR_ALWAYS_INLINE
|
321
|
+
int frt_bv_scan_next_unset_from(FrtBitVector *bv, const int bit)
|
322
|
+
{
|
323
|
+
frt_u32 pos = bit >> 5;
|
324
|
+
frt_u32 word = bv->bits[pos];
|
325
|
+
|
326
|
+
if (bit >= bv->size)
|
327
|
+
return -1;
|
328
|
+
|
329
|
+
/* Set all of the bits below this position */
|
330
|
+
word |= (1 << (bit & 31)) - 1;
|
331
|
+
if (~word) {
|
332
|
+
goto done;
|
333
|
+
}
|
334
|
+
else {
|
335
|
+
frt_u32 word_size = FRT_TO_WORD(bv->size);
|
336
|
+
for (pos++; pos < word_size; ++pos)
|
337
|
+
{
|
338
|
+
if ( ~(word = bv->bits[pos]) )
|
339
|
+
goto done;
|
340
|
+
}
|
341
|
+
}
|
342
|
+
return -1;
|
343
|
+
done:
|
344
|
+
return bv->curr_bit = (pos << 5) + frt_count_trailing_ones(word);
|
345
|
+
}
|
171
346
|
|
172
347
|
/**
|
173
|
-
* Scan the
|
174
|
-
*
|
348
|
+
* Scan the FrtBitVector for the next unset bit. Before using this
|
349
|
+
* function you should reset the FrtBitVector for scanning using
|
350
|
+
* +frt_bv_scan_reset+. You can the repeated call frt_bv_scan_next to
|
351
|
+
* get each unset bit until it finally returns -1.
|
175
352
|
*
|
176
|
-
* @param bv the
|
177
|
-
* @return the next unset
|
353
|
+
* @param bv the FrtBitVector to scan
|
354
|
+
* @return the next unset bits index or -1 if no more bits are unset
|
178
355
|
*/
|
179
|
-
|
356
|
+
static FRT_ATTR_ALWAYS_INLINE
|
357
|
+
int frt_bv_scan_next_unset(FrtBitVector *bv)
|
358
|
+
{
|
359
|
+
return frt_bv_scan_next_unset_from(bv, bv->curr_bit + 1);
|
360
|
+
}
|
180
361
|
|
181
362
|
/**
|
182
363
|
* Check whether the two BitVectors have the same bits set.
|
183
364
|
*
|
184
|
-
* @param bv1 first
|
365
|
+
* @param bv1 first FrtBitVector to compare
|
185
366
|
* @param bv2 second BitVectors to compare
|
186
367
|
* @return true if bv1 == bv2
|
187
368
|
*/
|
188
|
-
extern int
|
369
|
+
extern int frt_bv_eq(FrtBitVector *bv1, FrtBitVector *bv2);
|
189
370
|
|
190
371
|
/**
|
191
|
-
* Determines a hash value for the
|
372
|
+
* Determines a hash value for the FrtBitVector
|
192
373
|
*
|
193
|
-
* @param bv the
|
194
|
-
* @return A hash value for the
|
374
|
+
* @param bv the FrtBitVector to hash
|
375
|
+
* @return A hash value for the FrtBitVector
|
195
376
|
*/
|
196
|
-
extern unsigned long
|
377
|
+
extern unsigned long frt_bv_hash(FrtBitVector *bv);
|
378
|
+
|
379
|
+
static FRT_ATTR_ALWAYS_INLINE
|
380
|
+
void frt_bv_capa(FrtBitVector *bv, int capa, int size)
|
381
|
+
{
|
382
|
+
int word_size = FRT_TO_WORD(size);
|
383
|
+
if (bv->capa < capa)
|
384
|
+
{
|
385
|
+
FRT_REALLOC_N(bv->bits, frt_u32, capa);
|
386
|
+
bv->capa = capa;
|
387
|
+
memset(bv->bits + word_size, (bv->extends_as_ones ? 0xFF : 0),
|
388
|
+
sizeof(frt_u32) * (capa - word_size));
|
389
|
+
}
|
390
|
+
bv->size = size;
|
391
|
+
}
|
392
|
+
|
393
|
+
#define frt_bv_and_ext(dest, src, extends_as_ones, i, max) do { \
|
394
|
+
if (extends_as_ones) \
|
395
|
+
memcpy(&dest[i], &src[i], sizeof(*dest)*(max - i)); \
|
396
|
+
else memset(&dest[i], 0x00 , sizeof(*dest)*(max - i)); \
|
397
|
+
} while(0)
|
398
|
+
|
399
|
+
#define frt_bv_or_ext(dest, src, extends_as_ones, i, max) do { \
|
400
|
+
if (extends_as_ones) \
|
401
|
+
memset(&dest[i], 0xFF , sizeof(*dest)*(max - i)); \
|
402
|
+
else memcpy(&dest[i], &src[i], sizeof(*dest)*(max - i)); \
|
403
|
+
} while(0)
|
404
|
+
|
405
|
+
#define frt_bv_xor_ext(dest, src, extends_as_ones, i, max) do { \
|
406
|
+
frt_u32 n = (extends_as_ones ? 0xffffffff : 0); \
|
407
|
+
for (; i < max; ++i) \
|
408
|
+
dest[i] = src[i] ^ n; \
|
409
|
+
} while(0)
|
410
|
+
|
411
|
+
#define FRT_BV_OP(bv, a, b, op, ext_cb) do { \
|
412
|
+
int i; \
|
413
|
+
int a_wsz = FRT_TO_WORD(a->size); \
|
414
|
+
int b_wsz = FRT_TO_WORD(b->size); \
|
415
|
+
int max_size = frt_max2(a->size, b->size); \
|
416
|
+
int min_size = frt_min2(a->size, b->size); \
|
417
|
+
int max_word_size = FRT_TO_WORD(max_size); \
|
418
|
+
int min_word_size = FRT_TO_WORD(min_size); \
|
419
|
+
int capa = frt_max2(frt_round2(max_word_size), 4); \
|
420
|
+
\
|
421
|
+
bv->extends_as_ones = (a->extends_as_ones op b->extends_as_ones); \
|
422
|
+
frt_bv_capa(bv, capa, max_size); \
|
423
|
+
\
|
424
|
+
for (i = 0; i < min_word_size; ++i) \
|
425
|
+
bv->bits[i] = a->bits[i] op b->bits[i]; \
|
426
|
+
\
|
427
|
+
if (a_wsz != b_wsz) { \
|
428
|
+
frt_u32 *bits = a->bits; \
|
429
|
+
bool extends_as_ones = b->extends_as_ones; \
|
430
|
+
if (a_wsz < b_wsz) { \
|
431
|
+
bits = b->bits; \
|
432
|
+
extends_as_ones = a->extends_as_ones; \
|
433
|
+
} \
|
434
|
+
ext_cb(bv->bits, bits, extends_as_ones, i, max_word_size); \
|
435
|
+
} \
|
436
|
+
frt_bv_recount(bv); \
|
437
|
+
} while(0)
|
438
|
+
|
439
|
+
static FRT_ATTR_ALWAYS_INLINE
|
440
|
+
FrtBitVector *frt_bv_and_i(FrtBitVector *bv,
|
441
|
+
FrtBitVector *a, FrtBitVector *b)
|
442
|
+
{
|
443
|
+
FRT_BV_OP(bv, a, b, &, frt_bv_and_ext);
|
444
|
+
return bv;
|
445
|
+
}
|
446
|
+
|
447
|
+
static FRT_ATTR_ALWAYS_INLINE
|
448
|
+
FrtBitVector *frt_bv_or_i(FrtBitVector *bv,
|
449
|
+
FrtBitVector *a, FrtBitVector *b)
|
450
|
+
{
|
451
|
+
FRT_BV_OP(bv, a, b, |, frt_bv_or_ext);
|
452
|
+
return bv;
|
453
|
+
}
|
454
|
+
|
455
|
+
static FRT_ATTR_ALWAYS_INLINE
|
456
|
+
FrtBitVector *frt_bv_xor_i(FrtBitVector *bv,
|
457
|
+
FrtBitVector *a, FrtBitVector *b)
|
458
|
+
{
|
459
|
+
FRT_BV_OP(bv, a, b, ^, frt_bv_xor_ext);
|
460
|
+
return bv;
|
461
|
+
}
|
462
|
+
|
463
|
+
static FRT_ATTR_ALWAYS_INLINE
|
464
|
+
FrtBitVector *frt_bv_not_i(FrtBitVector *bv, FrtBitVector *bv1)
|
465
|
+
{
|
466
|
+
int i;
|
467
|
+
int word_size = FRT_TO_WORD(bv1->size);
|
468
|
+
int capa = frt_max2(frt_round2(word_size), 4);
|
469
|
+
|
470
|
+
bv->extends_as_ones = !bv1->extends_as_ones;
|
471
|
+
frt_bv_capa(bv, capa, bv1->size);
|
472
|
+
|
473
|
+
for (i = 0; i < word_size; i++)
|
474
|
+
bv->bits[i] = ~(bv1->bits[i]);
|
475
|
+
|
476
|
+
memset(bv->bits + word_size, (bv->extends_as_ones ? 0xFF : 0),
|
477
|
+
sizeof(frt_u32) * (bv->capa - word_size));
|
478
|
+
|
479
|
+
frt_bv_recount(bv);
|
480
|
+
return bv;
|
481
|
+
}
|
197
482
|
|
198
483
|
/**
|
199
484
|
* ANDs two BitVectors (+bv1+ and +bv2+) together and return the resultant
|
200
|
-
*
|
485
|
+
* FrtBitVector
|
201
486
|
*
|
202
|
-
* @param bv1 first
|
203
|
-
* @param bv2 second
|
204
|
-
* @return A
|
487
|
+
* @param bv1 first FrtBitVector to AND
|
488
|
+
* @param bv2 second FrtBitVector to AND
|
489
|
+
* @return A FrtBitVector with all bits set that are set in both bv1 and bv2
|
205
490
|
*/
|
206
|
-
|
491
|
+
static FRT_ATTR_ALWAYS_INLINE
|
492
|
+
FrtBitVector *frt_bv_and(FrtBitVector *bv1, FrtBitVector *bv2)
|
493
|
+
{
|
494
|
+
return frt_bv_and_i(frt_bv_new(), bv1, bv2);
|
495
|
+
}
|
207
496
|
|
208
497
|
/**
|
209
498
|
* ORs two BitVectors (+bv1+ and +bv2+) together and return the resultant
|
210
|
-
*
|
499
|
+
* FrtBitVector
|
211
500
|
*
|
212
|
-
* @param bv1 first
|
213
|
-
* @param bv2 second
|
214
|
-
* @return A
|
501
|
+
* @param bv1 first FrtBitVector to OR
|
502
|
+
* @param bv2 second FrtBitVector to OR
|
503
|
+
* @return A FrtBitVector with all bits set that are set in both bv1 and bv2
|
215
504
|
*/
|
216
|
-
|
505
|
+
static FRT_ATTR_ALWAYS_INLINE
|
506
|
+
FrtBitVector *frt_bv_or(FrtBitVector *bv1, FrtBitVector *bv2)
|
507
|
+
{
|
508
|
+
return frt_bv_or_i(frt_bv_new(), bv1, bv2);
|
509
|
+
}
|
510
|
+
|
217
511
|
|
218
512
|
/**
|
219
513
|
* XORs two BitVectors (+bv1+ and +bv2+) together and return the resultant
|
220
|
-
*
|
514
|
+
* FrtBitVector
|
221
515
|
*
|
222
|
-
* @param bv1 first
|
223
|
-
* @param bv2 second
|
224
|
-
* @return A
|
516
|
+
* @param bv1 first FrtBitVector to XOR
|
517
|
+
* @param bv2 second FrtBitVector to XOR
|
518
|
+
* @return A FrtBitVector with all bits set that are equal in bv1 and bv2
|
225
519
|
*/
|
226
|
-
|
520
|
+
static FRT_ATTR_ALWAYS_INLINE
|
521
|
+
FrtBitVector *frt_bv_xor(FrtBitVector *bv1, FrtBitVector *bv2)
|
522
|
+
{
|
523
|
+
return frt_bv_xor_i(frt_bv_new(), bv1, bv2);
|
524
|
+
}
|
227
525
|
|
228
526
|
/**
|
229
|
-
* Returns
|
527
|
+
* Returns FrtBitVector with all of +bv+'s bits flipped
|
230
528
|
*
|
231
|
-
* @param bv
|
232
|
-
* @return A
|
529
|
+
* @param bv FrtBitVector to flip
|
530
|
+
* @return A FrtBitVector with all bits set that are set in both bv1 and bv2
|
233
531
|
*/
|
234
|
-
|
532
|
+
static FRT_ATTR_ALWAYS_INLINE
|
533
|
+
FrtBitVector *frt_bv_not(FrtBitVector *bv)
|
534
|
+
{
|
535
|
+
return frt_bv_not_i(frt_bv_new(), bv);
|
536
|
+
}
|
235
537
|
|
236
538
|
/**
|
237
539
|
* ANDs two BitVectors together +bv1+ and +bv2+ in place of +bv1+
|
238
540
|
*
|
239
|
-
* @param bv1 first
|
240
|
-
* @param bv2 second
|
241
|
-
* @return A
|
541
|
+
* @param bv1 first FrtBitVector to AND
|
542
|
+
* @param bv2 second FrtBitVector to AND
|
543
|
+
* @return A FrtBitVector
|
242
544
|
* @return bv1 with all bits set that where set in both bv1 and bv2
|
243
545
|
*/
|
244
|
-
|
546
|
+
static FRT_ATTR_ALWAYS_INLINE
|
547
|
+
FrtBitVector *frt_bv_and_x(FrtBitVector *bv1, FrtBitVector *bv2)
|
548
|
+
{
|
549
|
+
return frt_bv_and_i(bv1, bv1, bv2);
|
550
|
+
}
|
245
551
|
|
246
552
|
/**
|
247
553
|
* ORs two BitVectors together
|
248
554
|
*
|
249
|
-
* @param bv1 first
|
250
|
-
* @param bv2 second
|
555
|
+
* @param bv1 first FrtBitVector to OR
|
556
|
+
* @param bv2 second FrtBitVector to OR
|
251
557
|
* @return bv1
|
252
558
|
*/
|
253
|
-
|
559
|
+
static FRT_ATTR_ALWAYS_INLINE
|
560
|
+
FrtBitVector *frt_bv_or_x(FrtBitVector *bv1, FrtBitVector *bv2)
|
561
|
+
{
|
562
|
+
return frt_bv_or_i(bv1, bv1, bv2);
|
563
|
+
}
|
254
564
|
|
255
565
|
/**
|
256
566
|
* XORs two BitVectors together +bv1+ and +bv2+ in place of +bv1+
|
257
567
|
*
|
258
|
-
* @param bv1 first
|
259
|
-
* @param bv2 second
|
568
|
+
* @param bv1 first FrtBitVector to XOR
|
569
|
+
* @param bv2 second FrtBitVector to XOR
|
260
570
|
* @return bv1
|
261
571
|
*/
|
262
|
-
|
572
|
+
static FRT_ATTR_ALWAYS_INLINE
|
573
|
+
FrtBitVector *frt_bv_xor_x(FrtBitVector *bv1, FrtBitVector *bv2)
|
574
|
+
{
|
575
|
+
return frt_bv_xor_i(bv1, bv1, bv2);
|
576
|
+
}
|
263
577
|
|
264
578
|
/**
|
265
|
-
* Flips all bits in the
|
579
|
+
* Flips all bits in the FrtBitVector +bv+
|
266
580
|
*
|
267
|
-
* @param bv
|
581
|
+
* @param bv FrtBitVector to flip
|
268
582
|
* @return A +bv+ with all it's bits flipped
|
269
583
|
*/
|
270
|
-
|
584
|
+
static FRT_ATTR_ALWAYS_INLINE
|
585
|
+
FrtBitVector *frt_bv_not_x(FrtBitVector *bv)
|
586
|
+
{
|
587
|
+
return frt_bv_not_i(bv, bv);
|
588
|
+
}
|
589
|
+
|
590
|
+
#ifdef __cplusplus
|
591
|
+
} // extern "C"
|
592
|
+
#endif
|
271
593
|
|
272
594
|
#endif
|