isomorfeus-ferret 0.12.6 → 0.12.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1 -4
- data/ext/isomorfeus_ferret_ext/brotli_common_constants.c +15 -0
- data/ext/isomorfeus_ferret_ext/brotli_common_constants.h +200 -0
- data/ext/isomorfeus_ferret_ext/brotli_common_context.c +156 -0
- data/ext/isomorfeus_ferret_ext/brotli_common_context.h +113 -0
- data/ext/isomorfeus_ferret_ext/brotli_common_dictionary.c +5914 -0
- data/ext/isomorfeus_ferret_ext/brotli_common_dictionary.h +64 -0
- data/ext/isomorfeus_ferret_ext/brotli_common_platform.c +22 -0
- data/ext/isomorfeus_ferret_ext/brotli_common_platform.h +594 -0
- data/ext/isomorfeus_ferret_ext/brotli_common_transform.c +291 -0
- data/ext/isomorfeus_ferret_ext/brotli_common_transform.h +85 -0
- data/ext/isomorfeus_ferret_ext/brotli_common_version.h +26 -0
- data/ext/isomorfeus_ferret_ext/brotli_dec_bit_reader.c +76 -0
- data/ext/isomorfeus_ferret_ext/brotli_dec_bit_reader.h +351 -0
- data/ext/isomorfeus_ferret_ext/brotli_dec_decode.c +2608 -0
- data/ext/isomorfeus_ferret_ext/brotli_dec_huffman.c +339 -0
- data/ext/isomorfeus_ferret_ext/brotli_dec_huffman.h +121 -0
- data/ext/isomorfeus_ferret_ext/brotli_dec_prefix.h +732 -0
- data/ext/isomorfeus_ferret_ext/brotli_dec_state.c +159 -0
- data/ext/isomorfeus_ferret_ext/brotli_dec_state.h +365 -0
- data/ext/isomorfeus_ferret_ext/brotli_decode.h +344 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_backward_references.c +145 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_backward_references.h +39 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_backward_references_hq.c +843 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_backward_references_hq.h +95 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_backward_references_inc.h +163 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_bit_cost.c +35 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_bit_cost.h +63 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_bit_cost_inc.h +127 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_block_encoder_inc.h +34 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_block_splitter.c +194 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_block_splitter.h +51 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_block_splitter_inc.h +440 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_brotli_bit_stream.c +1314 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_brotli_bit_stream.h +84 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_cluster.c +56 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_cluster.h +48 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_cluster_inc.h +320 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_command.c +28 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_command.h +190 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_compress_fragment.c +790 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_compress_fragment.h +61 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_compress_fragment_two_pass.c +645 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_compress_fragment_two_pass.h +54 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_dictionary_hash.c +1846 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_dictionary_hash.h +25 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_encode.c +1927 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_encoder_dict.c +33 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_encoder_dict.h +43 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_entropy_encode.c +503 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_entropy_encode.h +122 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_entropy_encode_static.h +539 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_fast_log.c +105 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_fast_log.h +66 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_find_match_length.h +79 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_hash.h +488 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_hash_composite_inc.h +125 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_hash_forgetful_chain_inc.h +293 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_hash_longest_match64_inc.h +267 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_hash_longest_match_inc.h +262 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_hash_longest_match_quickly_inc.h +266 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_hash_rolling_inc.h +212 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_hash_to_binary_tree_inc.h +329 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_histogram.c +100 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_histogram.h +63 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_histogram_inc.h +51 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_literal_cost.c +175 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_literal_cost.h +30 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_memory.c +170 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_memory.h +114 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_metablock.c +663 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_metablock.h +105 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_metablock_inc.h +183 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_params.h +46 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_prefix.h +53 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_quality.h +165 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_ringbuffer.h +167 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_static_dict.c +486 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_static_dict.h +40 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_static_dict_lut.h +5864 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_utf8_util.c +85 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_utf8_util.h +32 -0
- data/ext/isomorfeus_ferret_ext/brotli_enc_write_bits.h +87 -0
- data/ext/isomorfeus_ferret_ext/brotli_encode.h +448 -0
- data/ext/isomorfeus_ferret_ext/brotli_port.h +288 -0
- data/ext/isomorfeus_ferret_ext/brotli_types.h +83 -0
- data/ext/isomorfeus_ferret_ext/frb_index.c +35 -4
- data/ext/isomorfeus_ferret_ext/frt_document.h +1 -0
- data/ext/isomorfeus_ferret_ext/frt_fs_store.c +1 -0
- data/ext/isomorfeus_ferret_ext/frt_index.c +174 -25
- data/ext/isomorfeus_ferret_ext/frt_index.h +6 -3
- data/ext/isomorfeus_ferret_ext/frt_ram_store.c +1 -0
- data/ext/isomorfeus_ferret_ext/test_fields.c +57 -45
- data/ext/isomorfeus_ferret_ext/test_index.c +4 -1
- data/lib/isomorfeus/ferret/version.rb +1 -1
- metadata +88 -3
@@ -0,0 +1,190 @@
|
|
1
|
+
/* Copyright 2013 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
7
|
+
/* This class models a sequence of literals and a backward reference copy. */
|
8
|
+
|
9
|
+
#ifndef BROTLI_ENC_COMMAND_H_
|
10
|
+
#define BROTLI_ENC_COMMAND_H_
|
11
|
+
|
12
|
+
#include "brotli_common_constants.h"
|
13
|
+
#include "brotli_common_platform.h"
|
14
|
+
#include "brotli_types.h"
|
15
|
+
#include "brotli_enc_fast_log.h"
|
16
|
+
#include "brotli_enc_params.h"
|
17
|
+
#include "brotli_enc_prefix.h"
|
18
|
+
|
19
|
+
#if defined(__cplusplus) || defined(c_plusplus)
|
20
|
+
extern "C" {
|
21
|
+
#endif
|
22
|
+
|
23
|
+
BROTLI_INTERNAL extern const uint32_t
|
24
|
+
kBrotliInsBase[BROTLI_NUM_INS_COPY_CODES];
|
25
|
+
BROTLI_INTERNAL extern const uint32_t
|
26
|
+
kBrotliInsExtra[BROTLI_NUM_INS_COPY_CODES];
|
27
|
+
BROTLI_INTERNAL extern const uint32_t
|
28
|
+
kBrotliCopyBase[BROTLI_NUM_INS_COPY_CODES];
|
29
|
+
BROTLI_INTERNAL extern const uint32_t
|
30
|
+
kBrotliCopyExtra[BROTLI_NUM_INS_COPY_CODES];
|
31
|
+
|
32
|
+
static BROTLI_INLINE uint16_t GetInsertLengthCode(size_t insertlen) {
|
33
|
+
if (insertlen < 6) {
|
34
|
+
return (uint16_t)insertlen;
|
35
|
+
} else if (insertlen < 130) {
|
36
|
+
uint32_t nbits = Log2FloorNonZero(insertlen - 2) - 1u;
|
37
|
+
return (uint16_t)((nbits << 1) + ((insertlen - 2) >> nbits) + 2);
|
38
|
+
} else if (insertlen < 2114) {
|
39
|
+
return (uint16_t)(Log2FloorNonZero(insertlen - 66) + 10);
|
40
|
+
} else if (insertlen < 6210) {
|
41
|
+
return 21u;
|
42
|
+
} else if (insertlen < 22594) {
|
43
|
+
return 22u;
|
44
|
+
} else {
|
45
|
+
return 23u;
|
46
|
+
}
|
47
|
+
}
|
48
|
+
|
49
|
+
static BROTLI_INLINE uint16_t GetCopyLengthCode(size_t copylen) {
|
50
|
+
if (copylen < 10) {
|
51
|
+
return (uint16_t)(copylen - 2);
|
52
|
+
} else if (copylen < 134) {
|
53
|
+
uint32_t nbits = Log2FloorNonZero(copylen - 6) - 1u;
|
54
|
+
return (uint16_t)((nbits << 1) + ((copylen - 6) >> nbits) + 4);
|
55
|
+
} else if (copylen < 2118) {
|
56
|
+
return (uint16_t)(Log2FloorNonZero(copylen - 70) + 12);
|
57
|
+
} else {
|
58
|
+
return 23u;
|
59
|
+
}
|
60
|
+
}
|
61
|
+
|
62
|
+
static BROTLI_INLINE uint16_t CombineLengthCodes(
|
63
|
+
uint16_t inscode, uint16_t copycode, BROTLI_BOOL use_last_distance) {
|
64
|
+
uint16_t bits64 =
|
65
|
+
(uint16_t)((copycode & 0x7u) | ((inscode & 0x7u) << 3u));
|
66
|
+
if (use_last_distance && inscode < 8u && copycode < 16u) {
|
67
|
+
return (copycode < 8u) ? bits64 : (bits64 | 64u);
|
68
|
+
} else {
|
69
|
+
/* Specification: 5 Encoding of ... (last table) */
|
70
|
+
/* offset = 2 * index, where index is in range [0..8] */
|
71
|
+
uint32_t offset = 2u * ((copycode >> 3u) + 3u * (inscode >> 3u));
|
72
|
+
/* All values in specification are K * 64,
|
73
|
+
where K = [2, 3, 6, 4, 5, 8, 7, 9, 10],
|
74
|
+
i + 1 = [1, 2, 3, 4, 5, 6, 7, 8, 9],
|
75
|
+
K - i - 1 = [1, 1, 3, 0, 0, 2, 0, 1, 2] = D.
|
76
|
+
All values in D require only 2 bits to encode.
|
77
|
+
Magic constant is shifted 6 bits left, to avoid final multiplication. */
|
78
|
+
offset = (offset << 5u) + 0x40u + ((0x520D40u >> offset) & 0xC0u);
|
79
|
+
return (uint16_t)(offset | bits64);
|
80
|
+
}
|
81
|
+
}
|
82
|
+
|
83
|
+
static BROTLI_INLINE void GetLengthCode(size_t insertlen, size_t copylen,
|
84
|
+
BROTLI_BOOL use_last_distance,
|
85
|
+
uint16_t* code) {
|
86
|
+
uint16_t inscode = GetInsertLengthCode(insertlen);
|
87
|
+
uint16_t copycode = GetCopyLengthCode(copylen);
|
88
|
+
*code = CombineLengthCodes(inscode, copycode, use_last_distance);
|
89
|
+
}
|
90
|
+
|
91
|
+
static BROTLI_INLINE uint32_t GetInsertBase(uint16_t inscode) {
|
92
|
+
return kBrotliInsBase[inscode];
|
93
|
+
}
|
94
|
+
|
95
|
+
static BROTLI_INLINE uint32_t GetInsertExtra(uint16_t inscode) {
|
96
|
+
return kBrotliInsExtra[inscode];
|
97
|
+
}
|
98
|
+
|
99
|
+
static BROTLI_INLINE uint32_t GetCopyBase(uint16_t copycode) {
|
100
|
+
return kBrotliCopyBase[copycode];
|
101
|
+
}
|
102
|
+
|
103
|
+
static BROTLI_INLINE uint32_t GetCopyExtra(uint16_t copycode) {
|
104
|
+
return kBrotliCopyExtra[copycode];
|
105
|
+
}
|
106
|
+
|
107
|
+
typedef struct Command {
|
108
|
+
uint32_t insert_len_;
|
109
|
+
/* Stores copy_len in low 25 bits and copy_code - copy_len in high 7 bit. */
|
110
|
+
uint32_t copy_len_;
|
111
|
+
/* Stores distance extra bits. */
|
112
|
+
uint32_t dist_extra_;
|
113
|
+
uint16_t cmd_prefix_;
|
114
|
+
/* Stores distance code in low 10 bits
|
115
|
+
and number of extra bits in high 6 bits. */
|
116
|
+
uint16_t dist_prefix_;
|
117
|
+
} Command;
|
118
|
+
|
119
|
+
/* distance_code is e.g. 0 for same-as-last short code, or 16 for offset 1. */
|
120
|
+
static BROTLI_INLINE void InitCommand(Command* self,
|
121
|
+
const BrotliDistanceParams* dist, size_t insertlen,
|
122
|
+
size_t copylen, int copylen_code_delta, size_t distance_code) {
|
123
|
+
/* Don't rely on signed int representation, use honest casts. */
|
124
|
+
uint32_t delta = (uint8_t)((int8_t)copylen_code_delta);
|
125
|
+
self->insert_len_ = (uint32_t)insertlen;
|
126
|
+
self->copy_len_ = (uint32_t)(copylen | (delta << 25));
|
127
|
+
/* The distance prefix and extra bits are stored in this Command as if
|
128
|
+
npostfix and ndirect were 0, they are only recomputed later after the
|
129
|
+
clustering if needed. */
|
130
|
+
PrefixEncodeCopyDistance(
|
131
|
+
distance_code, dist->num_direct_distance_codes,
|
132
|
+
dist->distance_postfix_bits, &self->dist_prefix_, &self->dist_extra_);
|
133
|
+
GetLengthCode(
|
134
|
+
insertlen, (size_t)((int)copylen + copylen_code_delta),
|
135
|
+
TO_BROTLI_BOOL((self->dist_prefix_ & 0x3FF) == 0), &self->cmd_prefix_);
|
136
|
+
}
|
137
|
+
|
138
|
+
static BROTLI_INLINE void InitInsertCommand(Command* self, size_t insertlen) {
|
139
|
+
self->insert_len_ = (uint32_t)insertlen;
|
140
|
+
self->copy_len_ = 4 << 25;
|
141
|
+
self->dist_extra_ = 0;
|
142
|
+
self->dist_prefix_ = BROTLI_NUM_DISTANCE_SHORT_CODES;
|
143
|
+
GetLengthCode(insertlen, 4, BROTLI_FALSE, &self->cmd_prefix_);
|
144
|
+
}
|
145
|
+
|
146
|
+
static BROTLI_INLINE uint32_t CommandRestoreDistanceCode(
|
147
|
+
const Command* self, const BrotliDistanceParams* dist) {
|
148
|
+
if ((self->dist_prefix_ & 0x3FFu) <
|
149
|
+
BROTLI_NUM_DISTANCE_SHORT_CODES + dist->num_direct_distance_codes) {
|
150
|
+
return self->dist_prefix_ & 0x3FFu;
|
151
|
+
} else {
|
152
|
+
uint32_t dcode = self->dist_prefix_ & 0x3FFu;
|
153
|
+
uint32_t nbits = self->dist_prefix_ >> 10;
|
154
|
+
uint32_t extra = self->dist_extra_;
|
155
|
+
uint32_t postfix_mask = (1U << dist->distance_postfix_bits) - 1U;
|
156
|
+
uint32_t hcode = (dcode - dist->num_direct_distance_codes -
|
157
|
+
BROTLI_NUM_DISTANCE_SHORT_CODES) >>
|
158
|
+
dist->distance_postfix_bits;
|
159
|
+
uint32_t lcode = (dcode - dist->num_direct_distance_codes -
|
160
|
+
BROTLI_NUM_DISTANCE_SHORT_CODES) & postfix_mask;
|
161
|
+
uint32_t offset = ((2U + (hcode & 1U)) << nbits) - 4U;
|
162
|
+
return ((offset + extra) << dist->distance_postfix_bits) + lcode +
|
163
|
+
dist->num_direct_distance_codes + BROTLI_NUM_DISTANCE_SHORT_CODES;
|
164
|
+
}
|
165
|
+
}
|
166
|
+
|
167
|
+
static BROTLI_INLINE uint32_t CommandDistanceContext(const Command* self) {
|
168
|
+
uint32_t r = self->cmd_prefix_ >> 6;
|
169
|
+
uint32_t c = self->cmd_prefix_ & 7;
|
170
|
+
if ((r == 0 || r == 2 || r == 4 || r == 7) && (c <= 2)) {
|
171
|
+
return c;
|
172
|
+
}
|
173
|
+
return 3;
|
174
|
+
}
|
175
|
+
|
176
|
+
static BROTLI_INLINE uint32_t CommandCopyLen(const Command* self) {
|
177
|
+
return self->copy_len_ & 0x1FFFFFF;
|
178
|
+
}
|
179
|
+
|
180
|
+
static BROTLI_INLINE uint32_t CommandCopyLenCode(const Command* self) {
|
181
|
+
uint32_t modifier = self->copy_len_ >> 25;
|
182
|
+
int32_t delta = (int8_t)((uint8_t)(modifier | ((modifier & 0x40) << 1)));
|
183
|
+
return (uint32_t)((int32_t)(self->copy_len_ & 0x1FFFFFF) + delta);
|
184
|
+
}
|
185
|
+
|
186
|
+
#if defined(__cplusplus) || defined(c_plusplus)
|
187
|
+
} /* extern "C" */
|
188
|
+
#endif
|
189
|
+
|
190
|
+
#endif /* BROTLI_ENC_COMMAND_H_ */
|