brotli 0.4.0 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/main.yml +6 -3
- data/.github/workflows/publish.yml +7 -17
- data/.gitmodules +1 -1
- data/README.md +2 -2
- data/ext/brotli/brotli.c +1 -0
- data/ext/brotli/extconf.rb +6 -0
- data/lib/brotli/version.rb +1 -1
- data/test/brotli_test.rb +4 -1
- data/vendor/brotli/c/common/constants.c +1 -1
- data/vendor/brotli/c/common/constants.h +2 -1
- data/vendor/brotli/c/common/context.c +1 -1
- data/vendor/brotli/c/common/dictionary.c +5 -3
- data/vendor/brotli/c/common/platform.c +2 -1
- data/vendor/brotli/c/common/platform.h +60 -113
- data/vendor/brotli/c/common/shared_dictionary.c +521 -0
- data/vendor/brotli/c/common/shared_dictionary_internal.h +75 -0
- data/vendor/brotli/c/common/transform.c +1 -1
- data/vendor/brotli/c/common/version.h +31 -6
- data/vendor/brotli/c/dec/bit_reader.c +10 -8
- data/vendor/brotli/c/dec/bit_reader.h +172 -100
- data/vendor/brotli/c/dec/decode.c +467 -200
- data/vendor/brotli/c/dec/huffman.c +7 -4
- data/vendor/brotli/c/dec/huffman.h +2 -1
- data/vendor/brotli/c/dec/prefix.h +2 -1
- data/vendor/brotli/c/dec/state.c +33 -9
- data/vendor/brotli/c/dec/state.h +70 -35
- data/vendor/brotli/c/enc/backward_references.c +81 -19
- data/vendor/brotli/c/enc/backward_references.h +5 -4
- data/vendor/brotli/c/enc/backward_references_hq.c +148 -52
- data/vendor/brotli/c/enc/backward_references_hq.h +6 -5
- data/vendor/brotli/c/enc/backward_references_inc.h +31 -5
- data/vendor/brotli/c/enc/bit_cost.c +8 -7
- data/vendor/brotli/c/enc/bit_cost.h +5 -4
- data/vendor/brotli/c/enc/block_splitter.c +37 -14
- data/vendor/brotli/c/enc/block_splitter.h +5 -4
- data/vendor/brotli/c/enc/block_splitter_inc.h +86 -45
- data/vendor/brotli/c/enc/brotli_bit_stream.c +132 -110
- data/vendor/brotli/c/enc/brotli_bit_stream.h +11 -6
- data/vendor/brotli/c/enc/cluster.c +10 -9
- data/vendor/brotli/c/enc/cluster.h +7 -6
- data/vendor/brotli/c/enc/cluster_inc.h +25 -20
- data/vendor/brotli/c/enc/command.c +1 -1
- data/vendor/brotli/c/enc/command.h +5 -4
- data/vendor/brotli/c/enc/compound_dictionary.c +207 -0
- data/vendor/brotli/c/enc/compound_dictionary.h +74 -0
- data/vendor/brotli/c/enc/compress_fragment.c +93 -83
- data/vendor/brotli/c/enc/compress_fragment.h +32 -7
- data/vendor/brotli/c/enc/compress_fragment_two_pass.c +99 -87
- data/vendor/brotli/c/enc/compress_fragment_two_pass.h +21 -3
- data/vendor/brotli/c/enc/dictionary_hash.c +3 -1
- data/vendor/brotli/c/enc/encode.c +473 -404
- data/vendor/brotli/c/enc/encoder_dict.c +611 -4
- data/vendor/brotli/c/enc/encoder_dict.h +117 -3
- data/vendor/brotli/c/enc/entropy_encode.c +3 -2
- data/vendor/brotli/c/enc/entropy_encode.h +2 -1
- data/vendor/brotli/c/enc/entropy_encode_static.h +5 -2
- data/vendor/brotli/c/enc/fast_log.c +1 -1
- data/vendor/brotli/c/enc/fast_log.h +2 -1
- data/vendor/brotli/c/enc/find_match_length.h +15 -22
- data/vendor/brotli/c/enc/hash.h +285 -45
- data/vendor/brotli/c/enc/hash_composite_inc.h +26 -11
- data/vendor/brotli/c/enc/hash_forgetful_chain_inc.h +20 -18
- data/vendor/brotli/c/enc/hash_longest_match64_inc.h +34 -39
- data/vendor/brotli/c/enc/hash_longest_match_inc.h +6 -10
- data/vendor/brotli/c/enc/hash_longest_match_quickly_inc.h +4 -4
- data/vendor/brotli/c/enc/hash_rolling_inc.h +4 -4
- data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +6 -5
- data/vendor/brotli/c/enc/histogram.c +4 -4
- data/vendor/brotli/c/enc/histogram.h +7 -6
- data/vendor/brotli/c/enc/literal_cost.c +20 -15
- data/vendor/brotli/c/enc/literal_cost.h +4 -2
- data/vendor/brotli/c/enc/memory.c +29 -5
- data/vendor/brotli/c/enc/memory.h +19 -2
- data/vendor/brotli/c/enc/metablock.c +72 -58
- data/vendor/brotli/c/enc/metablock.h +9 -8
- data/vendor/brotli/c/enc/metablock_inc.h +8 -6
- data/vendor/brotli/c/enc/params.h +4 -3
- data/vendor/brotli/c/enc/prefix.h +3 -2
- data/vendor/brotli/c/enc/quality.h +40 -3
- data/vendor/brotli/c/enc/ringbuffer.h +4 -3
- data/vendor/brotli/c/enc/state.h +104 -0
- data/vendor/brotli/c/enc/static_dict.c +60 -4
- data/vendor/brotli/c/enc/static_dict.h +3 -2
- data/vendor/brotli/c/enc/static_dict_lut.h +2 -0
- data/vendor/brotli/c/enc/utf8_util.c +1 -1
- data/vendor/brotli/c/enc/utf8_util.h +2 -1
- data/vendor/brotli/c/enc/write_bits.h +2 -1
- data/vendor/brotli/c/include/brotli/decode.h +67 -2
- data/vendor/brotli/c/include/brotli/encode.h +55 -2
- data/vendor/brotli/c/include/brotli/port.h +28 -11
- data/vendor/brotli/c/include/brotli/shared_dictionary.h +100 -0
- metadata +9 -3
@@ -12,8 +12,8 @@
|
|
12
12
|
/* Computes the bit cost reduction by combining out[idx1] and out[idx2] and if
|
13
13
|
it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. */
|
14
14
|
BROTLI_INTERNAL void FN(BrotliCompareAndPushToQueue)(
|
15
|
-
const HistogramType* out, const uint32_t* cluster_size,
|
16
|
-
uint32_t idx2, size_t max_num_pairs, HistogramPair* pairs,
|
15
|
+
const HistogramType* out, HistogramType* tmp, const uint32_t* cluster_size,
|
16
|
+
uint32_t idx1, uint32_t idx2, size_t max_num_pairs, HistogramPair* pairs,
|
17
17
|
size_t* num_pairs) CODE({
|
18
18
|
BROTLI_BOOL is_good_pair = BROTLI_FALSE;
|
19
19
|
HistogramPair p;
|
@@ -42,10 +42,10 @@ BROTLI_INTERNAL void FN(BrotliCompareAndPushToQueue)(
|
|
42
42
|
} else {
|
43
43
|
double threshold = *num_pairs == 0 ? 1e99 :
|
44
44
|
BROTLI_MAX(double, 0.0, pairs[0].cost_diff);
|
45
|
-
HistogramType combo = out[idx1];
|
46
45
|
double cost_combo;
|
47
|
-
|
48
|
-
|
46
|
+
*tmp = out[idx1];
|
47
|
+
FN(HistogramAddHistogram)(tmp, &out[idx2]);
|
48
|
+
cost_combo = FN(BrotliPopulationCost)(tmp);
|
49
49
|
if (cost_combo < threshold - p.cost_diff) {
|
50
50
|
p.cost_combo = cost_combo;
|
51
51
|
is_good_pair = BROTLI_TRUE;
|
@@ -68,6 +68,7 @@ BROTLI_INTERNAL void FN(BrotliCompareAndPushToQueue)(
|
|
68
68
|
})
|
69
69
|
|
70
70
|
BROTLI_INTERNAL size_t FN(BrotliHistogramCombine)(HistogramType* out,
|
71
|
+
HistogramType* tmp,
|
71
72
|
uint32_t* cluster_size,
|
72
73
|
uint32_t* symbols,
|
73
74
|
uint32_t* clusters,
|
@@ -87,7 +88,7 @@ BROTLI_INTERNAL size_t FN(BrotliHistogramCombine)(HistogramType* out,
|
|
87
88
|
for (idx1 = 0; idx1 < num_clusters; ++idx1) {
|
88
89
|
size_t idx2;
|
89
90
|
for (idx2 = idx1 + 1; idx2 < num_clusters; ++idx2) {
|
90
|
-
FN(BrotliCompareAndPushToQueue)(out, cluster_size, clusters[idx1],
|
91
|
+
FN(BrotliCompareAndPushToQueue)(out, tmp, cluster_size, clusters[idx1],
|
91
92
|
clusters[idx2], max_num_pairs, &pairs[0], &num_pairs);
|
92
93
|
}
|
93
94
|
}
|
@@ -146,8 +147,8 @@ BROTLI_INTERNAL size_t FN(BrotliHistogramCombine)(HistogramType* out,
|
|
146
147
|
|
147
148
|
/* Push new pairs formed with the combined histogram to the heap. */
|
148
149
|
for (i = 0; i < num_clusters; ++i) {
|
149
|
-
FN(BrotliCompareAndPushToQueue)(out, cluster_size, best_idx1,
|
150
|
-
|
150
|
+
FN(BrotliCompareAndPushToQueue)(out, tmp, cluster_size, best_idx1,
|
151
|
+
clusters[i], max_num_pairs, &pairs[0], &num_pairs);
|
151
152
|
}
|
152
153
|
}
|
153
154
|
return num_clusters;
|
@@ -155,13 +156,14 @@ BROTLI_INTERNAL size_t FN(BrotliHistogramCombine)(HistogramType* out,
|
|
155
156
|
|
156
157
|
/* What is the bit cost of moving histogram from cur_symbol to candidate. */
|
157
158
|
BROTLI_INTERNAL double FN(BrotliHistogramBitCostDistance)(
|
158
|
-
const HistogramType* histogram, const HistogramType* candidate
|
159
|
+
const HistogramType* histogram, const HistogramType* candidate,
|
160
|
+
HistogramType* tmp) CODE({
|
159
161
|
if (histogram->total_count_ == 0) {
|
160
162
|
return 0.0;
|
161
163
|
} else {
|
162
|
-
|
163
|
-
FN(HistogramAddHistogram)(
|
164
|
-
return FN(BrotliPopulationCost)(
|
164
|
+
*tmp = *histogram;
|
165
|
+
FN(HistogramAddHistogram)(tmp, candidate);
|
166
|
+
return FN(BrotliPopulationCost)(tmp) - candidate->bit_cost_;
|
165
167
|
}
|
166
168
|
})
|
167
169
|
|
@@ -171,16 +173,16 @@ BROTLI_INTERNAL double FN(BrotliHistogramBitCostDistance)(
|
|
171
173
|
Note: we assume that out[]->bit_cost_ is already up-to-date. */
|
172
174
|
BROTLI_INTERNAL void FN(BrotliHistogramRemap)(const HistogramType* in,
|
173
175
|
size_t in_size, const uint32_t* clusters, size_t num_clusters,
|
174
|
-
HistogramType* out, uint32_t* symbols) CODE({
|
176
|
+
HistogramType* out, HistogramType* tmp, uint32_t* symbols) CODE({
|
175
177
|
size_t i;
|
176
178
|
for (i = 0; i < in_size; ++i) {
|
177
179
|
uint32_t best_out = i == 0 ? symbols[0] : symbols[i - 1];
|
178
180
|
double best_bits =
|
179
|
-
FN(BrotliHistogramBitCostDistance)(&in[i], &out[best_out]);
|
181
|
+
FN(BrotliHistogramBitCostDistance)(&in[i], &out[best_out], tmp);
|
180
182
|
size_t j;
|
181
183
|
for (j = 0; j < num_clusters; ++j) {
|
182
184
|
const double cur_bits =
|
183
|
-
FN(BrotliHistogramBitCostDistance)(&in[i], &out[clusters[j]]);
|
185
|
+
FN(BrotliHistogramBitCostDistance)(&in[i], &out[clusters[j]], tmp);
|
184
186
|
if (cur_bits < best_bits) {
|
185
187
|
best_bits = cur_bits;
|
186
188
|
best_out = clusters[j];
|
@@ -226,7 +228,7 @@ BROTLI_INTERNAL size_t FN(BrotliHistogramReindex)(MemoryManager* m,
|
|
226
228
|
++next_index;
|
227
229
|
}
|
228
230
|
}
|
229
|
-
/* TODO: by using idea of "cycle-sort" we can avoid allocation of
|
231
|
+
/* TODO(eustas): by using idea of "cycle-sort" we can avoid allocation of
|
230
232
|
tmp and reduce the number of copying by the factor of 2. */
|
231
233
|
tmp = BROTLI_ALLOC(m, HistogramType, next_index);
|
232
234
|
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(tmp)) return 0;
|
@@ -257,10 +259,12 @@ BROTLI_INTERNAL void FN(BrotliClusterHistograms)(
|
|
257
259
|
size_t pairs_capacity = max_input_histograms * max_input_histograms / 2;
|
258
260
|
/* For the first pass of clustering, we allow all pairs. */
|
259
261
|
HistogramPair* pairs = BROTLI_ALLOC(m, HistogramPair, pairs_capacity + 1);
|
262
|
+
/* TODO(eustas): move to "persistent" arena? */
|
263
|
+
HistogramType* tmp = BROTLI_ALLOC(m, HistogramType, 1);
|
260
264
|
size_t i;
|
261
265
|
|
262
266
|
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(cluster_size) ||
|
263
|
-
BROTLI_IS_NULL(clusters) || BROTLI_IS_NULL(pairs)) {
|
267
|
+
BROTLI_IS_NULL(clusters) || BROTLI_IS_NULL(pairs)|| BROTLI_IS_NULL(tmp)) {
|
264
268
|
return;
|
265
269
|
}
|
266
270
|
|
@@ -283,7 +287,7 @@ BROTLI_INTERNAL void FN(BrotliClusterHistograms)(
|
|
283
287
|
clusters[num_clusters + j] = (uint32_t)(i + j);
|
284
288
|
}
|
285
289
|
num_new_clusters =
|
286
|
-
FN(BrotliHistogramCombine)(out, cluster_size,
|
290
|
+
FN(BrotliHistogramCombine)(out, tmp, cluster_size,
|
287
291
|
&histogram_symbols[i],
|
288
292
|
&clusters[num_clusters], pairs,
|
289
293
|
num_to_combine, num_to_combine,
|
@@ -301,7 +305,7 @@ BROTLI_INTERNAL void FN(BrotliClusterHistograms)(
|
|
301
305
|
if (BROTLI_IS_OOM(m)) return;
|
302
306
|
|
303
307
|
/* Collapse similar histograms. */
|
304
|
-
num_clusters = FN(BrotliHistogramCombine)(out, cluster_size,
|
308
|
+
num_clusters = FN(BrotliHistogramCombine)(out, tmp, cluster_size,
|
305
309
|
histogram_symbols, clusters,
|
306
310
|
pairs, num_clusters, in_size,
|
307
311
|
max_histograms, max_num_pairs);
|
@@ -310,7 +314,8 @@ BROTLI_INTERNAL void FN(BrotliClusterHistograms)(
|
|
310
314
|
BROTLI_FREE(m, cluster_size);
|
311
315
|
/* Find the optimal map from original histograms to the final ones. */
|
312
316
|
FN(BrotliHistogramRemap)(in, in_size, clusters, num_clusters,
|
313
|
-
out, histogram_symbols);
|
317
|
+
out, tmp, histogram_symbols);
|
318
|
+
BROTLI_FREE(m, tmp);
|
314
319
|
BROTLI_FREE(m, clusters);
|
315
320
|
/* Convert the context map to a canonical form. */
|
316
321
|
*out_size = FN(BrotliHistogramReindex)(m, out, histogram_symbols, in_size);
|
@@ -9,12 +9,13 @@
|
|
9
9
|
#ifndef BROTLI_ENC_COMMAND_H_
|
10
10
|
#define BROTLI_ENC_COMMAND_H_
|
11
11
|
|
12
|
+
#include <brotli/types.h>
|
13
|
+
|
12
14
|
#include "../common/constants.h"
|
13
15
|
#include "../common/platform.h"
|
14
|
-
#include
|
15
|
-
#include "
|
16
|
-
#include "
|
17
|
-
#include "./prefix.h"
|
16
|
+
#include "fast_log.h"
|
17
|
+
#include "params.h"
|
18
|
+
#include "prefix.h"
|
18
19
|
|
19
20
|
#if defined(__cplusplus) || defined(c_plusplus)
|
20
21
|
extern "C" {
|
@@ -0,0 +1,207 @@
|
|
1
|
+
/* Copyright 2017 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
7
|
+
#include "compound_dictionary.h"
|
8
|
+
|
9
|
+
#include <brotli/types.h>
|
10
|
+
|
11
|
+
#include "../common/platform.h"
|
12
|
+
#include "memory.h"
|
13
|
+
#include "quality.h"
|
14
|
+
|
15
|
+
static PreparedDictionary* CreatePreparedDictionaryWithParams(MemoryManager* m,
|
16
|
+
const uint8_t* source, size_t source_size, uint32_t bucket_bits,
|
17
|
+
uint32_t slot_bits, uint32_t hash_bits, uint16_t bucket_limit) {
|
18
|
+
/* Step 1: create "bloated" hasher. */
|
19
|
+
uint32_t num_slots = 1u << slot_bits;
|
20
|
+
uint32_t num_buckets = 1u << bucket_bits;
|
21
|
+
uint32_t hash_shift = 64u - bucket_bits;
|
22
|
+
uint64_t hash_mask = (~((uint64_t)0U)) >> (64 - hash_bits);
|
23
|
+
uint32_t slot_mask = num_slots - 1;
|
24
|
+
size_t alloc_size = (sizeof(uint32_t) << slot_bits) +
|
25
|
+
(sizeof(uint32_t) << slot_bits) +
|
26
|
+
(sizeof(uint16_t) << bucket_bits) +
|
27
|
+
(sizeof(uint32_t) << bucket_bits) +
|
28
|
+
(sizeof(uint32_t) * source_size);
|
29
|
+
uint8_t* flat = NULL;
|
30
|
+
PreparedDictionary* result = NULL;
|
31
|
+
uint16_t* num = NULL;
|
32
|
+
uint32_t* bucket_heads = NULL;
|
33
|
+
uint32_t* next_bucket = NULL;
|
34
|
+
uint32_t* slot_offsets = NULL;
|
35
|
+
uint16_t* heads = NULL;
|
36
|
+
uint32_t* items = NULL;
|
37
|
+
uint8_t** source_ref = NULL;
|
38
|
+
uint32_t i;
|
39
|
+
uint32_t* slot_size = NULL;
|
40
|
+
uint32_t* slot_limit = NULL;
|
41
|
+
uint32_t total_items = 0;
|
42
|
+
if (slot_bits > 16) return NULL;
|
43
|
+
if (slot_bits > bucket_bits) return NULL;
|
44
|
+
if (bucket_bits - slot_bits >= 16) return NULL;
|
45
|
+
|
46
|
+
flat = BROTLI_ALLOC(m, uint8_t, alloc_size);
|
47
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(flat)) return NULL;
|
48
|
+
|
49
|
+
slot_size = (uint32_t*)flat;
|
50
|
+
slot_limit = (uint32_t*)(&slot_size[num_slots]);
|
51
|
+
num = (uint16_t*)(&slot_limit[num_slots]);
|
52
|
+
bucket_heads = (uint32_t*)(&num[num_buckets]);
|
53
|
+
next_bucket = (uint32_t*)(&bucket_heads[num_buckets]);
|
54
|
+
memset(num, 0, num_buckets * sizeof(num[0]));
|
55
|
+
|
56
|
+
/* TODO(eustas): apply custom "store" order. */
|
57
|
+
for (i = 0; i + 7 < source_size; ++i) {
|
58
|
+
const uint64_t h = (BROTLI_UNALIGNED_LOAD64LE(&source[i]) & hash_mask) *
|
59
|
+
kPreparedDictionaryHashMul64Long;
|
60
|
+
const uint32_t key = (uint32_t)(h >> hash_shift);
|
61
|
+
uint16_t count = num[key];
|
62
|
+
next_bucket[i] = (count == 0) ? ((uint32_t)(-1)) : bucket_heads[key];
|
63
|
+
bucket_heads[key] = i;
|
64
|
+
count++;
|
65
|
+
if (count > bucket_limit) count = bucket_limit;
|
66
|
+
num[key] = count;
|
67
|
+
}
|
68
|
+
|
69
|
+
/* Step 2: find slot limits. */
|
70
|
+
for (i = 0; i < num_slots; ++i) {
|
71
|
+
BROTLI_BOOL overflow = BROTLI_FALSE;
|
72
|
+
slot_limit[i] = bucket_limit;
|
73
|
+
while (BROTLI_TRUE) {
|
74
|
+
uint32_t limit = slot_limit[i];
|
75
|
+
size_t j;
|
76
|
+
uint32_t count = 0;
|
77
|
+
overflow = BROTLI_FALSE;
|
78
|
+
for (j = i; j < num_buckets; j += num_slots) {
|
79
|
+
uint32_t size = num[j];
|
80
|
+
/* Last chain may span behind 64K limit; overflow happens only if
|
81
|
+
we are about to use 0xFFFF+ as item offset. */
|
82
|
+
if (count >= 0xFFFF) {
|
83
|
+
overflow = BROTLI_TRUE;
|
84
|
+
break;
|
85
|
+
}
|
86
|
+
if (size > limit) size = limit;
|
87
|
+
count += size;
|
88
|
+
}
|
89
|
+
if (!overflow) {
|
90
|
+
slot_size[i] = count;
|
91
|
+
total_items += count;
|
92
|
+
break;
|
93
|
+
}
|
94
|
+
slot_limit[i]--;
|
95
|
+
}
|
96
|
+
}
|
97
|
+
|
98
|
+
/* Step 3: transfer data to "slim" hasher. */
|
99
|
+
alloc_size = sizeof(PreparedDictionary) + (sizeof(uint32_t) << slot_bits) +
|
100
|
+
(sizeof(uint16_t) << bucket_bits) + (sizeof(uint32_t) * total_items) +
|
101
|
+
sizeof(uint8_t*);
|
102
|
+
|
103
|
+
result = (PreparedDictionary*)BROTLI_ALLOC(m, uint8_t, alloc_size);
|
104
|
+
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(result)) {
|
105
|
+
BROTLI_FREE(m, flat);
|
106
|
+
return NULL;
|
107
|
+
}
|
108
|
+
slot_offsets = (uint32_t*)(&result[1]);
|
109
|
+
heads = (uint16_t*)(&slot_offsets[num_slots]);
|
110
|
+
items = (uint32_t*)(&heads[num_buckets]);
|
111
|
+
source_ref = (uint8_t**)(&items[total_items]);
|
112
|
+
|
113
|
+
result->magic = kLeanPreparedDictionaryMagic;
|
114
|
+
result->num_items = total_items;
|
115
|
+
result->source_size = (uint32_t)source_size;
|
116
|
+
result->hash_bits = hash_bits;
|
117
|
+
result->bucket_bits = bucket_bits;
|
118
|
+
result->slot_bits = slot_bits;
|
119
|
+
BROTLI_UNALIGNED_STORE_PTR(source_ref, source);
|
120
|
+
|
121
|
+
total_items = 0;
|
122
|
+
for (i = 0; i < num_slots; ++i) {
|
123
|
+
slot_offsets[i] = total_items;
|
124
|
+
total_items += slot_size[i];
|
125
|
+
slot_size[i] = 0;
|
126
|
+
}
|
127
|
+
for (i = 0; i < num_buckets; ++i) {
|
128
|
+
uint32_t slot = i & slot_mask;
|
129
|
+
uint32_t count = num[i];
|
130
|
+
uint32_t pos;
|
131
|
+
size_t j;
|
132
|
+
size_t cursor = slot_size[slot];
|
133
|
+
if (count > slot_limit[slot]) count = slot_limit[slot];
|
134
|
+
if (count == 0) {
|
135
|
+
heads[i] = 0xFFFF;
|
136
|
+
continue;
|
137
|
+
}
|
138
|
+
heads[i] = (uint16_t)cursor;
|
139
|
+
cursor += slot_offsets[slot];
|
140
|
+
slot_size[slot] += count;
|
141
|
+
pos = bucket_heads[i];
|
142
|
+
for (j = 0; j < count; j++) {
|
143
|
+
items[cursor++] = pos;
|
144
|
+
pos = next_bucket[pos];
|
145
|
+
}
|
146
|
+
items[cursor - 1] |= 0x80000000;
|
147
|
+
}
|
148
|
+
|
149
|
+
BROTLI_FREE(m, flat);
|
150
|
+
return result;
|
151
|
+
}
|
152
|
+
|
153
|
+
PreparedDictionary* CreatePreparedDictionary(MemoryManager* m,
|
154
|
+
const uint8_t* source, size_t source_size) {
|
155
|
+
uint32_t bucket_bits = 17;
|
156
|
+
uint32_t slot_bits = 7;
|
157
|
+
uint32_t hash_bits = 40;
|
158
|
+
uint16_t bucket_limit = 32;
|
159
|
+
size_t volume = 16u << bucket_bits;
|
160
|
+
/* Tune parameters to fit dictionary size. */
|
161
|
+
while (volume < source_size && bucket_bits < 22) {
|
162
|
+
bucket_bits++;
|
163
|
+
slot_bits++;
|
164
|
+
volume <<= 1;
|
165
|
+
}
|
166
|
+
return CreatePreparedDictionaryWithParams(m,
|
167
|
+
source, source_size, bucket_bits, slot_bits, hash_bits, bucket_limit);
|
168
|
+
}
|
169
|
+
|
170
|
+
void DestroyPreparedDictionary(MemoryManager* m,
|
171
|
+
PreparedDictionary* dictionary) {
|
172
|
+
if (!dictionary) return;
|
173
|
+
BROTLI_FREE(m, dictionary);
|
174
|
+
}
|
175
|
+
|
176
|
+
BROTLI_BOOL AttachPreparedDictionary(
|
177
|
+
CompoundDictionary* compound, const PreparedDictionary* dictionary) {
|
178
|
+
size_t length = 0;
|
179
|
+
size_t index = 0;
|
180
|
+
|
181
|
+
if (compound->num_chunks == SHARED_BROTLI_MAX_COMPOUND_DICTS) {
|
182
|
+
return BROTLI_FALSE;
|
183
|
+
}
|
184
|
+
|
185
|
+
if (!dictionary) return BROTLI_FALSE;
|
186
|
+
|
187
|
+
length = dictionary->source_size;
|
188
|
+
index = compound->num_chunks;
|
189
|
+
compound->total_size += length;
|
190
|
+
compound->chunks[index] = dictionary;
|
191
|
+
compound->chunk_offsets[index + 1] = compound->total_size;
|
192
|
+
{
|
193
|
+
uint32_t* slot_offsets = (uint32_t*)(&dictionary[1]);
|
194
|
+
uint16_t* heads = (uint16_t*)(&slot_offsets[1u << dictionary->slot_bits]);
|
195
|
+
uint32_t* items = (uint32_t*)(&heads[1u << dictionary->bucket_bits]);
|
196
|
+
const void* tail = (void*)&items[dictionary->num_items];
|
197
|
+
if (dictionary->magic == kPreparedDictionaryMagic) {
|
198
|
+
compound->chunk_source[index] = (const uint8_t*)tail;
|
199
|
+
} else {
|
200
|
+
/* dictionary->magic == kLeanPreparedDictionaryMagic */
|
201
|
+
compound->chunk_source[index] =
|
202
|
+
(const uint8_t*)BROTLI_UNALIGNED_LOAD_PTR((const uint8_t**)tail);
|
203
|
+
}
|
204
|
+
}
|
205
|
+
compound->num_chunks++;
|
206
|
+
return BROTLI_TRUE;
|
207
|
+
}
|
@@ -0,0 +1,74 @@
|
|
1
|
+
/* Copyright 2017 Google Inc. All Rights Reserved.
|
2
|
+
|
3
|
+
Distributed under MIT license.
|
4
|
+
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
5
|
+
*/
|
6
|
+
|
7
|
+
#ifndef BROTLI_ENC_PREPARED_DICTIONARY_H_
|
8
|
+
#define BROTLI_ENC_PREPARED_DICTIONARY_H_
|
9
|
+
|
10
|
+
#include <brotli/shared_dictionary.h>
|
11
|
+
#include <brotli/types.h>
|
12
|
+
|
13
|
+
#include "../common/platform.h"
|
14
|
+
#include "../common/constants.h"
|
15
|
+
#include "memory.h"
|
16
|
+
|
17
|
+
/* "Fat" prepared dictionary, could be cooked outside of C implementation,
|
18
|
+
* e.g. on Java side. LZ77 data is copied inside PreparedDictionary struct. */
|
19
|
+
static const uint32_t kPreparedDictionaryMagic = 0xDEBCEDE0;
|
20
|
+
|
21
|
+
static const uint32_t kSharedDictionaryMagic = 0xDEBCEDE1;
|
22
|
+
|
23
|
+
static const uint32_t kManagedDictionaryMagic = 0xDEBCEDE2;
|
24
|
+
|
25
|
+
/* "Lean" prepared dictionary. LZ77 data is referenced. It is the responsibility
|
26
|
+
* of caller of "prepare dictionary" to keep the LZ77 data while prepared
|
27
|
+
* dictionary is in use. */
|
28
|
+
static const uint32_t kLeanPreparedDictionaryMagic = 0xDEBCEDE3;
|
29
|
+
|
30
|
+
static const uint64_t kPreparedDictionaryHashMul64Long =
|
31
|
+
BROTLI_MAKE_UINT64_T(0x1FE35A7Bu, 0xD3579BD3u);
|
32
|
+
|
33
|
+
typedef struct PreparedDictionary {
|
34
|
+
uint32_t magic;
|
35
|
+
uint32_t num_items;
|
36
|
+
uint32_t source_size;
|
37
|
+
uint32_t hash_bits;
|
38
|
+
uint32_t bucket_bits;
|
39
|
+
uint32_t slot_bits;
|
40
|
+
|
41
|
+
/* --- Dynamic size members --- */
|
42
|
+
|
43
|
+
/* uint32_t slot_offsets[1 << slot_bits]; */
|
44
|
+
/* uint16_t heads[1 << bucket_bits]; */
|
45
|
+
/* uint32_t items[variable]; */
|
46
|
+
|
47
|
+
/* [maybe] uint8_t* source_ref, depending on magic. */
|
48
|
+
/* [maybe] uint8_t source[source_size], depending on magic. */
|
49
|
+
} PreparedDictionary;
|
50
|
+
|
51
|
+
BROTLI_INTERNAL PreparedDictionary* CreatePreparedDictionary(MemoryManager* m,
|
52
|
+
const uint8_t* source, size_t source_size);
|
53
|
+
|
54
|
+
BROTLI_INTERNAL void DestroyPreparedDictionary(MemoryManager* m,
|
55
|
+
PreparedDictionary* dictionary);
|
56
|
+
|
57
|
+
typedef struct CompoundDictionary {
|
58
|
+
/* LZ77 prefix, compound dictionary */
|
59
|
+
size_t num_chunks;
|
60
|
+
size_t total_size;
|
61
|
+
/* Client instances. */
|
62
|
+
const PreparedDictionary* chunks[SHARED_BROTLI_MAX_COMPOUND_DICTS + 1];
|
63
|
+
const uint8_t* chunk_source[SHARED_BROTLI_MAX_COMPOUND_DICTS + 1];
|
64
|
+
size_t chunk_offsets[SHARED_BROTLI_MAX_COMPOUND_DICTS + 1];
|
65
|
+
|
66
|
+
size_t num_prepared_instances_;
|
67
|
+
/* Owned instances. */
|
68
|
+
PreparedDictionary* prepared_instances_[SHARED_BROTLI_MAX_COMPOUND_DICTS + 1];
|
69
|
+
} CompoundDictionary;
|
70
|
+
|
71
|
+
BROTLI_INTERNAL BROTLI_BOOL AttachPreparedDictionary(
|
72
|
+
CompoundDictionary* compound, const PreparedDictionary* dictionary);
|
73
|
+
|
74
|
+
#endif /* BROTLI_ENC_PREPARED_DICTIONARY */
|