brotli 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. checksums.yaml +5 -5
  2. data/Gemfile +1 -0
  3. data/Rakefile +6 -1
  4. data/brotli.gemspec +1 -1
  5. data/docs/Brotli.html +485 -0
  6. data/docs/Brotli/Error.html +124 -0
  7. data/docs/_index.html +122 -0
  8. data/docs/class_list.html +51 -0
  9. data/docs/css/common.css +1 -0
  10. data/docs/css/full_list.css +58 -0
  11. data/docs/css/style.css +496 -0
  12. data/docs/file.README.html +127 -0
  13. data/docs/file_list.html +56 -0
  14. data/docs/frames.html +17 -0
  15. data/docs/index.html +127 -0
  16. data/docs/js/app.js +292 -0
  17. data/docs/js/full_list.js +216 -0
  18. data/docs/js/jquery.js +4 -0
  19. data/docs/method_list.html +67 -0
  20. data/docs/top-level-namespace.html +110 -0
  21. data/ext/brotli/brotli.c +20 -0
  22. data/lib/brotli/version.rb +1 -1
  23. data/vendor/brotli/c/common/constants.h +13 -6
  24. data/vendor/brotli/c/{dec → common}/context.h +182 -172
  25. data/vendor/brotli/c/common/dictionary.bin +0 -0
  26. data/vendor/brotli/c/common/dictionary.bin.br +0 -0
  27. data/vendor/brotli/c/common/dictionary.c +1 -1
  28. data/vendor/brotli/c/common/dictionary.h +4 -4
  29. data/vendor/brotli/c/common/platform.h +509 -0
  30. data/vendor/brotli/c/common/transform.c +235 -0
  31. data/vendor/brotli/c/common/transform.h +80 -0
  32. data/vendor/brotli/c/common/version.h +8 -1
  33. data/vendor/brotli/c/dec/bit_reader.c +1 -1
  34. data/vendor/brotli/c/dec/bit_reader.h +35 -86
  35. data/vendor/brotli/c/dec/decode.c +322 -205
  36. data/vendor/brotli/c/dec/huffman.c +35 -37
  37. data/vendor/brotli/c/dec/huffman.h +13 -9
  38. data/vendor/brotli/c/dec/prefix.h +3 -4
  39. data/vendor/brotli/c/dec/state.c +26 -34
  40. data/vendor/brotli/c/dec/state.h +34 -23
  41. data/vendor/brotli/c/enc/backward_references.c +25 -15
  42. data/vendor/brotli/c/enc/backward_references.h +5 -6
  43. data/vendor/brotli/c/enc/backward_references_hq.c +94 -68
  44. data/vendor/brotli/c/enc/backward_references_hq.h +22 -25
  45. data/vendor/brotli/c/enc/backward_references_inc.h +10 -10
  46. data/vendor/brotli/c/enc/bit_cost.c +1 -1
  47. data/vendor/brotli/c/enc/bit_cost.h +5 -5
  48. data/vendor/brotli/c/enc/block_encoder_inc.h +7 -6
  49. data/vendor/brotli/c/enc/block_splitter.c +2 -3
  50. data/vendor/brotli/c/enc/block_splitter.h +1 -1
  51. data/vendor/brotli/c/enc/block_splitter_inc.h +11 -11
  52. data/vendor/brotli/c/enc/brotli_bit_stream.c +102 -101
  53. data/vendor/brotli/c/enc/brotli_bit_stream.h +19 -38
  54. data/vendor/brotli/c/enc/cluster.c +1 -1
  55. data/vendor/brotli/c/enc/cluster.h +1 -1
  56. data/vendor/brotli/c/enc/command.h +40 -30
  57. data/vendor/brotli/c/enc/compress_fragment.c +21 -22
  58. data/vendor/brotli/c/enc/compress_fragment.h +1 -1
  59. data/vendor/brotli/c/enc/compress_fragment_two_pass.c +101 -68
  60. data/vendor/brotli/c/enc/compress_fragment_two_pass.h +1 -1
  61. data/vendor/brotli/c/enc/dictionary_hash.c +1 -1
  62. data/vendor/brotli/c/enc/encode.c +262 -162
  63. data/vendor/brotli/c/enc/encoder_dict.c +32 -0
  64. data/vendor/brotli/c/enc/encoder_dict.h +41 -0
  65. data/vendor/brotli/c/enc/entropy_encode.c +14 -14
  66. data/vendor/brotli/c/enc/entropy_encode.h +5 -5
  67. data/vendor/brotli/c/enc/entropy_encode_static.h +3 -3
  68. data/vendor/brotli/c/enc/fast_log.h +4 -2
  69. data/vendor/brotli/c/enc/find_match_length.h +3 -3
  70. data/vendor/brotli/c/enc/hash.h +75 -24
  71. data/vendor/brotli/c/enc/hash_composite_inc.h +133 -0
  72. data/vendor/brotli/c/enc/hash_forgetful_chain_inc.h +9 -8
  73. data/vendor/brotli/c/enc/hash_longest_match64_inc.h +8 -8
  74. data/vendor/brotli/c/enc/hash_longest_match_inc.h +8 -8
  75. data/vendor/brotli/c/enc/hash_longest_match_quickly_inc.h +10 -9
  76. data/vendor/brotli/c/enc/hash_rolling_inc.h +215 -0
  77. data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +9 -8
  78. data/vendor/brotli/c/enc/histogram.c +9 -6
  79. data/vendor/brotli/c/enc/histogram.h +6 -3
  80. data/vendor/brotli/c/enc/histogram_inc.h +1 -1
  81. data/vendor/brotli/c/enc/literal_cost.c +5 -5
  82. data/vendor/brotli/c/enc/literal_cost.h +2 -2
  83. data/vendor/brotli/c/enc/memory.c +5 -16
  84. data/vendor/brotli/c/enc/memory.h +40 -1
  85. data/vendor/brotli/c/enc/metablock.c +163 -25
  86. data/vendor/brotli/c/enc/metablock.h +13 -8
  87. data/vendor/brotli/c/enc/metablock_inc.h +1 -1
  88. data/vendor/brotli/c/enc/params.h +44 -0
  89. data/vendor/brotli/c/enc/prefix.h +3 -4
  90. data/vendor/brotli/c/enc/quality.h +29 -24
  91. data/vendor/brotli/c/enc/ringbuffer.h +15 -11
  92. data/vendor/brotli/c/enc/static_dict.c +49 -45
  93. data/vendor/brotli/c/enc/static_dict.h +4 -3
  94. data/vendor/brotli/c/enc/static_dict_lut.h +1 -1
  95. data/vendor/brotli/c/enc/utf8_util.c +20 -20
  96. data/vendor/brotli/c/enc/utf8_util.h +1 -1
  97. data/vendor/brotli/c/enc/write_bits.h +16 -21
  98. data/vendor/brotli/c/include/brotli/decode.h +13 -8
  99. data/vendor/brotli/c/include/brotli/encode.h +33 -8
  100. data/vendor/brotli/c/include/brotli/port.h +211 -83
  101. data/vendor/brotli/c/include/brotli/types.h +0 -7
  102. metadata +33 -12
  103. data/vendor/brotli/c/dec/port.h +0 -168
  104. data/vendor/brotli/c/dec/transform.h +0 -300
  105. data/vendor/brotli/c/enc/context.h +0 -184
  106. data/vendor/brotli/c/enc/port.h +0 -184
@@ -8,9 +8,9 @@
8
8
 
9
9
  #include "./histogram.h"
10
10
 
11
+ #include "../common/context.h"
11
12
  #include "./block_splitter.h"
12
13
  #include "./command.h"
13
- #include "./context.h"
14
14
 
15
15
  #if defined(__cplusplus) || defined(c_plusplus)
16
16
  extern "C" {
@@ -63,13 +63,16 @@ void BrotliBuildHistogramsWithContext(
63
63
  BlockSplitIteratorNext(&insert_and_copy_it);
64
64
  HistogramAddCommand(&insert_and_copy_histograms[insert_and_copy_it.type_],
65
65
  cmd->cmd_prefix_);
66
+ /* TODO: unwrap iterator blocks. */
66
67
  for (j = cmd->insert_len_; j != 0; --j) {
67
68
  size_t context;
68
69
  BlockSplitIteratorNext(&literal_it);
69
- context = context_modes ?
70
- ((literal_it.type_ << BROTLI_LITERAL_CONTEXT_BITS) +
71
- Context(prev_byte, prev_byte2, context_modes[literal_it.type_])) :
72
- literal_it.type_;
70
+ context = literal_it.type_;
71
+ if (context_modes) {
72
+ ContextLut lut = BROTLI_CONTEXT_LUT(context_modes[context]);
73
+ context = (context << BROTLI_LITERAL_CONTEXT_BITS) +
74
+ BROTLI_CONTEXT(prev_byte, prev_byte2, lut);
75
+ }
73
76
  HistogramAddLiteral(&literal_histograms[context],
74
77
  ringbuffer[pos & mask]);
75
78
  prev_byte2 = prev_byte;
@@ -86,7 +89,7 @@ void BrotliBuildHistogramsWithContext(
86
89
  context = (dist_it.type_ << BROTLI_DISTANCE_CONTEXT_BITS) +
87
90
  CommandDistanceContext(cmd);
88
91
  HistogramAddDistance(&copy_dist_histograms[context],
89
- cmd->dist_prefix_);
92
+ cmd->dist_prefix_ & 0x3FF);
90
93
  }
91
94
  }
92
95
  }
@@ -12,16 +12,19 @@
12
12
  #include <string.h> /* memset */
13
13
 
14
14
  #include "../common/constants.h"
15
+ #include "../common/context.h"
16
+ #include "../common/platform.h"
15
17
  #include <brotli/types.h>
16
18
  #include "./block_splitter.h"
17
19
  #include "./command.h"
18
- #include "./context.h"
19
- #include "./port.h"
20
20
 
21
21
  #if defined(__cplusplus) || defined(c_plusplus)
22
22
  extern "C" {
23
23
  #endif
24
24
 
25
+ /* The distance symbols effectively used by "Large Window Brotli" (32-bit). */
26
+ #define BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS 544
27
+
25
28
  #define FN(X) X ## Literal
26
29
  #define DATA_SIZE BROTLI_NUM_LITERAL_SYMBOLS
27
30
  #define DataType uint8_t
@@ -38,7 +41,7 @@ extern "C" {
38
41
  #undef FN
39
42
 
40
43
  #define FN(X) X ## Distance
41
- #define DATA_SIZE BROTLI_NUM_DISTANCE_SYMBOLS
44
+ #define DATA_SIZE BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS
42
45
  #include "./histogram_inc.h" /* NOLINT(build/include) */
43
46
  #undef DataType
44
47
  #undef DATA_SIZE
@@ -33,7 +33,7 @@ static BROTLI_INLINE void FN(HistogramAdd)(FN(Histogram)* self, size_t val) {
33
33
  }
34
34
 
35
35
  static BROTLI_INLINE void FN(HistogramAddVector)(FN(Histogram)* self,
36
- const DataType *p, size_t n) {
36
+ const DataType* p, size_t n) {
37
37
  self->total_count_ += n;
38
38
  n += 1;
39
39
  while (--n) ++self->data_[*p++];
@@ -9,9 +9,9 @@
9
9
 
10
10
  #include "./literal_cost.h"
11
11
 
12
+ #include "../common/platform.h"
12
13
  #include <brotli/types.h>
13
14
  #include "./fast_log.h"
14
- #include "./port.h"
15
15
  #include "./utf8_util.h"
16
16
 
17
17
  #if defined(__cplusplus) || defined(c_plusplus)
@@ -25,7 +25,7 @@ static size_t UTF8Position(size_t last, size_t c, size_t clamp) {
25
25
  return BROTLI_MIN(size_t, 1, clamp);
26
26
  } else {
27
27
  /* Let's decide over the last byte if this ends the sequence. */
28
- if (last < 0xe0) {
28
+ if (last < 0xE0) {
29
29
  return 0; /* Completed two or three byte coding. */
30
30
  } else { /* Next one is the 'Byte 3' of utf-8 encoding. */
31
31
  return BROTLI_MIN(size_t, 2, clamp);
@@ -34,7 +34,7 @@ static size_t UTF8Position(size_t last, size_t c, size_t clamp) {
34
34
  }
35
35
 
36
36
  static size_t DecideMultiByteStatsLevel(size_t pos, size_t len, size_t mask,
37
- const uint8_t *data) {
37
+ const uint8_t* data) {
38
38
  size_t counts[3] = { 0 };
39
39
  size_t max_utf8 = 1; /* should be 2, but 1 compresses better. */
40
40
  size_t last_c = 0;
@@ -54,7 +54,7 @@ static size_t DecideMultiByteStatsLevel(size_t pos, size_t len, size_t mask,
54
54
  }
55
55
 
56
56
  static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
57
- const uint8_t *data, float *cost) {
57
+ const uint8_t* data, float* cost) {
58
58
  /* max_utf8 is 0 (normal ASCII single byte modeling),
59
59
  1 (for 2-byte UTF-8 modeling), or 2 (for 3-byte UTF-8 modeling). */
60
60
  const size_t max_utf8 = DecideMultiByteStatsLevel(pos, len, mask, data);
@@ -125,7 +125,7 @@ static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
125
125
  }
126
126
 
127
127
  void BrotliEstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask,
128
- const uint8_t *data, float *cost) {
128
+ const uint8_t* data, float* cost) {
129
129
  if (BrotliIsMostlyUTF8(data, pos, mask, len, kMinUTF8Ratio)) {
130
130
  EstimateBitCostsForLiteralsUTF8(pos, len, mask, data, cost);
131
131
  return;
@@ -10,8 +10,8 @@
10
10
  #ifndef BROTLI_ENC_LITERAL_COST_H_
11
11
  #define BROTLI_ENC_LITERAL_COST_H_
12
12
 
13
+ #include "../common/platform.h"
13
14
  #include <brotli/types.h>
14
- #include "./port.h"
15
15
 
16
16
  #if defined(__cplusplus) || defined(c_plusplus)
17
17
  extern "C" {
@@ -21,7 +21,7 @@ extern "C" {
21
21
  ring-buffer (data, mask) will take entropy coded and writes these estimates
22
22
  to the cost[0..len) array. */
23
23
  BROTLI_INTERNAL void BrotliEstimateBitCostsForLiterals(
24
- size_t pos, size_t len, size_t mask, const uint8_t *data, float *cost);
24
+ size_t pos, size_t len, size_t mask, const uint8_t* data, float* cost);
25
25
 
26
26
  #if defined(__cplusplus) || defined(c_plusplus)
27
27
  } /* extern "C" */
@@ -9,12 +9,11 @@
9
9
 
10
10
  #include "./memory.h"
11
11
 
12
- #include <assert.h>
13
12
  #include <stdlib.h> /* exit, free, malloc */
14
13
  #include <string.h> /* memcpy */
15
14
 
15
+ #include "../common/platform.h"
16
16
  #include <brotli/types.h>
17
- #include "./port.h"
18
17
 
19
18
  #if defined(__cplusplus) || defined(c_plusplus)
20
19
  extern "C" {
@@ -28,22 +27,12 @@ extern "C" {
28
27
  #define NEW_ALLOCATED_OFFSET MAX_PERM_ALLOCATED
29
28
  #define NEW_FREED_OFFSET (MAX_PERM_ALLOCATED + MAX_NEW_ALLOCATED)
30
29
 
31
- static void* DefaultAllocFunc(void* opaque, size_t size) {
32
- BROTLI_UNUSED(opaque);
33
- return malloc(size);
34
- }
35
-
36
- static void DefaultFreeFunc(void* opaque, void* address) {
37
- BROTLI_UNUSED(opaque);
38
- free(address);
39
- }
40
-
41
30
  void BrotliInitMemoryManager(
42
31
  MemoryManager* m, brotli_alloc_func alloc_func, brotli_free_func free_func,
43
32
  void* opaque) {
44
33
  if (!alloc_func) {
45
- m->alloc_func = DefaultAllocFunc;
46
- m->free_func = DefaultFreeFunc;
34
+ m->alloc_func = BrotliDefaultAllocFunc;
35
+ m->free_func = BrotliDefaultFreeFunc;
47
36
  m->opaque = 0;
48
37
  } else {
49
38
  m->alloc_func = alloc_func;
@@ -132,11 +121,11 @@ static void CollectGarbagePointers(MemoryManager* m) {
132
121
  m->pointers + NEW_FREED_OFFSET, m->new_freed);
133
122
  m->perm_allocated -= annihilated;
134
123
  m->new_freed -= annihilated;
135
- assert(m->new_freed == 0);
124
+ BROTLI_DCHECK(m->new_freed == 0);
136
125
  }
137
126
 
138
127
  if (m->new_allocated != 0) {
139
- assert(m->perm_allocated + m->new_allocated <= MAX_PERM_ALLOCATED);
128
+ BROTLI_DCHECK(m->perm_allocated + m->new_allocated <= MAX_PERM_ALLOCATED);
140
129
  memcpy(m->pointers + PERM_ALLOCATED_OFFSET + m->perm_allocated,
141
130
  m->pointers + NEW_ALLOCATED_OFFSET,
142
131
  sizeof(void*) * m->new_allocated);
@@ -9,8 +9,10 @@
9
9
  #ifndef BROTLI_ENC_MEMORY_H_
10
10
  #define BROTLI_ENC_MEMORY_H_
11
11
 
12
+ #include <string.h> /* memcpy */
13
+
14
+ #include "../common/platform.h"
12
15
  #include <brotli/types.h>
13
- #include "./port.h"
14
16
 
15
17
  #if defined(__cplusplus) || defined(c_plusplus)
16
18
  extern "C" {
@@ -56,6 +58,43 @@ BROTLI_INTERNAL void BrotliFree(MemoryManager* m, void* p);
56
58
 
57
59
  BROTLI_INTERNAL void BrotliWipeOutMemoryManager(MemoryManager* m);
58
60
 
61
+ /*
62
+ Dynamically grows array capacity to at least the requested size
63
+ M: MemoryManager
64
+ T: data type
65
+ A: array
66
+ C: capacity
67
+ R: requested size
68
+ */
69
+ #define BROTLI_ENSURE_CAPACITY(M, T, A, C, R) { \
70
+ if (C < (R)) { \
71
+ size_t _new_size = (C == 0) ? (R) : C; \
72
+ T* new_array; \
73
+ while (_new_size < (R)) _new_size *= 2; \
74
+ new_array = BROTLI_ALLOC((M), T, _new_size); \
75
+ if (!BROTLI_IS_OOM(M) && C != 0) \
76
+ memcpy(new_array, A, C * sizeof(T)); \
77
+ BROTLI_FREE((M), A); \
78
+ A = new_array; \
79
+ C = _new_size; \
80
+ } \
81
+ }
82
+
83
+ /*
84
+ Appends value and dynamically grows array capacity when needed
85
+ M: MemoryManager
86
+ T: data type
87
+ A: array
88
+ C: array capacity
89
+ S: array size
90
+ V: value to append
91
+ */
92
+ #define BROTLI_ENSURE_CAPACITY_APPEND(M, T, A, C, S, V) { \
93
+ (S)++; \
94
+ BROTLI_ENSURE_CAPACITY(M, T, A, C, S); \
95
+ A[(S) - 1] = (V); \
96
+ }
97
+
59
98
  #if defined(__cplusplus) || defined(c_plusplus)
60
99
  } /* extern "C" */
61
100
  #endif
@@ -10,29 +10,131 @@
10
10
  #include "./metablock.h"
11
11
 
12
12
  #include "../common/constants.h"
13
+ #include "../common/context.h"
14
+ #include "../common/platform.h"
13
15
  #include <brotli/types.h>
14
16
  #include "./bit_cost.h"
15
17
  #include "./block_splitter.h"
16
18
  #include "./cluster.h"
17
- #include "./context.h"
18
19
  #include "./entropy_encode.h"
19
20
  #include "./histogram.h"
20
21
  #include "./memory.h"
21
- #include "./port.h"
22
22
  #include "./quality.h"
23
23
 
24
24
  #if defined(__cplusplus) || defined(c_plusplus)
25
25
  extern "C" {
26
26
  #endif
27
27
 
28
+ void BrotliInitDistanceParams(BrotliEncoderParams* params,
29
+ uint32_t npostfix, uint32_t ndirect) {
30
+ BrotliDistanceParams* dist_params = &params->dist;
31
+ uint32_t alphabet_size, max_distance;
32
+
33
+ dist_params->distance_postfix_bits = npostfix;
34
+ dist_params->num_direct_distance_codes = ndirect;
35
+
36
+ alphabet_size = BROTLI_DISTANCE_ALPHABET_SIZE(
37
+ npostfix, ndirect, BROTLI_MAX_DISTANCE_BITS);
38
+ max_distance = ndirect + (1U << (BROTLI_MAX_DISTANCE_BITS + npostfix + 2)) -
39
+ (1U << (npostfix + 2));
40
+
41
+ if (params->large_window) {
42
+ static const uint32_t bound[BROTLI_MAX_NPOSTFIX + 1] = {0, 4, 12, 28};
43
+ uint32_t postfix = 1U << npostfix;
44
+ alphabet_size = BROTLI_DISTANCE_ALPHABET_SIZE(
45
+ npostfix, ndirect, BROTLI_LARGE_MAX_DISTANCE_BITS);
46
+ /* The maximum distance is set so that no distance symbol used can encode
47
+ a distance larger than BROTLI_MAX_ALLOWED_DISTANCE with all
48
+ its extra bits set. */
49
+ if (ndirect < bound[npostfix]) {
50
+ max_distance = BROTLI_MAX_ALLOWED_DISTANCE - (bound[npostfix] - ndirect);
51
+ } else if (ndirect >= bound[npostfix] + postfix) {
52
+ max_distance = (3U << 29) - 4 + (ndirect - bound[npostfix]);
53
+ } else {
54
+ max_distance = BROTLI_MAX_ALLOWED_DISTANCE;
55
+ }
56
+ }
57
+
58
+ dist_params->alphabet_size = alphabet_size;
59
+ dist_params->max_distance = max_distance;
60
+ }
61
+
62
+ static void RecomputeDistancePrefixes(Command* cmds,
63
+ size_t num_commands,
64
+ const BrotliDistanceParams* orig_params,
65
+ const BrotliDistanceParams* new_params) {
66
+ size_t i;
67
+
68
+ if (orig_params->distance_postfix_bits == new_params->distance_postfix_bits &&
69
+ orig_params->num_direct_distance_codes ==
70
+ new_params->num_direct_distance_codes) {
71
+ return;
72
+ }
73
+
74
+ for (i = 0; i < num_commands; ++i) {
75
+ Command* cmd = &cmds[i];
76
+ if (CommandCopyLen(cmd) && cmd->cmd_prefix_ >= 128) {
77
+ PrefixEncodeCopyDistance(CommandRestoreDistanceCode(cmd, orig_params),
78
+ new_params->num_direct_distance_codes,
79
+ new_params->distance_postfix_bits,
80
+ &cmd->dist_prefix_,
81
+ &cmd->dist_extra_);
82
+ }
83
+ }
84
+ }
85
+
86
+ static BROTLI_BOOL ComputeDistanceCost(const Command* cmds,
87
+ size_t num_commands,
88
+ const BrotliDistanceParams* orig_params,
89
+ const BrotliDistanceParams* new_params,
90
+ double* cost) {
91
+ size_t i;
92
+ BROTLI_BOOL equal_params = BROTLI_FALSE;
93
+ uint16_t dist_prefix;
94
+ uint32_t dist_extra;
95
+ double extra_bits = 0.0;
96
+ HistogramDistance histo;
97
+ HistogramClearDistance(&histo);
98
+
99
+ if (orig_params->distance_postfix_bits == new_params->distance_postfix_bits &&
100
+ orig_params->num_direct_distance_codes ==
101
+ new_params->num_direct_distance_codes) {
102
+ equal_params = BROTLI_TRUE;
103
+ }
104
+
105
+ for (i = 0; i < num_commands; i++) {
106
+ const Command* cmd = &cmds[i];
107
+ if (CommandCopyLen(cmd) && cmd->cmd_prefix_ >= 128) {
108
+ if (equal_params) {
109
+ dist_prefix = cmd->dist_prefix_;
110
+ } else {
111
+ uint32_t distance = CommandRestoreDistanceCode(cmd, orig_params);
112
+ if (distance > new_params->max_distance) {
113
+ return BROTLI_FALSE;
114
+ }
115
+ PrefixEncodeCopyDistance(distance,
116
+ new_params->num_direct_distance_codes,
117
+ new_params->distance_postfix_bits,
118
+ &dist_prefix,
119
+ &dist_extra);
120
+ }
121
+ HistogramAddDistance(&histo, dist_prefix & 0x3FF);
122
+ extra_bits += dist_prefix >> 10;
123
+ }
124
+ }
125
+
126
+ *cost = BrotliPopulationCostDistance(&histo) + extra_bits;
127
+ return BROTLI_TRUE;
128
+ }
129
+
28
130
  void BrotliBuildMetaBlock(MemoryManager* m,
29
131
  const uint8_t* ringbuffer,
30
132
  const size_t pos,
31
133
  const size_t mask,
32
- const BrotliEncoderParams* params,
134
+ BrotliEncoderParams* params,
33
135
  uint8_t prev_byte,
34
136
  uint8_t prev_byte2,
35
- const Command* cmds,
137
+ Command* cmds,
36
138
  size_t num_commands,
37
139
  ContextType literal_context_mode,
38
140
  MetaBlockSplit* mb) {
@@ -45,6 +147,46 @@ void BrotliBuildMetaBlock(MemoryManager* m,
45
147
  size_t distance_histograms_size;
46
148
  size_t i;
47
149
  size_t literal_context_multiplier = 1;
150
+ uint32_t npostfix;
151
+ uint32_t ndirect_msb = 0;
152
+ BROTLI_BOOL check_orig = BROTLI_TRUE;
153
+ double best_dist_cost = 1e99;
154
+ BrotliEncoderParams orig_params = *params;
155
+ BrotliEncoderParams new_params = *params;
156
+
157
+ for (npostfix = 0; npostfix <= BROTLI_MAX_NPOSTFIX; npostfix++) {
158
+ for (; ndirect_msb < 16; ndirect_msb++) {
159
+ uint32_t ndirect = ndirect_msb << npostfix;
160
+ BROTLI_BOOL skip;
161
+ double dist_cost;
162
+ BrotliInitDistanceParams(&new_params, npostfix, ndirect);
163
+ if (npostfix == orig_params.dist.distance_postfix_bits &&
164
+ ndirect == orig_params.dist.num_direct_distance_codes) {
165
+ check_orig = BROTLI_FALSE;
166
+ }
167
+ skip = !ComputeDistanceCost(
168
+ cmds, num_commands,
169
+ &orig_params.dist, &new_params.dist, &dist_cost);
170
+ if (skip || (dist_cost > best_dist_cost)) {
171
+ break;
172
+ }
173
+ best_dist_cost = dist_cost;
174
+ params->dist = new_params.dist;
175
+ }
176
+ if (ndirect_msb > 0) ndirect_msb--;
177
+ ndirect_msb /= 2;
178
+ }
179
+ if (check_orig) {
180
+ double dist_cost;
181
+ ComputeDistanceCost(cmds, num_commands,
182
+ &orig_params.dist, &orig_params.dist, &dist_cost);
183
+ if (dist_cost < best_dist_cost) {
184
+ best_dist_cost = dist_cost;
185
+ params->dist = orig_params.dist;
186
+ }
187
+ }
188
+ RecomputeDistancePrefixes(cmds, num_commands,
189
+ &orig_params.dist, &params->dist);
48
190
 
49
191
  BrotliSplitBlock(m, cmds, num_commands,
50
192
  ringbuffer, pos, mask, params,
@@ -77,7 +219,7 @@ void BrotliBuildMetaBlock(MemoryManager* m,
77
219
  if (BROTLI_IS_OOM(m)) return;
78
220
  ClearHistogramsDistance(distance_histograms, distance_histograms_size);
79
221
 
80
- assert(mb->command_histograms == 0);
222
+ BROTLI_DCHECK(mb->command_histograms == 0);
81
223
  mb->command_histograms_size = mb->command_split.num_types;
82
224
  mb->command_histograms =
83
225
  BROTLI_ALLOC(m, HistogramCommand, mb->command_histograms_size);
@@ -90,14 +232,14 @@ void BrotliBuildMetaBlock(MemoryManager* m,
90
232
  literal_histograms, mb->command_histograms, distance_histograms);
91
233
  BROTLI_FREE(m, literal_context_modes);
92
234
 
93
- assert(mb->literal_context_map == 0);
235
+ BROTLI_DCHECK(mb->literal_context_map == 0);
94
236
  mb->literal_context_map_size =
95
237
  mb->literal_split.num_types << BROTLI_LITERAL_CONTEXT_BITS;
96
238
  mb->literal_context_map =
97
239
  BROTLI_ALLOC(m, uint32_t, mb->literal_context_map_size);
98
240
  if (BROTLI_IS_OOM(m)) return;
99
241
 
100
- assert(mb->literal_histograms == 0);
242
+ BROTLI_DCHECK(mb->literal_histograms == 0);
101
243
  mb->literal_histograms_size = mb->literal_context_map_size;
102
244
  mb->literal_histograms =
103
245
  BROTLI_ALLOC(m, HistogramLiteral, mb->literal_histograms_size);
@@ -121,14 +263,14 @@ void BrotliBuildMetaBlock(MemoryManager* m,
121
263
  }
122
264
  }
123
265
 
124
- assert(mb->distance_context_map == 0);
266
+ BROTLI_DCHECK(mb->distance_context_map == 0);
125
267
  mb->distance_context_map_size =
126
268
  mb->distance_split.num_types << BROTLI_DISTANCE_CONTEXT_BITS;
127
269
  mb->distance_context_map =
128
270
  BROTLI_ALLOC(m, uint32_t, mb->distance_context_map_size);
129
271
  if (BROTLI_IS_OOM(m)) return;
130
272
 
131
- assert(mb->distance_histograms == 0);
273
+ BROTLI_DCHECK(mb->distance_histograms == 0);
132
274
  mb->distance_histograms_size = mb->distance_context_map_size;
133
275
  mb->distance_histograms =
134
276
  BROTLI_ALLOC(m, HistogramDistance, mb->distance_histograms_size);
@@ -200,7 +342,7 @@ static void InitContextBlockSplitter(
200
342
  size_t* histograms_size) {
201
343
  size_t max_num_blocks = num_symbols / min_block_size + 1;
202
344
  size_t max_num_types;
203
- assert(num_contexts <= BROTLI_MAX_STATIC_CONTEXTS);
345
+ BROTLI_DCHECK(num_contexts <= BROTLI_MAX_STATIC_CONTEXTS);
204
346
 
205
347
  self->alphabet_size_ = alphabet_size;
206
348
  self->num_contexts_ = num_contexts;
@@ -226,7 +368,7 @@ static void InitContextBlockSplitter(
226
368
  if (BROTLI_IS_OOM(m)) return;
227
369
  split->num_blocks = max_num_blocks;
228
370
  if (BROTLI_IS_OOM(m)) return;
229
- assert(*histograms == 0);
371
+ BROTLI_DCHECK(*histograms == 0);
230
372
  *histograms_size = max_num_types * num_contexts;
231
373
  *histograms = BROTLI_ALLOC(m, HistogramLiteral, *histograms_size);
232
374
  self->histograms_ = *histograms;
@@ -379,7 +521,7 @@ static void MapStaticContexts(MemoryManager* m,
379
521
  const uint32_t* static_context_map,
380
522
  MetaBlockSplit* mb) {
381
523
  size_t i;
382
- assert(mb->literal_context_map == 0);
524
+ BROTLI_DCHECK(mb->literal_context_map == 0);
383
525
  mb->literal_context_map_size =
384
526
  mb->literal_split.num_types << BROTLI_LITERAL_CONTEXT_BITS;
385
527
  mb->literal_context_map =
@@ -398,9 +540,9 @@ static void MapStaticContexts(MemoryManager* m,
398
540
 
399
541
  static BROTLI_INLINE void BrotliBuildMetaBlockGreedyInternal(
400
542
  MemoryManager* m, const uint8_t* ringbuffer, size_t pos, size_t mask,
401
- uint8_t prev_byte, uint8_t prev_byte2, ContextType literal_context_mode,
543
+ uint8_t prev_byte, uint8_t prev_byte2, ContextLut literal_context_lut,
402
544
  const size_t num_contexts, const uint32_t* static_context_map,
403
- const Command *commands, size_t n_commands, MetaBlockSplit* mb) {
545
+ const Command* commands, size_t n_commands, MetaBlockSplit* mb) {
404
546
  union {
405
547
  BlockSplitterLiteral plain;
406
548
  ContextBlockSplitter ctx;
@@ -441,7 +583,8 @@ static BROTLI_INLINE void BrotliBuildMetaBlockGreedyInternal(
441
583
  if (num_contexts == 1) {
442
584
  BlockSplitterAddSymbolLiteral(&lit_blocks.plain, literal);
443
585
  } else {
444
- size_t context = Context(prev_byte, prev_byte2, literal_context_mode);
586
+ size_t context =
587
+ BROTLI_CONTEXT(prev_byte, prev_byte2, literal_context_lut);
445
588
  ContextBlockSplitterAddSymbol(&lit_blocks.ctx, m, literal,
446
589
  static_context_map[context]);
447
590
  if (BROTLI_IS_OOM(m)) return;
@@ -455,7 +598,7 @@ static BROTLI_INLINE void BrotliBuildMetaBlockGreedyInternal(
455
598
  prev_byte2 = ringbuffer[(pos - 2) & mask];
456
599
  prev_byte = ringbuffer[(pos - 1) & mask];
457
600
  if (cmd.cmd_prefix_ >= 128) {
458
- BlockSplitterAddSymbolDistance(&dist_blocks, cmd.dist_prefix_);
601
+ BlockSplitterAddSymbolDistance(&dist_blocks, cmd.dist_prefix_ & 0x3FF);
459
602
  }
460
603
  }
461
604
  }
@@ -482,7 +625,7 @@ void BrotliBuildMetaBlockGreedy(MemoryManager* m,
482
625
  size_t mask,
483
626
  uint8_t prev_byte,
484
627
  uint8_t prev_byte2,
485
- ContextType literal_context_mode,
628
+ ContextLut literal_context_lut,
486
629
  size_t num_contexts,
487
630
  const uint32_t* static_context_map,
488
631
  const Command* commands,
@@ -490,19 +633,17 @@ void BrotliBuildMetaBlockGreedy(MemoryManager* m,
490
633
  MetaBlockSplit* mb) {
491
634
  if (num_contexts == 1) {
492
635
  BrotliBuildMetaBlockGreedyInternal(m, ringbuffer, pos, mask, prev_byte,
493
- prev_byte2, literal_context_mode, 1, NULL, commands, n_commands, mb);
636
+ prev_byte2, literal_context_lut, 1, NULL, commands, n_commands, mb);
494
637
  } else {
495
638
  BrotliBuildMetaBlockGreedyInternal(m, ringbuffer, pos, mask, prev_byte,
496
- prev_byte2, literal_context_mode, num_contexts, static_context_map,
639
+ prev_byte2, literal_context_lut, num_contexts, static_context_map,
497
640
  commands, n_commands, mb);
498
641
  }
499
642
  }
500
643
 
501
- void BrotliOptimizeHistograms(size_t num_direct_distance_codes,
502
- size_t distance_postfix_bits,
644
+ void BrotliOptimizeHistograms(uint32_t num_distance_codes,
503
645
  MetaBlockSplit* mb) {
504
646
  uint8_t good_for_rle[BROTLI_NUM_COMMAND_SYMBOLS];
505
- size_t num_distance_codes;
506
647
  size_t i;
507
648
  for (i = 0; i < mb->literal_histograms_size; ++i) {
508
649
  BrotliOptimizeHuffmanCountsForRle(256, mb->literal_histograms[i].data_,
@@ -513,9 +654,6 @@ void BrotliOptimizeHistograms(size_t num_direct_distance_codes,
513
654
  mb->command_histograms[i].data_,
514
655
  good_for_rle);
515
656
  }
516
- num_distance_codes = BROTLI_NUM_DISTANCE_SHORT_CODES +
517
- num_direct_distance_codes +
518
- ((2 * BROTLI_MAX_DISTANCE_BITS) << distance_postfix_bits);
519
657
  for (i = 0; i < mb->distance_histograms_size; ++i) {
520
658
  BrotliOptimizeHuffmanCountsForRle(num_distance_codes,
521
659
  mb->distance_histograms[i].data_,