brotli 0.2.3 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/main.yml +37 -0
  3. data/.github/workflows/publish.yml +24 -0
  4. data/.gitmodules +1 -1
  5. data/Gemfile +6 -3
  6. data/README.md +2 -2
  7. data/Rakefile +16 -9
  8. data/brotli.gemspec +7 -13
  9. data/ext/brotli/brotli.c +210 -31
  10. data/ext/brotli/buffer.c +1 -7
  11. data/ext/brotli/buffer.h +1 -1
  12. data/ext/brotli/extconf.rb +25 -17
  13. data/lib/brotli/version.rb +1 -1
  14. data/test/brotli_test.rb +107 -0
  15. data/test/brotli_writer_test.rb +36 -0
  16. data/test/test_helper.rb +8 -0
  17. data/vendor/brotli/c/common/constants.c +15 -0
  18. data/vendor/brotli/c/common/constants.h +137 -0
  19. data/vendor/brotli/c/common/context.c +156 -0
  20. data/vendor/brotli/c/common/context.h +4 -152
  21. data/vendor/brotli/c/common/dictionary.bin.br +0 -0
  22. data/vendor/brotli/c/common/dictionary.c +14 -3
  23. data/vendor/brotli/c/common/platform.c +23 -0
  24. data/vendor/brotli/c/common/platform.h +95 -122
  25. data/vendor/brotli/c/common/shared_dictionary.c +521 -0
  26. data/vendor/brotli/c/common/shared_dictionary_internal.h +75 -0
  27. data/vendor/brotli/c/common/transform.c +60 -4
  28. data/vendor/brotli/c/common/transform.h +5 -0
  29. data/vendor/brotli/c/common/version.h +31 -6
  30. data/vendor/brotli/c/dec/bit_reader.c +34 -4
  31. data/vendor/brotli/c/dec/bit_reader.h +221 -107
  32. data/vendor/brotli/c/dec/decode.c +772 -403
  33. data/vendor/brotli/c/dec/huffman.c +7 -4
  34. data/vendor/brotli/c/dec/huffman.h +8 -13
  35. data/vendor/brotli/c/dec/prefix.h +1 -18
  36. data/vendor/brotli/c/dec/state.c +40 -21
  37. data/vendor/brotli/c/dec/state.h +201 -59
  38. data/vendor/brotli/c/enc/backward_references.c +88 -25
  39. data/vendor/brotli/c/enc/backward_references.h +10 -8
  40. data/vendor/brotli/c/enc/backward_references_hq.c +194 -80
  41. data/vendor/brotli/c/enc/backward_references_hq.h +17 -13
  42. data/vendor/brotli/c/enc/backward_references_inc.h +52 -16
  43. data/vendor/brotli/c/enc/bit_cost.c +8 -7
  44. data/vendor/brotli/c/enc/bit_cost.h +5 -4
  45. data/vendor/brotli/c/enc/block_splitter.c +40 -17
  46. data/vendor/brotli/c/enc/block_splitter.h +5 -4
  47. data/vendor/brotli/c/enc/block_splitter_inc.h +99 -49
  48. data/vendor/brotli/c/enc/brotli_bit_stream.c +142 -137
  49. data/vendor/brotli/c/enc/brotli_bit_stream.h +11 -6
  50. data/vendor/brotli/c/enc/cluster.c +10 -9
  51. data/vendor/brotli/c/enc/cluster.h +7 -6
  52. data/vendor/brotli/c/enc/cluster_inc.h +30 -22
  53. data/vendor/brotli/c/enc/command.c +28 -0
  54. data/vendor/brotli/c/enc/command.h +17 -16
  55. data/vendor/brotli/c/enc/compound_dictionary.c +207 -0
  56. data/vendor/brotli/c/enc/compound_dictionary.h +74 -0
  57. data/vendor/brotli/c/enc/compress_fragment.c +93 -83
  58. data/vendor/brotli/c/enc/compress_fragment.h +32 -7
  59. data/vendor/brotli/c/enc/compress_fragment_two_pass.c +100 -88
  60. data/vendor/brotli/c/enc/compress_fragment_two_pass.h +21 -3
  61. data/vendor/brotli/c/enc/dictionary_hash.c +1829 -1101
  62. data/vendor/brotli/c/enc/dictionary_hash.h +2 -1
  63. data/vendor/brotli/c/enc/encode.c +550 -416
  64. data/vendor/brotli/c/enc/encoder_dict.c +613 -5
  65. data/vendor/brotli/c/enc/encoder_dict.h +120 -4
  66. data/vendor/brotli/c/enc/entropy_encode.c +5 -2
  67. data/vendor/brotli/c/enc/entropy_encode.h +4 -3
  68. data/vendor/brotli/c/enc/entropy_encode_static.h +5 -2
  69. data/vendor/brotli/c/enc/fast_log.c +105 -0
  70. data/vendor/brotli/c/enc/fast_log.h +21 -101
  71. data/vendor/brotli/c/enc/find_match_length.h +17 -25
  72. data/vendor/brotli/c/enc/hash.h +350 -120
  73. data/vendor/brotli/c/enc/hash_composite_inc.h +71 -67
  74. data/vendor/brotli/c/enc/hash_forgetful_chain_inc.h +92 -51
  75. data/vendor/brotli/c/enc/hash_longest_match64_inc.h +79 -84
  76. data/vendor/brotli/c/enc/hash_longest_match_inc.h +53 -54
  77. data/vendor/brotli/c/enc/hash_longest_match_quickly_inc.h +93 -62
  78. data/vendor/brotli/c/enc/hash_rolling_inc.h +25 -29
  79. data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h +42 -40
  80. data/vendor/brotli/c/enc/histogram.c +4 -4
  81. data/vendor/brotli/c/enc/histogram.h +7 -6
  82. data/vendor/brotli/c/enc/literal_cost.c +20 -15
  83. data/vendor/brotli/c/enc/literal_cost.h +4 -2
  84. data/vendor/brotli/c/enc/memory.c +29 -5
  85. data/vendor/brotli/c/enc/memory.h +43 -14
  86. data/vendor/brotli/c/enc/metablock.c +95 -85
  87. data/vendor/brotli/c/enc/metablock.h +9 -8
  88. data/vendor/brotli/c/enc/metablock_inc.h +9 -7
  89. data/vendor/brotli/c/enc/params.h +7 -4
  90. data/vendor/brotli/c/enc/prefix.h +3 -2
  91. data/vendor/brotli/c/enc/quality.h +40 -3
  92. data/vendor/brotli/c/enc/ringbuffer.h +8 -4
  93. data/vendor/brotli/c/enc/state.h +104 -0
  94. data/vendor/brotli/c/enc/static_dict.c +60 -4
  95. data/vendor/brotli/c/enc/static_dict.h +3 -2
  96. data/vendor/brotli/c/enc/static_dict_lut.h +2 -0
  97. data/vendor/brotli/c/enc/utf8_util.c +2 -2
  98. data/vendor/brotli/c/enc/utf8_util.h +2 -1
  99. data/vendor/brotli/c/enc/write_bits.h +29 -26
  100. data/vendor/brotli/c/include/brotli/decode.h +67 -2
  101. data/vendor/brotli/c/include/brotli/encode.h +77 -3
  102. data/vendor/brotli/c/include/brotli/port.h +34 -3
  103. data/vendor/brotli/c/include/brotli/shared_dictionary.h +100 -0
  104. metadata +23 -97
  105. data/.travis.yml +0 -31
  106. data/docs/Brotli/Error.html +0 -124
  107. data/docs/Brotli.html +0 -485
  108. data/docs/_index.html +0 -122
  109. data/docs/class_list.html +0 -51
  110. data/docs/css/common.css +0 -1
  111. data/docs/css/full_list.css +0 -58
  112. data/docs/css/style.css +0 -496
  113. data/docs/file.README.html +0 -127
  114. data/docs/file_list.html +0 -56
  115. data/docs/frames.html +0 -17
  116. data/docs/index.html +0 -127
  117. data/docs/js/app.js +0 -292
  118. data/docs/js/full_list.js +0 -216
  119. data/docs/js/jquery.js +0 -4
  120. data/docs/method_list.html +0 -67
  121. data/docs/top-level-namespace.html +0 -110
  122. data/spec/brotli_spec.rb +0 -88
  123. data/spec/inflate_spec.rb +0 -75
  124. data/spec/spec_helper.rb +0 -4
@@ -0,0 +1,104 @@
1
+ /* Copyright 2022 Google Inc. All Rights Reserved.
2
+
3
+ Distributed under MIT license.
4
+ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
5
+ */
6
+
7
+ /* Encoder state. */
8
+
9
+ #ifndef BROTLI_ENC_STATE_H_
10
+ #define BROTLI_ENC_STATE_H_
11
+
12
+ #include <brotli/types.h>
13
+
14
+ #include "command.h"
15
+ #include "compress_fragment.h"
16
+ #include "compress_fragment_two_pass.h"
17
+ #include "hash.h"
18
+ #include "memory.h"
19
+ #include "params.h"
20
+ #include "ringbuffer.h"
21
+
22
+ typedef enum BrotliEncoderStreamState {
23
+ /* Default state. */
24
+ BROTLI_STREAM_PROCESSING = 0,
25
+ /* Intermediate state; after next block is emitted, byte-padding should be
26
+ performed before getting back to default state. */
27
+ BROTLI_STREAM_FLUSH_REQUESTED = 1,
28
+ /* Last metablock was produced; no more input is acceptable. */
29
+ BROTLI_STREAM_FINISHED = 2,
30
+ /* Flushing compressed block and writing meta-data block header. */
31
+ BROTLI_STREAM_METADATA_HEAD = 3,
32
+ /* Writing metadata block body. */
33
+ BROTLI_STREAM_METADATA_BODY = 4
34
+ } BrotliEncoderStreamState;
35
+
36
+ typedef enum BrotliEncoderFlintState {
37
+ BROTLI_FLINT_NEEDS_2_BYTES = 2,
38
+ BROTLI_FLINT_NEEDS_1_BYTE = 1,
39
+ BROTLI_FLINT_WAITING_FOR_PROCESSING = 0,
40
+ BROTLI_FLINT_WAITING_FOR_FLUSHING = -1,
41
+ BROTLI_FLINT_DONE = -2
42
+ } BrotliEncoderFlintState;
43
+
44
+ typedef struct BrotliEncoderStateStruct {
45
+ BrotliEncoderParams params;
46
+
47
+ MemoryManager memory_manager_;
48
+
49
+ uint64_t input_pos_;
50
+ RingBuffer ringbuffer_;
51
+ size_t cmd_alloc_size_;
52
+ Command* commands_;
53
+ size_t num_commands_;
54
+ size_t num_literals_;
55
+ size_t last_insert_len_;
56
+ uint64_t last_flush_pos_;
57
+ uint64_t last_processed_pos_;
58
+ int dist_cache_[BROTLI_NUM_DISTANCE_SHORT_CODES];
59
+ int saved_dist_cache_[4];
60
+ uint16_t last_bytes_;
61
+ uint8_t last_bytes_bits_;
62
+ /* "Flint" is a tiny uncompressed block emitted before the continuation
63
+ block to unwire literal context from previous data. Despite being int8_t,
64
+ field is actually BrotliEncoderFlintState enum. */
65
+ int8_t flint_;
66
+ uint8_t prev_byte_;
67
+ uint8_t prev_byte2_;
68
+ size_t storage_size_;
69
+ uint8_t* storage_;
70
+
71
+ Hasher hasher_;
72
+
73
+ /* Hash table for FAST_ONE_PASS_COMPRESSION_QUALITY mode. */
74
+ int small_table_[1 << 10]; /* 4KiB */
75
+ int* large_table_; /* Allocated only when needed */
76
+ size_t large_table_size_;
77
+
78
+ BrotliOnePassArena* one_pass_arena_;
79
+ BrotliTwoPassArena* two_pass_arena_;
80
+
81
+ /* Command and literal buffers for FAST_TWO_PASS_COMPRESSION_QUALITY. */
82
+ uint32_t* command_buf_;
83
+ uint8_t* literal_buf_;
84
+
85
+ uint64_t total_in_;
86
+ uint8_t* next_out_;
87
+ size_t available_out_;
88
+ uint64_t total_out_;
89
+ /* Temporary buffer for padding flush bits or metadata block header / body. */
90
+ union {
91
+ uint64_t u64[2];
92
+ uint8_t u8[16];
93
+ } tiny_buf_;
94
+ uint32_t remaining_metadata_bytes_;
95
+ BrotliEncoderStreamState stream_state_;
96
+
97
+ BROTLI_BOOL is_last_block_emitted_;
98
+ BROTLI_BOOL is_initialized_;
99
+ } BrotliEncoderStateStruct;
100
+
101
+ typedef struct BrotliEncoderStateStruct BrotliEncoderStateInternal;
102
+ #define BrotliEncoderState BrotliEncoderStateInternal
103
+
104
+ #endif // BROTLI_ENC_STATE_H_
@@ -4,13 +4,13 @@
4
4
  See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
5
5
  */
6
6
 
7
- #include "./static_dict.h"
7
+ #include "static_dict.h"
8
8
 
9
9
  #include "../common/dictionary.h"
10
10
  #include "../common/platform.h"
11
11
  #include "../common/transform.h"
12
- #include "./encoder_dict.h"
13
- #include "./find_match_length.h"
12
+ #include "encoder_dict.h"
13
+ #include "find_match_length.h"
14
14
 
15
15
  #if defined(__cplusplus) || defined(c_plusplus)
16
16
  extern "C" {
@@ -74,10 +74,27 @@ static BROTLI_INLINE BROTLI_BOOL IsMatch(const BrotliDictionary* dictionary,
74
74
  }
75
75
  }
76
76
 
77
- BROTLI_BOOL BrotliFindAllStaticDictionaryMatches(
77
+ /* Finds matches for a single static dictionary */
78
+ static BROTLI_BOOL BrotliFindAllStaticDictionaryMatchesFor(
78
79
  const BrotliEncoderDictionary* dictionary, const uint8_t* data,
79
80
  size_t min_length, size_t max_length, uint32_t* matches) {
80
81
  BROTLI_BOOL has_found_match = BROTLI_FALSE;
82
+ #if defined(BROTLI_EXPERIMENTAL)
83
+ if (dictionary->has_words_heavy) {
84
+ const BrotliTrieNode* node = &dictionary->trie.root;
85
+ size_t l = 0;
86
+ while (node && l < max_length) {
87
+ uint8_t c;
88
+ if (l >= min_length && node->len_) {
89
+ AddMatch(node->idx_, l, node->len_, matches);
90
+ has_found_match = BROTLI_TRUE;
91
+ }
92
+ c = data[l++];
93
+ node = BrotliTrieSub(&dictionary->trie, node, c);
94
+ }
95
+ return has_found_match;
96
+ }
97
+ #endif /* BROTLI_EXPERIMENTAL */
81
98
  {
82
99
  size_t offset = dictionary->buckets[Hash(data)];
83
100
  BROTLI_BOOL end = !offset;
@@ -481,6 +498,45 @@ BROTLI_BOOL BrotliFindAllStaticDictionaryMatches(
481
498
  return has_found_match;
482
499
  }
483
500
 
501
+ /* Finds matches for one or more dictionaries, if multiple are present
502
+ in the contextual dictionary */
503
+ BROTLI_BOOL BrotliFindAllStaticDictionaryMatches(
504
+ const BrotliEncoderDictionary* dictionary, const uint8_t* data,
505
+ size_t min_length, size_t max_length, uint32_t* matches) {
506
+ BROTLI_BOOL has_found_match =
507
+ BrotliFindAllStaticDictionaryMatchesFor(
508
+ dictionary, data, min_length, max_length, matches);
509
+
510
+ if (!!dictionary->parent && dictionary->parent->num_dictionaries > 1) {
511
+ uint32_t matches2[BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN + 1];
512
+ int l;
513
+ const BrotliEncoderDictionary* dictionary2 = dictionary->parent->dict[0];
514
+ if (dictionary2 == dictionary) {
515
+ dictionary2 = dictionary->parent->dict[1];
516
+ }
517
+
518
+ for (l = 0; l < BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN + 1; l++) {
519
+ matches2[l] = kInvalidMatch;
520
+ }
521
+
522
+ has_found_match |= BrotliFindAllStaticDictionaryMatchesFor(
523
+ dictionary2, data, min_length, max_length, matches2);
524
+
525
+ for (l = 0; l < BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN + 1; l++) {
526
+ if (matches2[l] != kInvalidMatch) {
527
+ uint32_t dist = (uint32_t)(matches2[l] >> 5);
528
+ uint32_t len_code = matches2[l] & 31;
529
+ uint32_t skipdist = (uint32_t)((uint32_t)(1 << dictionary->words->
530
+ size_bits_by_length[len_code]) & ~1u) *
531
+ (uint32_t)dictionary->num_transforms;
532
+ /* TODO(lode): check for dist overflow */
533
+ dist += skipdist;
534
+ AddMatch(dist, (size_t)l, len_code, matches);
535
+ }
536
+ }
537
+ }
538
+ return has_found_match;
539
+ }
484
540
  #if defined(__cplusplus) || defined(c_plusplus)
485
541
  } /* extern "C" */
486
542
  #endif
@@ -9,10 +9,11 @@
9
9
  #ifndef BROTLI_ENC_STATIC_DICT_H_
10
10
  #define BROTLI_ENC_STATIC_DICT_H_
11
11
 
12
+ #include <brotli/types.h>
13
+
12
14
  #include "../common/dictionary.h"
13
15
  #include "../common/platform.h"
14
- #include <brotli/types.h>
15
- #include "./encoder_dict.h"
16
+ #include "encoder_dict.h"
16
17
 
17
18
  #if defined(__cplusplus) || defined(c_plusplus)
18
19
  extern "C" {
@@ -22,6 +22,7 @@ typedef struct DictWord {
22
22
  uint16_t idx;
23
23
  } DictWord;
24
24
 
25
+ /* GENERATED CODE START */
25
26
  static const int kDictNumBits = 15;
26
27
  static const uint32_t kDictHashMul32 = 0x1E35A7BD;
27
28
 
@@ -5856,6 +5857,7 @@ static const DictWord kStaticDictionaryWords[31705] = {
5856
5857
  ,0,1735},{5,0,598},{7,0,791},{8,0,108},{9,0,123},{7,10,1570},{140,10,542},{142,
5857
5858
  11,410},{9,11,660},{138,11,347}
5858
5859
  };
5860
+ /* GENERATED CODE END */
5859
5861
 
5860
5862
  #if defined(__cplusplus) || defined(c_plusplus)
5861
5863
  } /* extern "C" */
@@ -6,7 +6,7 @@
6
6
 
7
7
  /* Heuristics for deciding about the UTF8-ness of strings. */
8
8
 
9
- #include "./utf8_util.h"
9
+ #include "utf8_util.h"
10
10
 
11
11
  #include <brotli/types.h>
12
12
 
@@ -77,7 +77,7 @@ BROTLI_BOOL BrotliIsMostlyUTF8(
77
77
  i += bytes_read;
78
78
  if (symbol < 0x110000) size_utf8 += bytes_read;
79
79
  }
80
- return TO_BROTLI_BOOL(size_utf8 > min_fraction * (double)length);
80
+ return TO_BROTLI_BOOL((double)size_utf8 > min_fraction * (double)length);
81
81
  }
82
82
 
83
83
  #if defined(__cplusplus) || defined(c_plusplus)
@@ -9,9 +9,10 @@
9
9
  #ifndef BROTLI_ENC_UTF8_UTIL_H_
10
10
  #define BROTLI_ENC_UTF8_UTIL_H_
11
11
 
12
- #include "../common/platform.h"
13
12
  #include <brotli/types.h>
14
13
 
14
+ #include "../common/platform.h"
15
+
15
16
  #if defined(__cplusplus) || defined(c_plusplus)
16
17
  extern "C" {
17
18
  #endif
@@ -9,15 +9,14 @@
9
9
  #ifndef BROTLI_ENC_WRITE_BITS_H_
10
10
  #define BROTLI_ENC_WRITE_BITS_H_
11
11
 
12
- #include "../common/platform.h"
13
12
  #include <brotli/types.h>
14
13
 
14
+ #include "../common/platform.h"
15
+
15
16
  #if defined(__cplusplus) || defined(c_plusplus)
16
17
  extern "C" {
17
18
  #endif
18
19
 
19
- /*#define BIT_WRITER_DEBUG */
20
-
21
20
  /* This function writes bits into bytes in increasing addresses, and within
22
21
  a byte least-significant-bit first.
23
22
 
@@ -28,7 +27,7 @@ extern "C" {
28
27
 
29
28
  0000 0RRR 0000 0000 0000 0000
30
29
 
31
- Now, we could write 5 or less bits in MSB by just sifting by 3
30
+ Now, we could write 5 or less bits in MSB by just shifting by 3
32
31
  and OR'ing to BYTE-0.
33
32
 
34
33
  For n bits, we take the last 5 bits, OR that with high bits in BYTE-0,
@@ -37,37 +36,41 @@ static BROTLI_INLINE void BrotliWriteBits(size_t n_bits,
37
36
  uint64_t bits,
38
37
  size_t* BROTLI_RESTRICT pos,
39
38
  uint8_t* BROTLI_RESTRICT array) {
39
+ BROTLI_LOG(("WriteBits %2d 0x%08x%08x %10d\n", (int)n_bits,
40
+ (uint32_t)(bits >> 32), (uint32_t)(bits & 0xFFFFFFFF),
41
+ (int)*pos));
42
+ BROTLI_DCHECK((bits >> n_bits) == 0);
43
+ BROTLI_DCHECK(n_bits <= 56);
40
44
  #if defined(BROTLI_LITTLE_ENDIAN)
41
45
  /* This branch of the code can write up to 56 bits at a time,
42
46
  7 bits are lost by being perhaps already in *p and at least
43
47
  1 bit is needed to initialize the bit-stream ahead (i.e. if 7
44
48
  bits are in *p and we write 57 bits, then the next write will
45
49
  access a byte that was never initialized). */
46
- uint8_t* p = &array[*pos >> 3];
47
- uint64_t v = (uint64_t)(*p); /* Zero-extend 8 to 64 bits. */
48
- BROTLI_LOG(("WriteBits %2d 0x%08x%08x %10d\n", (int)n_bits,
49
- (uint32_t)(bits >> 32), (uint32_t)(bits & 0xFFFFFFFF),
50
- (int)*pos));
51
- BROTLI_DCHECK((bits >> n_bits) == 0);
52
- BROTLI_DCHECK(n_bits <= 56);
53
- v |= bits << (*pos & 7);
54
- BROTLI_UNALIGNED_STORE64LE(p, v); /* Set some bits. */
55
- *pos += n_bits;
50
+ {
51
+ uint8_t* p = &array[*pos >> 3];
52
+ uint64_t v = (uint64_t)(*p); /* Zero-extend 8 to 64 bits. */
53
+ v |= bits << (*pos & 7);
54
+ BROTLI_UNALIGNED_STORE64LE(p, v); /* Set some bits. */
55
+ *pos += n_bits;
56
+ }
56
57
  #else
57
58
  /* implicit & 0xFF is assumed for uint8_t arithmetics */
58
- uint8_t* array_pos = &array[*pos >> 3];
59
- const size_t bits_reserved_in_first_byte = (*pos & 7);
60
- size_t bits_left_to_write;
61
- bits <<= bits_reserved_in_first_byte;
62
- *array_pos++ |= (uint8_t)bits;
63
- for (bits_left_to_write = n_bits + bits_reserved_in_first_byte;
64
- bits_left_to_write >= 9;
65
- bits_left_to_write -= 8) {
66
- bits >>= 8;
67
- *array_pos++ = (uint8_t)bits;
59
+ {
60
+ uint8_t* array_pos = &array[*pos >> 3];
61
+ const size_t bits_reserved_in_first_byte = (*pos & 7);
62
+ size_t bits_left_to_write;
63
+ bits <<= bits_reserved_in_first_byte;
64
+ *array_pos++ |= (uint8_t)bits;
65
+ for (bits_left_to_write = n_bits + bits_reserved_in_first_byte;
66
+ bits_left_to_write >= 9;
67
+ bits_left_to_write -= 8) {
68
+ bits >>= 8;
69
+ *array_pos++ = (uint8_t)bits;
70
+ }
71
+ *array_pos = 0;
72
+ *pos += n_bits;
68
73
  }
69
- *array_pos = 0;
70
- *pos += n_bits;
71
74
  #endif
72
75
  }
73
76
 
@@ -13,6 +13,7 @@
13
13
  #define BROTLI_DEC_DECODE_H_
14
14
 
15
15
  #include <brotli/port.h>
16
+ #include <brotli/shared_dictionary.h>
16
17
  #include <brotli/types.h>
17
18
 
18
19
  #if defined(__cplusplus) || defined(c_plusplus)
@@ -85,8 +86,9 @@ typedef enum {
85
86
  BROTLI_ERROR_CODE(_ERROR_FORMAT_, PADDING_2, -15) SEPARATOR \
86
87
  BROTLI_ERROR_CODE(_ERROR_FORMAT_, DISTANCE, -16) SEPARATOR \
87
88
  \
88
- /* -17..-18 codes are reserved */ \
89
+ /* -17 code is reserved */ \
89
90
  \
91
+ BROTLI_ERROR_CODE(_ERROR_, COMPOUND_DICTIONARY, -18) SEPARATOR \
90
92
  BROTLI_ERROR_CODE(_ERROR_, DICTIONARY_NOT_SET, -19) SEPARATOR \
91
93
  BROTLI_ERROR_CODE(_ERROR_, INVALID_ARGUMENTS, -20) SEPARATOR \
92
94
  \
@@ -154,6 +156,28 @@ typedef enum BrotliDecoderParameter {
154
156
  BROTLI_DEC_API BROTLI_BOOL BrotliDecoderSetParameter(
155
157
  BrotliDecoderState* state, BrotliDecoderParameter param, uint32_t value);
156
158
 
159
+ /**
160
+ * Adds LZ77 prefix dictionary, adds or replaces built-in static dictionary and
161
+ * transforms.
162
+ *
163
+ * Attached dictionary ownership is not transferred.
164
+ * Data provided to this method should be kept accessible until
165
+ * decoding is finished and decoder instance is destroyed.
166
+ *
167
+ * @note Dictionaries can NOT be attached after actual decoding is started.
168
+ *
169
+ * @param state decoder instance
170
+ * @param type dictionary data format
171
+ * @param data_size length of memory region pointed by @p data
172
+ * @param data dictionary data in format corresponding to @p type
173
+ * @returns ::BROTLI_FALSE if dictionary is corrupted,
174
+ * or dictionary count limit is reached
175
+ * @returns ::BROTLI_TRUE if dictionary is accepted / attached
176
+ */
177
+ BROTLI_DEC_API BROTLI_BOOL BrotliDecoderAttachDictionary(
178
+ BrotliDecoderState* state, BrotliSharedDictionaryType type,
179
+ size_t data_size, const uint8_t data[BROTLI_ARRAY_PARAM(data_size)]);
180
+
157
181
  /**
158
182
  * Creates an instance of ::BrotliDecoderState and initializes it.
159
183
  *
@@ -333,10 +357,51 @@ BROTLI_DEC_API const char* BrotliDecoderErrorString(BrotliDecoderErrorCode c);
333
357
  /**
334
358
  * Gets a decoder library version.
335
359
  *
336
- * Look at BROTLI_VERSION for more information.
360
+ * Look at BROTLI_MAKE_HEX_VERSION for more information.
337
361
  */
338
362
  BROTLI_DEC_API uint32_t BrotliDecoderVersion(void);
339
363
 
364
+ /**
365
+ * Callback to fire on metadata block start.
366
+ *
367
+ * After this callback is fired, if @p size is not @c 0, it is followed by
368
+ * ::brotli_decoder_metadata_chunk_func as more metadata block contents become
369
+ * accessible.
370
+ *
371
+ * @param opaque callback handle
372
+ * @param size size of metadata block
373
+ */
374
+ typedef void (*brotli_decoder_metadata_start_func)(void* opaque, size_t size);
375
+
376
+ /**
377
+ * Callback to fire on metadata block chunk becomes available.
378
+ *
379
+ * This function can be invoked multiple times per metadata block; block should
380
+ * be considered finished when sum of @p size matches the announced metadata
381
+ * block size. Chunks contents pointed by @p data are transient and shouln not
382
+ * be accessed after leaving the callback.
383
+ *
384
+ * @param opaque callback handle
385
+ * @param data pointer to metadata contents
386
+ * @param size size of metadata block chunk, at least @c 1
387
+ */
388
+ typedef void (*brotli_decoder_metadata_chunk_func)(void* opaque,
389
+ const uint8_t* data,
390
+ size_t size);
391
+
392
+ /**
393
+ * Sets callback for receiving metadata blocks.
394
+ *
395
+ * @param state decoder instance
396
+ * @param start_func callback on metadata block start
397
+ * @param chunk_func callback on metadata block chunk
398
+ * @param opaque callback handle
399
+ */
400
+ BROTLI_DEC_API void BrotliDecoderSetMetadataCallbacks(
401
+ BrotliDecoderState* state,
402
+ brotli_decoder_metadata_start_func start_func,
403
+ brotli_decoder_metadata_chunk_func chunk_func, void* opaque);
404
+
340
405
  #if defined(__cplusplus) || defined(c_plusplus)
341
406
  } /* extern "C" */
342
407
  #endif
@@ -13,6 +13,7 @@
13
13
  #define BROTLI_ENC_ENCODE_H_
14
14
 
15
15
  #include <brotli/port.h>
16
+ #include <brotli/shared_dictionary.h>
16
17
  #include <brotli/types.h>
17
18
 
18
19
  #if defined(__cplusplus) || defined(c_plusplus)
@@ -201,7 +202,23 @@ typedef enum BrotliEncoderParameter {
201
202
  *
202
203
  * Range is from 0 to (15 << NPOSTFIX) in steps of (1 << NPOSTFIX).
203
204
  */
204
- BROTLI_PARAM_NDIRECT = 8
205
+ BROTLI_PARAM_NDIRECT = 8,
206
+ /**
207
+ * Number of bytes of input stream already processed by a different instance.
208
+ *
209
+ * @note It is important to configure all the encoder instances with same
210
+ * parameters (except this one) in order to allow all the encoded parts
211
+ * obey the same restrictions implied by header.
212
+ *
213
+ * If offset is not 0, then stream header is omitted.
214
+ * In any case output start is byte aligned, so for proper streams stitching
215
+ * "predecessor" stream must be flushed.
216
+ *
217
+ * Range is not artificially limited, but all the values greater or equal to
218
+ * maximal window size have the same effect. Values greater than 2**30 are not
219
+ * allowed.
220
+ */
221
+ BROTLI_PARAM_STREAM_OFFSET = 9
205
222
  } BrotliEncoderParameter;
206
223
 
207
224
  /**
@@ -253,6 +270,51 @@ BROTLI_ENC_API BrotliEncoderState* BrotliEncoderCreateInstance(
253
270
  */
254
271
  BROTLI_ENC_API void BrotliEncoderDestroyInstance(BrotliEncoderState* state);
255
272
 
273
+ /* Opaque type for pointer to different possible internal structures containing
274
+ dictionary prepared for the encoder */
275
+ typedef struct BrotliEncoderPreparedDictionaryStruct
276
+ BrotliEncoderPreparedDictionary;
277
+
278
+ /**
279
+ * Prepares a shared dictionary from the given file format for the encoder.
280
+ *
281
+ * @p alloc_func and @p free_func @b MUST be both zero or both non-zero. In the
282
+ * case they are both zero, default memory allocators are used. @p opaque is
283
+ * passed to @p alloc_func and @p free_func when they are called. @p free_func
284
+ * has to return without doing anything when asked to free a NULL pointer.
285
+ *
286
+ * @param type type of dictionary stored in data
287
+ * @param data_size size of @p data buffer
288
+ * @param data pointer to the dictionary data
289
+ * @param quality the maximum Brotli quality to prepare the dictionary for,
290
+ * use BROTLI_MAX_QUALITY by default
291
+ * @param alloc_func custom memory allocation function
292
+ * @param free_func custom memory free function
293
+ * @param opaque custom memory manager handle
294
+ */
295
+ BROTLI_ENC_API BrotliEncoderPreparedDictionary*
296
+ BrotliEncoderPrepareDictionary(BrotliSharedDictionaryType type,
297
+ size_t data_size, const uint8_t data[BROTLI_ARRAY_PARAM(data_size)],
298
+ int quality,
299
+ brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque);
300
+
301
+ BROTLI_ENC_API void BrotliEncoderDestroyPreparedDictionary(
302
+ BrotliEncoderPreparedDictionary* dictionary);
303
+
304
+ /**
305
+ * Attaches a prepared dictionary of any type to the encoder. Can be used
306
+ * multiple times to attach multiple dictionaries. The dictionary type was
307
+ * determined by BrotliEncoderPrepareDictionary. Multiple raw prefix
308
+ * dictionaries and/or max 1 serialized dictionary with custom words can be
309
+ * attached.
310
+ *
311
+ * @returns ::BROTLI_FALSE in case of error
312
+ * @returns ::BROTLI_TRUE otherwise
313
+ */
314
+ BROTLI_ENC_API BROTLI_BOOL BrotliEncoderAttachPreparedDictionary(
315
+ BrotliEncoderState* state,
316
+ const BrotliEncoderPreparedDictionary* dictionary);
317
+
256
318
  /**
257
319
  * Calculates the output size bound for the given @p input_size.
258
320
  *
@@ -274,6 +336,11 @@ BROTLI_ENC_API size_t BrotliEncoderMaxCompressedSize(size_t input_size);
274
336
  * @note If ::BrotliEncoderMaxCompressedSize(@p input_size) returns non-zero
275
337
  * value, then output is guaranteed to be no longer than that.
276
338
  *
339
+ * @note If @p lgwin is greater than ::BROTLI_MAX_WINDOW_BITS then resulting
340
+ * stream might be incompatible with RFC 7932; to decode such streams,
341
+ * decoder should be configured with
342
+ * ::BROTLI_DECODER_PARAM_LARGE_WINDOW = @c 1
343
+ *
277
344
  * @param quality quality parameter value, e.g. ::BROTLI_DEFAULT_QUALITY
278
345
  * @param lgwin lgwin parameter value, e.g. ::BROTLI_DEFAULT_WINDOW
279
346
  * @param mode mode parameter value, e.g. ::BROTLI_DEFAULT_MODE
@@ -386,7 +453,7 @@ BROTLI_ENC_API BROTLI_BOOL BrotliEncoderHasMoreOutput(
386
453
  *
387
454
  * This method is used to make language bindings easier and more efficient:
388
455
  * -# push data to ::BrotliEncoderCompressStream,
389
- * until ::BrotliEncoderHasMoreOutput returns BROTL_TRUE
456
+ * until ::BrotliEncoderHasMoreOutput returns BROTLI_TRUE
390
457
  * -# use ::BrotliEncoderTakeOutput to peek bytes and copy to language-specific
391
458
  * entity
392
459
  *
@@ -412,11 +479,18 @@ BROTLI_ENC_API BROTLI_BOOL BrotliEncoderHasMoreOutput(
412
479
  BROTLI_ENC_API const uint8_t* BrotliEncoderTakeOutput(
413
480
  BrotliEncoderState* state, size_t* size);
414
481
 
482
+ /* Returns the estimated peak memory usage (in bytes) of the BrotliCompress()
483
+ function, not counting the memory needed for the input and output. */
484
+ BROTLI_ENC_EXTRA_API size_t BrotliEncoderEstimatePeakMemoryUsage(
485
+ int quality, int lgwin, size_t input_size);
486
+ /* Returns 0 if dictionary is not valid; otherwise returns allocation size. */
487
+ BROTLI_ENC_EXTRA_API size_t BrotliEncoderGetPreparedDictionarySize(
488
+ const BrotliEncoderPreparedDictionary* dictionary);
415
489
 
416
490
  /**
417
491
  * Gets an encoder library version.
418
492
  *
419
- * Look at BROTLI_VERSION for more information.
493
+ * Look at BROTLI_MAKE_HEX_VERSION for more information.
420
494
  */
421
495
  BROTLI_ENC_API uint32_t BrotliEncoderVersion(void);
422
496
 
@@ -218,6 +218,12 @@
218
218
  BROTLI_GNUC_VERSION_CHECK(major, minor, patch)
219
219
  #endif
220
220
 
221
+ #if defined(__has_feature)
222
+ #define BROTLI_HAS_FEATURE(feature) __has_feature(feature)
223
+ #else
224
+ #define BROTLI_HAS_FEATURE(feature) (0)
225
+ #endif
226
+
221
227
  #if defined(_WIN32) || defined(__CYGWIN__)
222
228
  #define BROTLI_PUBLIC
223
229
  #elif BROTLI_GNUC_VERSION_CHECK(3, 3, 0) || \
@@ -233,9 +239,28 @@
233
239
  #define BROTLI_PUBLIC
234
240
  #endif
235
241
 
236
- #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \
237
- !defined(__STDC_NO_VLA__) && !defined(__cplusplus) && \
238
- !defined(__PGI) && !defined(__PGIC__) && !defined(__TINYC__)
242
+ /* BROTLI_INTERNAL could be defined to override visibility, e.g. for tests. */
243
+ #if !defined(BROTLI_INTERNAL)
244
+ #if defined(_WIN32) || defined(__CYGWIN__)
245
+ #define BROTLI_INTERNAL
246
+ #elif BROTLI_GNUC_VERSION_CHECK(3, 3, 0) || \
247
+ BROTLI_TI_VERSION_CHECK(8, 0, 0) || \
248
+ BROTLI_INTEL_VERSION_CHECK(16, 0, 0) || \
249
+ BROTLI_ARM_VERSION_CHECK(4, 1, 0) || \
250
+ BROTLI_IBM_VERSION_CHECK(13, 1, 0) || \
251
+ BROTLI_SUNPRO_VERSION_CHECK(5, 11, 0) || \
252
+ (BROTLI_TI_VERSION_CHECK(7, 3, 0) && \
253
+ defined(__TI_GNU_ATTRIBUTE_SUPPORT__) && defined(__TI_EABI__))
254
+ #define BROTLI_INTERNAL __attribute__ ((visibility ("hidden")))
255
+ #else
256
+ #define BROTLI_INTERNAL
257
+ #endif
258
+ #endif
259
+
260
+ #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \
261
+ !defined(__STDC_NO_VLA__) && !defined(__cplusplus) && \
262
+ !defined(__PGI) && !defined(__PGIC__) && !defined(__TINYC__) && \
263
+ !defined(__clang__)
239
264
  #define BROTLI_ARRAY_PARAM(name) (name)
240
265
  #else
241
266
  #define BROTLI_ARRAY_PARAM(name)
@@ -271,4 +296,10 @@
271
296
  #define BROTLI_ENC_API
272
297
  #endif
273
298
 
299
+ #if defined(BROTLI_BUILD_ENC_EXTRA_API)
300
+ #define BROTLI_ENC_EXTRA_API BROTLI_ENC_API
301
+ #else
302
+ #define BROTLI_ENC_EXTRA_API BROTLI_INTERNAL
303
+ #endif
304
+
274
305
  #endif /* BROTLI_COMMON_PORT_H_ */