brotli 0.4.0 → 0.5.0

data/vendor/brotli/c/dec/huffman.c

@@ -6,13 +6,14 @@
 
 /* Utilities for building Huffman decoding tables. */
 
-#include "./huffman.h"
+#include "huffman.h"
 
 #include <string.h>  /* memcpy, memset */
 
+#include <brotli/types.h>
+
 #include "../common/constants.h"
 #include "../common/platform.h"
-#include <brotli/types.h>
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
@@ -117,11 +118,13 @@ void BrotliBuildCodeLengthsHuffmanTable(HuffmanCode* table,
   int bits_count;
   BROTLI_DCHECK(BROTLI_HUFFMAN_MAX_CODE_LENGTH_CODE_LENGTH <=
                 BROTLI_REVERSE_BITS_MAX);
+  BROTLI_DCHECK(BROTLI_HUFFMAN_MAX_CODE_LENGTH_CODE_LENGTH == 5);
 
   /* Generate offsets into sorted symbol table by code length. */
   symbol = -1;
   bits = 1;
-  BROTLI_REPEAT(BROTLI_HUFFMAN_MAX_CODE_LENGTH_CODE_LENGTH, {
+  /* BROTLI_HUFFMAN_MAX_CODE_LENGTH_CODE_LENGTH == 5 */
+  BROTLI_REPEAT_5({
     symbol += count[bits];
     offset[bits] = symbol;
     bits++;
@@ -132,7 +135,7 @@ void BrotliBuildCodeLengthsHuffmanTable(HuffmanCode* table,
   /* Sort symbols by length, by symbol order within each length. */
   symbol = BROTLI_CODE_LENGTH_CODES;
   do {
-    BROTLI_REPEAT(6, {
+    BROTLI_REPEAT_6({
       symbol--;
       sorted[offset[code_lengths[symbol]]--] = symbol;
     });

data/vendor/brotli/c/dec/huffman.h

@@ -9,9 +9,10 @@
 #ifndef BROTLI_DEC_HUFFMAN_H_
 #define BROTLI_DEC_HUFFMAN_H_
 
-#include "../common/platform.h"
 #include <brotli/types.h>
 
+#include "../common/platform.h"
+
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
 #endif

data/vendor/brotli/c/dec/prefix.h

@@ -10,9 +10,10 @@
 #ifndef BROTLI_DEC_PREFIX_H_
 #define BROTLI_DEC_PREFIX_H_
 
-#include "../common/constants.h"
 #include <brotli/types.h>
 
+#include "../common/constants.h"
+
 typedef struct CmdLutElement {
   uint8_t insert_len_extra_bits;
   uint8_t copy_len_extra_bits;

data/vendor/brotli/c/dec/state.c

@@ -4,12 +4,14 @@
    See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
 */
 
-#include "./state.h"
+#include "state.h"
 
 #include <stdlib.h>  /* free, malloc */
 
 #include <brotli/types.h>
-#include "./huffman.h"
+
+#include "../common/dictionary.h"
+#include "huffman.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
@@ -42,6 +44,7 @@ BROTLI_BOOL BrotliDecoderStateInit(BrotliDecoderState* s,
   s->pos = 0;
   s->rb_roundtrips = 0;
   s->partial_pos_out = 0;
+  s->used_input = 0;
 
   s->block_type_trees = NULL;
   s->block_len_trees = NULL;
@@ -81,17 +84,23 @@ BROTLI_BOOL BrotliDecoderStateInit(BrotliDecoderState* s,
 
   s->mtf_upper_bound = 63;
 
-  s->dictionary = BrotliGetDictionary();
-  s->transforms = BrotliGetTransforms();
+  s->compound_dictionary = NULL;
+  s->dictionary =
+      BrotliSharedDictionaryCreateInstance(alloc_func, free_func, opaque);
+  if (!s->dictionary) return BROTLI_FALSE;
+
+  s->metadata_start_func = NULL;
+  s->metadata_chunk_func = NULL;
+  s->metadata_callback_opaque = 0;
 
   return BROTLI_TRUE;
 }
 
 void BrotliDecoderStateMetablockBegin(BrotliDecoderState* s) {
   s->meta_block_remaining_len = 0;
-  s->block_length[0] = 1U << 24;
-  s->block_length[1] = 1U << 24;
-  s->block_length[2] = 1U << 24;
+  s->block_length[0] = BROTLI_BLOCK_SIZE_CAP;
+  s->block_length[1] = BROTLI_BLOCK_SIZE_CAP;
+  s->block_length[2] = BROTLI_BLOCK_SIZE_CAP;
   s->num_block_types[0] = 1;
   s->num_block_types[1] = 1;
   s->num_block_types[2] = 1;
@@ -126,16 +135,31 @@ void BrotliDecoderStateCleanupAfterMetablock(BrotliDecoderState* s) {
   BROTLI_DECODER_FREE(s, s->distance_hgroup.htrees);
 }
 
+#ifdef BROTLI_REPORTING
+/* When BROTLI_REPORTING is defined extra reporting module have to be linked. */
+void BrotliDecoderOnFinish(const BrotliDecoderState* s);
+#define BROTLI_DECODER_ON_FINISH(s) BrotliDecoderOnFinish(s);
+#else
+#if !defined(BROTLI_DECODER_ON_FINISH)
+#define BROTLI_DECODER_ON_FINISH(s) (void)(s);
+#endif
+#endif
+
 void BrotliDecoderStateCleanup(BrotliDecoderState* s) {
   BrotliDecoderStateCleanupAfterMetablock(s);
 
+  BROTLI_DECODER_ON_FINISH(s);
+
+  BROTLI_DECODER_FREE(s, s->compound_dictionary);
+  BrotliSharedDictionaryDestroyInstance(s->dictionary);
+  s->dictionary = NULL;
   BROTLI_DECODER_FREE(s, s->ringbuffer);
   BROTLI_DECODER_FREE(s, s->block_type_trees);
 }
 
 BROTLI_BOOL BrotliDecoderHuffmanTreeGroupInit(BrotliDecoderState* s,
-    HuffmanTreeGroup* group, uint32_t alphabet_size_max,
-    uint32_t alphabet_size_limit, uint32_t ntrees) {
+    HuffmanTreeGroup* group, brotli_reg_t alphabet_size_max,
+    brotli_reg_t alphabet_size_limit, brotli_reg_t ntrees) {
   /* 376 = 256 (1-st level table) + 4 + 7 + 15 + 31 + 63 (2-nd level mix-tables)
      This number is discovered "unlimited" "enough" calculator; it is actually
      a wee bigger than required in several cases (especially for alphabets with

data/vendor/brotli/c/dec/state.h

@@ -9,13 +9,16 @@
 #ifndef BROTLI_DEC_STATE_H_
 #define BROTLI_DEC_STATE_H_
 
+#include <brotli/decode.h>
+#include <brotli/shared_dictionary.h>
+#include <brotli/types.h>
+
 #include "../common/constants.h"
 #include "../common/dictionary.h"
 #include "../common/platform.h"
 #include "../common/transform.h"
-#include <brotli/types.h>
-#include "./bit_reader.h"
-#include "./huffman.h"
+#include "bit_reader.h"
+#include "huffman.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
@@ -189,20 +192,34 @@ typedef enum {
   BROTLI_STATE_READ_BLOCK_LENGTH_SUFFIX
 } BrotliRunningReadBlockLengthState;
 
+/* BrotliDecoderState addon, used for Compound Dictionary functionality. */
+typedef struct BrotliDecoderCompoundDictionary {
+  int num_chunks;
+  int total_size;
+  int br_index;
+  int br_offset;
+  int br_length;
+  int br_copied;
+  const uint8_t* chunks[16];
+  int chunk_offsets[16];
+  int block_bits;
+  uint8_t block_map[256];
+} BrotliDecoderCompoundDictionary;
+
 typedef struct BrotliMetablockHeaderArena {
   BrotliRunningTreeGroupState substate_tree_group;
   BrotliRunningContextMapState substate_context_map;
   BrotliRunningHuffmanState substate_huffman;
 
-  uint32_t sub_loop_counter;
+  brotli_reg_t sub_loop_counter;
 
-  uint32_t repeat_code_len;
-  uint32_t prev_code_len;
+  brotli_reg_t repeat_code_len;
+  brotli_reg_t prev_code_len;
 
   /* For ReadHuffmanCode. */
-  uint32_t symbol;
-  uint32_t repeat;
-  uint32_t space;
+  brotli_reg_t symbol;
+  brotli_reg_t repeat;
+  brotli_reg_t space;
 
   /* Huffman table for "histograms". */
   HuffmanCode table[32];
@@ -216,21 +233,22 @@ typedef struct BrotliMetablockHeaderArena {
   uint8_t code_length_code_lengths[BROTLI_CODE_LENGTH_CODES];
   /* Population counts for the code lengths. */
   uint16_t code_length_histo[16];
+  /* TODO(eustas): +2 bytes padding */
 
   /* For HuffmanTreeGroupDecode. */
   int htree_index;
   HuffmanCode* next;
 
   /* For DecodeContextMap. */
-  uint32_t context_index;
-  uint32_t max_run_length_prefix;
-  uint32_t code;
+  brotli_reg_t context_index;
+  brotli_reg_t max_run_length_prefix;
+  brotli_reg_t code;
   HuffmanCode context_map_table[BROTLI_HUFFMAN_MAX_SIZE_272];
 } BrotliMetablockHeaderArena;
 
 typedef struct BrotliMetablockBodyArena {
   uint8_t dist_extra_bits[544];
-  uint32_t dist_offset[544];
+  brotli_reg_t dist_offset[544];
 } BrotliMetablockBodyArena;
 
 struct BrotliDecoderStateStruct {
@@ -251,7 +269,7 @@ struct BrotliDecoderStateStruct {
     uint64_t u64;
     uint8_t u8[8];
   } buffer;
-  uint32_t buffer_length;
+  brotli_reg_t buffer_length;
 
   int pos;
   int max_backward_distance;
@@ -261,6 +279,8 @@ struct BrotliDecoderStateStruct {
   int dist_rb_idx;
   int dist_rb[4];
   int error_code;
+  int meta_block_remaining_len;
+
   uint8_t* ringbuffer;
   uint8_t* ringbuffer_end;
   HuffmanCode* htree_command;
@@ -281,54 +301,64 @@ struct BrotliDecoderStateStruct {
   /* Distance context is actual after command is decoded and before distance is
      computed. After distance computation it is used as a temporary variable. */
   int distance_context;
-  int meta_block_remaining_len;
-  uint32_t block_length_index;
-  uint32_t block_length[3];
-  uint32_t num_block_types[3];
-  uint32_t block_type_rb[6];
-  uint32_t distance_postfix_bits;
-  uint32_t num_direct_distance_codes;
-  uint32_t num_dist_htrees;
+  brotli_reg_t block_length[3];
+  brotli_reg_t block_length_index;
+  brotli_reg_t num_block_types[3];
+  brotli_reg_t block_type_rb[6];
+  brotli_reg_t distance_postfix_bits;
+  brotli_reg_t num_direct_distance_codes;
+  brotli_reg_t num_dist_htrees;
   uint8_t* dist_context_map;
   HuffmanCode* literal_htree;
-  uint8_t dist_htree_index;
-
-  int copy_length;
-  int distance_code;
 
   /* For partial write operations. */
   size_t rb_roundtrips;  /* how many times we went around the ring-buffer */
   size_t partial_pos_out;  /* how much output to the user in total */
 
   /* For InverseMoveToFrontTransform. */
-  uint32_t mtf_upper_bound;
+  brotli_reg_t mtf_upper_bound;
   uint32_t mtf[64 + 1];
 
+  int copy_length;
+  int distance_code;
+
+  uint8_t dist_htree_index;
+  /* TODO(eustas): +3 bytes padding */
+
   /* Less used attributes are at the end of this struct. */
 
+  brotli_decoder_metadata_start_func metadata_start_func;
+  brotli_decoder_metadata_chunk_func metadata_chunk_func;
+  void* metadata_callback_opaque;
+
+  /* For reporting. */
+  uint64_t used_input;  /* how many bytes of input are consumed */
+
   /* States inside function calls. */
   BrotliRunningMetablockHeaderState substate_metablock_header;
   BrotliRunningUncompressedState substate_uncompressed;
   BrotliRunningDecodeUint8State substate_decode_uint8;
   BrotliRunningReadBlockLengthState substate_read_block_length;
 
+  int new_ringbuffer_size;
+  /* TODO(eustas): +4 bytes padding */
+
   unsigned int is_last_metablock : 1;
   unsigned int is_uncompressed : 1;
   unsigned int is_metadata : 1;
   unsigned int should_wrap_ringbuffer : 1;
   unsigned int canny_ringbuffer_allocation : 1;
   unsigned int large_window : 1;
+  unsigned int window_bits : 6;
   unsigned int size_nibbles : 8;
-  uint32_t window_bits;
+  /* TODO(eustas): +12 bits padding */
 
-  int new_ringbuffer_size;
-
-  uint32_t num_literal_htrees;
+  brotli_reg_t num_literal_htrees;
   uint8_t* context_map;
   uint8_t* context_modes;
 
-  const BrotliDictionary* dictionary;
-  const BrotliTransforms* transforms;
+  BrotliSharedDictionary* dictionary;
+  BrotliDecoderCompoundDictionary* compound_dictionary;
 
   uint32_t trivial_literal_contexts[8];  /* 256 bits */
 
@@ -348,8 +378,9 @@ BROTLI_INTERNAL void BrotliDecoderStateMetablockBegin(BrotliDecoderState* s);
 BROTLI_INTERNAL void BrotliDecoderStateCleanupAfterMetablock(
     BrotliDecoderState* s);
 BROTLI_INTERNAL BROTLI_BOOL BrotliDecoderHuffmanTreeGroupInit(
-    BrotliDecoderState* s, HuffmanTreeGroup* group, uint32_t alphabet_size_max,
-    uint32_t alphabet_size_limit, uint32_t ntrees);
+    BrotliDecoderState* s, HuffmanTreeGroup* group,
+    brotli_reg_t alphabet_size_max, brotli_reg_t alphabet_size_limit,
+    brotli_reg_t ntrees);
 
 #define BROTLI_DECODER_ALLOC(S, L) S->alloc_func(S->memory_manager_opaque, L)
 
@@ -358,6 +389,10 @@ BROTLI_INTERNAL BROTLI_BOOL BrotliDecoderHuffmanTreeGroupInit(
   X = NULL;                                  \
 }
 
+/* Literal/Command/Distance block size maximum; same as maximum metablock size;
+   used as block size when there is no block switching. */
+#define BROTLI_BLOCK_SIZE_CAP (1U << 24)
+
 #if defined(__cplusplus) || defined(c_plusplus)
 }  /* extern "C" */
 #endif

data/vendor/brotli/c/enc/backward_references.c

@@ -6,17 +6,19 @@
 
 /* Function to find backward reference copies. */
 
-#include "./backward_references.h"
+#include "backward_references.h"
+
+#include <brotli/types.h>
 
 #include "../common/constants.h"
-#include "../common/context.h"
 #include "../common/dictionary.h"
 #include "../common/platform.h"
-#include <brotli/types.h>
-#include "./command.h"
-#include "./dictionary_hash.h"
-#include "./memory.h"
-#include "./quality.h"
+#include "command.h"
+#include "compound_dictionary.h"
+#include "dictionary_hash.h"
+#include "encoder_dict.h"
+#include "memory.h"
+#include "quality.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
@@ -52,67 +54,103 @@ static BROTLI_INLINE size_t ComputeDistanceCode(size_t distance,
 #define EXPORT_FN(X) EXPAND_CAT(X, EXPAND_CAT(PREFIX(), HASHER()))
 
 #define PREFIX() N
+#define ENABLE_COMPOUND_DICTIONARY 0
 
 #define HASHER() H2
 /* NOLINTNEXTLINE(build/include) */
-#include "./backward_references_inc.h"
+#include "backward_references_inc.h"
 #undef HASHER
 
 #define HASHER() H3
 /* NOLINTNEXTLINE(build/include) */
-#include "./backward_references_inc.h"
+#include "backward_references_inc.h"
 #undef HASHER
 
 #define HASHER() H4
 /* NOLINTNEXTLINE(build/include) */
-#include "./backward_references_inc.h"
+#include "backward_references_inc.h"
 #undef HASHER
 
 #define HASHER() H5
 /* NOLINTNEXTLINE(build/include) */
-#include "./backward_references_inc.h"
+#include "backward_references_inc.h"
 #undef HASHER
 
 #define HASHER() H6
 /* NOLINTNEXTLINE(build/include) */
-#include "./backward_references_inc.h"
+#include "backward_references_inc.h"
 #undef HASHER
 
 #define HASHER() H40
 /* NOLINTNEXTLINE(build/include) */
-#include "./backward_references_inc.h"
+#include "backward_references_inc.h"
 #undef HASHER
 
 #define HASHER() H41
 /* NOLINTNEXTLINE(build/include) */
-#include "./backward_references_inc.h"
+#include "backward_references_inc.h"
 #undef HASHER
 
 #define HASHER() H42
 /* NOLINTNEXTLINE(build/include) */
-#include "./backward_references_inc.h"
+#include "backward_references_inc.h"
 #undef HASHER
 
 #define HASHER() H54
 /* NOLINTNEXTLINE(build/include) */
-#include "./backward_references_inc.h"
+#include "backward_references_inc.h"
 #undef HASHER
 
 #define HASHER() H35
 /* NOLINTNEXTLINE(build/include) */
-#include "./backward_references_inc.h"
+#include "backward_references_inc.h"
 #undef HASHER
 
 #define HASHER() H55
 /* NOLINTNEXTLINE(build/include) */
-#include "./backward_references_inc.h"
+#include "backward_references_inc.h"
 #undef HASHER
 
 #define HASHER() H65
 /* NOLINTNEXTLINE(build/include) */
-#include "./backward_references_inc.h"
+#include "backward_references_inc.h"
 #undef HASHER
 
+#undef ENABLE_COMPOUND_DICTIONARY
+#undef PREFIX
+#define PREFIX() D
+#define ENABLE_COMPOUND_DICTIONARY 1
+
+#define HASHER() H5
+/* NOLINTNEXTLINE(build/include) */
+#include "backward_references_inc.h"
+#undef HASHER
+#define HASHER() H6
+/* NOLINTNEXTLINE(build/include) */
+#include "backward_references_inc.h"
+#undef HASHER
+#define HASHER() H40
+/* NOLINTNEXTLINE(build/include) */
+#include "backward_references_inc.h"
+#undef HASHER
+#define HASHER() H41
+/* NOLINTNEXTLINE(build/include) */
+#include "backward_references_inc.h"
+#undef HASHER
+#define HASHER() H42
+/* NOLINTNEXTLINE(build/include) */
+#include "backward_references_inc.h"
+#undef HASHER
+#define HASHER() H55
+/* NOLINTNEXTLINE(build/include) */
+#include "backward_references_inc.h"
+#undef HASHER
+#define HASHER() H65
+/* NOLINTNEXTLINE(build/include) */
+#include "backward_references_inc.h"
+#undef HASHER
+
+#undef ENABLE_COMPOUND_DICTIONARY
 #undef PREFIX
 
 #undef EXPORT_FN
@@ -125,6 +163,29 @@ void BrotliCreateBackwardReferences(size_t num_bytes,
     ContextLut literal_context_lut, const BrotliEncoderParams* params,
     Hasher* hasher, int* dist_cache, size_t* last_insert_len,
     Command* commands, size_t* num_commands, size_t* num_literals) {
+  if (params->dictionary.compound.num_chunks != 0) {
+    switch (params->hasher.type) {
+#define CASE_(N)                                                    \
+      case N:                                                       \
+        CreateBackwardReferencesDH ## N(num_bytes,                  \
+            position, ringbuffer, ringbuffer_mask,                  \
+            literal_context_lut, params, hasher, dist_cache,        \
+            last_insert_len, commands, num_commands, num_literals); \
+        return;
+      CASE_(5)
+      CASE_(6)
+      CASE_(40)
+      CASE_(41)
+      CASE_(42)
+      CASE_(55)
+      CASE_(65)
+#undef CASE_
+      default:
+        BROTLI_DCHECK(false);
+        break;
+    }
+  }
+
   switch (params->hasher.type) {
 #define CASE_(N)                                                  \
     case N:                                                       \
@@ -136,6 +197,7 @@ void BrotliCreateBackwardReferences(size_t num_bytes,
     FOR_GENERIC_HASHERS(CASE_)
 #undef CASE_
     default:
+      BROTLI_DCHECK(false);
       break;
   }
 }

data/vendor/brotli/c/enc/backward_references.h

@@ -9,14 +9,15 @@
 #ifndef BROTLI_ENC_BACKWARD_REFERENCES_H_
 #define BROTLI_ENC_BACKWARD_REFERENCES_H_
 
+#include <brotli/types.h>
+
 #include "../common/constants.h"
 #include "../common/context.h"
 #include "../common/dictionary.h"
 #include "../common/platform.h"
-#include <brotli/types.h>
-#include "./command.h"
-#include "./hash.h"
-#include "./quality.h"
+#include "command.h"
+#include "hash.h"
+#include "quality.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {