brotli 0.2.0 → 0.4.0

data/vendor/brotli/c/enc/hash_to_binary_tree_inc.h

@@ -24,8 +24,8 @@ static BROTLI_INLINE size_t FN(StoreLookahead)(void) {
   return MAX_TREE_COMP_LENGTH;
 }
 
-static uint32_t FN(HashBytes)(const uint8_t *data) {
-  uint32_t h = BROTLI_UNALIGNED_LOAD32(data) * kHashMul32;
+static uint32_t FN(HashBytes)(const uint8_t* BROTLI_RESTRICT data) {
+  uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
   /* The higher bits contain more mixture from the multiplication,
      so we take our results from there. */
   return h >> (32 - BUCKET_BITS);
@@ -38,7 +38,7 @@ typedef struct HashToBinaryTree {
   /* Hash table that maps the 4-byte hashes of the sequence to the last
      position where this hash was found, which is the root of the binary
      tree of sequences that share this hash bucket. */
-  uint32_t buckets_[BUCKET_SIZE];
+  uint32_t* buckets_;  /* uint32_t[BUCKET_SIZE]; */
 
   /* A position used to mark a non-existent sequence, i.e. a tree is empty if
      its root is at invalid_pos_ and a node is a leaf if both its children
@@ -51,34 +51,30 @@ typedef struct HashToBinaryTree {
      corresponding to a hash is a sequence starting at buckets_[hash] and
      the left and right children of a sequence starting at pos are
      forest_[2 * pos] and forest_[2 * pos + 1]. */
-  /* uint32_t forest[2 * num_nodes] */
+  uint32_t* forest_;  /* uint32_t[2 * num_nodes] */
 } HashToBinaryTree;
 
-static BROTLI_INLINE HashToBinaryTree* FN(Self)(HasherHandle handle) {
-  return (HashToBinaryTree*)&(GetHasherCommon(handle)[1]);
-}
-
-static BROTLI_INLINE uint32_t* FN(Forest)(HashToBinaryTree* self) {
-  return (uint32_t*)(&self[1]);
-}
-
 static void FN(Initialize)(
-    HasherHandle handle, const BrotliEncoderParams* params) {
-  HashToBinaryTree* self = FN(Self)(handle);
+    HasherCommon* common, HashToBinaryTree* BROTLI_RESTRICT self,
+    const BrotliEncoderParams* params) {
+  self->buckets_ = (uint32_t*)common->extra;
+  self->forest_ = &self->buckets_[BUCKET_SIZE];
+
   self->window_mask_ = (1u << params->lgwin) - 1u;
   self->invalid_pos_ = (uint32_t)(0 - self->window_mask_);
 }
 
-static void FN(Prepare)(HasherHandle handle, BROTLI_BOOL one_shot,
-    size_t input_size, const uint8_t* data) {
-  HashToBinaryTree* self = FN(Self)(handle);
+static void FN(Prepare)
+    (HashToBinaryTree* BROTLI_RESTRICT self, BROTLI_BOOL one_shot,
+    size_t input_size, const uint8_t* BROTLI_RESTRICT data) {
   uint32_t invalid_pos = self->invalid_pos_;
   uint32_t i;
+  uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
   BROTLI_UNUSED(data);
   BROTLI_UNUSED(one_shot);
   BROTLI_UNUSED(input_size);
   for (i = 0; i < BUCKET_SIZE; i++) {
-    self->buckets_[i] = invalid_pos;
+    buckets[i] = invalid_pos;
   }
 }
 
@@ -89,15 +85,17 @@ static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
   if (one_shot && input_size < num_nodes) {
     num_nodes = input_size;
   }
-  return sizeof(HashToBinaryTree) + 2 * sizeof(uint32_t) * num_nodes;
+  return sizeof(uint32_t) * BUCKET_SIZE + 2 * sizeof(uint32_t) * num_nodes;
 }
 
-static BROTLI_INLINE size_t FN(LeftChildIndex)(HashToBinaryTree* self,
+static BROTLI_INLINE size_t FN(LeftChildIndex)(
+    HashToBinaryTree* BROTLI_RESTRICT self,
     const size_t pos) {
   return 2 * (pos & self->window_mask_);
 }
 
-static BROTLI_INLINE size_t FN(RightChildIndex)(HashToBinaryTree* self,
+static BROTLI_INLINE size_t FN(RightChildIndex)(
+    HashToBinaryTree* BROTLI_RESTRICT self,
     const size_t pos) {
   return 2 * (pos & self->window_mask_) + 1;
 }
@@ -113,7 +111,7 @@ static BROTLI_INLINE size_t FN(RightChildIndex)(HashToBinaryTree* self,
 
    This function must be called with increasing cur_ix positions. */
 static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
-    HashToBinaryTree* self, const uint8_t* const BROTLI_RESTRICT data,
+    HashToBinaryTree* BROTLI_RESTRICT self, const uint8_t* BROTLI_RESTRICT data,
     const size_t cur_ix, const size_t ring_buffer_mask, const size_t max_length,
     const size_t max_backward, size_t* const BROTLI_RESTRICT best_len,
     BackwardMatch* BROTLI_RESTRICT matches) {
@@ -123,8 +121,9 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
   const BROTLI_BOOL should_reroot_tree =
       TO_BROTLI_BOOL(max_length >= MAX_TREE_COMP_LENGTH);
   const uint32_t key = FN(HashBytes)(&data[cur_ix_masked]);
-  uint32_t* forest = FN(Forest)(self);
-  size_t prev_ix = self->buckets_[key];
+  uint32_t* BROTLI_RESTRICT buckets = self->buckets_;
+  uint32_t* BROTLI_RESTRICT forest = self->forest_;
+  size_t prev_ix = buckets[key];
   /* The forest index of the rightmost node of the left subtree of the new
      root, updated as we traverse and re-root the tree of the hash bucket. */
   size_t node_left = FN(LeftChildIndex)(self, cur_ix);
@@ -139,7 +138,7 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
   size_t best_len_right = 0;
   size_t depth_remaining;
   if (should_reroot_tree) {
-    self->buckets_[key] = (uint32_t)cur_ix;
+    buckets[key] = (uint32_t)cur_ix;
   }
   for (depth_remaining = MAX_TREE_SEARCH_DEPTH; ; --depth_remaining) {
     const size_t backward = cur_ix - prev_ix;
@@ -154,12 +153,13 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
     {
       const size_t cur_len = BROTLI_MIN(size_t, best_len_left, best_len_right);
       size_t len;
-      assert(cur_len <= MAX_TREE_COMP_LENGTH);
+      BROTLI_DCHECK(cur_len <= MAX_TREE_COMP_LENGTH);
       len = cur_len +
           FindMatchLengthWithLimit(&data[cur_ix_masked + cur_len],
                                    &data[prev_ix_masked + cur_len],
                                    max_length - cur_len);
-      assert(0 == memcmp(&data[cur_ix_masked], &data[prev_ix_masked], len));
+      BROTLI_DCHECK(
+          0 == memcmp(&data[cur_ix_masked], &data[prev_ix_masked], len));
       if (matches && len > *best_len) {
         *best_len = len;
         InitBackwardMatch(matches++, backward, len);
@@ -198,11 +198,14 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
    matches in matches[0] to matches[*num_matches - 1]. The matches will be
    sorted by strictly increasing length and (non-strictly) increasing
    distance. */
-static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
-    const BrotliDictionary* dictionary, const uint8_t* data,
+static BROTLI_INLINE size_t FN(FindAllMatches)(
+    HashToBinaryTree* BROTLI_RESTRICT self,
+    const BrotliEncoderDictionary* dictionary,
+    const uint8_t* BROTLI_RESTRICT data,
     const size_t ring_buffer_mask, const size_t cur_ix,
-    const size_t max_length, const size_t max_backward, const size_t gap,
-    const BrotliEncoderParams* params, BackwardMatch* matches) {
+    const size_t max_length, const size_t max_backward,
+    const size_t dictionary_distance, const BrotliEncoderParams* params,
+    BackwardMatch* matches) {
   BackwardMatch* const orig_matches = matches;
   const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
   size_t best_len = 1;
@@ -234,7 +237,7 @@ static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
     }
   }
   if (best_len < max_length) {
-    matches = FN(StoreAndFindMatches)(FN(Self)(handle), data, cur_ix,
+    matches = FN(StoreAndFindMatches)(self, data, cur_ix,
         ring_buffer_mask, max_length, max_backward, &best_len, matches);
   }
   for (i = 0; i <= BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN; ++i) {
@@ -250,8 +253,8 @@ static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
       for (l = minlen; l <= maxlen; ++l) {
         uint32_t dict_id = dict_matches[l];
         if (dict_id < kInvalidMatch) {
-          size_t distance = max_backward + gap + (dict_id >> 5) + 1;
-          if (distance < BROTLI_MAX_DISTANCE) {
+          size_t distance = dictionary_distance + (dict_id >> 5) + 1;
+          if (distance <= params->dist.max_distance) {
             InitDictionaryBackwardMatch(matches++, distance, l, dict_id & 31);
           }
         }
@@ -264,18 +267,18 @@ static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
 /* Stores the hash of the next 4 bytes and re-roots the binary tree at the
    current sequence, without returning any matches.
    REQUIRES: ix + MAX_TREE_COMP_LENGTH <= end-of-current-block */
-static BROTLI_INLINE void FN(Store)(HasherHandle handle, const uint8_t *data,
+static BROTLI_INLINE void FN(Store)(HashToBinaryTree* BROTLI_RESTRICT self,
+    const uint8_t* BROTLI_RESTRICT data,
     const size_t mask, const size_t ix) {
-  HashToBinaryTree* self = FN(Self)(handle);
   /* Maximum distance is window size - 16, see section 9.1. of the spec. */
   const size_t max_backward = self->window_mask_ - BROTLI_WINDOW_GAP + 1;
   FN(StoreAndFindMatches)(self, data, ix, mask, MAX_TREE_COMP_LENGTH,
       max_backward, NULL, NULL);
 }
 
-static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
-    const uint8_t *data, const size_t mask, const size_t ix_start,
-    const size_t ix_end) {
+static BROTLI_INLINE void FN(StoreRange)(HashToBinaryTree* BROTLI_RESTRICT self,
+    const uint8_t* BROTLI_RESTRICT data, const size_t mask,
+    const size_t ix_start, const size_t ix_end) {
   size_t i = ix_start;
   size_t j = ix_start;
   if (ix_start + 63 <= ix_end) {
@@ -283,18 +286,18 @@ static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
   }
   if (ix_start + 512 <= i) {
     for (; j < i; j += 8) {
-      FN(Store)(handle, data, mask, j);
+      FN(Store)(self, data, mask, j);
     }
   }
   for (; i < ix_end; ++i) {
-    FN(Store)(handle, data, mask, i);
+    FN(Store)(self, data, mask, i);
   }
 }
 
-static BROTLI_INLINE void FN(StitchToPreviousBlock)(HasherHandle handle,
+static BROTLI_INLINE void FN(StitchToPreviousBlock)(
+    HashToBinaryTree* BROTLI_RESTRICT self,
     size_t num_bytes, size_t position, const uint8_t* ringbuffer,
     size_t ringbuffer_mask) {
-  HashToBinaryTree* self = FN(Self)(handle);
   if (num_bytes >= FN(HashTypeLength)() - 1 &&
       position >= MAX_TREE_COMP_LENGTH) {
     /* Store the last `MAX_TREE_COMP_LENGTH - 1` positions in the hasher.

data/vendor/brotli/c/enc/histogram.c

@@ -8,9 +8,9 @@
 
 #include "./histogram.h"
 
+#include "../common/context.h"
 #include "./block_splitter.h"
 #include "./command.h"
-#include "./context.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
@@ -63,13 +63,16 @@ void BrotliBuildHistogramsWithContext(
     BlockSplitIteratorNext(&insert_and_copy_it);
     HistogramAddCommand(&insert_and_copy_histograms[insert_and_copy_it.type_],
         cmd->cmd_prefix_);
+    /* TODO: unwrap iterator blocks. */
     for (j = cmd->insert_len_; j != 0; --j) {
       size_t context;
       BlockSplitIteratorNext(&literal_it);
-      context = context_modes ?
-          ((literal_it.type_ << BROTLI_LITERAL_CONTEXT_BITS) +
-              Context(prev_byte, prev_byte2, context_modes[literal_it.type_])) :
-          literal_it.type_;
+      context = literal_it.type_;
+      if (context_modes) {
+        ContextLut lut = BROTLI_CONTEXT_LUT(context_modes[context]);
+        context = (context << BROTLI_LITERAL_CONTEXT_BITS) +
+            BROTLI_CONTEXT(prev_byte, prev_byte2, lut);
+      }
       HistogramAddLiteral(&literal_histograms[context],
           ringbuffer[pos & mask]);
       prev_byte2 = prev_byte;
@@ -86,7 +89,7 @@ void BrotliBuildHistogramsWithContext(
         context = (dist_it.type_ << BROTLI_DISTANCE_CONTEXT_BITS) +
             CommandDistanceContext(cmd);
         HistogramAddDistance(&copy_dist_histograms[context],
-            cmd->dist_prefix_);
+            cmd->dist_prefix_ & 0x3FF);
       }
     }
   }

data/vendor/brotli/c/enc/histogram.h

@@ -12,16 +12,19 @@
 #include <string.h>  /* memset */
 
 #include "../common/constants.h"
+#include "../common/context.h"
+#include "../common/platform.h"
 #include <brotli/types.h>
 #include "./block_splitter.h"
 #include "./command.h"
-#include "./context.h"
-#include "./port.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
 #endif
 
+/* The distance symbols effectively used by "Large Window Brotli" (32-bit). */
+#define BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS 544
+
 #define FN(X) X ## Literal
 #define DATA_SIZE BROTLI_NUM_LITERAL_SYMBOLS
 #define DataType uint8_t
@@ -38,7 +41,7 @@ extern "C" {
 #undef FN
 
 #define FN(X) X ## Distance
-#define DATA_SIZE BROTLI_NUM_DISTANCE_SYMBOLS
+#define DATA_SIZE BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS
 #include "./histogram_inc.h"  /* NOLINT(build/include) */
 #undef DataType
 #undef DATA_SIZE

data/vendor/brotli/c/enc/histogram_inc.h

@@ -33,7 +33,7 @@ static BROTLI_INLINE void FN(HistogramAdd)(FN(Histogram)* self, size_t val) {
 }
 
 static BROTLI_INLINE void FN(HistogramAddVector)(FN(Histogram)* self,
-    const DataType *p, size_t n) {
+    const DataType* p, size_t n) {
   self->total_count_ += n;
   n += 1;
   while (--n) ++self->data_[*p++];

data/vendor/brotli/c/enc/literal_cost.c

@@ -9,9 +9,9 @@
 
 #include "./literal_cost.h"
 
+#include "../common/platform.h"
 #include <brotli/types.h>
 #include "./fast_log.h"
-#include "./port.h"
 #include "./utf8_util.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
@@ -25,7 +25,7 @@ static size_t UTF8Position(size_t last, size_t c, size_t clamp) {
     return BROTLI_MIN(size_t, 1, clamp);
   } else {
     /* Let's decide over the last byte if this ends the sequence. */
-    if (last < 0xe0) {
+    if (last < 0xE0) {
       return 0;  /* Completed two or three byte coding. */
     } else {  /* Next one is the 'Byte 3' of utf-8 encoding. */
       return BROTLI_MIN(size_t, 2, clamp);
@@ -34,7 +34,7 @@ static size_t UTF8Position(size_t last, size_t c, size_t clamp) {
 }
 
 static size_t DecideMultiByteStatsLevel(size_t pos, size_t len, size_t mask,
-                                        const uint8_t *data) {
+                                        const uint8_t* data) {
   size_t counts[3] = { 0 };
   size_t max_utf8 = 1;  /* should be 2, but 1 compresses better. */
   size_t last_c = 0;
@@ -54,7 +54,7 @@ static size_t DecideMultiByteStatsLevel(size_t pos, size_t len, size_t mask,
 }
 
 static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
-                                            const uint8_t *data, float *cost) {
+                                            const uint8_t* data, float* cost) {
   /* max_utf8 is 0 (normal ASCII single byte modeling),
      1 (for 2-byte UTF-8 modeling), or 2 (for 3-byte UTF-8 modeling). */
   const size_t max_utf8 = DecideMultiByteStatsLevel(pos, len, mask, data);
@@ -125,7 +125,7 @@ static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
 }
 
 void BrotliEstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask,
-                                       const uint8_t *data, float *cost) {
+                                       const uint8_t* data, float* cost) {
   if (BrotliIsMostlyUTF8(data, pos, mask, len, kMinUTF8Ratio)) {
     EstimateBitCostsForLiteralsUTF8(pos, len, mask, data, cost);
     return;

data/vendor/brotli/c/enc/literal_cost.h

@@ -10,8 +10,8 @@
 #ifndef BROTLI_ENC_LITERAL_COST_H_
 #define BROTLI_ENC_LITERAL_COST_H_
 
+#include "../common/platform.h"
 #include <brotli/types.h>
-#include "./port.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
@@ -21,7 +21,7 @@ extern "C" {
    ring-buffer (data, mask) will take entropy coded and writes these estimates
    to the cost[0..len) array. */
 BROTLI_INTERNAL void BrotliEstimateBitCostsForLiterals(
-    size_t pos, size_t len, size_t mask, const uint8_t *data, float *cost);
+    size_t pos, size_t len, size_t mask, const uint8_t* data, float* cost);
 
 #if defined(__cplusplus) || defined(c_plusplus)
 }  /* extern "C" */

data/vendor/brotli/c/enc/memory.c

@@ -9,12 +9,11 @@
 
 #include "./memory.h"
 
-#include <assert.h>
 #include <stdlib.h>  /* exit, free, malloc */
 #include <string.h>  /* memcpy */
 
+#include "../common/platform.h"
 #include <brotli/types.h>
-#include "./port.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
@@ -28,22 +27,12 @@ extern "C" {
 #define NEW_ALLOCATED_OFFSET MAX_PERM_ALLOCATED
 #define NEW_FREED_OFFSET (MAX_PERM_ALLOCATED + MAX_NEW_ALLOCATED)
 
-static void* DefaultAllocFunc(void* opaque, size_t size) {
-  BROTLI_UNUSED(opaque);
-  return malloc(size);
-}
-
-static void DefaultFreeFunc(void* opaque, void* address) {
-  BROTLI_UNUSED(opaque);
-  free(address);
-}
-
 void BrotliInitMemoryManager(
     MemoryManager* m, brotli_alloc_func alloc_func, brotli_free_func free_func,
     void* opaque) {
   if (!alloc_func) {
-    m->alloc_func = DefaultAllocFunc;
-    m->free_func = DefaultFreeFunc;
+    m->alloc_func = BrotliDefaultAllocFunc;
+    m->free_func = BrotliDefaultFreeFunc;
     m->opaque = 0;
   } else {
     m->alloc_func = alloc_func;
@@ -132,11 +121,11 @@ static void CollectGarbagePointers(MemoryManager* m) {
         m->pointers + NEW_FREED_OFFSET, m->new_freed);
     m->perm_allocated -= annihilated;
     m->new_freed -= annihilated;
-    assert(m->new_freed == 0);
+    BROTLI_DCHECK(m->new_freed == 0);
   }
 
   if (m->new_allocated != 0) {
-    assert(m->perm_allocated + m->new_allocated <= MAX_PERM_ALLOCATED);
+    BROTLI_DCHECK(m->perm_allocated + m->new_allocated <= MAX_PERM_ALLOCATED);
     memcpy(m->pointers + PERM_ALLOCATED_OFFSET + m->perm_allocated,
            m->pointers + NEW_ALLOCATED_OFFSET,
            sizeof(void*) * m->new_allocated);

data/vendor/brotli/c/enc/memory.h

@@ -9,8 +9,10 @@
 #ifndef BROTLI_ENC_MEMORY_H_
 #define BROTLI_ENC_MEMORY_H_
 
+#include <string.h>  /* memcpy */
+
+#include "../common/platform.h"
 #include <brotli/types.h>
-#include "./port.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
@@ -54,8 +56,57 @@ BROTLI_INTERNAL void BrotliFree(MemoryManager* m, void* p);
 #define BROTLI_IS_OOM(M) (!!(M)->is_oom)
 #endif  /* BROTLI_ENCODER_EXIT_ON_OOM */
 
+/*
+BROTLI_IS_NULL is a fake check, BROTLI_IS_OOM does the heavy lifting.
+The only purpose of it is to explain static analyzers the state of things.
+NB: use ONLY together with BROTLI_IS_OOM
+    AND ONLY for allocations in the current scope.
+ */
+#if defined(__clang_analyzer__) && !defined(BROTLI_ENCODER_EXIT_ON_OOM)
+#define BROTLI_IS_NULL(A) ((A) == nullptr)
+#else  /* defined(__clang_analyzer__) */
+#define BROTLI_IS_NULL(A) (!!0)
+#endif  /* defined(__clang_analyzer__) */
+
 BROTLI_INTERNAL void BrotliWipeOutMemoryManager(MemoryManager* m);
 
+/*
+Dynamically grows array capacity to at least the requested size
+M: MemoryManager
+T: data type
+A: array
+C: capacity
+R: requested size
+*/
+#define BROTLI_ENSURE_CAPACITY(M, T, A, C, R) {                    \
+  if (C < (R)) {                                                   \
+    size_t _new_size = (C == 0) ? (R) : C;                         \
+    T* new_array;                                                  \
+    while (_new_size < (R)) _new_size *= 2;                        \
+    new_array = BROTLI_ALLOC((M), T, _new_size);                   \
+    if (!BROTLI_IS_OOM(M) && !BROTLI_IS_NULL(new_array) && C != 0) \
+      memcpy(new_array, A, C * sizeof(T));                         \
+    BROTLI_FREE((M), A);                                           \
+    A = new_array;                                                 \
+    C = _new_size;                                                 \
+  }                                                                \
+}
+
+/*
+Appends value and dynamically grows array capacity when needed
+M: MemoryManager
+T: data type
+A: array
+C: array capacity
+S: array size
+V: value to append
+*/
+#define BROTLI_ENSURE_CAPACITY_APPEND(M, T, A, C, S, V) { \
+  (S)++;                                                  \
+  BROTLI_ENSURE_CAPACITY(M, T, A, C, S);                  \
+  A[(S) - 1] = (V);                                       \
+}
+
 #if defined(__cplusplus) || defined(c_plusplus)
 }  /* extern "C" */
 #endif