brotli 0.2.0 → 0.4.0

data/vendor/brotli/c/enc/backward_references_hq.h

@@ -10,42 +10,42 @@
 #define BROTLI_ENC_BACKWARD_REFERENCES_HQ_H_
 
 #include "../common/constants.h"
+#include "../common/context.h"
 #include "../common/dictionary.h"
+#include "../common/platform.h"
 #include <brotli/types.h>
 #include "./command.h"
 #include "./hash.h"
 #include "./memory.h"
-#include "./port.h"
 #include "./quality.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
 #endif
 
-BROTLI_INTERNAL void BrotliCreateZopfliBackwardReferences(
-    MemoryManager* m, const BrotliDictionary* dictionary, size_t num_bytes,
+BROTLI_INTERNAL void BrotliCreateZopfliBackwardReferences(MemoryManager* m,
+    size_t num_bytes,
     size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
-    const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
-    size_t* last_insert_len, Command* commands, size_t* num_commands,
-    size_t* num_literals);
+    ContextLut literal_context_lut, const BrotliEncoderParams* params,
+    Hasher* hasher, int* dist_cache, size_t* last_insert_len,
+    Command* commands, size_t* num_commands, size_t* num_literals);
 
-BROTLI_INTERNAL void BrotliCreateHqZopfliBackwardReferences(
-    MemoryManager* m, const BrotliDictionary* dictionary, size_t num_bytes,
+BROTLI_INTERNAL void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m,
+    size_t num_bytes,
     size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
-    const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
-    size_t* last_insert_len, Command* commands, size_t* num_commands,
-    size_t* num_literals);
+    ContextLut literal_context_lut, const BrotliEncoderParams* params,
+    Hasher* hasher, int* dist_cache, size_t* last_insert_len,
+    Command* commands, size_t* num_commands, size_t* num_literals);
 
 typedef struct ZopfliNode {
-  /* best length to get up to this byte (not including this byte itself)
-     highest 8 bit is used to reconstruct the length code */
+  /* Best length to get up to this byte (not including this byte itself)
+     highest 7 bit is used to reconstruct the length code. */
   uint32_t length;
-  /* distance associated with the length
-     highest 7 bit contains distance short code + 1 (or zero if no short code)
-  */
+  /* Distance associated with the length. */
   uint32_t distance;
-  /* number of literal inserts before this copy */
-  uint32_t insert_length;
+  /* Number of literal inserts before this copy; highest 5 bits contain
+     distance short code + 1 (or zero if no short code). */
+  uint32_t dcode_insert_length;
 
   /* This union holds information used by dynamic-programming. During forward
      pass |cost| it used to store the goal function. When node is processed its
@@ -78,14 +78,13 @@ BROTLI_INTERNAL void BrotliInitZopfliNodes(ZopfliNode* array, size_t length);
      (2) nodes[i].command_length() <= i and
      (3) nodes[i - nodes[i].command_length()].cost < kInfinity */
 BROTLI_INTERNAL size_t BrotliZopfliComputeShortestPath(
-    MemoryManager* m, const BrotliDictionary* dictionary, size_t num_bytes,
+    MemoryManager* m, size_t num_bytes,
     size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
-    const BrotliEncoderParams* params, const size_t max_backward_limit,
-    const int* dist_cache, HasherHandle hasher, ZopfliNode* nodes);
+    ContextLut literal_context_lut, const BrotliEncoderParams* params,
+    const int* dist_cache, Hasher* hasher, ZopfliNode* nodes);
 
 BROTLI_INTERNAL void BrotliZopfliCreateCommands(
-    const size_t num_bytes, const size_t block_start,
-    const size_t max_backward_limit, const ZopfliNode* nodes,
+    const size_t num_bytes, const size_t block_start, const ZopfliNode* nodes,
     int* dist_cache, size_t* last_insert_len, const BrotliEncoderParams* params,
     Command* commands, size_t* num_literals);
 

data/vendor/brotli/c/enc/backward_references_inc.h

@@ -8,14 +8,15 @@
 /* template parameters: EXPORT_FN, FN */
 
 static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
-    const BrotliDictionary* dictionary,
-    const uint16_t* dictionary_hash, size_t num_bytes, size_t position,
+    size_t num_bytes, size_t position,
     const uint8_t* ringbuffer, size_t ringbuffer_mask,
-    const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
-    size_t* last_insert_len, Command* commands, size_t* num_commands,
-    size_t* num_literals) {
+    ContextLut literal_context_lut, const BrotliEncoderParams* params,
+    Hasher* hasher, int* dist_cache, size_t* last_insert_len,
+    Command* commands, size_t* num_commands, size_t* num_literals) {
+  HASHER()* privat = &hasher->privat.FN(_);
   /* Set maximum distance, see section 9.1. of the spec. */
   const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin);
+  const size_t position_offset = params->stream_offset;
 
   const Command* const orig_commands = commands;
   size_t insert_length = *last_insert_len;
@@ -32,19 +33,23 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
   /* Minimum score to accept a backward reference. */
   const score_t kMinScore = BROTLI_SCORE_BASE + 100;
 
-  FN(PrepareDistanceCache)(hasher, dist_cache);
+  BROTLI_UNUSED(literal_context_lut);
+
+  FN(PrepareDistanceCache)(privat, dist_cache);
 
   while (position + FN(HashTypeLength)() < pos_end) {
     size_t max_length = pos_end - position;
     size_t max_distance = BROTLI_MIN(size_t, position, max_backward_limit);
+    size_t dictionary_start = BROTLI_MIN(size_t,
+        position + position_offset, max_backward_limit);
     HasherSearchResult sr;
     sr.len = 0;
     sr.len_code_delta = 0;
     sr.distance = 0;
     sr.score = kMinScore;
-    FN(FindLongestMatch)(hasher, dictionary, dictionary_hash, ringbuffer,
-                         ringbuffer_mask, dist_cache, position,
-                         max_length, max_distance, gap, &sr);
+    FN(FindLongestMatch)(privat, &params->dictionary,
+        ringbuffer, ringbuffer_mask, dist_cache, position, max_length,
+        max_distance, dictionary_start + gap, params->dist.max_distance, &sr);
     if (sr.score > kMinScore) {
       /* Found a match. Let's look for something even better ahead. */
       int delayed_backward_references_in_row = 0;
@@ -58,9 +63,13 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
         sr2.distance = 0;
         sr2.score = kMinScore;
         max_distance = BROTLI_MIN(size_t, position + 1, max_backward_limit);
-        FN(FindLongestMatch)(hasher, dictionary, dictionary_hash, ringbuffer,
-                             ringbuffer_mask, dist_cache, position + 1,
-                             max_length, max_distance, gap, &sr2);
+        dictionary_start = BROTLI_MIN(size_t,
+            position + 1 + position_offset, max_backward_limit);
+        FN(FindLongestMatch)(privat,
+            &params->dictionary,
+            ringbuffer, ringbuffer_mask, dist_cache, position + 1, max_length,
+            max_distance, dictionary_start + gap, params->dist.max_distance,
+            &sr2);
         if (sr2.score >= sr.score + cost_diff_lazy) {
           /* Ok, let's just write one byte for now and start a match from the
              next byte. */
@@ -76,21 +85,22 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
       }
       apply_random_heuristics =
           position + 2 * sr.len + random_heuristics_window_size;
-      max_distance = BROTLI_MIN(size_t, position, max_backward_limit);
+      dictionary_start = BROTLI_MIN(size_t,
+          position + position_offset, max_backward_limit);
       {
         /* The first 16 codes are special short-codes,
            and the minimum offset is 1. */
-        size_t distance_code =
-            ComputeDistanceCode(sr.distance, max_distance + gap, dist_cache);
-        if ((sr.distance <= (max_distance + gap)) && distance_code > 0) {
+        size_t distance_code = ComputeDistanceCode(
+            sr.distance, dictionary_start + gap, dist_cache);
+        if ((sr.distance <= (dictionary_start + gap)) && distance_code > 0) {
           dist_cache[3] = dist_cache[2];
           dist_cache[2] = dist_cache[1];
           dist_cache[1] = dist_cache[0];
           dist_cache[0] = (int)sr.distance;
-          FN(PrepareDistanceCache)(hasher, dist_cache);
+          FN(PrepareDistanceCache)(privat, dist_cache);
         }
-        InitCommand(commands++, insert_length, sr.len, sr.len_code_delta,
-            distance_code);
+        InitCommand(commands++, &params->dist, insert_length,
+            sr.len, sr.len_code_delta, distance_code);
       }
       *num_literals += insert_length;
       insert_length = 0;
@@ -105,7 +115,7 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
           range_start = BROTLI_MIN(size_t, range_end, BROTLI_MAX(size_t,
               range_start, position + sr.len - (sr.distance << 2)));
         }
-        FN(StoreRange)(hasher, ringbuffer, ringbuffer_mask, range_start,
+        FN(StoreRange)(privat, ringbuffer, ringbuffer_mask, range_start,
                        range_end);
       }
       position += sr.len;
@@ -131,7 +141,7 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
           size_t pos_jump =
               BROTLI_MIN(size_t, position + 16, pos_end - kMargin);
           for (; position < pos_jump; position += 4) {
-            FN(Store)(hasher, ringbuffer, ringbuffer_mask, position);
+            FN(Store)(privat, ringbuffer, ringbuffer_mask, position);
             insert_length += 4;
           }
         } else {
@@ -140,7 +150,7 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
           size_t pos_jump =
               BROTLI_MIN(size_t, position + 8, pos_end - kMargin);
           for (; position < pos_jump; position += 2) {
-            FN(Store)(hasher, ringbuffer, ringbuffer_mask, position);
+            FN(Store)(privat, ringbuffer, ringbuffer_mask, position);
             insert_length += 2;
           }
         }

data/vendor/brotli/c/enc/bit_cost.c

@@ -9,10 +9,10 @@
 #include "./bit_cost.h"
 
 #include "../common/constants.h"
+#include "../common/platform.h"
 #include <brotli/types.h>
 #include "./fast_log.h"
 #include "./histogram.h"
-#include "./port.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {

data/vendor/brotli/c/enc/bit_cost.h

@@ -9,20 +9,20 @@
 #ifndef BROTLI_ENC_BIT_COST_H_
 #define BROTLI_ENC_BIT_COST_H_
 
+#include "../common/platform.h"
 #include <brotli/types.h>
 #include "./fast_log.h"
 #include "./histogram.h"
-#include "./port.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
 #endif
 
-static BROTLI_INLINE double ShannonEntropy(const uint32_t *population,
-                                           size_t size, size_t *total) {
+static BROTLI_INLINE double ShannonEntropy(
+    const uint32_t* population, size_t size, size_t* total) {
   size_t sum = 0;
   double retval = 0;
-  const uint32_t *population_end = population + size;
+  const uint32_t* population_end = population + size;
   size_t p;
   if (size & 1) {
     goto odd_number_of_elements_left;
@@ -42,7 +42,7 @@ static BROTLI_INLINE double ShannonEntropy(const uint32_t *population,
 }
 
 static BROTLI_INLINE double BitsEntropy(
-    const uint32_t *population, size_t size) {
+    const uint32_t* population, size_t size) {
   size_t sum;
   double retval = ShannonEntropy(population, size, &sum);
   if (retval < sum) {

data/vendor/brotli/c/enc/block_encoder_inc.h

@@ -13,9 +13,9 @@
    stream. */
 static void FN(BuildAndStoreEntropyCodes)(MemoryManager* m, BlockEncoder* self,
     const HistogramType* histograms, const size_t histograms_size,
-    HuffmanTree* tree, size_t* storage_ix, uint8_t* storage) {
-  const size_t alphabet_size = self->alphabet_size_;
-  const size_t table_size = histograms_size * alphabet_size;
+    const size_t alphabet_size, HuffmanTree* tree,
+    size_t* storage_ix, uint8_t* storage) {
+  const size_t table_size = histograms_size * self->histogram_length_;
   self->depths_ = BROTLI_ALLOC(m, uint8_t, table_size);
   self->bits_ = BROTLI_ALLOC(m, uint16_t, table_size);
   if (BROTLI_IS_OOM(m)) return;
@@ -23,9 +23,10 @@ static void FN(BuildAndStoreEntropyCodes)(MemoryManager* m, BlockEncoder* self,
   {
     size_t i;
     for (i = 0; i < histograms_size; ++i) {
-      size_t ix = i * alphabet_size;
-      BuildAndStoreHuffmanTree(&histograms[i].data_[0], alphabet_size, tree,
-          &self->depths_[ix], &self->bits_[ix], storage_ix, storage);
+      size_t ix = i * self->histogram_length_;
+      BuildAndStoreHuffmanTree(&histograms[i].data_[0], self->histogram_length_,
+          alphabet_size, tree, &self->depths_[ix], &self->bits_[ix],
+          storage_ix, storage);
     }
   }
 }

data/vendor/brotli/c/enc/block_splitter.c

@@ -8,16 +8,15 @@
 
 #include "./block_splitter.h"
 
-#include <assert.h>
 #include <string.h>  /* memcpy, memset */
 
+#include "../common/platform.h"
 #include "./bit_cost.h"
 #include "./cluster.h"
 #include "./command.h"
 #include "./fast_log.h"
 #include "./histogram.h"
 #include "./memory.h"
-#include "./port.h"
 #include "./quality.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
@@ -133,7 +132,7 @@ void BrotliSplitBlock(MemoryManager* m,
   {
     size_t literals_count = CountLiterals(cmds, num_commands);
     uint8_t* literals = BROTLI_ALLOC(m, uint8_t, literals_count);
-    if (BROTLI_IS_OOM(m)) return;
+    if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(literals)) return;
     /* Create a continuous array of literals. */
     CopyLiteralsToByteArray(cmds, num_commands, data, pos, mask, literals);
     /* Create the block split on the array of literals.
@@ -151,7 +150,7 @@ void BrotliSplitBlock(MemoryManager* m,
     /* Compute prefix codes for commands. */
     uint16_t* insert_and_copy_codes = BROTLI_ALLOC(m, uint16_t, num_commands);
     size_t i;
-    if (BROTLI_IS_OOM(m)) return;
+    if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(insert_and_copy_codes)) return;
     for (i = 0; i < num_commands; ++i) {
       insert_and_copy_codes[i] = cmds[i].cmd_prefix_;
     }
@@ -171,11 +170,11 @@ void BrotliSplitBlock(MemoryManager* m,
     uint16_t* distance_prefixes = BROTLI_ALLOC(m, uint16_t, num_commands);
     size_t j = 0;
     size_t i;
-    if (BROTLI_IS_OOM(m)) return;
+    if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(distance_prefixes)) return;
     for (i = 0; i < num_commands; ++i) {
       const Command* cmd = &cmds[i];
       if (CommandCopyLen(cmd) && cmd->cmd_prefix_ >= 128) {
-        distance_prefixes[j++] = cmd->dist_prefix_;
+        distance_prefixes[j++] = cmd->dist_prefix_ & 0x3FF;
       }
     }
     /* Create the block split on the array of distance prefixes. */

data/vendor/brotli/c/enc/block_splitter.h

@@ -9,10 +9,10 @@
 #ifndef BROTLI_ENC_BLOCK_SPLITTER_H_
 #define BROTLI_ENC_BLOCK_SPLITTER_H_
 
+#include "../common/platform.h"
 #include <brotli/types.h>
 #include "./command.h"
 #include "./memory.h"
-#include "./port.h"
 #include "./quality.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)

data/vendor/brotli/c/enc/block_splitter_inc.h

@@ -70,13 +70,13 @@ static size_t FN(FindBlocks)(const DataType* data, const size_t length,
                              double* insert_cost,
                              double* cost,
                              uint8_t* switch_signal,
-                             uint8_t *block_id) {
+                             uint8_t* block_id) {
   const size_t data_size = FN(HistogramDataSize)();
   const size_t bitmaplen = (num_histograms + 7) >> 3;
   size_t num_blocks = 1;
   size_t i;
   size_t j;
-  assert(num_histograms <= 256);
+  BROTLI_DCHECK(num_histograms <= 256);
   if (num_histograms <= 1) {
     for (i = 0; i < length; ++i) {
       block_id[i] = 0;
@@ -126,7 +126,7 @@ static size_t FN(FindBlocks)(const DataType* data, const size_t length,
       if (cost[k] >= block_switch_cost) {
         const uint8_t mask = (uint8_t)(1u << (k & 7));
         cost[k] = block_switch_cost;
-        assert((k >> 3) < bitmaplen);
+        BROTLI_DCHECK((k >> 3) < bitmaplen);
         switch_signal[ix + (k >> 3)] |= mask;
       }
     }
@@ -137,7 +137,7 @@ static size_t FN(FindBlocks)(const DataType* data, const size_t length,
     uint8_t cur_id = block_id[byte_ix];
     while (byte_ix > 0) {
       const uint8_t mask = (uint8_t)(1u << (cur_id & 7));
-      assert(((size_t)cur_id >> 3) < bitmaplen);
+      BROTLI_DCHECK(((size_t)cur_id >> 3) < bitmaplen);
       --byte_ix;
       ix -= bitmaplen;
       if (switch_signal[ix + (cur_id >> 3)] & mask) {
@@ -161,16 +161,16 @@ static size_t FN(RemapBlockIds)(uint8_t* block_ids, const size_t length,
     new_id[i] = kInvalidId;
   }
   for (i = 0; i < length; ++i) {
-    assert(block_ids[i] < num_histograms);
+    BROTLI_DCHECK(block_ids[i] < num_histograms);
     if (new_id[block_ids[i]] == kInvalidId) {
       new_id[block_ids[i]] = next_id++;
     }
   }
   for (i = 0; i < length; ++i) {
     block_ids[i] = (uint8_t)new_id[block_ids[i]];
-    assert(block_ids[i] < num_histograms);
+    BROTLI_DCHECK(block_ids[i] < num_histograms);
   }
-  assert(next_id <= num_histograms);
+  BROTLI_DCHECK(next_id <= num_histograms);
   return next_id;
 }
 
@@ -219,20 +219,25 @@ static void FN(ClusterBlocks)(MemoryManager* m,
   uint32_t symbols[HISTOGRAMS_PER_BATCH] = { 0 };
   uint32_t remap[HISTOGRAMS_PER_BATCH] = { 0 };
 
-  if (BROTLI_IS_OOM(m)) return;
+  if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(histogram_symbols) ||
+      BROTLI_IS_NULL(block_lengths) || BROTLI_IS_NULL(all_histograms) ||
+      BROTLI_IS_NULL(cluster_size) || BROTLI_IS_NULL(histograms) ||
+      BROTLI_IS_NULL(pairs)) {
+    return;
+  }
 
   memset(block_lengths, 0, num_blocks * sizeof(uint32_t));
 
   {
     size_t block_idx = 0;
     for (i = 0; i < length; ++i) {
-      assert(block_idx < num_blocks);
+      BROTLI_DCHECK(block_idx < num_blocks);
       ++block_lengths[block_idx];
       if (i + 1 == length || block_ids[i] != block_ids[i + 1]) {
         ++block_idx;
       }
     }
-    assert(block_idx == num_blocks);
+    BROTLI_DCHECK(block_idx == num_blocks);
   }
 
   for (i = 0; i < num_blocks; i += HISTOGRAMS_PER_BATCH) {
@@ -268,8 +273,8 @@ static void FN(ClusterBlocks)(MemoryManager* m,
       histogram_symbols[i + j] = (uint32_t)num_clusters + remap[symbols[j]];
     }
     num_clusters += num_new_clusters;
-    assert(num_clusters == cluster_size_size);
-    assert(num_clusters == all_histograms_size);
+    BROTLI_DCHECK(num_clusters == cluster_size_size);
+    BROTLI_DCHECK(num_clusters == all_histograms_size);
   }
   BROTLI_FREE(m, histograms);
 
@@ -278,11 +283,11 @@ static void FN(ClusterBlocks)(MemoryManager* m,
   if (pairs_capacity < max_num_pairs + 1) {
     BROTLI_FREE(m, pairs);
     pairs = BROTLI_ALLOC(m, HistogramPair, max_num_pairs + 1);
-    if (BROTLI_IS_OOM(m)) return;
+    if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(pairs)) return;
   }
 
   clusters = BROTLI_ALLOC(m, uint32_t, num_clusters);
-  if (BROTLI_IS_OOM(m)) return;
+  if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(clusters)) return;
   for (i = 0; i < num_clusters; ++i) {
     clusters[i] = (uint32_t)i;
   }
@@ -294,7 +299,7 @@ static void FN(ClusterBlocks)(MemoryManager* m,
   BROTLI_FREE(m, cluster_size);
 
   new_index = BROTLI_ALLOC(m, uint32_t, num_clusters);
-  if (BROTLI_IS_OOM(m)) return;
+  if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(new_index)) return;
   for (i = 0; i < num_clusters; ++i) new_index[i] = kInvalidIndex;
   pos = 0;
   {
@@ -386,7 +391,7 @@ static void FN(SplitByteVector)(MemoryManager* m,
     return;
   }
   histograms = BROTLI_ALLOC(m, HistogramType, num_histograms);
-  if (BROTLI_IS_OOM(m)) return;
+  if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(histograms)) return;
   /* Find good entropy codes. */
   FN(InitialEntropyCodes)(data, length,
                           sampling_stride_length,
@@ -405,7 +410,11 @@ static void FN(SplitByteVector)(MemoryManager* m,
     uint16_t* new_id = BROTLI_ALLOC(m, uint16_t, num_histograms);
     const size_t iters = params->quality < HQ_ZOPFLIFICATION_QUALITY ? 3 : 10;
     size_t i;
-    if (BROTLI_IS_OOM(m)) return;
+    if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(block_ids) ||
+        BROTLI_IS_NULL(insert_cost) || BROTLI_IS_NULL(cost) ||
+        BROTLI_IS_NULL(switch_signal) || BROTLI_IS_NULL(new_id)) {
+      return;
+    }
     for (i = 0; i < iters; ++i) {
       num_blocks = FN(FindBlocks)(data, length,
                                   block_switch_cost,