brotli 0.1.8 → 0.2.0

data/vendor/brotli/{enc/backward_references.h → c/enc/backward_references_hq.h}

@@ -6,10 +6,12 @@
 
 /* Function to find backward reference copies. */
 
-#ifndef BROTLI_ENC_BACKWARD_REFERENCES_H_
-#define BROTLI_ENC_BACKWARD_REFERENCES_H_
+#ifndef BROTLI_ENC_BACKWARD_REFERENCES_HQ_H_
+#define BROTLI_ENC_BACKWARD_REFERENCES_HQ_H_
 
-#include "../common/types.h"
+#include "../common/constants.h"
+#include "../common/dictionary.h"
+#include <brotli/types.h>
 #include "./command.h"
 #include "./hash.h"
 #include "./memory.h"
@@ -20,14 +22,17 @@
 extern "C" {
 #endif
 
-/* "commands" points to the next output command to write to, "*num_commands" is
-   initially the total amount of commands output by previous
-   CreateBackwardReferences calls, and must be incremented by the amount written
-   by this call. */
-BROTLI_INTERNAL void BrotliCreateBackwardReferences(
-    MemoryManager* m, size_t num_bytes, size_t position, BROTLI_BOOL is_last,
-    const uint8_t* ringbuffer, size_t ringbuffer_mask,
-    const BrotliEncoderParams* params, Hashers* hashers, int* dist_cache,
+BROTLI_INTERNAL void BrotliCreateZopfliBackwardReferences(
+    MemoryManager* m, const BrotliDictionary* dictionary, size_t num_bytes,
+    size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
+    const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
+    size_t* last_insert_len, Command* commands, size_t* num_commands,
+    size_t* num_literals);
+
+BROTLI_INTERNAL void BrotliCreateHqZopfliBackwardReferences(
+    MemoryManager* m, const BrotliDictionary* dictionary, size_t num_bytes,
+    size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
+    const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
     size_t* last_insert_len, Command* commands, size_t* num_commands,
     size_t* num_literals);
 
@@ -44,7 +49,7 @@ typedef struct ZopfliNode {
 
   /* This union holds information used by dynamic-programming. During forward
      pass |cost| it used to store the goal function. When node is processed its
-     |cost| is invalidated in favor of |shortcut|. On path backtracing pass
+     |cost| is invalidated in favor of |shortcut|. On path back-tracing pass
      |next| is assigned the offset to next node on the path. */
   union {
     /* Smallest cost to get to this byte from the beginning, as found so far. */
@@ -63,7 +68,7 @@ BROTLI_INTERNAL void BrotliInitZopfliNodes(ZopfliNode* array, size_t length);
    position + num_bytes.
 
    On return, path->size() is the number of commands found and path[i] is the
-   length of the ith command (copy length plus insert length).
+   length of the i-th command (copy length plus insert length).
    Note that the sum of the lengths of all commands can be less than num_bytes.
 
    On return, the nodes[0..num_bytes] array will have the following
@@ -73,27 +78,19 @@ BROTLI_INTERNAL void BrotliInitZopfliNodes(ZopfliNode* array, size_t length);
      (2) nodes[i].command_length() <= i and
      (3) nodes[i - nodes[i].command_length()].cost < kInfinity */
 BROTLI_INTERNAL size_t BrotliZopfliComputeShortestPath(
-    MemoryManager* m, size_t num_bytes, size_t position,
-    const uint8_t* ringbuffer, size_t ringbuffer_mask,
+    MemoryManager* m, const BrotliDictionary* dictionary, size_t num_bytes,
+    size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
     const BrotliEncoderParams* params, const size_t max_backward_limit,
-    const int* dist_cache, H10* hasher, ZopfliNode* nodes);
-
-BROTLI_INTERNAL void BrotliZopfliCreateCommands(const size_t num_bytes,
-                                                const size_t block_start,
-                                                const size_t max_backward_limit,
-                                                const ZopfliNode* nodes,
-                                                int* dist_cache,
-                                                size_t* last_insert_len,
-                                                Command* commands,
-                                                size_t* num_literals);
-
-/* Maximum distance, see section 9.1. of the spec. */
-static BROTLI_INLINE size_t MaxBackwardLimit(int lgwin) {
-  return (1u << lgwin) - 16;
-}
+    const int* dist_cache, HasherHandle hasher, ZopfliNode* nodes);
+
+BROTLI_INTERNAL void BrotliZopfliCreateCommands(
+    const size_t num_bytes, const size_t block_start,
+    const size_t max_backward_limit, const ZopfliNode* nodes,
+    int* dist_cache, size_t* last_insert_len, const BrotliEncoderParams* params,
+    Command* commands, size_t* num_literals);
 
 #if defined(__cplusplus) || defined(c_plusplus)
 }  /* extern "C" */
 #endif
 
-#endif  /* BROTLI_ENC_BACKWARD_REFERENCES_H_ */
+#endif  /* BROTLI_ENC_BACKWARD_REFERENCES_HQ_H_ */

data/vendor/brotli/{enc → c/enc}/backward_references_inc.h

@@ -5,18 +5,17 @@
    See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
 */
 
-/* template parameters: FN */
+/* template parameters: EXPORT_FN, FN */
 
-#define Hasher HASHER()
-
-static BROTLI_NOINLINE void FN(CreateBackwardReferences)(
-    MemoryManager* m, size_t num_bytes, size_t position, BROTLI_BOOL is_last,
+static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
+    const BrotliDictionary* dictionary,
+    const uint16_t* dictionary_hash, size_t num_bytes, size_t position,
     const uint8_t* ringbuffer, size_t ringbuffer_mask,
-    const BrotliEncoderParams* params, Hasher* hasher, int* dist_cache,
+    const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
     size_t* last_insert_len, Command* commands, size_t* num_commands,
     size_t* num_literals) {
   /* Set maximum distance, see section 9.1. of the spec. */
-  const size_t max_backward_limit = MaxBackwardLimit(params->lgwin);
+  const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin);
 
   const Command* const orig_commands = commands;
   size_t insert_length = *last_insert_len;
@@ -28,42 +27,41 @@ static BROTLI_NOINLINE void FN(CreateBackwardReferences)(
   const size_t random_heuristics_window_size =
       LiteralSpreeLengthForSparseSearch(params);
   size_t apply_random_heuristics = position + random_heuristics_window_size;
+  const size_t gap = 0;
 
   /* Minimum score to accept a backward reference. */
-  const score_t kMinScore = BROTLI_SCORE_BASE + 400;
+  const score_t kMinScore = BROTLI_SCORE_BASE + 100;
 
-  FN(Init)(m, hasher, ringbuffer, params, position, num_bytes, is_last);
-  if (BROTLI_IS_OOM(m)) return;
-  FN(StitchToPreviousBlock)(hasher, num_bytes, position,
-                            ringbuffer, ringbuffer_mask);
+  FN(PrepareDistanceCache)(hasher, dist_cache);
 
   while (position + FN(HashTypeLength)() < pos_end) {
     size_t max_length = pos_end - position;
     size_t max_distance = BROTLI_MIN(size_t, position, max_backward_limit);
     HasherSearchResult sr;
     sr.len = 0;
-    sr.len_x_code = 0;
+    sr.len_code_delta = 0;
     sr.distance = 0;
     sr.score = kMinScore;
-    if (FN(FindLongestMatch)(hasher, ringbuffer, ringbuffer_mask, dist_cache,
-                             position, max_length, max_distance, &sr)) {
+    FN(FindLongestMatch)(hasher, dictionary, dictionary_hash, ringbuffer,
+                         ringbuffer_mask, dist_cache, position,
+                         max_length, max_distance, gap, &sr);
+    if (sr.score > kMinScore) {
       /* Found a match. Let's look for something even better ahead. */
       int delayed_backward_references_in_row = 0;
       --max_length;
       for (;; --max_length) {
-        const score_t cost_diff_lazy = 700;
-        BROTLI_BOOL is_match_found;
+        const score_t cost_diff_lazy = 175;
         HasherSearchResult sr2;
         sr2.len = params->quality < MIN_QUALITY_FOR_EXTENSIVE_REFERENCE_SEARCH ?
             BROTLI_MIN(size_t, sr.len - 1, max_length) : 0;
-        sr2.len_x_code = 0;
+        sr2.len_code_delta = 0;
         sr2.distance = 0;
         sr2.score = kMinScore;
         max_distance = BROTLI_MIN(size_t, position + 1, max_backward_limit);
-        is_match_found = FN(FindLongestMatch)(hasher, ringbuffer,
-            ringbuffer_mask, dist_cache, position + 1, max_length, max_distance,
-            &sr2);
-        if (is_match_found && sr2.score >= sr.score + cost_diff_lazy) {
+        FN(FindLongestMatch)(hasher, dictionary, dictionary_hash, ringbuffer,
+                             ringbuffer_mask, dist_cache, position + 1,
+                             max_length, max_distance, gap, &sr2);
+        if (sr2.score >= sr.score + cost_diff_lazy) {
           /* Ok, let's just write one byte for now and start a match from the
              next byte. */
           ++position;
@@ -80,26 +78,36 @@ static BROTLI_NOINLINE void FN(CreateBackwardReferences)(
           position + 2 * sr.len + random_heuristics_window_size;
       max_distance = BROTLI_MIN(size_t, position, max_backward_limit);
       {
-        /* The first 16 codes are special shortcodes,
+        /* The first 16 codes are special short-codes,
            and the minimum offset is 1. */
         size_t distance_code =
-            ComputeDistanceCode(sr.distance, max_distance, dist_cache);
-        if (sr.distance <= max_distance && distance_code > 0) {
+            ComputeDistanceCode(sr.distance, max_distance + gap, dist_cache);
+        if ((sr.distance <= (max_distance + gap)) && distance_code > 0) {
           dist_cache[3] = dist_cache[2];
           dist_cache[2] = dist_cache[1];
           dist_cache[1] = dist_cache[0];
           dist_cache[0] = (int)sr.distance;
+          FN(PrepareDistanceCache)(hasher, dist_cache);
         }
-        InitCommand(commands++, insert_length, sr.len, sr.len ^ sr.len_x_code,
+        InitCommand(commands++, insert_length, sr.len, sr.len_code_delta,
             distance_code);
       }
       *num_literals += insert_length;
       insert_length = 0;
       /* Put the hash keys into the table, if there are enough bytes left.
          Depending on the hasher implementation, it can push all positions
-         in the given range or only a subset of them. */
-      FN(StoreRange)(hasher, ringbuffer, ringbuffer_mask, position + 2,
-                     BROTLI_MIN(size_t, position + sr.len, store_end));
+         in the given range or only a subset of them.
+         Avoid hash poisoning with RLE data. */
+      {
+        size_t range_start = position + 2;
+        size_t range_end = BROTLI_MIN(size_t, position + sr.len, store_end);
+        if (sr.distance < (sr.len >> 2)) {
+          range_start = BROTLI_MIN(size_t, range_end, BROTLI_MAX(size_t,
+              range_start, position + sr.len - (sr.distance << 2)));
+        }
+        FN(StoreRange)(hasher, ringbuffer, ringbuffer_mask, range_start,
+                       range_end);
+      }
       position += sr.len;
     } else {
       ++insert_length;
@@ -143,5 +151,3 @@ static BROTLI_NOINLINE void FN(CreateBackwardReferences)(
   *last_insert_len = insert_length;
   *num_commands += (size_t)(commands - orig_commands);
 }
-
-#undef Hasher

data/vendor/brotli/{enc → c/enc}/bit_cost.c

@@ -9,7 +9,7 @@
 #include "./bit_cost.h"
 
 #include "../common/constants.h"
-#include "../common/types.h"
+#include <brotli/types.h>
 #include "./fast_log.h"
 #include "./histogram.h"
 #include "./port.h"

data/vendor/brotli/{enc → c/enc}/bit_cost.h

@@ -9,7 +9,7 @@
 #ifndef BROTLI_ENC_BIT_COST_H_
 #define BROTLI_ENC_BIT_COST_H_
 
-#include "../common/types.h"
+#include <brotli/types.h>
 #include "./fast_log.h"
 #include "./histogram.h"
 #include "./port.h"

data/vendor/brotli/{enc → c/enc}/block_splitter.c

@@ -74,11 +74,9 @@ static void CopyLiteralsToByteArray(const Command* cmds,
   }
 }
 
-static BROTLI_INLINE unsigned int MyRand(unsigned int* seed) {
+static BROTLI_INLINE uint32_t MyRand(uint32_t* seed) {
+  /* Initial seed should be 7. In this case, loop length is (1 << 29). */
   *seed *= 16807U;
-  if (*seed == 0) {
-    *seed = 1;
-  }
   return *seed;
 }
 

data/vendor/brotli/{enc → c/enc}/block_splitter.h

@@ -9,7 +9,7 @@
 #ifndef BROTLI_ENC_BLOCK_SPLITTER_H_
 #define BROTLI_ENC_BLOCK_SPLITTER_H_
 
-#include "../common/types.h"
+#include <brotli/types.h>
 #include "./command.h"
 #include "./memory.h"
 #include "./port.h"

data/vendor/brotli/{enc → c/enc}/block_splitter_inc.h

@@ -13,7 +13,7 @@ static void FN(InitialEntropyCodes)(const DataType* data, size_t length,
                                     size_t stride,
                                     size_t num_histograms,
                                     HistogramType* histograms) {
-  unsigned int seed = 7;
+  uint32_t seed = 7;
   size_t block_length = length / num_histograms;
   size_t i;
   FN(ClearHistograms)(histograms, num_histograms);
@@ -29,14 +29,13 @@ static void FN(InitialEntropyCodes)(const DataType* data, size_t length,
   }
 }
 
-static void FN(RandomSample)(unsigned int* seed,
+static void FN(RandomSample)(uint32_t* seed,
                              const DataType* data,
                              size_t length,
                              size_t stride,
                              HistogramType* sample) {
   size_t pos = 0;
   if (stride >= length) {
-    pos = 0;
     stride = length;
   } else {
     pos = MyRand(seed) % (length - stride + 1);
@@ -50,7 +49,7 @@ static void FN(RefineEntropyCodes)(const DataType* data, size_t length,
                                    HistogramType* histograms) {
   size_t iters =
       kIterMulForRefining * length / stride + kMinItersForRefining;
-  unsigned int seed = 7;
+  uint32_t seed = 7;
   size_t iter;
   iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms;
   for (iter = 0; iter < iters; ++iter) {
@@ -61,7 +60,7 @@ static void FN(RefineEntropyCodes)(const DataType* data, size_t length,
   }
 }
 
-/* Assigns a block id from the range [0, vec.size()) to each data element
+/* Assigns a block id from the range [0, num_histograms) to each data element
    in data[0..length) and fills in block_id[0..length) with the assigned values.
    Returns the number of blocks, i.e. one plus the number of block switches. */
 static size_t FN(FindBlocks)(const DataType* data, const size_t length,
@@ -214,7 +213,6 @@ static void FN(ClusterBlocks)(MemoryManager* m,
   size_t num_final_clusters;
   static const uint32_t kInvalidIndex = BROTLI_UINT32_MAX;
   uint32_t* new_index;
-  uint8_t max_type = 0;
   size_t i;
   uint32_t sizes[HISTOGRAMS_PER_BATCH] = { 0 };
   uint32_t new_clusters[HISTOGRAMS_PER_BATCH] = { 0 };
@@ -337,6 +335,7 @@ static void FN(ClusterBlocks)(MemoryManager* m,
   {
     uint32_t cur_length = 0;
     size_t block_idx = 0;
+    uint8_t max_type = 0;
     for (i = 0; i < num_blocks; ++i) {
       cur_length += block_lengths[i];
       if (i + 1 == num_blocks ||
@@ -398,7 +397,7 @@ static void FN(SplitByteVector)(MemoryManager* m,
   {
     /* Find a good path through literals with the good entropy codes. */
     uint8_t* block_ids = BROTLI_ALLOC(m, uint8_t, length);
-    size_t num_blocks;
+    size_t num_blocks = 0;
     const size_t bitmaplen = (num_histograms + 7) >> 3;
     double* insert_cost = BROTLI_ALLOC(m, double, data_size * num_histograms);
     double* cost = BROTLI_ALLOC(m, double, num_histograms);

data/vendor/brotli/{enc → c/enc}/brotli_bit_stream.c

@@ -13,7 +13,7 @@
 #include <string.h>  /* memcpy, memset */
 
 #include "../common/constants.h"
-#include "../common/types.h"
+#include <brotli/types.h>
 #include "./context.h"
 #include "./entropy_encode.h"
 #include "./entropy_encode_static.h"
@@ -27,6 +27,12 @@ extern "C" {
 #endif
 
 #define MAX_HUFFMAN_TREE_SIZE (2 * BROTLI_NUM_COMMAND_SYMBOLS + 1)
+/* The size of Huffman dictionary for distances assuming that NPOSTFIX = 0 and
+ NDIRECT = 0. */
+#define SIMPLE_DISTANCE_ALPHABET_SIZE (BROTLI_NUM_DISTANCE_SHORT_CODES + \
+                                       (2 * BROTLI_MAX_DISTANCE_BITS))
+/* SIMPLE_DISTANCE_ALPHABET_SIZE == 64 */
+#define SIMPLE_DISTANCE_ALPHABET_BITS 6
 
 /* Represents the range of values belonging to a prefix code:
    [offset, offset + 2^nbits) */
@@ -76,7 +82,7 @@ static BROTLI_INLINE size_t NextBlockTypeCode(
   return type_code;
 }
 
-/* nibblesbits represents the 2 bits to encode MNIBBLES (0-3)
+/* |nibblesbits| represents the 2 bits to encode MNIBBLES (0-3)
    REQUIRES: length > 0
    REQUIRES: length <= (1 << 24) */
 static void BrotliEncodeMlen(size_t length, uint64_t* bits,
@@ -343,7 +349,7 @@ void BrotliStoreHuffmanTree(const uint8_t* depths, size_t num,
     code_length_bitdepth[code] = 0;
   }
 
-  /* Store the real huffman tree now. */
+  /* Store the real Huffman tree now. */
   BrotliStoreHuffmanTreeToBitMask(huffman_tree_size,
                                   huffman_tree,
                                   huffman_tree_extra_bits,
@@ -450,7 +456,7 @@ void BrotliBuildAndStoreHuffmanTreeFast(MemoryManager* m,
       for (l = length; l != 0;) {
         --l;
         if (histogram[l]) {
-          if (PREDICT_TRUE(histogram[l] >= count_limit)) {
+          if (BROTLI_PREDICT_TRUE(histogram[l] >= count_limit)) {
             InitHuffmanTree(node, histogram[l], -1, (int16_t)l);
           } else {
             InitHuffmanTree(node, count_limit, -1, (int16_t)l);
@@ -548,7 +554,7 @@ void BrotliBuildAndStoreHuffmanTreeFast(MemoryManager* m,
     /* Complex Huffman Tree */
     StoreStaticCodeLengthCode(storage_ix, storage);
 
-    /* Actual rle coding. */
+    /* Actual RLE coding. */
     for (i = 0; i < length;) {
       const uint8_t value = depth[i];
       size_t reps = 1;
@@ -1151,12 +1157,12 @@ void BrotliStoreMetaBlockTrivial(MemoryManager* m,
   HistogramLiteral lit_histo;
   HistogramCommand cmd_histo;
   HistogramDistance dist_histo;
-  uint8_t lit_depth[256];
-  uint16_t lit_bits[256];
+  uint8_t lit_depth[BROTLI_NUM_LITERAL_SYMBOLS];
+  uint16_t lit_bits[BROTLI_NUM_LITERAL_SYMBOLS];
   uint8_t cmd_depth[BROTLI_NUM_COMMAND_SYMBOLS];
   uint16_t cmd_bits[BROTLI_NUM_COMMAND_SYMBOLS];
-  uint8_t dist_depth[64];
-  uint16_t dist_bits[64];
+  uint8_t dist_depth[SIMPLE_DISTANCE_ALPHABET_SIZE];
+  uint16_t dist_bits[SIMPLE_DISTANCE_ALPHABET_SIZE];
   HuffmanTree* tree;
 
   StoreCompressedMetaBlockHeader(is_last, length, storage_ix, storage);
@@ -1172,13 +1178,14 @@ void BrotliStoreMetaBlockTrivial(MemoryManager* m,
 
   tree = BROTLI_ALLOC(m, HuffmanTree, MAX_HUFFMAN_TREE_SIZE);
   if (BROTLI_IS_OOM(m)) return;
-  BuildAndStoreHuffmanTree(lit_histo.data_, 256, tree,
+  BuildAndStoreHuffmanTree(lit_histo.data_, BROTLI_NUM_LITERAL_SYMBOLS, tree,
                            lit_depth, lit_bits,
                            storage_ix, storage);
   BuildAndStoreHuffmanTree(cmd_histo.data_, BROTLI_NUM_COMMAND_SYMBOLS, tree,
                            cmd_depth, cmd_bits,
                            storage_ix, storage);
-  BuildAndStoreHuffmanTree(dist_histo.data_, 64, tree,
+  BuildAndStoreHuffmanTree(dist_histo.data_, SIMPLE_DISTANCE_ALPHABET_SIZE,
+                           tree,
                            dist_depth, dist_bits,
                            storage_ix, storage);
   BROTLI_FREE(m, tree);
@@ -1245,8 +1252,8 @@ void BrotliStoreMetaBlockFast(MemoryManager* m,
     uint16_t lit_bits[BROTLI_NUM_LITERAL_SYMBOLS];
     uint8_t cmd_depth[BROTLI_NUM_COMMAND_SYMBOLS];
     uint16_t cmd_bits[BROTLI_NUM_COMMAND_SYMBOLS];
-    uint8_t dist_depth[64];
-    uint16_t dist_bits[64];
+    uint8_t dist_depth[SIMPLE_DISTANCE_ALPHABET_SIZE];
+    uint16_t dist_bits[SIMPLE_DISTANCE_ALPHABET_SIZE];
     HistogramClearLiteral(&lit_histo);
     HistogramClearCommand(&cmd_histo);
     HistogramClearDistance(&dist_histo);
@@ -1266,7 +1273,8 @@ void BrotliStoreMetaBlockFast(MemoryManager* m,
     if (BROTLI_IS_OOM(m)) return;
     BrotliBuildAndStoreHuffmanTreeFast(m, dist_histo.data_,
                                        dist_histo.total_count_,
-                                       /* max_bits = */ 6,
+                                       /* max_bits = */
+                                       SIMPLE_DISTANCE_ALPHABET_BITS,
                                        dist_depth, dist_bits,
                                        storage_ix, storage);
     if (BROTLI_IS_OOM(m)) return;
@@ -1317,18 +1325,6 @@ void BrotliStoreUncompressedMetaBlock(BROTLI_BOOL is_final_block,
   }
 }
 
-void BrotliStoreSyncMetaBlock(size_t* BROTLI_RESTRICT storage_ix,
-                              uint8_t* BROTLI_RESTRICT storage) {
-  /* Empty metadata meta-block bit pattern:
-       1 bit:  is_last (0)
-       2 bits: num nibbles (3)
-       1 bit:  reserved (0)
-       2 bits: metadata length bytes (0) */
-  BrotliWriteBits(6, 6, storage_ix, storage);
-  JumpToByteBoundary(storage_ix, storage);
-}
-
-
 #if defined(__cplusplus) || defined(c_plusplus)
 }  /* extern "C" */
 #endif