brotli 0.2.3 → 0.5.0

data/vendor/brotli/c/enc/backward_references_inc.h

@@ -10,11 +10,13 @@
 static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
     size_t num_bytes, size_t position,
     const uint8_t* ringbuffer, size_t ringbuffer_mask,
-    const BrotliEncoderParams* params,
-    HasherHandle hasher, int* dist_cache, size_t* last_insert_len,
+    ContextLut literal_context_lut, const BrotliEncoderParams* params,
+    Hasher* hasher, int* dist_cache, size_t* last_insert_len,
     Command* commands, size_t* num_commands, size_t* num_literals) {
+  HASHER()* privat = &hasher->privat.FN(_);
   /* Set maximum distance, see section 9.1. of the spec. */
   const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin);
+  const size_t position_offset = params->stream_offset;
 
   const Command* const orig_commands = commands;
   size_t insert_length = *last_insert_len;
@@ -26,24 +28,42 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
   const size_t random_heuristics_window_size =
       LiteralSpreeLengthForSparseSearch(params);
   size_t apply_random_heuristics = position + random_heuristics_window_size;
-  const size_t gap = 0;
+  const size_t gap = params->dictionary.compound.total_size;
 
   /* Minimum score to accept a backward reference. */
   const score_t kMinScore = BROTLI_SCORE_BASE + 100;
 
-  FN(PrepareDistanceCache)(hasher, dist_cache);
+  FN(PrepareDistanceCache)(privat, dist_cache);
 
   while (position + FN(HashTypeLength)() < pos_end) {
     size_t max_length = pos_end - position;
     size_t max_distance = BROTLI_MIN(size_t, position, max_backward_limit);
+    size_t dictionary_start = BROTLI_MIN(size_t,
+        position + position_offset, max_backward_limit);
     HasherSearchResult sr;
+    int dict_id = 0;
+    uint8_t p1 = 0;
+    uint8_t p2 = 0;
+    if (params->dictionary.contextual.context_based) {
+      p1 = position >= 1 ?
+          ringbuffer[(size_t)(position - 1) & ringbuffer_mask] : 0;
+      p2 = position >= 2 ?
+          ringbuffer[(size_t)(position - 2) & ringbuffer_mask] : 0;
+      dict_id = params->dictionary.contextual.context_map[
+          BROTLI_CONTEXT(p1, p2, literal_context_lut)];
+    }
     sr.len = 0;
     sr.len_code_delta = 0;
     sr.distance = 0;
     sr.score = kMinScore;
-    FN(FindLongestMatch)(hasher, &params->dictionary,
+    FN(FindLongestMatch)(privat, params->dictionary.contextual.dict[dict_id],
         ringbuffer, ringbuffer_mask, dist_cache, position, max_length,
-        max_distance, gap, params->dist.max_distance, &sr);
+        max_distance, dictionary_start + gap, params->dist.max_distance, &sr);
+    if (ENABLE_COMPOUND_DICTIONARY) {
+      LookupCompoundDictionaryMatch(&params->dictionary.compound, ringbuffer,
+          ringbuffer_mask, dist_cache, position, max_length,
+          dictionary_start, params->dist.max_distance, &sr);
+    }
     if (sr.score > kMinScore) {
       /* Found a match. Let's look for something even better ahead. */
       int delayed_backward_references_in_row = 0;
@@ -57,10 +77,25 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
         sr2.distance = 0;
         sr2.score = kMinScore;
         max_distance = BROTLI_MIN(size_t, position + 1, max_backward_limit);
-        FN(FindLongestMatch)(hasher,
-            &params->dictionary,
+        dictionary_start = BROTLI_MIN(size_t,
+            position + 1 + position_offset, max_backward_limit);
+        if (params->dictionary.contextual.context_based) {
+          p2 = p1;
+          p1 = ringbuffer[position & ringbuffer_mask];
+          dict_id = params->dictionary.contextual.context_map[
+              BROTLI_CONTEXT(p1, p2, literal_context_lut)];
+        }
+        FN(FindLongestMatch)(privat,
+            params->dictionary.contextual.dict[dict_id],
             ringbuffer, ringbuffer_mask, dist_cache, position + 1, max_length,
-            max_distance, gap, params->dist.max_distance, &sr2);
+            max_distance, dictionary_start + gap, params->dist.max_distance,
+            &sr2);
+        if (ENABLE_COMPOUND_DICTIONARY) {
+          LookupCompoundDictionaryMatch(
+              &params->dictionary.compound, ringbuffer,
+              ringbuffer_mask, dist_cache, position + 1, max_length,
+              dictionary_start, params->dist.max_distance, &sr2);
+        }
         if (sr2.score >= sr.score + cost_diff_lazy) {
           /* Ok, let's just write one byte for now and start a match from the
              next byte. */
@@ -76,18 +111,19 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
       }
       apply_random_heuristics =
           position + 2 * sr.len + random_heuristics_window_size;
-      max_distance = BROTLI_MIN(size_t, position, max_backward_limit);
+      dictionary_start = BROTLI_MIN(size_t,
+          position + position_offset, max_backward_limit);
       {
         /* The first 16 codes are special short-codes,
            and the minimum offset is 1. */
         size_t distance_code = ComputeDistanceCode(
-            sr.distance, max_distance + gap, dist_cache);
-        if ((sr.distance <= (max_distance + gap)) && distance_code > 0) {
+            sr.distance, dictionary_start + gap, dist_cache);
+        if ((sr.distance <= (dictionary_start + gap)) && distance_code > 0) {
           dist_cache[3] = dist_cache[2];
           dist_cache[2] = dist_cache[1];
           dist_cache[1] = dist_cache[0];
           dist_cache[0] = (int)sr.distance;
-          FN(PrepareDistanceCache)(hasher, dist_cache);
+          FN(PrepareDistanceCache)(privat, dist_cache);
         }
         InitCommand(commands++, &params->dist, insert_length,
             sr.len, sr.len_code_delta, distance_code);
@@ -105,7 +141,7 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
           range_start = BROTLI_MIN(size_t, range_end, BROTLI_MAX(size_t,
               range_start, position + sr.len - (sr.distance << 2)));
         }
-        FN(StoreRange)(hasher, ringbuffer, ringbuffer_mask, range_start,
+        FN(StoreRange)(privat, ringbuffer, ringbuffer_mask, range_start,
                        range_end);
       }
       position += sr.len;
@@ -131,7 +167,7 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
           size_t pos_jump =
               BROTLI_MIN(size_t, position + 16, pos_end - kMargin);
           for (; position < pos_jump; position += 4) {
-            FN(Store)(hasher, ringbuffer, ringbuffer_mask, position);
+            FN(Store)(privat, ringbuffer, ringbuffer_mask, position);
             insert_length += 4;
           }
         } else {
@@ -140,7 +176,7 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
           size_t pos_jump =
               BROTLI_MIN(size_t, position + 8, pos_end - kMargin);
           for (; position < pos_jump; position += 2) {
-            FN(Store)(hasher, ringbuffer, ringbuffer_mask, position);
+            FN(Store)(privat, ringbuffer, ringbuffer_mask, position);
             insert_length += 2;
           }
         }

data/vendor/brotli/c/enc/bit_cost.c

@@ -6,28 +6,29 @@
 
 /* Functions to estimate the bit cost of Huffman trees. */
 
-#include "./bit_cost.h"
+#include "bit_cost.h"
+
+#include <brotli/types.h>
 
 #include "../common/constants.h"
 #include "../common/platform.h"
-#include <brotli/types.h>
-#include "./fast_log.h"
-#include "./histogram.h"
+#include "fast_log.h"
+#include "histogram.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
 #endif
 
 #define FN(X) X ## Literal
-#include "./bit_cost_inc.h"  /* NOLINT(build/include) */
+#include "bit_cost_inc.h"  /* NOLINT(build/include) */
 #undef FN
 
 #define FN(X) X ## Command
-#include "./bit_cost_inc.h"  /* NOLINT(build/include) */
+#include "bit_cost_inc.h"  /* NOLINT(build/include) */
 #undef FN
 
 #define FN(X) X ## Distance
-#include "./bit_cost_inc.h"  /* NOLINT(build/include) */
+#include "bit_cost_inc.h"  /* NOLINT(build/include) */
 #undef FN
 
 #if defined(__cplusplus) || defined(c_plusplus)

data/vendor/brotli/c/enc/bit_cost.h

@@ -9,10 +9,11 @@
 #ifndef BROTLI_ENC_BIT_COST_H_
 #define BROTLI_ENC_BIT_COST_H_
 
-#include "../common/platform.h"
 #include <brotli/types.h>
-#include "./fast_log.h"
-#include "./histogram.h"
+
+#include "../common/platform.h"
+#include "fast_log.h"
+#include "histogram.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
@@ -45,7 +46,7 @@ static BROTLI_INLINE double BitsEntropy(
     const uint32_t* population, size_t size) {
   size_t sum;
   double retval = ShannonEntropy(population, size, &sum);
-  if (retval < sum) {
+  if (retval < (double)sum) {
     /* At least one bit per literal is needed. */
     retval = (double)sum;
   }

data/vendor/brotli/c/enc/block_splitter.c

@@ -6,18 +6,18 @@
 
 /* Block split point selection utilities. */
 
-#include "./block_splitter.h"
+#include "block_splitter.h"
 
 #include <string.h>  /* memcpy, memset */
 
 #include "../common/platform.h"
-#include "./bit_cost.h"
-#include "./cluster.h"
-#include "./command.h"
-#include "./fast_log.h"
-#include "./histogram.h"
-#include "./memory.h"
-#include "./quality.h"
+#include "bit_cost.h"
+#include "cluster.h"
+#include "command.h"
+#include "fast_log.h"
+#include "histogram.h"
+#include "memory.h"
+#include "quality.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
@@ -30,6 +30,7 @@ static const double kCommandBlockSwitchCost = 13.5;
 static const double kDistanceBlockSwitchCost = 14.6;
 static const size_t kLiteralStrideLength = 70;
 static const size_t kCommandStrideLength = 40;
+static const size_t kDistanceStrideLength = 40;
 static const size_t kSymbolsPerLiteralHistogram = 544;
 static const size_t kSymbolsPerCommandHistogram = 530;
 static const size_t kSymbolsPerDistanceHistogram = 544;
@@ -89,19 +90,19 @@ static BROTLI_INLINE double BitCost(size_t count) {
 #define FN(X) X ## Literal
 #define DataType uint8_t
 /* NOLINTNEXTLINE(build/include) */
-#include "./block_splitter_inc.h"
+#include "block_splitter_inc.h"
 #undef DataType
 #undef FN
 
 #define FN(X) X ## Command
 #define DataType uint16_t
 /* NOLINTNEXTLINE(build/include) */
-#include "./block_splitter_inc.h"
+#include "block_splitter_inc.h"
 #undef FN
 
 #define FN(X) X ## Distance
 /* NOLINTNEXTLINE(build/include) */
-#include "./block_splitter_inc.h"
+#include "block_splitter_inc.h"
 #undef DataType
 #undef FN
 
@@ -119,6 +120,8 @@ void BrotliDestroyBlockSplit(MemoryManager* m, BlockSplit* self) {
   BROTLI_FREE(m, self->lengths);
 }
 
+/* Extracts literals, command distance and prefix codes, then applies
+ * SplitByteVector to create partitioning. */
 void BrotliSplitBlock(MemoryManager* m,
                       const Command* cmds,
                       const size_t num_commands,
@@ -132,11 +135,13 @@ void BrotliSplitBlock(MemoryManager* m,
   {
     size_t literals_count = CountLiterals(cmds, num_commands);
     uint8_t* literals = BROTLI_ALLOC(m, uint8_t, literals_count);
-    if (BROTLI_IS_OOM(m)) return;
+    if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(literals)) return;
     /* Create a continuous array of literals. */
     CopyLiteralsToByteArray(cmds, num_commands, data, pos, mask, literals);
     /* Create the block split on the array of literals.
-       Literal histograms have alphabet size 256. */
+     * Literal histograms can have alphabet size up to 256.
+     * Though, to accomodate context modeling, less than half of maximum size
+     * is allowed. */
     SplitByteVectorLiteral(
         m, literals, literals_count,
         kSymbolsPerLiteralHistogram, kMaxLiteralHistograms,
@@ -144,13 +149,17 @@ void BrotliSplitBlock(MemoryManager* m,
         literal_split);
     if (BROTLI_IS_OOM(m)) return;
     BROTLI_FREE(m, literals);
+    /* NB: this might be a good place for injecting extra splitting without
+     *     increasing encoder complexity; however, output parition would be less
+     *     optimal than one produced with forced splitting inside
+     *     SplitByteVector (FindBlocks / ClusterBlocks). */
   }
 
   {
     /* Compute prefix codes for commands. */
     uint16_t* insert_and_copy_codes = BROTLI_ALLOC(m, uint16_t, num_commands);
     size_t i;
-    if (BROTLI_IS_OOM(m)) return;
+    if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(insert_and_copy_codes)) return;
     for (i = 0; i < num_commands; ++i) {
       insert_and_copy_codes[i] = cmds[i].cmd_prefix_;
     }
@@ -161,7 +170,7 @@ void BrotliSplitBlock(MemoryManager* m,
         kCommandStrideLength, kCommandBlockSwitchCost, params,
         insert_and_copy_split);
     if (BROTLI_IS_OOM(m)) return;
-    /* TODO: reuse for distances? */
+    /* TODO(eustas): reuse for distances? */
     BROTLI_FREE(m, insert_and_copy_codes);
   }
 
@@ -170,7 +179,7 @@ void BrotliSplitBlock(MemoryManager* m,
     uint16_t* distance_prefixes = BROTLI_ALLOC(m, uint16_t, num_commands);
     size_t j = 0;
     size_t i;
-    if (BROTLI_IS_OOM(m)) return;
+    if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(distance_prefixes)) return;
     for (i = 0; i < num_commands; ++i) {
       const Command* cmd = &cmds[i];
       if (CommandCopyLen(cmd) && cmd->cmd_prefix_ >= 128) {
@@ -181,13 +190,27 @@ void BrotliSplitBlock(MemoryManager* m,
     SplitByteVectorDistance(
         m, distance_prefixes, j,
         kSymbolsPerDistanceHistogram, kMaxCommandHistograms,
-        kCommandStrideLength, kDistanceBlockSwitchCost, params,
+        kDistanceStrideLength, kDistanceBlockSwitchCost, params,
         dist_split);
     if (BROTLI_IS_OOM(m)) return;
     BROTLI_FREE(m, distance_prefixes);
   }
 }
 
+#if defined(BROTLI_TEST)
+size_t CountLiteralsForTest(const Command*, const size_t);
+size_t CountLiteralsForTest(const Command* cmds, const size_t num_commands) {
+  return CountLiterals(cmds, num_commands);
+}
+
+void CopyLiteralsToByteArrayForTest(const Command*,
+    const size_t, const uint8_t*, const size_t, const size_t, uint8_t*);
+void CopyLiteralsToByteArrayForTest(const Command* cmds,
+    const size_t num_commands, const uint8_t* data, const size_t offset,
+    const size_t mask, uint8_t* literals) {
+  CopyLiteralsToByteArray(cmds, num_commands, data, offset, mask, literals);
+}
+#endif
 
 #if defined(__cplusplus) || defined(c_plusplus)
 }  /* extern "C" */

data/vendor/brotli/c/enc/block_splitter.h

@@ -9,11 +9,12 @@
 #ifndef BROTLI_ENC_BLOCK_SPLITTER_H_
 #define BROTLI_ENC_BLOCK_SPLITTER_H_
 
-#include "../common/platform.h"
 #include <brotli/types.h>
-#include "./command.h"
-#include "./memory.h"
-#include "./quality.h"
+
+#include "../common/platform.h"
+#include "command.h"
+#include "memory.h"
+#include "quality.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {