brotli 0.1.3 → 0.1.4

data/vendor/brotli/enc/backward_references.h

@@ -4,113 +4,96 @@
    See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
 */
 
-// Function to find backward reference copies.
+/* Function to find backward reference copies. */
 
 #ifndef BROTLI_ENC_BACKWARD_REFERENCES_H_
 #define BROTLI_ENC_BACKWARD_REFERENCES_H_
 
-#include <vector>
-
-#include "./hash.h"
+#include "../common/types.h"
 #include "./command.h"
-#include "./types.h"
-
-namespace brotli {
-
-// "commands" points to the next output command to write to, "*num_commands" is
-// initially the total amount of commands output by previous
-// CreateBackwardReferences calls, and must be incremented by the amount written
-// by this call.
-void CreateBackwardReferences(size_t num_bytes,
-                              size_t position,
-                              bool is_last,
-                              const uint8_t* ringbuffer,
-                              size_t ringbuffer_mask,
-                              const int quality,
-                              const int lgwin,
-                              Hashers* hashers,
-                              int hash_type,
-                              int* dist_cache,
-                              size_t* last_insert_len,
-                              Command* commands,
-                              size_t* num_commands,
-                              size_t* num_literals);
-
-static const float kInfinity = std::numeric_limits<float>::infinity();
-
-struct ZopfliNode {
-  ZopfliNode(void) : length(1),
-                     distance(0),
-                     insert_length(0),
-                     cost(kInfinity) {}
-
-  inline uint32_t copy_length() const {
-    return length & 0xffffff;
-  }
-
-  inline uint32_t length_code() const {
-    const uint32_t modifier = length >> 24;
-    return copy_length() + 9u - modifier;
-  }
-
-  inline uint32_t copy_distance() const {
-    return distance & 0x1ffffff;
-  }
-
-  inline uint32_t distance_code() const {
-    const uint32_t short_code = distance >> 25;
-    return short_code == 0 ? copy_distance() + 15 : short_code - 1;
-  }
-
-  inline uint32_t command_length() const {
-    return copy_length() + insert_length;
-  }
-
-  // best length to get up to this byte (not including this byte itself)
-  // highest 8 bit is used to reconstruct the length code
+#include "./hash.h"
+#include "./memory.h"
+#include "./port.h"
+#include "./quality.h"
+
+#if defined(__cplusplus) || defined(c_plusplus)
+extern "C" {
+#endif
+
+/* "commands" points to the next output command to write to, "*num_commands" is
+   initially the total amount of commands output by previous
+   CreateBackwardReferences calls, and must be incremented by the amount written
+   by this call. */
+BROTLI_INTERNAL void BrotliCreateBackwardReferences(
+    MemoryManager* m, size_t num_bytes, size_t position, BROTLI_BOOL is_last,
+    const uint8_t* ringbuffer, size_t ringbuffer_mask,
+    const BrotliEncoderParams* params, Hashers* hashers, int* dist_cache,
+    size_t* last_insert_len, Command* commands, size_t* num_commands,
+    size_t* num_literals);
+
+typedef struct ZopfliNode {
+  /* best length to get up to this byte (not including this byte itself)
+     highest 8 bit is used to reconstruct the length code */
   uint32_t length;
-  // distance associated with the length
-  // highest 7 bit contains distance short code + 1 (or zero if no short code)
+  /* distance associated with the length
+     highest 7 bit contains distance short code + 1 (or zero if no short code)
+  */
   uint32_t distance;
-  // number of literal inserts before this copy
+  /* number of literal inserts before this copy */
   uint32_t insert_length;
-  // smallest cost to get to this byte from the beginning, as found so far
-  float cost;
-};
-
-// Computes the shortest path of commands from position to at most
-// position + num_bytes.
-//
-// On return, path->size() is the number of commands found and path[i] is the
-// length of the ith command (copy length plus insert length).
-// Note that the sum of the lengths of all commands can be less than num_bytes.
-//
-// On return, the nodes[0..num_bytes] array will have the following
-// "ZopfliNode array invariant":
-// For each i in [1..num_bytes], if nodes[i].cost < kInfinity, then
-//   (1) nodes[i].copy_length() >= 2
-//   (2) nodes[i].command_length() <= i and
-//   (3) nodes[i - nodes[i].command_length()].cost < kInfinity
-void ZopfliComputeShortestPath(size_t num_bytes,
-                               size_t position,
-                               const uint8_t* ringbuffer,
-                               size_t ringbuffer_mask,
-                               const size_t max_backward_limit,
-                               const int* dist_cache,
-                               Hashers::H10* hasher,
-                               ZopfliNode* nodes,
-                               std::vector<uint32_t>* path);
-
-void ZopfliCreateCommands(const size_t num_bytes,
-                          const size_t block_start,
-                          const size_t max_backward_limit,
-                          const std::vector<uint32_t>& path,
-                          const ZopfliNode* nodes,
-                          int* dist_cache,
-                          size_t* last_insert_len,
-                          Command* commands,
-                          size_t* num_literals);
-
-}  // namespace brotli
-
-#endif  // BROTLI_ENC_BACKWARD_REFERENCES_H_
+
+  /* This union holds information used by dynamic-programming. During forward
+     pass |cost| it used to store the goal function. When node is processed its
+     |cost| is invalidated in favor of |shortcut|. On path backtracing pass
+     |next| is assigned the offset to next node on the path. */
+  union {
+    /* Smallest cost to get to this byte from the beginning, as found so far. */
+    float cost;
+    /* Offset to the next node on the path. Equals to command_length() of the
+       next node on the path. For last node equals to BROTLI_UINT32_MAX */
+    uint32_t next;
+    /* Node position that provides next distance for distance cache. */
+    uint32_t shortcut;
+  } u;
+} ZopfliNode;
+
+BROTLI_INTERNAL void BrotliInitZopfliNodes(ZopfliNode* array, size_t length);
+
+/* Computes the shortest path of commands from position to at most
+   position + num_bytes.
+
+   On return, path->size() is the number of commands found and path[i] is the
+   length of the ith command (copy length plus insert length).
+   Note that the sum of the lengths of all commands can be less than num_bytes.
+
+   On return, the nodes[0..num_bytes] array will have the following
+   "ZopfliNode array invariant":
+   For each i in [1..num_bytes], if nodes[i].cost < kInfinity, then
+     (1) nodes[i].copy_length() >= 2
+     (2) nodes[i].command_length() <= i and
+     (3) nodes[i - nodes[i].command_length()].cost < kInfinity */
+BROTLI_INTERNAL size_t BrotliZopfliComputeShortestPath(
+    MemoryManager* m, size_t num_bytes, size_t position,
+    const uint8_t* ringbuffer, size_t ringbuffer_mask,
+    const BrotliEncoderParams* params, const size_t max_backward_limit,
+    const int* dist_cache, H10* hasher, ZopfliNode* nodes);
+
+BROTLI_INTERNAL void BrotliZopfliCreateCommands(const size_t num_bytes,
+                                                const size_t block_start,
+                                                const size_t max_backward_limit,
+                                                const ZopfliNode* nodes,
+                                                int* dist_cache,
+                                                size_t* last_insert_len,
+                                                Command* commands,
+                                                size_t* num_literals);
+
+/* Maximum distance, see section 9.1. of the spec. */
+static BROTLI_INLINE size_t MaxBackwardLimit(int lgwin) {
+  return (1u << lgwin) - 16;
+}
+
+#if defined(__cplusplus) || defined(c_plusplus)
+}  /* extern "C" */
+#endif
+
+#endif  /* BROTLI_ENC_BACKWARD_REFERENCES_H_ */

data/vendor/brotli/enc/backward_references_inc.h

@@ -0,0 +1,147 @@
+/* NOLINT(build/header_guard) */
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* template parameters: FN */
+
+#define Hasher HASHER()
+
+static BROTLI_NOINLINE void FN(CreateBackwardReferences)(
+    MemoryManager* m, size_t num_bytes, size_t position, BROTLI_BOOL is_last,
+    const uint8_t* ringbuffer, size_t ringbuffer_mask,
+    const BrotliEncoderParams* params, Hasher* hasher, int* dist_cache,
+    size_t* last_insert_len, Command* commands, size_t* num_commands,
+    size_t* num_literals) {
+  /* Set maximum distance, see section 9.1. of the spec. */
+  const size_t max_backward_limit = MaxBackwardLimit(params->lgwin);
+
+  const Command* const orig_commands = commands;
+  size_t insert_length = *last_insert_len;
+  const size_t pos_end = position + num_bytes;
+  const size_t store_end = num_bytes >= FN(StoreLookahead)() ?
+      position + num_bytes - FN(StoreLookahead)() + 1 : position;
+
+  /* For speed up heuristics for random data. */
+  const size_t random_heuristics_window_size =
+      LiteralSpreeLengthForSparseSearch(params);
+  size_t apply_random_heuristics = position + random_heuristics_window_size;
+
+  /* Minimum score to accept a backward reference. */
+  const score_t kMinScore = BROTLI_SCORE_BASE + 400;
+
+  FN(Init)(m, hasher, ringbuffer, params, position, num_bytes, is_last);
+  if (BROTLI_IS_OOM(m)) return;
+  FN(StitchToPreviousBlock)(hasher, num_bytes, position,
+                            ringbuffer, ringbuffer_mask);
+
+  while (position + FN(HashTypeLength)() < pos_end) {
+    size_t max_length = pos_end - position;
+    size_t max_distance = BROTLI_MIN(size_t, position, max_backward_limit);
+    HasherSearchResult sr;
+    sr.len = 0;
+    sr.len_x_code = 0;
+    sr.distance = 0;
+    sr.score = kMinScore;
+    if (FN(FindLongestMatch)(hasher, ringbuffer, ringbuffer_mask, dist_cache,
+                             position, max_length, max_distance, &sr)) {
+      /* Found a match. Let's look for something even better ahead. */
+      int delayed_backward_references_in_row = 0;
+      --max_length;
+      for (;; --max_length) {
+        const score_t cost_diff_lazy = 700;
+        BROTLI_BOOL is_match_found;
+        HasherSearchResult sr2;
+        sr2.len = params->quality < MIN_QUALITY_FOR_EXTENSIVE_REFERENCE_SEARCH ?
+            BROTLI_MIN(size_t, sr.len - 1, max_length) : 0;
+        sr2.len_x_code = 0;
+        sr2.distance = 0;
+        sr2.score = kMinScore;
+        max_distance = BROTLI_MIN(size_t, position + 1, max_backward_limit);
+        is_match_found = FN(FindLongestMatch)(hasher, ringbuffer,
+            ringbuffer_mask, dist_cache, position + 1, max_length, max_distance,
+            &sr2);
+        if (is_match_found && sr2.score >= sr.score + cost_diff_lazy) {
+          /* Ok, let's just write one byte for now and start a match from the
+             next byte. */
+          ++position;
+          ++insert_length;
+          sr = sr2;
+          if (++delayed_backward_references_in_row < 4 &&
+              position + FN(HashTypeLength)() < pos_end) {
+            continue;
+          }
+        }
+        break;
+      }
+      apply_random_heuristics =
+          position + 2 * sr.len + random_heuristics_window_size;
+      max_distance = BROTLI_MIN(size_t, position, max_backward_limit);
+      {
+        /* The first 16 codes are special shortcodes,
+           and the minimum offset is 1. */
+        size_t distance_code =
+            ComputeDistanceCode(sr.distance, max_distance, dist_cache);
+        if (sr.distance <= max_distance && distance_code > 0) {
+          dist_cache[3] = dist_cache[2];
+          dist_cache[2] = dist_cache[1];
+          dist_cache[1] = dist_cache[0];
+          dist_cache[0] = (int)sr.distance;
+        }
+        InitCommand(commands++, insert_length, sr.len, sr.len ^ sr.len_x_code,
+            distance_code);
+      }
+      *num_literals += insert_length;
+      insert_length = 0;
+      /* Put the hash keys into the table, if there are enough bytes left.
+         Depending on the hasher implementation, it can push all positions
+         in the given range or only a subset of them. */
+      FN(StoreRange)(hasher, ringbuffer, ringbuffer_mask, position + 2,
+                     BROTLI_MIN(size_t, position + sr.len, store_end));
+      position += sr.len;
+    } else {
+      ++insert_length;
+      ++position;
+      /* If we have not seen matches for a long time, we can skip some
+         match lookups. Unsuccessful match lookups are very very expensive
+         and this kind of a heuristic speeds up compression quite
+         a lot. */
+      if (position > apply_random_heuristics) {
+        /* Going through uncompressible data, jump. */
+        if (position >
+            apply_random_heuristics + 4 * random_heuristics_window_size) {
+          /* It is quite a long time since we saw a copy, so we assume
+             that this data is not compressible, and store hashes less
+             often. Hashes of non compressible data are less likely to
+             turn out to be useful in the future, too, so we store less of
+             them to not to flood out the hash table of good compressible
+             data. */
+          const size_t kMargin =
+              BROTLI_MAX(size_t, FN(StoreLookahead)() - 1, 4);
+          size_t pos_jump =
+              BROTLI_MIN(size_t, position + 16, pos_end - kMargin);
+          for (; position < pos_jump; position += 4) {
+            FN(Store)(hasher, ringbuffer, ringbuffer_mask, position);
+            insert_length += 4;
+          }
+        } else {
+          const size_t kMargin =
+              BROTLI_MAX(size_t, FN(StoreLookahead)() - 1, 2);
+          size_t pos_jump =
+              BROTLI_MIN(size_t, position + 8, pos_end - kMargin);
+          for (; position < pos_jump; position += 2) {
+            FN(Store)(hasher, ringbuffer, ringbuffer_mask, position);
+            insert_length += 2;
+          }
+        }
+      }
+    }
+  }
+  insert_length += pos_end - position;
+  *last_insert_len = insert_length;
+  *num_commands += (size_t)(commands - orig_commands);
+}
+
+#undef Hasher

data/vendor/brotli/enc/bit_cost.c

@@ -0,0 +1,35 @@
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Functions to estimate the bit cost of Huffman trees. */
+
+#include "./bit_cost.h"
+
+#include "../common/constants.h"
+#include "../common/types.h"
+#include "./fast_log.h"
+#include "./histogram.h"
+#include "./port.h"
+
+#if defined(__cplusplus) || defined(c_plusplus)
+extern "C" {
+#endif
+
+#define FN(X) X ## Literal
+#include "./bit_cost_inc.h"  /* NOLINT(build/include) */
+#undef FN
+
+#define FN(X) X ## Command
+#include "./bit_cost_inc.h"  /* NOLINT(build/include) */
+#undef FN
+
+#define FN(X) X ## Distance
+#include "./bit_cost_inc.h"  /* NOLINT(build/include) */
+#undef FN
+
+#if defined(__cplusplus) || defined(c_plusplus)
+}  /* extern "C" */
+#endif

data/vendor/brotli/enc/bit_cost.h

@@ -4,19 +4,22 @@
    See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
 */
 
-// Functions to estimate the bit cost of Huffman trees.
+/* Functions to estimate the bit cost of Huffman trees. */
 
 #ifndef BROTLI_ENC_BIT_COST_H_
 #define BROTLI_ENC_BIT_COST_H_
 
-#include "./entropy_encode.h"
+#include "../common/types.h"
 #include "./fast_log.h"
-#include "./types.h"
+#include "./histogram.h"
+#include "./port.h"
 
-namespace brotli {
+#if defined(__cplusplus) || defined(c_plusplus)
+extern "C" {
+#endif
 
-static inline double ShannonEntropy(const uint32_t *population, size_t size,
-                                    size_t *total) {
+static BROTLI_INLINE double ShannonEntropy(const uint32_t *population,
+                                           size_t size, size_t *total) {
   size_t sum = 0;
   double retval = 0;
   const uint32_t *population_end = population + size;
@@ -27,135 +30,34 @@ static inline double ShannonEntropy(const uint32_t *population, size_t size,
   while (population < population_end) {
     p = *population++;
     sum += p;
-    retval -= static_cast<double>(p) * FastLog2(p);
+    retval -= (double)p * FastLog2(p);
  odd_number_of_elements_left:
     p = *population++;
     sum += p;
-    retval -= static_cast<double>(p) * FastLog2(p);
+    retval -= (double)p * FastLog2(p);
   }
-  if (sum) retval += static_cast<double>(sum) * FastLog2(sum);
+  if (sum) retval += (double)sum * FastLog2(sum);
   *total = sum;
   return retval;
 }
 
-static inline double BitsEntropy(const uint32_t *population, size_t size) {
+static BROTLI_INLINE double BitsEntropy(
+    const uint32_t *population, size_t size) {
   size_t sum;
   double retval = ShannonEntropy(population, size, &sum);
   if (retval < sum) {
-    // At least one bit per literal is needed.
-    retval = static_cast<double>(sum);
+    /* At least one bit per literal is needed. */
+    retval = (double)sum;
   }
   return retval;
 }
 
-template<int kSize>
-double PopulationCost(const Histogram<kSize>& histogram) {
-  static const double kOneSymbolHistogramCost = 12;
-  static const double kTwoSymbolHistogramCost = 20;
-  static const double kThreeSymbolHistogramCost = 28;
-  static const double kFourSymbolHistogramCost = 37;
-  if (histogram.total_count_ == 0) {
-    return kOneSymbolHistogramCost;
-  }
-  int count = 0;
-  int s[5];
-  for (int i = 0; i < kSize; ++i) {
-    if (histogram.data_[i] > 0) {
-      s[count] = i;
-      ++count;
-      if (count > 4) break;
-    }
-  }
-  if (count == 1) {
-    return kOneSymbolHistogramCost;
-  }
-  if (count == 2) {
-    return (kTwoSymbolHistogramCost +
-            static_cast<double>(histogram.total_count_));
-  }
-  if (count == 3) {
-    const uint32_t histo0 = histogram.data_[s[0]];
-    const uint32_t histo1 = histogram.data_[s[1]];
-    const uint32_t histo2 = histogram.data_[s[2]];
-    const uint32_t histomax = std::max(histo0, std::max(histo1, histo2));
-    return (kThreeSymbolHistogramCost +
-            2 * (histo0 + histo1 + histo2) - histomax);
-  }
-  if (count == 4) {
-    uint32_t histo[4];
-    for (int i = 0; i < 4; ++i) {
-      histo[i] = histogram.data_[s[i]];
-    }
-    // Sort
-    for (int i = 0; i < 4; ++i) {
-      for (int j = i + 1; j < 4; ++j) {
-        if (histo[j] > histo[i]) {
-          std::swap(histo[j], histo[i]);
-        }
-      }
-    }
-    const uint32_t h23 = histo[2] + histo[3];
-    const uint32_t histomax = std::max(h23, histo[0]);
-    return (kFourSymbolHistogramCost +
-            3 * h23 + 2 * (histo[0] + histo[1]) - histomax);
-  }
-
-  // In this loop we compute the entropy of the histogram and simultaneously
-  // build a simplified histogram of the code length codes where we use the
-  // zero repeat code 17, but we don't use the non-zero repeat code 16.
-  double bits = 0;
-  size_t max_depth = 1;
-  uint32_t depth_histo[kCodeLengthCodes] = { 0 };
-  const double log2total = FastLog2(histogram.total_count_);
-  for (size_t i = 0; i < kSize;) {
-    if (histogram.data_[i] > 0) {
-      // Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) =
-      //                          =  log2(total_count) - log2(count(symbol))
-      double log2p = log2total - FastLog2(histogram.data_[i]);
-      // Approximate the bit depth by round(-log2(P(symbol)))
-      size_t depth = static_cast<size_t>(log2p + 0.5);
-      bits += histogram.data_[i] * log2p;
-      if (depth > 15) {
-        depth = 15;
-      }
-      if (depth > max_depth) {
-        max_depth = depth;
-      }
-      ++depth_histo[depth];
-      ++i;
-    } else {
-      // Compute the run length of zeros and add the appropriate number of 0 and
-      // 17 code length codes to the code length code histogram.
-      uint32_t reps = 1;
-      for (size_t k = i + 1; k < kSize && histogram.data_[k] == 0; ++k) {
-        ++reps;
-      }
-      i += reps;
-      if (i == kSize) {
-        // Don't add any cost for the last zero run, since these are encoded
-        // only implicitly.
-        break;
-      }
-      if (reps < 3) {
-        depth_histo[0] += reps;
-      } else {
-        reps -= 2;
-        while (reps > 0) {
-          ++depth_histo[17];
-          // Add the 3 extra bits for the 17 code length code.
-          bits += 3;
-          reps >>= 3;
-        }
-      }
-    }
-  }
-  // Add the estimated encoding cost of the code length code histogram.
-  bits += static_cast<double>(18 + 2 * max_depth);
-  // Add the entropy of the code length code histogram.
-  bits += BitsEntropy(depth_histo, kCodeLengthCodes);
-  return bits;
-}
+BROTLI_INTERNAL double BrotliPopulationCostLiteral(const HistogramLiteral*);
+BROTLI_INTERNAL double BrotliPopulationCostCommand(const HistogramCommand*);
+BROTLI_INTERNAL double BrotliPopulationCostDistance(const HistogramDistance*);
 
-}  // namespace brotli
+#if defined(__cplusplus) || defined(c_plusplus)
+}  /* extern "C" */
+#endif
 
-#endif  // BROTLI_ENC_BIT_COST_H_
+#endif  /* BROTLI_ENC_BIT_COST_H_ */