brotli 0.1.1 → 0.1.2

data/vendor/brotli/enc/backward_references.cc

@@ -18,12 +18,13 @@
 
 namespace brotli {
 
-static const double kInfinity = std::numeric_limits<double>::infinity();
+// The maximum length for which the zopflification uses distinct distances.
+static const uint16_t kMaxZopfliLen = 325;
 
 // Histogram based cost model for zopflification.
 class ZopfliCostModel {
  public:
-  ZopfliCostModel() : min_cost_cmd_(kInfinity) {}
+  ZopfliCostModel(void) : min_cost_cmd_(kInfinity) {}
 
   void SetFromCommands(size_t num_bytes,
                        size_t position,
@@ -31,34 +32,34 @@ class ZopfliCostModel {
                        size_t ringbuffer_mask,
                        const Command* commands,
                        size_t num_commands,
-                       int last_insert_len) {
-    std::vector<int> histogram_literal(256, 0);
-    std::vector<int> histogram_cmd(kNumCommandPrefixes, 0);
-    std::vector<int> histogram_dist(kNumDistancePrefixes, 0);
+                       size_t last_insert_len) {
+    std::vector<uint32_t> histogram_literal(256, 0);
+    std::vector<uint32_t> histogram_cmd(kNumCommandPrefixes, 0);
+    std::vector<uint32_t> histogram_dist(kNumDistancePrefixes, 0);
 
     size_t pos = position - last_insert_len;
     for (size_t i = 0; i < num_commands; i++) {
-      int inslength = commands[i].insert_len_;
-      int copylength = commands[i].copy_len_;
-      int distcode = commands[i].dist_prefix_;
-      int cmdcode = commands[i].cmd_prefix_;
+      size_t inslength = commands[i].insert_len_;
+      size_t copylength = commands[i].copy_len();
+      size_t distcode = commands[i].dist_prefix_;
+      size_t cmdcode = commands[i].cmd_prefix_;
 
       histogram_cmd[cmdcode]++;
       if (cmdcode >= 128) histogram_dist[distcode]++;
 
-      for (int j = 0; j < inslength; j++) {
+      for (size_t j = 0; j < inslength; j++) {
         histogram_literal[ringbuffer[(pos + j) & ringbuffer_mask]]++;
       }
 
       pos += inslength + copylength;
     }
 
-    std::vector<double> cost_literal;
+    std::vector<float> cost_literal;
     Set(histogram_literal, &cost_literal);
     Set(histogram_cmd, &cost_cmd_);
     Set(histogram_dist, &cost_dist_);
 
-    for (int i = 0; i < kNumCommandPrefixes; ++i) {
+    for (uint32_t i = 0; i < kNumCommandPrefixes; ++i) {
       min_cost_cmd_ = std::min(min_cost_cmd_, cost_cmd_[i]);
     }
 
@@ -74,27 +75,26 @@ class ZopfliCostModel {
                            size_t position,
                            const uint8_t* ringbuffer,
                            size_t ringbuffer_mask) {
-    std::vector<float> literal_cost(num_bytes + 1);
+    literal_costs_.resize(num_bytes + 2);
     EstimateBitCostsForLiterals(position, num_bytes, ringbuffer_mask,
-                                ringbuffer, &literal_cost[0]);
-    literal_costs_.resize(num_bytes + 1);
+                                ringbuffer, &literal_costs_[1]);
     literal_costs_[0] = 0.0;
     for (size_t i = 0; i < num_bytes; ++i) {
-      literal_costs_[i + 1] = literal_costs_[i] + literal_cost[i];
+      literal_costs_[i + 1] += literal_costs_[i];
     }
     cost_cmd_.resize(kNumCommandPrefixes);
     cost_dist_.resize(kNumDistancePrefixes);
-    for (int i = 0; i < kNumCommandPrefixes; ++i) {
-      cost_cmd_[i] = FastLog2(11 + i);
+    for (uint32_t i = 0; i < kNumCommandPrefixes; ++i) {
+      cost_cmd_[i] = static_cast<float>(FastLog2(11 + i));
     }
-    for (int i = 0; i < kNumDistancePrefixes; ++i) {
-      cost_dist_[i] = FastLog2(20 + i);
+    for (uint32_t i = 0; i < kNumDistancePrefixes; ++i) {
+      cost_dist_[i] = static_cast<float>(FastLog2(20 + i));
     }
-    min_cost_cmd_ = FastLog2(11);
+    min_cost_cmd_ = static_cast<float>(FastLog2(11));
   }
 
-  double GetCommandCost(
-      int dist_code, int length_code, int insert_length) const {
+  float GetCommandCost(
+      size_t dist_code, size_t length_code, size_t insert_length) const {
     uint16_t inscode = GetInsertLengthCode(insert_length);
     uint16_t copycode = GetCopyLengthCode(length_code);
     uint16_t cmdcode = CombineLengthCodes(inscode, copycode, dist_code == 0);
@@ -103,28 +103,29 @@ class ZopfliCostModel {
     PrefixEncodeCopyDistance(dist_code, 0, 0, &dist_symbol, &distextra);
     uint32_t distnumextra = distextra >> 24;
 
-    double result = insextra[inscode] + copyextra[copycode] + distnumextra;
+    float result = static_cast<float>(
+        GetInsertExtra(inscode) + GetCopyExtra(copycode) + distnumextra);
     result += cost_cmd_[cmdcode];
     if (cmdcode >= 128) result += cost_dist_[dist_symbol];
     return result;
   }
 
-  double GetLiteralCosts(size_t from, size_t to) const {
+  float GetLiteralCosts(size_t from, size_t to) const {
     return literal_costs_[to] - literal_costs_[from];
   }
 
-  double GetMinCostCmd() const {
+  float GetMinCostCmd(void) const {
     return min_cost_cmd_;
   }
 
  private:
-  void Set(const std::vector<int>& histogram, std::vector<double>* cost) {
+  void Set(const std::vector<uint32_t>& histogram, std::vector<float>* cost) {
     cost->resize(histogram.size());
-    int sum = 0;
+    size_t sum = 0;
     for (size_t i = 0; i < histogram.size(); i++) {
       sum += histogram[i];
     }
-    double log2sum = FastLog2(sum);
+    float log2sum = static_cast<float>(FastLog2(sum));
     for (size_t i = 0; i < histogram.size(); i++) {
       if (histogram[i] == 0) {
         (*cost)[i] = log2sum + 2;
@@ -132,54 +133,42 @@ class ZopfliCostModel {
       }
 
       // Shannon bits for this symbol.
-      (*cost)[i] = log2sum - FastLog2(histogram[i]);
+      (*cost)[i] = log2sum - static_cast<float>(FastLog2(histogram[i]));
 
       // Cannot be coded with less than 1 bit
       if ((*cost)[i] < 1) (*cost)[i] = 1;
     }
   }
 
-  std::vector<double> cost_cmd_;  // The insert and copy length symbols.
-  std::vector<double> cost_dist_;
+  std::vector<float> cost_cmd_;  // The insert and copy length symbols.
+  std::vector<float> cost_dist_;
   // Cumulative costs of literals per position in the stream.
-  std::vector<double> literal_costs_;
-  double min_cost_cmd_;
+  std::vector<float> literal_costs_;
+  float min_cost_cmd_;
 };
 
-inline void SetDistanceCache(int distance,
-                             int distance_code,
-                             int max_distance,
-                             const int* dist_cache,
-                             int* result_dist_cache) {
-  if (distance <= max_distance && distance_code > 0) {
-    result_dist_cache[0] = distance;
-    memcpy(&result_dist_cache[1], dist_cache, 3 * sizeof(dist_cache[0]));
-  } else {
-    memcpy(result_dist_cache, dist_cache, 4 * sizeof(dist_cache[0]));
-  }
-}
-
-inline int ComputeDistanceCode(int distance,
-                               int max_distance,
-                               int quality,
-                               const int* dist_cache) {
+inline size_t ComputeDistanceCode(size_t distance,
+                                  size_t max_distance,
+                                  int quality,
+                                  const int* dist_cache) {
   if (distance <= max_distance) {
-    if (distance == dist_cache[0]) {
+    if (distance == static_cast<size_t>(dist_cache[0])) {
       return 0;
-    } else if (distance == dist_cache[1]) {
+    } else if (distance == static_cast<size_t>(dist_cache[1])) {
       return 1;
-    } else if (distance == dist_cache[2]) {
+    } else if (distance == static_cast<size_t>(dist_cache[2])) {
       return 2;
-    } else if (distance == dist_cache[3]) {
+    } else if (distance == static_cast<size_t>(dist_cache[3])) {
       return 3;
     } else if (quality > 3 && distance >= 6) {
-      for (int k = 4; k < kNumDistanceShortCodes; ++k) {
-        int idx = kDistanceCacheIndex[k];
-        int candidate = dist_cache[idx] + kDistanceCacheOffset[k];
-        static const int kLimits[16] = { 0, 0, 0, 0,
-                                         6, 6, 11, 11,
-                                         11, 11, 11, 11,
-                                         12, 12, 12, 12 };
+      for (size_t k = 4; k < kNumDistanceShortCodes; ++k) {
+        size_t idx = kDistanceCacheIndex[k];
+        size_t candidate =
+            static_cast<size_t>(dist_cache[idx] + kDistanceCacheOffset[k]);
+        static const size_t kLimits[16] = {  0,  0,  0,  0,
+                                             6,  6, 11, 11,
+                                            11, 11, 11, 11,
+                                            12, 12, 12, 12 };
         if (distance == candidate && distance >= kLimits[k]) {
           return k;
         }
@@ -189,99 +178,78 @@ inline int ComputeDistanceCode(int distance,
   return distance + 15;
 }
 
-struct ZopfliNode {
-  ZopfliNode() : length(1),
-                 distance(0),
-                 distance_code(0),
-                 length_code(0),
-                 insert_length(0),
-                 cost(kInfinity) {}
-
-  // best length to get up to this byte (not including this byte itself)
-  int length;
-  // distance associated with the length
-  int distance;
-  int distance_code;
-  int distance_cache[4];
-  // length code associated with the length - usually the same as length,
-  // except in case of length-changing dictionary transformation.
-  int length_code;
-  // number of literal inserts before this copy
-  int insert_length;
-  // smallest cost to get to this byte from the beginning, as found so far
-  double cost;
-};
-
+// REQUIRES: len >= 2, start_pos <= pos
+// REQUIRES: cost < kInfinity, nodes[start_pos].cost < kInfinity
+// Maintains the "ZopfliNode array invariant".
 inline void UpdateZopfliNode(ZopfliNode* nodes, size_t pos, size_t start_pos,
-                             int len, int len_code, int dist, int dist_code,
-                             int max_dist, const int* dist_cache,
-                             double cost) {
+                             size_t len, size_t len_code, size_t dist,
+                             size_t short_code, float cost) {
   ZopfliNode& next = nodes[pos + len];
-  next.length = len;
-  next.length_code = len_code;
-  next.distance = dist;
-  next.distance_code = dist_code;
-  next.insert_length = static_cast<int>(pos - start_pos);
+  next.length = static_cast<uint32_t>(len | ((len + 9u - len_code) << 24));
+  next.distance = static_cast<uint32_t>(dist | (short_code << 25));
+  next.insert_length = static_cast<uint32_t>(pos - start_pos);
   next.cost = cost;
-  SetDistanceCache(dist, dist_code, max_dist, dist_cache,
-                   &next.distance_cache[0]);
 }
 
 // Maintains the smallest 2^k cost difference together with their positions
 class StartPosQueue {
  public:
+  struct PosData {
+    size_t pos;
+    int distance_cache[4];
+    float costdiff;
+  };
+
   explicit StartPosQueue(int bits)
-      : mask_((1 << bits) - 1), q_(1 << bits), idx_(0) {}
+      : mask_((1u << bits) - 1), q_(1 << bits), idx_(0) {}
 
-  void Clear() {
+  void Clear(void) {
     idx_ = 0;
   }
 
-  void Push(size_t pos, double costdiff) {
-    if (costdiff == kInfinity) {
-      // We can't start a command from an unreachable start position.
-      // E.g. position 1 in a stream is always unreachable, because all commands
-      // have a copy of at least length 2.
-      return;
-    }
-    q_[idx_ & mask_] = std::make_pair(pos, costdiff);
-    // Restore the sorted order.
-    for (int i = idx_; i > 0 && i > idx_ - mask_; --i) {
-      if (q_[i & mask_].second > q_[(i - 1) & mask_].second) {
-        std::swap(q_[i & mask_], q_[(i - 1) & mask_]);
+  void Push(const StartPosQueue::PosData& posdata) {
+    size_t offset = ~idx_ & mask_;
+    ++idx_;
+    size_t len = size();
+    q_[offset] = posdata;
+    /* Restore the sorted order. In the list of |len| items at most |len - 1|
+       adjacent element comparisons / swaps are required. */
+    for (size_t i = 1; i < len; ++i) {
+      if (q_[offset & mask_].costdiff > q_[(offset + 1) & mask_].costdiff) {
+        std::swap(q_[offset & mask_], q_[(offset + 1) & mask_]);
       }
+      ++offset;
     }
-    ++idx_;
   }
 
-  int size() const { return std::min(idx_, mask_ + 1); }
+  size_t size(void) const { return std::min(idx_, mask_ + 1); }
 
-  size_t GetStartPos(int k) const {
-    return q_[(idx_ - k - 1) & mask_].first;
+  const StartPosQueue::PosData& GetStartPosData(size_t k) const {
+    return q_[(k - idx_) & mask_];
   }
 
  private:
-  const int mask_;
-  std::vector<std::pair<size_t, double> > q_;
-  int idx_;
+  const size_t mask_;
+  std::vector<PosData> q_;
+  size_t idx_;
 };
 
 // Returns the minimum possible copy length that can improve the cost of any
 // future position.
-int ComputeMinimumCopyLength(const StartPosQueue& queue,
-                             const std::vector<ZopfliNode>& nodes,
-                             const ZopfliCostModel& model,
-                             size_t pos,
-                             double min_cost_cmd) {
+static size_t ComputeMinimumCopyLength(const StartPosQueue& queue,
+                                       const ZopfliNode* nodes,
+                                       const ZopfliCostModel& model,
+                                       const size_t num_bytes,
+                                       const size_t pos) {
   // Compute the minimum possible cost of reaching any future position.
-  const size_t start0 = queue.GetStartPos(0);
-  double min_cost = (nodes[start0].cost +
-                     model.GetLiteralCosts(start0, pos) +
-                     min_cost_cmd);
-  int len = 2;
-  int next_len_bucket = 4;
-  int next_len_offset = 10;
-  while (pos + len < nodes.size() && nodes[pos + len].cost <= min_cost) {
+  const size_t start0 = queue.GetStartPosData(0).pos;
+  float min_cost = (nodes[start0].cost +
+                    model.GetLiteralCosts(start0, pos) +
+                    model.GetMinCostCmd());
+  size_t len = 2;
+  size_t next_len_bucket = 4;
+  size_t next_len_offset = 10;
+  while (pos + len <= num_bytes && nodes[pos + len].cost <= min_cost) {
     // We already reached (pos + len) with no more cost than the minimum
     // possible cost of reaching anything from this pos, so there is no point in
     // looking for lengths <= len.
@@ -289,7 +257,7 @@ int ComputeMinimumCopyLength(const StartPosQueue& queue,
     if (len == next_len_offset) {
       // We reached the next copy length code bucket, so we add one more
       // extra bit to the minimum cost.
-      min_cost += 1.0;
+      min_cost += static_cast<float>(1.0);
       next_len_offset += next_len_bucket;
       next_len_bucket *= 2;
     }
@@ -297,193 +265,309 @@ int ComputeMinimumCopyLength(const StartPosQueue& queue,
   return len;
 }
 
-void ZopfliIterate(size_t num_bytes,
-                   size_t position,
-                   const uint8_t* ringbuffer,
-                   size_t ringbuffer_mask,
-                   const size_t max_backward_limit,
-                   const ZopfliCostModel& model,
-                   const std::vector<int>& num_matches,
-                   const std::vector<BackwardMatch>& matches,
-                   int* dist_cache,
-                   int* last_insert_len,
-                   Command* commands,
-                   size_t* num_commands,
-                   int* num_literals) {
-  const Command * const orig_commands = commands;
-
-  std::vector<ZopfliNode> nodes(num_bytes + 1);
-  nodes[0].length = 0;
-  nodes[0].cost = 0;
-  memcpy(nodes[0].distance_cache, dist_cache, 4 * sizeof(dist_cache[0]));
-
-  StartPosQueue queue(3);
-  const double min_cost_cmd = model.GetMinCostCmd();
+// Fills in dist_cache[0..3] with the last four distances (as defined by
+// Section 4. of the Spec) that would be used at (block_start + pos) if we
+// used the shortest path of commands from block_start, computed from
+// nodes[0..pos]. The last four distances at block_start are in
+// starting_dist_cach[0..3].
+// REQUIRES: nodes[pos].cost < kInfinity
+// REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant".
+static void ComputeDistanceCache(const size_t block_start,
+                                 const size_t pos,
+                                 const size_t max_backward,
+                                 const int* starting_dist_cache,
+                                 const ZopfliNode* nodes,
+                                 int* dist_cache) {
+  int idx = 0;
+  size_t p = pos;
+  // Because of prerequisite, does at most (pos + 1) / 2 iterations.
+  while (idx < 4 && p > 0) {
+    const size_t clen = nodes[p].copy_length();
+    const size_t ilen = nodes[p].insert_length;
+    const size_t dist = nodes[p].copy_distance();
+    // Since block_start + p is the end position of the command, the copy part
+    // starts from block_start + p - clen. Distances that are greater than this
+    // or greater than max_backward are static dictionary references, and do
+    // not update the last distances. Also distance code 0 (last distance)
+    // does not update the last distances.
+    if (dist + clen <= block_start + p && dist <= max_backward &&
+        nodes[p].distance_code() > 0) {
+      dist_cache[idx++] = static_cast<int>(dist);
+    }
+    // Because of prerequisite, p >= clen + ilen >= 2.
+    p -= clen + ilen;
+  }
+  for (; idx < 4; ++idx) {
+    dist_cache[idx] = *starting_dist_cache++;
+  }
+}
 
-  size_t cur_match_pos = 0;
-  for (size_t i = 0; i + 3 < num_bytes; i++) {
-    size_t cur_ix = position + i;
-    size_t cur_ix_masked = cur_ix & ringbuffer_mask;
-    int max_distance = static_cast<int>(std::min(cur_ix, max_backward_limit));
-    int max_length = static_cast<int>(num_bytes - i);
-
-    queue.Push(i, nodes[i].cost - model.GetLiteralCosts(0, i));
-
-    const int min_len = ComputeMinimumCopyLength(queue, nodes, model,
-                                                 i, min_cost_cmd);
-
-    // Go over the command starting positions in order of increasing cost
-    // difference.
-    for (int k = 0; k < 5 && k < queue.size(); ++k) {
-      const size_t start = queue.GetStartPos(k);
-      const double start_costdiff =
-          nodes[start].cost - model.GetLiteralCosts(0, start);
-      const int* dist_cache2 = &nodes[start].distance_cache[0];
-
-      // Look for last distance matches using the distance cache from this
-      // starting position.
-      int best_len = min_len - 1;
-      for (int j = 0; j < kNumDistanceShortCodes; ++j) {
-        const int idx = kDistanceCacheIndex[j];
-        const int backward = dist_cache2[idx] + kDistanceCacheOffset[j];
-        size_t prev_ix = cur_ix - backward;
-        if (prev_ix >= cur_ix) {
-          continue;
-        }
-        if (PREDICT_FALSE(backward > max_distance)) {
-          continue;
-        }
-        prev_ix &= ringbuffer_mask;
+static void UpdateNodes(const size_t num_bytes,
+                        const size_t block_start,
+                        const size_t pos,
+                        const uint8_t* ringbuffer,
+                        const size_t ringbuffer_mask,
+                        const size_t max_backward_limit,
+                        const int* starting_dist_cache,
+                        const size_t num_matches,
+                        const BackwardMatch* matches,
+                        const ZopfliCostModel* model,
+                        StartPosQueue* queue,
+                        ZopfliNode* nodes) {
+  size_t cur_ix = block_start + pos;
+  size_t cur_ix_masked = cur_ix & ringbuffer_mask;
+  size_t max_distance = std::min(cur_ix, max_backward_limit);
+
+  if (nodes[pos].cost <= model->GetLiteralCosts(0, pos)) {
+    StartPosQueue::PosData posdata;
+    posdata.pos = pos;
+    posdata.costdiff = nodes[pos].cost - model->GetLiteralCosts(0, pos);
+    ComputeDistanceCache(block_start, pos, max_backward_limit,
+                         starting_dist_cache, nodes, posdata.distance_cache);
+    queue->Push(posdata);
+  }
 
-        if (cur_ix_masked + best_len > ringbuffer_mask ||
-            prev_ix + best_len > ringbuffer_mask ||
-            ringbuffer[cur_ix_masked + best_len] !=
-            ringbuffer[prev_ix + best_len]) {
-          continue;
-        }
-        const int len =
-            FindMatchLengthWithLimit(&ringbuffer[prev_ix],
-                                     &ringbuffer[cur_ix_masked],
-                                     max_length);
-        for (int l = best_len + 1; l <= len; ++l) {
-          const int inslen = static_cast<int>(i - start);
-          double cmd_cost = model.GetCommandCost(j, l, inslen);
-          double cost = start_costdiff + cmd_cost + model.GetLiteralCosts(0, i);
-          if (cost < nodes[i + l].cost) {
-            UpdateZopfliNode(&nodes[0], i, start, l, l, backward, j,
-                             max_distance, dist_cache2, cost);
-          }
-          best_len = l;
-        }
+  const size_t min_len = ComputeMinimumCopyLength(
+      *queue, nodes, *model, num_bytes, pos);
+
+  // Go over the command starting positions in order of increasing cost
+  // difference.
+  for (size_t k = 0; k < 5 && k < queue->size(); ++k) {
+    const StartPosQueue::PosData& posdata = queue->GetStartPosData(k);
+    const size_t start = posdata.pos;
+    const float start_costdiff = posdata.costdiff;
+
+    // Look for last distance matches using the distance cache from this
+    // starting position.
+    size_t best_len = min_len - 1;
+    for (size_t j = 0; j < kNumDistanceShortCodes; ++j) {
+      const size_t idx = kDistanceCacheIndex[j];
+      const size_t backward = static_cast<size_t>(posdata.distance_cache[idx] +
+                                                  kDistanceCacheOffset[j]);
+      size_t prev_ix = cur_ix - backward;
+      if (prev_ix >= cur_ix) {
+        continue;
+      }
+      if (PREDICT_FALSE(backward > max_distance)) {
+        continue;
       }
+      prev_ix &= ringbuffer_mask;
 
-      // At higher iterations look only for new last distance matches, since
-      // looking only for new command start positions with the same distances
-      // does not help much.
-      if (k >= 2) continue;
-
-      // Loop through all possible copy lengths at this position.
-      int len = min_len;
-      for (int j = 0; j < num_matches[i]; ++j) {
-        BackwardMatch match = matches[cur_match_pos + j];
-        int dist = match.distance;
-        bool is_dictionary_match = dist > max_distance;
-        // We already tried all possible last distance matches, so we can use
-        // normal distance code here.
-        int dist_code = dist + 15;
-        // Try all copy lengths up until the maximum copy length corresponding
-        // to this distance. If the distance refers to the static dictionary, or
-        // the maximum length is long enough, try only one maximum length.
-        int max_len = match.length();
-        if (len < max_len && (is_dictionary_match || max_len > kMaxZopfliLen)) {
-          len = max_len;
-        }
-        for (; len <= max_len; ++len) {
-          int len_code = is_dictionary_match ? match.length_code() : len;
-          const int inslen = static_cast<int>(i - start);
-          double cmd_cost = model.GetCommandCost(dist_code, len_code, inslen);
-          double cost = start_costdiff + cmd_cost + model.GetLiteralCosts(0, i);
-          if (cost < nodes[i + len].cost) {
-            UpdateZopfliNode(&nodes[0], i, start, len, len_code, dist,
-                             dist_code, max_distance, dist_cache2, cost);
-          }
+      if (cur_ix_masked + best_len > ringbuffer_mask ||
+          prev_ix + best_len > ringbuffer_mask ||
+          ringbuffer[cur_ix_masked + best_len] !=
+          ringbuffer[prev_ix + best_len]) {
+        continue;
+      }
+      const size_t len =
+          FindMatchLengthWithLimit(&ringbuffer[prev_ix],
+                                   &ringbuffer[cur_ix_masked],
+                                   num_bytes - pos);
+      for (size_t l = best_len + 1; l <= len; ++l) {
+        const size_t inslen = pos - start;
+        float cmd_cost = model->GetCommandCost(j, l, inslen);
+        float cost = start_costdiff + cmd_cost + model->GetLiteralCosts(0, pos);
+        if (cost < nodes[pos + l].cost) {
+          UpdateZopfliNode(&nodes[0], pos, start, l, l, backward, j + 1, cost);
         }
+        best_len = l;
       }
     }
 
-    cur_match_pos += num_matches[i];
-
-    // The zopflification can be too slow in case of very long lengths, so in
-    // such case skip it all, it does not cost a lot of compression ratio.
-    if (num_matches[i] == 1 &&
-        matches[cur_match_pos - 1].length() > kMaxZopfliLen) {
-      i += matches[cur_match_pos - 1].length() - 1;
-      queue.Clear();
+    // At higher iterations look only for new last distance matches, since
+    // looking only for new command start positions with the same distances
+    // does not help much.
+    if (k >= 2) continue;
+
+    // Loop through all possible copy lengths at this position.
+    size_t len = min_len;
+    for (size_t j = 0; j < num_matches; ++j) {
+      BackwardMatch match = matches[j];
+      size_t dist = match.distance;
+      bool is_dictionary_match = dist > max_distance;
+      // We already tried all possible last distance matches, so we can use
+      // normal distance code here.
+      size_t dist_code = dist + 15;
+      // Try all copy lengths up until the maximum copy length corresponding
+      // to this distance. If the distance refers to the static dictionary, or
+      // the maximum length is long enough, try only one maximum length.
+      size_t max_len = match.length();
+      if (len < max_len && (is_dictionary_match || max_len > kMaxZopfliLen)) {
+        len = max_len;
+      }
+      for (; len <= max_len; ++len) {
+        size_t len_code = is_dictionary_match ? match.length_code() : len;
+        const size_t inslen = pos - start;
+        float cmd_cost = model->GetCommandCost(dist_code, len_code, inslen);
+        float cost = start_costdiff + cmd_cost + model->GetLiteralCosts(0, pos);
+        if (cost < nodes[pos + len].cost) {
+          UpdateZopfliNode(&nodes[0], pos, start, len, len_code, dist, 0, cost);
+        }
+      }
     }
   }
+}
 
-  std::vector<int> backwards;
+static void ComputeShortestPathFromNodes(size_t num_bytes,
+                                         const ZopfliNode* nodes,
+                                         std::vector<uint32_t>* path) {
+  std::vector<uint32_t> backwards(num_bytes / 2 + 1);
   size_t index = num_bytes;
   while (nodes[index].cost == kInfinity) --index;
-  while (index > 0) {
-    int len = nodes[index].length + nodes[index].insert_length;
-    backwards.push_back(len);
+  size_t num_commands = 0;
+  while (index != 0) {
+    size_t len = nodes[index].command_length();
+    backwards[num_commands++] = static_cast<uint32_t>(len);
     index -= len;
   }
-
-  std::vector<int> path;
-  for (size_t i = backwards.size(); i > 0; i--) {
-    path.push_back(backwards[i - 1]);
+  path->resize(num_commands);
+  for (size_t i = num_commands, j = 0; i > 0; --i, ++j) {
+    (*path)[j] = backwards[i - 1];
   }
+}
 
+void ZopfliCreateCommands(const size_t num_bytes,
+                          const size_t block_start,
+                          const size_t max_backward_limit,
+                          const std::vector<uint32_t>& path,
+                          const ZopfliNode* nodes,
+                          int* dist_cache,
+                          size_t* last_insert_len,
+                          Command* commands,
+                          size_t* num_literals) {
   size_t pos = 0;
   for (size_t i = 0; i < path.size(); i++) {
     const ZopfliNode& next = nodes[pos + path[i]];
-    int copy_length = next.length;
-    int insert_length = next.insert_length;
+    size_t copy_length = next.copy_length();
+    size_t insert_length = next.insert_length;
     pos += insert_length;
     if (i == 0) {
       insert_length += *last_insert_len;
       *last_insert_len = 0;
     }
-    int distance = next.distance;
-    int len_code = next.length_code;
-    int max_distance =
-        static_cast<int>(std::min(position + pos, max_backward_limit));
+    size_t distance = next.copy_distance();
+    size_t len_code = next.length_code();
+    size_t max_distance = std::min(block_start + pos, max_backward_limit);
     bool is_dictionary = (distance > max_distance);
-    int dist_code = next.distance_code;
+    size_t dist_code = next.distance_code();
 
     Command cmd(insert_length, copy_length, len_code, dist_code);
-    *commands++ = cmd;
+    commands[i] = cmd;
 
     if (!is_dictionary && dist_code > 0) {
       dist_cache[3] = dist_cache[2];
       dist_cache[2] = dist_cache[1];
       dist_cache[1] = dist_cache[0];
-      dist_cache[0] = distance;
+      dist_cache[0] = static_cast<int>(distance);
     }
 
     *num_literals += insert_length;
-    insert_length = 0;
     pos += copy_length;
   }
-  *last_insert_len += static_cast<int>(num_bytes - pos);
-  *num_commands += (commands - orig_commands);
+  *last_insert_len += num_bytes - pos;
+}
+
+static void ZopfliIterate(size_t num_bytes,
+                          size_t position,
+                          const uint8_t* ringbuffer,
+                          size_t ringbuffer_mask,
+                          const size_t max_backward_limit,
+                          const int* dist_cache,
+                          const ZopfliCostModel& model,
+                          const std::vector<uint32_t>& num_matches,
+                          const std::vector<BackwardMatch>& matches,
+                          ZopfliNode* nodes,
+                          std::vector<uint32_t>* path) {
+  nodes[0].length = 0;
+  nodes[0].cost = 0;
+  StartPosQueue queue(3);
+  size_t cur_match_pos = 0;
+  for (size_t i = 0; i + 3 < num_bytes; i++) {
+    UpdateNodes(num_bytes, position, i, ringbuffer, ringbuffer_mask,
+                max_backward_limit, dist_cache, num_matches[i],
+                &matches[cur_match_pos], &model, &queue, &nodes[0]);
+    cur_match_pos += num_matches[i];
+    // The zopflification can be too slow in case of very long lengths, so in
+    // such case skip it all, it does not cost a lot of compression ratio.
+    if (num_matches[i] == 1 &&
+        matches[cur_match_pos - 1].length() > kMaxZopfliLen) {
+      i += matches[cur_match_pos - 1].length() - 1;
+      queue.Clear();
+    }
+  }
+  ComputeShortestPathFromNodes(num_bytes, &nodes[0], path);
+}
+
+
+void ZopfliComputeShortestPath(size_t num_bytes,
+                               size_t position,
+                               const uint8_t* ringbuffer,
+                               size_t ringbuffer_mask,
+                               const size_t max_backward_limit,
+                               const int* dist_cache,
+                               Hashers::H10* hasher,
+                               ZopfliNode* nodes,
+                               std::vector<uint32_t>* path) {
+  nodes[0].length = 0;
+  nodes[0].cost = 0;
+  ZopfliCostModel* model = new ZopfliCostModel;
+  model->SetFromLiteralCosts(num_bytes, position,
+                             ringbuffer, ringbuffer_mask);
+  StartPosQueue queue(3);
+  BackwardMatch matches[Hashers::H10::kMaxNumMatches];
+  for (size_t i = 0; i + 3 < num_bytes; i++) {
+    const size_t max_distance = std::min(position + i, max_backward_limit);
+    size_t num_matches = hasher->FindAllMatches(
+        ringbuffer, ringbuffer_mask, position + i, num_bytes - i, max_distance,
+        matches);
+    if (num_matches > 0 &&
+        matches[num_matches - 1].length() > kMaxZopfliLen) {
+      matches[0] = matches[num_matches - 1];
+      num_matches = 1;
+    }
+    UpdateNodes(num_bytes, position, i, ringbuffer, ringbuffer_mask,
+                max_backward_limit, dist_cache, num_matches, matches,
+                model, &queue, nodes);
+    if (num_matches == 1 && matches[0].length() > kMaxZopfliLen) {
+      for (size_t j = 1; j < matches[0].length() && i + 4 < num_bytes; ++j) {
+        ++i;
+        if (matches[0].length() - j < 64 &&
+            num_bytes - i >= kMaxTreeCompLength) {
+          hasher->Store(ringbuffer, ringbuffer_mask, position + i);
+        }
+      }
+      queue.Clear();
+    }
+  }
+  delete model;
+  ComputeShortestPathFromNodes(num_bytes, nodes, path);
 }
 
 template<typename Hasher>
 void CreateBackwardReferences(size_t num_bytes,
                               size_t position,
+                              bool is_last,
                               const uint8_t* ringbuffer,
                               size_t ringbuffer_mask,
-                              const size_t max_backward_limit,
                               const int quality,
+                              const int lgwin,
                               Hasher* hasher,
                               int* dist_cache,
-                              int* last_insert_len,
+                              size_t* last_insert_len,
                               Command* commands,
                               size_t* num_commands,
-                              int* num_literals) {
+                              size_t* num_literals) {
+  // Set maximum distance, see section 9.1. of the spec.
+  const size_t max_backward_limit = (1 << lgwin) - 16;
+
+  // Choose which init method is faster.
+  // memset is about 100 times faster than hasher->InitForData().
+  const size_t kMaxBytesForPartialHashInit = Hasher::kHashMapSize >> 7;
+  if (position == 0 && is_last && num_bytes <= kMaxBytesForPartialHashInit) {
+    hasher->InitForData(ringbuffer, num_bytes);
+  } else {
+    hasher->Init();
+  }
   if (num_bytes >= 3 && position >= 3) {
     // Prepare the hashes for three last bytes of the last write.
     // These could not be calculated before, since they require knowledge
@@ -496,7 +580,7 @@ void CreateBackwardReferences(size_t num_bytes,
                   static_cast<uint32_t>(position - 1));
   }
   const Command * const orig_commands = commands;
-  int insert_length = *last_insert_len;
+  size_t insert_length = *last_insert_len;
   size_t i = position & ringbuffer_mask;
   const size_t i_diff = position - i;
   const size_t i_end = i + num_bytes;
@@ -506,15 +590,14 @@ void CreateBackwardReferences(size_t num_bytes,
   size_t apply_random_heuristics = i + random_heuristics_window_size;
 
   // Minimum score to accept a backward reference.
-  const int kMinScore = 4.0;
+  const double kMinScore = 4.0;
 
   while (i + Hasher::kHashTypeLength - 1 < i_end) {
-    int max_length = static_cast<int>(i_end - i);
-    int max_distance =
-        static_cast<int>(std::min(i + i_diff, max_backward_limit));
-    int best_len = 0;
-    int best_len_code = 0;
-    int best_dist = 0;
+    size_t max_length = i_end - i;
+    size_t max_distance = std::min(i + i_diff, max_backward_limit);
+    size_t best_len = 0;
+    size_t best_len_code = 0;
+    size_t best_dist = 0;
     double best_score = kMinScore;
     bool match_found = hasher->FindLongestMatch(
         ringbuffer, ringbuffer_mask,
@@ -525,13 +608,12 @@ void CreateBackwardReferences(size_t num_bytes,
       int delayed_backward_references_in_row = 0;
       for (;;) {
         --max_length;
-        int best_len_2 = quality < 5 ? std::min(best_len - 1, max_length) : 0;
-        int best_len_code_2 = 0;
-        int best_dist_2 = 0;
+        size_t best_len_2 =
+            quality < 5 ? std::min(best_len - 1, max_length) : 0;
+        size_t best_len_code_2 = 0;
+        size_t best_dist_2 = 0;
         double best_score_2 = kMinScore;
-        max_distance =
-            static_cast<int>(std::min(i + i_diff + 1, max_backward_limit));
-        hasher->Store(ringbuffer + i, static_cast<uint32_t>(i + i_diff));
+        max_distance = std::min(i + i_diff + 1, max_backward_limit);
         match_found = hasher->FindLongestMatch(
             ringbuffer, ringbuffer_mask,
             dist_cache, static_cast<uint32_t>(i + i_diff + 1),
@@ -555,15 +637,15 @@ void CreateBackwardReferences(size_t num_bytes,
       }
       apply_random_heuristics =
           i + 2 * best_len + random_heuristics_window_size;
-      max_distance = static_cast<int>(std::min(i + i_diff, max_backward_limit));
+      max_distance = std::min(i + i_diff, max_backward_limit);
       // The first 16 codes are special shortcodes, and the minimum offset is 1.
-      int distance_code =
+      size_t distance_code =
           ComputeDistanceCode(best_dist, max_distance, quality, dist_cache);
       if (best_dist <= max_distance && distance_code > 0) {
         dist_cache[3] = dist_cache[2];
         dist_cache[2] = dist_cache[1];
         dist_cache[1] = dist_cache[0];
-        dist_cache[0] = best_dist;
+        dist_cache[0] = static_cast<int>(best_dist);
       }
       Command cmd(insert_length, best_len, best_len_code, distance_code);
       *commands++ = cmd;
@@ -571,14 +653,13 @@ void CreateBackwardReferences(size_t num_bytes,
       insert_length = 0;
       // Put the hash keys into the table, if there are enough
       // bytes left.
-      for (int j = 1; j < best_len; ++j) {
+      for (size_t j = 2; j < best_len; ++j) {
         hasher->Store(&ringbuffer[i + j],
                       static_cast<uint32_t>(i + i_diff + j));
       }
       i += best_len;
     } else {
       ++insert_length;
-      hasher->Store(ringbuffer + i, static_cast<uint32_t>(i + i_diff));
       ++i;
       // If we have not seen matches for a long time, we can skip some
       // match lookups. Unsuccessful match lookups are very very expensive
@@ -608,76 +689,91 @@ void CreateBackwardReferences(size_t num_bytes,
       }
     }
   }
-  insert_length += static_cast<int>(i_end - i);
+  insert_length += i_end - i;
   *last_insert_len = insert_length;
-  *num_commands += commands - orig_commands;
+  *num_commands += static_cast<size_t>(commands - orig_commands);
 }
 
 void CreateBackwardReferences(size_t num_bytes,
                               size_t position,
+                              bool is_last,
                               const uint8_t* ringbuffer,
                               size_t ringbuffer_mask,
-                              const size_t max_backward_limit,
                               const int quality,
+                              const int lgwin,
                               Hashers* hashers,
                               int hash_type,
                               int* dist_cache,
-                              int* last_insert_len,
+                              size_t* last_insert_len,
                               Command* commands,
                               size_t* num_commands,
-                              int* num_literals) {
+                              size_t* num_literals) {
   bool zopflify = quality > 9;
   if (zopflify) {
-    Hashers::H9* hasher = hashers->hash_h9;
-    if (num_bytes >= 3 && position >= 3) {
-      // Prepare the hashes for three last bytes of the last write.
-      // These could not be calculated before, since they require knowledge
-      // of both the previous and the current block.
-      hasher->Store(&ringbuffer[(position - 3) & ringbuffer_mask],
-                    static_cast<uint32_t>(position - 3));
-      hasher->Store(&ringbuffer[(position - 2) & ringbuffer_mask],
-                    static_cast<uint32_t>(position - 2));
-      hasher->Store(&ringbuffer[(position - 1) & ringbuffer_mask],
-                    static_cast<uint32_t>(position - 1));
+    Hashers::H10* hasher = hashers->hash_h10;
+    hasher->Init(lgwin, position, num_bytes, is_last);
+    hasher->StitchToPreviousBlock(num_bytes, position,
+                                  ringbuffer, ringbuffer_mask);
+    // Set maximum distance, see section 9.1. of the spec.
+    const size_t max_backward_limit = (1 << lgwin) - 16;
+    if (quality == 10) {
+      std::vector<ZopfliNode> nodes(num_bytes + 1);
+      std::vector<uint32_t> path;
+      ZopfliComputeShortestPath(num_bytes, position,
+                                ringbuffer, ringbuffer_mask,
+                                max_backward_limit, dist_cache, hasher,
+                                &nodes[0], &path);
+      ZopfliCreateCommands(num_bytes, position, max_backward_limit, path,
+                           &nodes[0], dist_cache, last_insert_len, commands,
+                           num_literals);
+      *num_commands += path.size();
+      return;
     }
-    std::vector<int> num_matches(num_bytes);
-    std::vector<BackwardMatch> matches(3 * num_bytes);
+    std::vector<uint32_t> num_matches(num_bytes);
+    std::vector<BackwardMatch> matches(4 * num_bytes);
     size_t cur_match_pos = 0;
     for (size_t i = 0; i + 3 < num_bytes; ++i) {
-      int max_distance =
-          static_cast<int>(std::min(position + i, max_backward_limit));
-      int max_length = static_cast<int>(num_bytes - i);
-      // Ensure that we have at least kMaxZopfliLen free slots.
-      if (matches.size() < cur_match_pos + kMaxZopfliLen) {
-        matches.resize(cur_match_pos + kMaxZopfliLen);
+      size_t max_distance = std::min(position + i, max_backward_limit);
+      size_t max_length = num_bytes - i;
+      // Ensure that we have enough free slots.
+      if (matches.size() < cur_match_pos + Hashers::H10::kMaxNumMatches) {
+        matches.resize(cur_match_pos + Hashers::H10::kMaxNumMatches);
+      }
+      size_t num_found_matches = hasher->FindAllMatches(
+          ringbuffer, ringbuffer_mask, position + i, max_length, max_distance,
+          &matches[cur_match_pos]);
+      const size_t cur_match_end = cur_match_pos + num_found_matches;
+      for (size_t j = cur_match_pos; j + 1 < cur_match_end; ++j) {
+        assert(matches[j].length() < matches[j + 1].length());
+        assert(matches[j].distance > max_distance ||
+               matches[j].distance <= matches[j + 1].distance);
       }
-      hasher->FindAllMatches(
-          ringbuffer, ringbuffer_mask,
-          static_cast<uint32_t>(position + i), max_length, max_distance,
-          &num_matches[i], &matches[cur_match_pos]);
-      hasher->Store(&ringbuffer[(position + i) & ringbuffer_mask],
-                    static_cast<uint32_t>(position + i));
-      cur_match_pos += num_matches[i];
-      if (num_matches[i] == 1) {
-        const int match_len = matches[cur_match_pos - 1].length();
+      num_matches[i] = static_cast<uint32_t>(num_found_matches);
+      if (num_found_matches > 0) {
+        const size_t match_len = matches[cur_match_end - 1].length();
         if (match_len > kMaxZopfliLen) {
-          for (int j = 1; j < match_len; ++j) {
+          matches[cur_match_pos++] = matches[cur_match_end - 1];
+          num_matches[i] = 1;
+          for (size_t j = 1; j < match_len; ++j) {
             ++i;
-            hasher->Store(&ringbuffer[(position + i) & ringbuffer_mask],
-                          static_cast<uint32_t>(position + i));
+            if (match_len - j < 64 && num_bytes - i >= kMaxTreeCompLength) {
+              hasher->Store(ringbuffer, ringbuffer_mask, position + i);
+            }
             num_matches[i] = 0;
           }
+        } else {
+          cur_match_pos = cur_match_end;
         }
       }
     }
-    int orig_num_literals = *num_literals;
-    int orig_last_insert_len = *last_insert_len;
+    size_t orig_num_literals = *num_literals;
+    size_t orig_last_insert_len = *last_insert_len;
     int orig_dist_cache[4] = {
       dist_cache[0], dist_cache[1], dist_cache[2], dist_cache[3]
     };
     size_t orig_num_commands = *num_commands;
-    static const int kIterations = 2;
-    for (int i = 0; i < kIterations; i++) {
+    static const size_t kIterations = 2;
+    for (size_t i = 0; i < kIterations; i++) {
       ZopfliCostModel model;
       if (i == 0) {
         model.SetFromLiteralCosts(num_bytes, position,
@@ -692,67 +788,67 @@ void CreateBackwardReferences(size_t num_bytes,
       *num_literals = orig_num_literals;
       *last_insert_len = orig_last_insert_len;
       memcpy(dist_cache, orig_dist_cache, 4 * sizeof(dist_cache[0]));
+      std::vector<ZopfliNode> nodes(num_bytes + 1);
+      std::vector<uint32_t> path;
       ZopfliIterate(num_bytes, position, ringbuffer, ringbuffer_mask,
-                    max_backward_limit, model, num_matches, matches, dist_cache,
-                    last_insert_len, commands, num_commands, num_literals);
+                    max_backward_limit, dist_cache, model, num_matches, matches,
+                    &nodes[0], &path);
+      ZopfliCreateCommands(num_bytes, position, max_backward_limit, path,
+                           &nodes[0], dist_cache, last_insert_len, commands,
+                           num_literals);
+      *num_commands += path.size();
     }
     return;
   }
 
   switch (hash_type) {
-    case 1:
-      CreateBackwardReferences<Hashers::H1>(
-          num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
-          quality, hashers->hash_h1, dist_cache, last_insert_len,
-          commands, num_commands, num_literals);
-      break;
     case 2:
       CreateBackwardReferences<Hashers::H2>(
-          num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
-          quality, hashers->hash_h2, dist_cache, last_insert_len,
-          commands, num_commands, num_literals);
+          num_bytes, position, is_last, ringbuffer, ringbuffer_mask,
+          quality, lgwin, hashers->hash_h2, dist_cache,
+          last_insert_len, commands, num_commands, num_literals);
       break;
     case 3:
       CreateBackwardReferences<Hashers::H3>(
-          num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
-          quality, hashers->hash_h3, dist_cache, last_insert_len,
-          commands, num_commands, num_literals);
+          num_bytes, position, is_last, ringbuffer, ringbuffer_mask,
+          quality, lgwin, hashers->hash_h3, dist_cache,
+          last_insert_len, commands, num_commands, num_literals);
       break;
     case 4:
       CreateBackwardReferences<Hashers::H4>(
-          num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
-          quality, hashers->hash_h4, dist_cache, last_insert_len,
-          commands, num_commands, num_literals);
+          num_bytes, position, is_last, ringbuffer, ringbuffer_mask,
+          quality, lgwin, hashers->hash_h4, dist_cache,
+          last_insert_len, commands, num_commands, num_literals);
       break;
     case 5:
       CreateBackwardReferences<Hashers::H5>(
-          num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
-          quality, hashers->hash_h5, dist_cache, last_insert_len,
-          commands, num_commands, num_literals);
+          num_bytes, position, is_last, ringbuffer, ringbuffer_mask,
+          quality, lgwin, hashers->hash_h5, dist_cache,
+          last_insert_len, commands, num_commands, num_literals);
       break;
     case 6:
       CreateBackwardReferences<Hashers::H6>(
-          num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
-          quality, hashers->hash_h6, dist_cache, last_insert_len,
-          commands, num_commands, num_literals);
+          num_bytes, position, is_last, ringbuffer, ringbuffer_mask,
+          quality, lgwin, hashers->hash_h6, dist_cache,
+          last_insert_len, commands, num_commands, num_literals);
       break;
     case 7:
       CreateBackwardReferences<Hashers::H7>(
-          num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
-          quality, hashers->hash_h7, dist_cache, last_insert_len,
-          commands, num_commands, num_literals);
+          num_bytes, position, is_last, ringbuffer, ringbuffer_mask,
+          quality, lgwin, hashers->hash_h7, dist_cache,
+          last_insert_len, commands, num_commands, num_literals);
       break;
     case 8:
       CreateBackwardReferences<Hashers::H8>(
-          num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
-          quality, hashers->hash_h8, dist_cache, last_insert_len,
-          commands, num_commands, num_literals);
+          num_bytes, position, is_last, ringbuffer, ringbuffer_mask,
+          quality, lgwin, hashers->hash_h8, dist_cache,
+          last_insert_len, commands, num_commands, num_literals);
       break;
     case 9:
       CreateBackwardReferences<Hashers::H9>(
-          num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
-          quality, hashers->hash_h9, dist_cache, last_insert_len,
-          commands, num_commands, num_literals);
+          num_bytes, position, is_last, ringbuffer, ringbuffer_mask,
+          quality, lgwin, hashers->hash_h9, dist_cache,
+          last_insert_len, commands, num_commands, num_literals);
       break;
     default:
       break;