brotli 0.2.0 → 0.4.0

data/vendor/brotli/c/enc/backward_references.c

@@ -9,12 +9,13 @@
 #include "./backward_references.h"
 
 #include "../common/constants.h"
+#include "../common/context.h"
 #include "../common/dictionary.h"
+#include "../common/platform.h"
 #include <brotli/types.h>
 #include "./command.h"
 #include "./dictionary_hash.h"
 #include "./memory.h"
-#include "./port.h"
 #include "./quality.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
@@ -49,6 +50,7 @@ static BROTLI_INLINE size_t ComputeDistanceCode(size_t distance,
 #define CAT(a, b) a ## b
 #define FN(X) EXPAND_CAT(X, HASHER())
 #define EXPORT_FN(X) EXPAND_CAT(X, EXPAND_CAT(PREFIX(), HASHER()))
+
 #define PREFIX() N
 
 #define HASHER() H2
@@ -96,30 +98,39 @@ static BROTLI_INLINE size_t ComputeDistanceCode(size_t distance,
 #include "./backward_references_inc.h"
 #undef HASHER
 
+#define HASHER() H35
+/* NOLINTNEXTLINE(build/include) */
+#include "./backward_references_inc.h"
+#undef HASHER
+
+#define HASHER() H55
+/* NOLINTNEXTLINE(build/include) */
+#include "./backward_references_inc.h"
+#undef HASHER
+
+#define HASHER() H65
+/* NOLINTNEXTLINE(build/include) */
+#include "./backward_references_inc.h"
+#undef HASHER
+
 #undef PREFIX
+
 #undef EXPORT_FN
 #undef FN
 #undef CAT
 #undef EXPAND_CAT
 
-void BrotliCreateBackwardReferences(const BrotliDictionary* dictionary,
-                                    size_t num_bytes,
-                                    size_t position,
-                                    const uint8_t* ringbuffer,
-                                    size_t ringbuffer_mask,
-                                    const BrotliEncoderParams* params,
-                                    HasherHandle hasher,
-                                    int* dist_cache,
-                                    size_t* last_insert_len,
-                                    Command* commands,
-                                    size_t* num_commands,
-                                    size_t* num_literals) {
+void BrotliCreateBackwardReferences(size_t num_bytes,
+    size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
+    ContextLut literal_context_lut, const BrotliEncoderParams* params,
+    Hasher* hasher, int* dist_cache, size_t* last_insert_len,
+    Command* commands, size_t* num_commands, size_t* num_literals) {
   switch (params->hasher.type) {
 #define CASE_(N)                                                  \
     case N:                                                       \
-      CreateBackwardReferencesNH ## N(dictionary,                 \
-          kStaticDictionaryHash, num_bytes, position, ringbuffer, \
-          ringbuffer_mask, params, hasher, dist_cache,            \
+      CreateBackwardReferencesNH ## N(num_bytes,                  \
+          position, ringbuffer, ringbuffer_mask,                  \
+          literal_context_lut, params, hasher, dist_cache,        \
           last_insert_len, commands, num_commands, num_literals); \
       return;
     FOR_GENERIC_HASHERS(CASE_)

data/vendor/brotli/c/enc/backward_references.h

@@ -10,11 +10,12 @@
 #define BROTLI_ENC_BACKWARD_REFERENCES_H_
 
 #include "../common/constants.h"
+#include "../common/context.h"
 #include "../common/dictionary.h"
+#include "../common/platform.h"
 #include <brotli/types.h>
 #include "./command.h"
 #include "./hash.h"
-#include "./port.h"
 #include "./quality.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
@@ -25,12 +26,11 @@ extern "C" {
    initially the total amount of commands output by previous
    CreateBackwardReferences calls, and must be incremented by the amount written
    by this call. */
-BROTLI_INTERNAL void BrotliCreateBackwardReferences(
-    const BrotliDictionary* dictionary, size_t num_bytes, size_t position,
-    const uint8_t* ringbuffer, size_t ringbuffer_mask,
-    const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
-    size_t* last_insert_len, Command* commands, size_t* num_commands,
-    size_t* num_literals);
+BROTLI_INTERNAL void BrotliCreateBackwardReferences(size_t num_bytes,
+    size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
+    ContextLut literal_context_lut, const BrotliEncoderParams* params,
+    Hasher* hasher, int* dist_cache, size_t* last_insert_len,
+    Command* commands, size_t* num_commands, size_t* num_literals);
 
 #if defined(__cplusplus) || defined(c_plusplus)
 }  /* extern "C" */

data/vendor/brotli/c/enc/backward_references_hq.c

@@ -11,13 +11,15 @@
 #include <string.h>  /* memcpy, memset */
 
 #include "../common/constants.h"
+#include "../common/context.h"
+#include "../common/platform.h"
 #include <brotli/types.h>
 #include "./command.h"
 #include "./fast_log.h"
 #include "./find_match_length.h"
 #include "./literal_cost.h"
 #include "./memory.h"
-#include "./port.h"
+#include "./params.h"
 #include "./prefix.h"
 #include "./quality.h"
 
@@ -25,6 +27,9 @@
 extern "C" {
 #endif
 
+/* BrotliCalculateDistanceCodeLimit(BROTLI_MAX_ALLOWED_DISTANCE, 3, 120). */
+#define BROTLI_MAX_EFFECTIVE_DISTANCE_ALPHABET_SIZE 544
+
 static const float kInfinity = 1.7e38f;  /* ~= 2 ^ 127 */
 
 static const uint32_t kDistanceCacheIndex[] = {
@@ -39,40 +44,41 @@ void BrotliInitZopfliNodes(ZopfliNode* array, size_t length) {
   size_t i;
   stub.length = 1;
   stub.distance = 0;
-  stub.insert_length = 0;
+  stub.dcode_insert_length = 0;
   stub.u.cost = kInfinity;
   for (i = 0; i < length; ++i) array[i] = stub;
 }
 
 static BROTLI_INLINE uint32_t ZopfliNodeCopyLength(const ZopfliNode* self) {
-  return self->length & 0xffffff;
+  return self->length & 0x1FFFFFF;
 }
 
 static BROTLI_INLINE uint32_t ZopfliNodeLengthCode(const ZopfliNode* self) {
-  const uint32_t modifier = self->length >> 24;
+  const uint32_t modifier = self->length >> 25;
   return ZopfliNodeCopyLength(self) + 9u - modifier;
 }
 
 static BROTLI_INLINE uint32_t ZopfliNodeCopyDistance(const ZopfliNode* self) {
-  return self->distance & 0x1ffffff;
+  return self->distance;
 }
 
 static BROTLI_INLINE uint32_t ZopfliNodeDistanceCode(const ZopfliNode* self) {
-  const uint32_t short_code = self->distance >> 25;
+  const uint32_t short_code = self->dcode_insert_length >> 27;
   return short_code == 0 ?
       ZopfliNodeCopyDistance(self) + BROTLI_NUM_DISTANCE_SHORT_CODES - 1 :
       short_code - 1;
 }
 
 static BROTLI_INLINE uint32_t ZopfliNodeCommandLength(const ZopfliNode* self) {
-  return ZopfliNodeCopyLength(self) + self->insert_length;
+  return ZopfliNodeCopyLength(self) + (self->dcode_insert_length & 0x7FFFFFF);
 }
 
 /* Histogram based cost model for zopflification. */
 typedef struct ZopfliCostModel {
   /* The insert and copy length symbols. */
   float cost_cmd_[BROTLI_NUM_COMMAND_SYMBOLS];
-  float cost_dist_[BROTLI_NUM_DISTANCE_SYMBOLS];
+  float* cost_dist_;
+  uint32_t distance_histogram_size;
   /* Cumulative costs of literals per position in the stream. */
   float* literal_costs_;
   float min_cost_cmd_;
@@ -80,28 +86,41 @@ typedef struct ZopfliCostModel {
 } ZopfliCostModel;
 
 static void InitZopfliCostModel(
-    MemoryManager* m, ZopfliCostModel* self, size_t num_bytes) {
+    MemoryManager* m, ZopfliCostModel* self, const BrotliDistanceParams* dist,
+    size_t num_bytes) {
   self->num_bytes_ = num_bytes;
   self->literal_costs_ = BROTLI_ALLOC(m, float, num_bytes + 2);
+  self->cost_dist_ = BROTLI_ALLOC(m, float, dist->alphabet_size_limit);
+  self->distance_histogram_size = dist->alphabet_size_limit;
   if (BROTLI_IS_OOM(m)) return;
 }
 
 static void CleanupZopfliCostModel(MemoryManager* m, ZopfliCostModel* self) {
   BROTLI_FREE(m, self->literal_costs_);
+  BROTLI_FREE(m, self->cost_dist_);
 }
 
 static void SetCost(const uint32_t* histogram, size_t histogram_size,
-                    float* cost) {
+                    BROTLI_BOOL literal_histogram, float* cost) {
   size_t sum = 0;
+  size_t missing_symbol_sum;
   float log2sum;
+  float missing_symbol_cost;
   size_t i;
   for (i = 0; i < histogram_size; i++) {
     sum += histogram[i];
   }
   log2sum = (float)FastLog2(sum);
+  missing_symbol_sum = sum;
+  if (!literal_histogram) {
+    for (i = 0; i < histogram_size; i++) {
+      if (histogram[i] == 0) missing_symbol_sum++;
+    }
+  }
+  missing_symbol_cost = (float)FastLog2(missing_symbol_sum) + 2;
   for (i = 0; i < histogram_size; i++) {
     if (histogram[i] == 0) {
-      cost[i] = log2sum + 2;
+      cost[i] = missing_symbol_cost;
       continue;
     }
 
@@ -122,7 +141,7 @@ static void ZopfliCostModelSetFromCommands(ZopfliCostModel* self,
                                            size_t last_insert_len) {
   uint32_t histogram_literal[BROTLI_NUM_LITERAL_SYMBOLS];
   uint32_t histogram_cmd[BROTLI_NUM_COMMAND_SYMBOLS];
-  uint32_t histogram_dist[BROTLI_NUM_DISTANCE_SYMBOLS];
+  uint32_t histogram_dist[BROTLI_MAX_EFFECTIVE_DISTANCE_ALPHABET_SIZE];
   float cost_literal[BROTLI_NUM_LITERAL_SYMBOLS];
   size_t pos = position - last_insert_len;
   float min_cost_cmd = kInfinity;
@@ -136,7 +155,7 @@ static void ZopfliCostModelSetFromCommands(ZopfliCostModel* self,
   for (i = 0; i < num_commands; i++) {
     size_t inslength = commands[i].insert_len_;
     size_t copylength = CommandCopyLen(&commands[i]);
-    size_t distcode = commands[i].dist_prefix_;
+    size_t distcode = commands[i].dist_prefix_ & 0x3FF;
     size_t cmdcode = commands[i].cmd_prefix_;
     size_t j;
 
@@ -150,9 +169,12 @@ static void ZopfliCostModelSetFromCommands(ZopfliCostModel* self,
     pos += inslength + copylength;
   }
 
-  SetCost(histogram_literal, BROTLI_NUM_LITERAL_SYMBOLS, cost_literal);
-  SetCost(histogram_cmd, BROTLI_NUM_COMMAND_SYMBOLS, cost_cmd);
-  SetCost(histogram_dist, BROTLI_NUM_DISTANCE_SYMBOLS, self->cost_dist_);
+  SetCost(histogram_literal, BROTLI_NUM_LITERAL_SYMBOLS, BROTLI_TRUE,
+          cost_literal);
+  SetCost(histogram_cmd, BROTLI_NUM_COMMAND_SYMBOLS, BROTLI_FALSE,
+          cost_cmd);
+  SetCost(histogram_dist, self->distance_histogram_size, BROTLI_FALSE,
+          self->cost_dist_);
 
   for (i = 0; i < BROTLI_NUM_COMMAND_SYMBOLS; ++i) {
     min_cost_cmd = BROTLI_MIN(float, min_cost_cmd, cost_cmd[i]);
@@ -161,11 +183,14 @@ static void ZopfliCostModelSetFromCommands(ZopfliCostModel* self,
 
   {
     float* literal_costs = self->literal_costs_;
+    float literal_carry = 0.0;
     size_t num_bytes = self->num_bytes_;
     literal_costs[0] = 0.0;
     for (i = 0; i < num_bytes; ++i) {
-      literal_costs[i + 1] = literal_costs[i] +
+      literal_carry +=
           cost_literal[ringbuffer[(position + i) & ringbuffer_mask]];
+      literal_costs[i + 1] = literal_costs[i] + literal_carry;
+      literal_carry -= literal_costs[i + 1] - literal_costs[i];
     }
   }
 }
@@ -175,6 +200,7 @@ static void ZopfliCostModelSetFromLiteralCosts(ZopfliCostModel* self,
                                                const uint8_t* ringbuffer,
                                                size_t ringbuffer_mask) {
   float* literal_costs = self->literal_costs_;
+  float literal_carry = 0.0;
   float* cost_dist = self->cost_dist_;
   float* cost_cmd = self->cost_cmd_;
   size_t num_bytes = self->num_bytes_;
@@ -183,12 +209,14 @@ static void ZopfliCostModelSetFromLiteralCosts(ZopfliCostModel* self,
                                     ringbuffer, &literal_costs[1]);
   literal_costs[0] = 0.0;
   for (i = 0; i < num_bytes; ++i) {
-    literal_costs[i + 1] += literal_costs[i];
+    literal_carry += literal_costs[i + 1];
+    literal_costs[i + 1] = literal_costs[i] + literal_carry;
+    literal_carry -= literal_costs[i + 1] - literal_costs[i];
   }
   for (i = 0; i < BROTLI_NUM_COMMAND_SYMBOLS; ++i) {
     cost_cmd[i] = (float)FastLog2(11 + (uint32_t)i);
   }
-  for (i = 0; i < BROTLI_NUM_DISTANCE_SYMBOLS; ++i) {
+  for (i = 0; i < self->distance_histogram_size; ++i) {
     cost_dist[i] = (float)FastLog2(20 + (uint32_t)i);
   }
   self->min_cost_cmd_ = (float)FastLog2(11);
@@ -221,9 +249,10 @@ static BROTLI_INLINE void UpdateZopfliNode(ZopfliNode* nodes, size_t pos,
     size_t start_pos, size_t len, size_t len_code, size_t dist,
     size_t short_code, float cost) {
   ZopfliNode* next = &nodes[pos + len];
-  next->length = (uint32_t)(len | ((len + 9u - len_code) << 24));
-  next->distance = (uint32_t)(dist | (short_code << 25));
-  next->insert_length = (uint32_t)(pos - start_pos);
+  next->length = (uint32_t)(len | ((len + 9u - len_code) << 25));
+  next->distance = (uint32_t)dist;
+  next->dcode_insert_length = (uint32_t)(
+      (short_code << 27) | (pos - start_pos));
   next->u.cost = cost;
 }
 
@@ -299,21 +328,21 @@ static size_t ComputeMinimumCopyLength(const float start_cost,
    REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */
 static uint32_t ComputeDistanceShortcut(const size_t block_start,
                                         const size_t pos,
-                                        const size_t max_backward,
+                                        const size_t max_backward_limit,
                                         const size_t gap,
                                         const ZopfliNode* nodes) {
   const size_t clen = ZopfliNodeCopyLength(&nodes[pos]);
-  const size_t ilen = nodes[pos].insert_length;
+  const size_t ilen = nodes[pos].dcode_insert_length & 0x7FFFFFF;
   const size_t dist = ZopfliNodeCopyDistance(&nodes[pos]);
   /* Since |block_start + pos| is the end position of the command, the copy part
      starts from |block_start + pos - clen|. Distances that are greater than
-     this or greater than |max_backward| are static dictionary references, and
-     do not update the last distances. Also distance code 0 (last distance)
-     does not update the last distances. */
+     this or greater than |max_backward_limit| + |gap| are static dictionary
+     references, and do not update the last distances.
+     Also distance code 0 (last distance) does not update the last distances. */
   if (pos == 0) {
     return 0;
   } else if (dist + clen <= block_start + pos + gap &&
-             dist <= max_backward + gap &&
+             dist <= max_backward_limit + gap &&
              ZopfliNodeDistanceCode(&nodes[pos]) > 0) {
     return (uint32_t)pos;
   } else {
@@ -335,7 +364,7 @@ static void ComputeDistanceCache(const size_t pos,
   int idx = 0;
   size_t p = nodes[pos].u.shortcut;
   while (idx < 4 && p > 0) {
-    const size_t ilen = nodes[p].insert_length;
+    const size_t ilen = nodes[p].dcode_insert_length & 0x7FFFFFF;
     const size_t clen = ZopfliNodeCopyLength(&nodes[p]);
     const size_t dist = ZopfliNodeCopyDistance(&nodes[p]);
     dist_cache[idx++] = (int)dist;
@@ -377,9 +406,12 @@ static size_t UpdateNodes(
     const int* starting_dist_cache, const size_t num_matches,
     const BackwardMatch* matches, const ZopfliCostModel* model,
     StartPosQueue* queue, ZopfliNode* nodes) {
+  const size_t stream_offset = params->stream_offset;
   const size_t cur_ix = block_start + pos;
   const size_t cur_ix_masked = cur_ix & ringbuffer_mask;
   const size_t max_distance = BROTLI_MIN(size_t, cur_ix, max_backward_limit);
+  const size_t dictionary_start = BROTLI_MIN(size_t,
+      cur_ix + stream_offset, max_backward_limit);
   const size_t max_len = num_bytes - pos;
   const size_t max_zopfli_len = MaxZopfliLen(params);
   const size_t max_iters = MaxZopfliCandidates(params);
@@ -388,8 +420,8 @@ static size_t UpdateNodes(
   size_t k;
   size_t gap = 0;
 
-  EvaluateNode(block_start, pos, max_backward_limit, gap, starting_dist_cache,
-      model, queue, nodes);
+  EvaluateNode(block_start + stream_offset, pos, max_backward_limit, gap,
+      starting_dist_cache, model, queue, nodes);
 
   {
     const PosData* posdata = StartPosQueueAt(queue, 0);
@@ -422,10 +454,12 @@ static size_t UpdateNodes(
       if (cur_ix_masked + best_len > ringbuffer_mask) {
         break;
       }
-      if (BROTLI_PREDICT_FALSE(backward > max_distance + gap)) {
+      if (BROTLI_PREDICT_FALSE(backward > dictionary_start + gap)) {
+        /* Word dictionary -> ignore. */
         continue;
       }
       if (backward <= max_distance) {
+        /* Regular backward reference. */
         if (prev_ix >= cur_ix) {
           continue;
         }
@@ -439,6 +473,8 @@ static size_t UpdateNodes(
                                        &ringbuffer[cur_ix_masked],
                                        max_len);
       } else {
+        /* "Gray" area. It is addressable by decoder, but this encoder
+           instance does not have that data -> should not touch it. */
         continue;
       }
       {
@@ -473,7 +509,7 @@ static size_t UpdateNodes(
         BackwardMatch match = matches[j];
         size_t dist = match.distance;
         BROTLI_BOOL is_dictionary_match =
-            TO_BROTLI_BOOL(dist > max_distance + gap);
+            TO_BROTLI_BOOL(dist > dictionary_start + gap);
         /* We already tried all possible last distance matches, so we can use
            normal distance code here. */
         size_t dist_code = dist + BROTLI_NUM_DISTANCE_SHORT_CODES - 1;
@@ -482,10 +518,12 @@ static size_t UpdateNodes(
         uint32_t distnumextra;
         float dist_cost;
         size_t max_match_len;
-        PrefixEncodeCopyDistance(dist_code, 0, 0, &dist_symbol, &distextra);
-        distnumextra = distextra >> 24;
+        PrefixEncodeCopyDistance(
+            dist_code, params->dist.num_direct_distance_codes,
+            params->dist.distance_postfix_bits, &dist_symbol, &distextra);
+        distnumextra = dist_symbol >> 10;
         dist_cost = base_cost + (float)distnumextra +
-            ZopfliCostModelGetDistanceCost(model, dist_symbol);
+            ZopfliCostModelGetDistanceCost(model, dist_symbol & 0x3FF);
 
         /* Try all copy lengths up until the maximum copy length corresponding
            to this distance. If the distance refers to the static dictionary, or
@@ -517,7 +555,8 @@ static size_t ComputeShortestPathFromNodes(size_t num_bytes,
     ZopfliNode* nodes) {
   size_t index = num_bytes;
   size_t num_commands = 0;
-  while (nodes[index].insert_length == 0 && nodes[index].length == 1) --index;
+  while ((nodes[index].dcode_insert_length & 0x7FFFFFF) == 0 &&
+      nodes[index].length == 1) --index;
   nodes[index].u.next = BROTLI_UINT32_MAX;
   while (index != 0) {
     size_t len = ZopfliNodeCommandLength(&nodes[index]);
@@ -530,23 +569,19 @@ static size_t ComputeShortestPathFromNodes(size_t num_bytes,
 
 /* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */
 void BrotliZopfliCreateCommands(const size_t num_bytes,
-                                const size_t block_start,
-                                const size_t max_backward_limit,
-                                const ZopfliNode* nodes,
-                                int* dist_cache,
-                                size_t* last_insert_len,
-                                const BrotliEncoderParams* params,
-                                Command* commands,
-                                size_t* num_literals) {
+    const size_t block_start, const ZopfliNode* nodes, int* dist_cache,
+    size_t* last_insert_len, const BrotliEncoderParams* params,
+    Command* commands, size_t* num_literals) {
+  const size_t stream_offset = params->stream_offset;
+  const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin);
   size_t pos = 0;
   uint32_t offset = nodes[0].u.next;
   size_t i;
   size_t gap = 0;
-  BROTLI_UNUSED(params);
   for (i = 0; offset != BROTLI_UINT32_MAX; i++) {
     const ZopfliNode* next = &nodes[pos + offset];
     size_t copy_length = ZopfliNodeCopyLength(next);
-    size_t insert_length = next->insert_length;
+    size_t insert_length = next->dcode_insert_length & 0x7FFFFFF;
     pos += insert_length;
     offset = next->u.next;
     if (i == 0) {
@@ -556,12 +591,12 @@ void BrotliZopfliCreateCommands(const size_t num_bytes,
     {
       size_t distance = ZopfliNodeCopyDistance(next);
       size_t len_code = ZopfliNodeLengthCode(next);
-      size_t max_distance =
-          BROTLI_MIN(size_t, block_start + pos, max_backward_limit);
-      BROTLI_BOOL is_dictionary = TO_BROTLI_BOOL(distance > max_distance + gap);
+      size_t dictionary_start = BROTLI_MIN(size_t,
+          block_start + pos + stream_offset, max_backward_limit);
+      BROTLI_BOOL is_dictionary =
+          TO_BROTLI_BOOL(distance > dictionary_start + gap);
       size_t dist_code = ZopfliNodeDistanceCode(next);
-
-      InitCommand(&commands[i], insert_length,
+      InitCommand(&commands[i], &params->dist, insert_length,
           copy_length, (int)len_code - (int)copy_length, dist_code);
 
       if (!is_dictionary && dist_code > 0) {
@@ -578,18 +613,13 @@ void BrotliZopfliCreateCommands(const size_t num_bytes,
   *last_insert_len += num_bytes - pos;
 }
 
-static size_t ZopfliIterate(size_t num_bytes,
-                            size_t position,
-                            const uint8_t* ringbuffer,
-                            size_t ringbuffer_mask,
-                            const BrotliEncoderParams* params,
-                            const size_t max_backward_limit,
-                            const size_t gap,
-                            const int* dist_cache,
-                            const ZopfliCostModel* model,
-                            const uint32_t* num_matches,
-                            const BackwardMatch* matches,
-                            ZopfliNode* nodes) {
+static size_t ZopfliIterate(size_t num_bytes, size_t position,
+    const uint8_t* ringbuffer, size_t ringbuffer_mask,
+    const BrotliEncoderParams* params, const size_t gap, const int* dist_cache,
+    const ZopfliCostModel* model, const uint32_t* num_matches,
+    const BackwardMatch* matches, ZopfliNode* nodes) {
+  const size_t stream_offset = params->stream_offset;
+  const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin);
   const size_t max_zopfli_len = MaxZopfliLen(params);
   StartPosQueue queue;
   size_t cur_match_pos = 0;
@@ -613,8 +643,8 @@ static size_t ZopfliIterate(size_t num_bytes,
       while (skip) {
         i++;
         if (i + 3 >= num_bytes) break;
-        EvaluateNode(position, i, max_backward_limit, gap, dist_cache, model,
-            &queue, nodes);
+        EvaluateNode(position + stream_offset, i, max_backward_limit, gap,
+            dist_cache, model, &queue, nodes);
         cur_match_pos += num_matches[i];
         skip--;
       }
@@ -624,28 +654,25 @@ static size_t ZopfliIterate(size_t num_bytes,
 }
 
 /* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */
-size_t BrotliZopfliComputeShortestPath(MemoryManager* m,
-                                       const BrotliDictionary* dictionary,
-                                       size_t num_bytes,
-                                       size_t position,
-                                       const uint8_t* ringbuffer,
-                                       size_t ringbuffer_mask,
-                                       const BrotliEncoderParams* params,
-                                       const size_t max_backward_limit,
-                                       const int* dist_cache,
-                                       HasherHandle hasher,
-                                       ZopfliNode* nodes) {
+size_t BrotliZopfliComputeShortestPath(MemoryManager* m, size_t num_bytes,
+    size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
+    ContextLut literal_context_lut, const BrotliEncoderParams* params,
+    const int* dist_cache, Hasher* hasher, ZopfliNode* nodes) {
+  const size_t stream_offset = params->stream_offset;
+  const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin);
   const size_t max_zopfli_len = MaxZopfliLen(params);
   ZopfliCostModel model;
   StartPosQueue queue;
-  BackwardMatch matches[MAX_NUM_MATCHES_H10];
+  BackwardMatch matches[2 * (MAX_NUM_MATCHES_H10 + 64)];
   const size_t store_end = num_bytes >= StoreLookaheadH10() ?
       position + num_bytes - StoreLookaheadH10() + 1 : position;
   size_t i;
   size_t gap = 0;
+  size_t lz_matches_offset = 0;
+  BROTLI_UNUSED(literal_context_lut);
   nodes[0].length = 0;
   nodes[0].u.cost = 0;
-  InitZopfliCostModel(m, &model, num_bytes);
+  InitZopfliCostModel(m, &model, &params->dist, num_bytes);
   if (BROTLI_IS_OOM(m)) return 0;
   ZopfliCostModelSetFromLiteralCosts(
       &model, position, ringbuffer, ringbuffer_mask);
@@ -653,10 +680,14 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m,
   for (i = 0; i + HashTypeLengthH10() - 1 < num_bytes; i++) {
     const size_t pos = position + i;
     const size_t max_distance = BROTLI_MIN(size_t, pos, max_backward_limit);
-    size_t num_matches = FindAllMatchesH10(hasher, dictionary, ringbuffer,
-        ringbuffer_mask, pos, num_bytes - i, max_distance, gap, params,
-        matches);
+    const size_t dictionary_start = BROTLI_MIN(size_t,
+        pos + stream_offset, max_backward_limit);
     size_t skip;
+    size_t num_matches;
+    num_matches = FindAllMatchesH10(&hasher->privat._H10,
+        &params->dictionary,
+        ringbuffer, ringbuffer_mask, pos, num_bytes - i, max_distance,
+        dictionary_start + gap, params, &matches[lz_matches_offset]);
     if (num_matches > 0 &&
         BackwardMatchLength(&matches[num_matches - 1]) > max_zopfli_len) {
       matches[0] = matches[num_matches - 1];
@@ -671,14 +702,15 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m,
     }
     if (skip > 1) {
       /* Add the tail of the copy to the hasher. */
-      StoreRangeH10(hasher, ringbuffer, ringbuffer_mask, pos + 1, BROTLI_MIN(
+      StoreRangeH10(&hasher->privat._H10,
+          ringbuffer, ringbuffer_mask, pos + 1, BROTLI_MIN(
           size_t, pos + skip, store_end));
       skip--;
       while (skip) {
         i++;
         if (i + HashTypeLengthH10() - 1 >= num_bytes) break;
-        EvaluateNode(position, i, max_backward_limit, gap, dist_cache, &model,
-            &queue, nodes);
+        EvaluateNode(position + stream_offset, i, max_backward_limit, gap,
+            dist_cache, &model, &queue, nodes);
         skip--;
       }
     }
@@ -687,32 +719,29 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m,
   return ComputeShortestPathFromNodes(num_bytes, nodes);
 }
 
-void BrotliCreateZopfliBackwardReferences(
-    MemoryManager* m, const BrotliDictionary* dictionary, size_t num_bytes,
+void BrotliCreateZopfliBackwardReferences(MemoryManager* m, size_t num_bytes,
     size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
-    const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
-    size_t* last_insert_len, Command* commands, size_t* num_commands,
-    size_t* num_literals) {
-  const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin);
-  ZopfliNode* nodes;
-  nodes = BROTLI_ALLOC(m, ZopfliNode, num_bytes + 1);
-  if (BROTLI_IS_OOM(m)) return;
+    ContextLut literal_context_lut, const BrotliEncoderParams* params,
+    Hasher* hasher, int* dist_cache, size_t* last_insert_len,
+    Command* commands, size_t* num_commands, size_t* num_literals) {
+  ZopfliNode* nodes = BROTLI_ALLOC(m, ZopfliNode, num_bytes + 1);
+  if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(nodes)) return;
   BrotliInitZopfliNodes(nodes, num_bytes + 1);
-  *num_commands += BrotliZopfliComputeShortestPath(m, dictionary, num_bytes,
-      position, ringbuffer, ringbuffer_mask, params, max_backward_limit,
+  *num_commands += BrotliZopfliComputeShortestPath(m, num_bytes,
+      position, ringbuffer, ringbuffer_mask, literal_context_lut, params,
       dist_cache, hasher, nodes);
   if (BROTLI_IS_OOM(m)) return;
-  BrotliZopfliCreateCommands(num_bytes, position, max_backward_limit, nodes,
-      dist_cache, last_insert_len, params, commands, num_literals);
+  BrotliZopfliCreateCommands(num_bytes, position, nodes, dist_cache,
+      last_insert_len, params, commands, num_literals);
   BROTLI_FREE(m, nodes);
 }
 
-void BrotliCreateHqZopfliBackwardReferences(
-    MemoryManager* m, const BrotliDictionary* dictionary, size_t num_bytes,
+void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m, size_t num_bytes,
     size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
-    const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
-    size_t* last_insert_len, Command* commands, size_t* num_commands,
-    size_t* num_literals) {
+    ContextLut literal_context_lut, const BrotliEncoderParams* params,
+    Hasher* hasher, int* dist_cache, size_t* last_insert_len,
+    Command* commands, size_t* num_commands, size_t* num_literals) {
+  const size_t stream_offset = params->stream_offset;
   const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin);
   uint32_t* num_matches = BROTLI_ALLOC(m, uint32_t, num_bytes);
   size_t matches_size = 4 * num_bytes;
@@ -728,24 +757,33 @@ void BrotliCreateHqZopfliBackwardReferences(
   ZopfliNode* nodes;
   BackwardMatch* matches = BROTLI_ALLOC(m, BackwardMatch, matches_size);
   size_t gap = 0;
-  if (BROTLI_IS_OOM(m)) return;
+  size_t shadow_matches = 0;
+  BROTLI_UNUSED(literal_context_lut);
+  if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(num_matches) ||
+      BROTLI_IS_NULL(matches)) {
+    return;
+  }
   for (i = 0; i + HashTypeLengthH10() - 1 < num_bytes; ++i) {
     const size_t pos = position + i;
     size_t max_distance = BROTLI_MIN(size_t, pos, max_backward_limit);
+    size_t dictionary_start = BROTLI_MIN(size_t,
+        pos + stream_offset, max_backward_limit);
     size_t max_length = num_bytes - i;
     size_t num_found_matches;
     size_t cur_match_end;
     size_t j;
     /* Ensure that we have enough free slots. */
     BROTLI_ENSURE_CAPACITY(m, BackwardMatch, matches, matches_size,
-        cur_match_pos + MAX_NUM_MATCHES_H10);
+        cur_match_pos + MAX_NUM_MATCHES_H10 + shadow_matches);
     if (BROTLI_IS_OOM(m)) return;
-    num_found_matches = FindAllMatchesH10(hasher, dictionary, ringbuffer,
-        ringbuffer_mask, pos, max_length, max_distance, gap, params,
-        &matches[cur_match_pos]);
+    num_found_matches = FindAllMatchesH10(&hasher->privat._H10,
+        &params->dictionary,
+        ringbuffer, ringbuffer_mask, pos, max_length,
+        max_distance, dictionary_start + gap, params,
+        &matches[cur_match_pos + shadow_matches]);
     cur_match_end = cur_match_pos + num_found_matches;
     for (j = cur_match_pos; j + 1 < cur_match_end; ++j) {
-      assert(BackwardMatchLength(&matches[j]) <=
+      BROTLI_DCHECK(BackwardMatchLength(&matches[j]) <=
           BackwardMatchLength(&matches[j + 1]));
     }
     num_matches[i] = (uint32_t)num_found_matches;
@@ -756,7 +794,8 @@ void BrotliCreateHqZopfliBackwardReferences(
         matches[cur_match_pos++] = matches[cur_match_end - 1];
         num_matches[i] = 1;
         /* Add the tail of the copy to the hasher. */
-        StoreRangeH10(hasher, ringbuffer, ringbuffer_mask, pos + 1,
+        StoreRangeH10(&hasher->privat._H10,
+                      ringbuffer, ringbuffer_mask, pos + 1,
                       BROTLI_MIN(size_t, pos + match_len, store_end));
         memset(&num_matches[i + 1], 0, skip * sizeof(num_matches[0]));
         i += skip;
@@ -770,8 +809,8 @@ void BrotliCreateHqZopfliBackwardReferences(
   memcpy(orig_dist_cache, dist_cache, 4 * sizeof(dist_cache[0]));
   orig_num_commands = *num_commands;
   nodes = BROTLI_ALLOC(m, ZopfliNode, num_bytes + 1);
-  if (BROTLI_IS_OOM(m)) return;
-  InitZopfliCostModel(m, &model, num_bytes);
+  if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(nodes)) return;
+  InitZopfliCostModel(m, &model, &params->dist, num_bytes);
   if (BROTLI_IS_OOM(m)) return;
   for (i = 0; i < 2; i++) {
     BrotliInitZopfliNodes(nodes, num_bytes + 1);
@@ -788,10 +827,10 @@ void BrotliCreateHqZopfliBackwardReferences(
     *last_insert_len = orig_last_insert_len;
     memcpy(dist_cache, orig_dist_cache, 4 * sizeof(dist_cache[0]));
     *num_commands += ZopfliIterate(num_bytes, position, ringbuffer,
-        ringbuffer_mask, params, max_backward_limit, gap, dist_cache,
-        &model, num_matches, matches, nodes);
-    BrotliZopfliCreateCommands(num_bytes, position, max_backward_limit,
-        nodes, dist_cache, last_insert_len, params, commands, num_literals);
+        ringbuffer_mask, params, gap, dist_cache, &model, num_matches, matches,
+        nodes);
+    BrotliZopfliCreateCommands(num_bytes, position, nodes, dist_cache,
+        last_insert_len, params, commands, num_literals);
   }
   CleanupZopfliCostModel(m, &model);
   BROTLI_FREE(m, nodes);