brotli 0.2.3 → 0.4.0

data/vendor/brotli/c/dec/huffman.h

@@ -18,12 +18,6 @@ extern "C" {
 
 #define BROTLI_HUFFMAN_MAX_CODE_LENGTH 15
 
-/* Maximum possible Huffman table size for an alphabet size of (index * 32),
-   max code length 15 and root table bits 8. */
-static const uint16_t kMaxHuffmanTableSize[] = {
-  256, 402, 436, 468, 500, 534, 566, 598, 630, 662, 694, 726, 758, 790, 822,
-  854, 886, 920, 952, 984, 1016, 1048, 1080, 1112, 1144, 1176, 1208, 1240, 1272,
-  1304, 1336, 1368, 1400, 1432, 1464, 1496, 1528};
 /* BROTLI_NUM_BLOCK_LEN_SYMBOLS == 26 */
 #define BROTLI_HUFFMAN_MAX_SIZE_26 396
 /* BROTLI_MAX_BLOCK_TYPE_SYMBOLS == 258 */
@@ -82,7 +76,7 @@ typedef BROTLI_ALIGNED(4) uint32_t HuffmanCode;
 
 static BROTLI_INLINE HuffmanCode ConstructHuffmanCode(const uint8_t bits,
     const uint16_t value) {
-  return ((value & 0xFFFF) << 16) | (bits & 0xFF);
+  return (HuffmanCode) ((value & 0xFFFF) << 16) | (bits & 0xFF);
 }
 
 #define BROTLI_HC_MARK_TABLE_FOR_FAST_LOAD(H) uint32_t __fastload_##H = (*H)
@@ -100,7 +94,7 @@ BROTLI_INTERNAL void BrotliBuildCodeLengthsHuffmanTable(HuffmanCode* root_table,
 /* Builds Huffman lookup table assuming code lengths are in symbol order.
    Returns size of resulting table. */
 BROTLI_INTERNAL uint32_t BrotliBuildHuffmanTable(HuffmanCode* root_table,
-    int root_bits, const uint16_t* const symbol_lists, uint16_t* count_arg);
+    int root_bits, const uint16_t* const symbol_lists, uint16_t* count);
 
 /* Builds a simple Huffman table. The |num_symbols| parameter is to be
    interpreted as follows: 0 means 1 symbol, 1 means 2 symbols,
@@ -110,13 +104,13 @@ BROTLI_INTERNAL uint32_t BrotliBuildSimpleHuffmanTable(HuffmanCode* table,
     int root_bits, uint16_t* symbols, uint32_t num_symbols);
 
 /* Contains a collection of Huffman trees with the same alphabet size. */
-/* max_symbol is needed due to simple codes since log2(alphabet_size) could be
-   greater than log2(max_symbol). */
+/* alphabet_size_limit is needed due to simple codes, since
+   log2(alphabet_size_max) could be greater than log2(alphabet_size_limit). */
 typedef struct {
   HuffmanCode** htrees;
   HuffmanCode* codes;
-  uint16_t alphabet_size;
-  uint16_t max_symbol;
+  uint16_t alphabet_size_max;
+  uint16_t alphabet_size_limit;
   uint16_t num_htrees;
 } HuffmanTreeGroup;
 

data/vendor/brotli/c/dec/prefix.h

@@ -13,24 +13,6 @@
 #include "../common/constants.h"
 #include <brotli/types.h>
 
-/* Represents the range of values belonging to a prefix code:
-   [offset, offset + 2^nbits) */
-struct PrefixCodeRange {
-  uint16_t offset;
-  uint8_t nbits;
-};
-
-static const struct PrefixCodeRange
-    kBlockLengthPrefixCode[BROTLI_NUM_BLOCK_LEN_SYMBOLS] = {
-  {   1,  2}, {    5,  2}, {  9,   2}, {  13,  2},
-  {  17,  3}, {   25,  3}, {  33,  3}, {  41,  3},
-  {  49,  4}, {   65,  4}, {  81,  4}, {  97,  4},
-  { 113,  5}, {  145,  5}, { 177,  5}, { 209,  5},
-  { 241,  6}, {  305,  6}, { 369,  7}, { 497,  8},
-  { 753,  9}, { 1265, 10}, {2289, 11}, {4337, 12},
-  {8433, 13}, {16625, 24}
-};
-
 typedef struct CmdLutElement {
   uint8_t insert_len_extra_bits;
   uint8_t copy_len_extra_bits;

data/vendor/brotli/c/dec/state.c

@@ -33,10 +33,7 @@ BROTLI_BOOL BrotliDecoderStateInit(BrotliDecoderState* s,
   s->state = BROTLI_STATE_UNINITED;
   s->large_window = 0;
   s->substate_metablock_header = BROTLI_STATE_METABLOCK_HEADER_NONE;
-  s->substate_tree_group = BROTLI_STATE_TREE_GROUP_NONE;
-  s->substate_context_map = BROTLI_STATE_CONTEXT_MAP_NONE;
   s->substate_uncompressed = BROTLI_STATE_UNCOMPRESSED_NONE;
-  s->substate_huffman = BROTLI_STATE_HUFFMAN_NONE;
   s->substate_decode_uint8 = BROTLI_STATE_DECODE_UINT8_NONE;
   s->substate_read_block_length = BROTLI_STATE_READ_BLOCK_LENGTH_NONE;
 
@@ -59,8 +56,6 @@ BROTLI_BOOL BrotliDecoderStateInit(BrotliDecoderState* s,
   s->context_map_slice = NULL;
   s->dist_context_map_slice = NULL;
 
-  s->sub_loop_counter = 0;
-
   s->literal_hgroup.codes = NULL;
   s->literal_hgroup.htrees = NULL;
   s->insert_copy_hgroup.codes = NULL;
@@ -84,9 +79,6 @@ BROTLI_BOOL BrotliDecoderStateInit(BrotliDecoderState* s,
   s->block_type_trees = NULL;
   s->block_len_trees = NULL;
 
-  /* Make small negative indexes addressable. */
-  s->symbol_lists = &s->symbols_lists_array[BROTLI_HUFFMAN_MAX_CODE_LENGTH + 1];
-
   s->mtf_upper_bound = 63;
 
   s->dictionary = BrotliGetDictionary();
@@ -142,17 +134,20 @@ void BrotliDecoderStateCleanup(BrotliDecoderState* s) {
 }
 
 BROTLI_BOOL BrotliDecoderHuffmanTreeGroupInit(BrotliDecoderState* s,
-    HuffmanTreeGroup* group, uint32_t alphabet_size, uint32_t max_symbol,
-    uint32_t ntrees) {
-  /* Pack two allocations into one */
-  const size_t max_table_size = kMaxHuffmanTableSize[(alphabet_size + 31) >> 5];
+    HuffmanTreeGroup* group, uint32_t alphabet_size_max,
+    uint32_t alphabet_size_limit, uint32_t ntrees) {
+  /* 376 = 256 (1-st level table) + 4 + 7 + 15 + 31 + 63 (2-nd level mix-tables)
+     This number is discovered "unlimited" "enough" calculator; it is actually
+     a wee bigger than required in several cases (especially for alphabets with
+     less than 16 symbols). */
+  const size_t max_table_size = alphabet_size_limit + 376;
   const size_t code_size = sizeof(HuffmanCode) * ntrees * max_table_size;
   const size_t htree_size = sizeof(HuffmanCode*) * ntrees;
   /* Pointer alignment is, hopefully, wider than sizeof(HuffmanCode). */
   HuffmanCode** p = (HuffmanCode**)BROTLI_DECODER_ALLOC(s,
       code_size + htree_size);
-  group->alphabet_size = (uint16_t)alphabet_size;
-  group->max_symbol = (uint16_t)max_symbol;
+  group->alphabet_size_max = (uint16_t)alphabet_size_max;
+  group->alphabet_size_limit = (uint16_t)alphabet_size_limit;
   group->num_htrees = (uint16_t)ntrees;
   group->htrees = p;
   group->codes = (HuffmanCode*)(&p[ntrees]);

data/vendor/brotli/c/dec/state.h

@@ -21,6 +21,95 @@
 extern "C" {
 #endif
 
+/* Graphviz diagram that describes state transitions:
+
+digraph States {
+  graph [compound=true]
+  concentrate=true
+  node [shape="box"]
+
+  UNINITED -> {LARGE_WINDOW_BITS -> INITIALIZE}
+  subgraph cluster_metablock_workflow {
+    style="rounded"
+    label=< <B>METABLOCK CYCLE</B> >
+    METABLOCK_BEGIN -> METABLOCK_HEADER
+    METABLOCK_HEADER:sw -> METADATA
+    METABLOCK_HEADER:s -> UNCOMPRESSED
+    METABLOCK_HEADER:se -> METABLOCK_DONE:ne
+    METADATA:s -> METABLOCK_DONE:w
+    UNCOMPRESSED:s -> METABLOCK_DONE:n
+    METABLOCK_DONE:e -> METABLOCK_BEGIN:e [constraint="false"]
+  }
+  INITIALIZE -> METABLOCK_BEGIN
+  METABLOCK_DONE -> DONE
+
+  subgraph cluster_compressed_metablock {
+    style="rounded"
+    label=< <B>COMPRESSED METABLOCK</B> >
+
+    subgraph cluster_command {
+      style="rounded"
+      label=< <B>HOT LOOP</B> >
+
+      _METABLOCK_DONE_PORT_ [shape=point style=invis]
+
+      {
+        // Set different shape for nodes returning from "compressed metablock".
+        node [shape=invhouse]; CMD_INNER CMD_POST_DECODE_LITERALS;
+        CMD_POST_WRAP_COPY; CMD_INNER_WRITE; CMD_POST_WRITE_1;
+      }
+
+      CMD_BEGIN -> CMD_INNER -> CMD_POST_DECODE_LITERALS -> CMD_POST_WRAP_COPY
+
+      // IO ("write") nodes are not in the hot loop!
+      CMD_INNER_WRITE [style=dashed]
+      CMD_INNER -> CMD_INNER_WRITE
+      CMD_POST_WRITE_1 [style=dashed]
+      CMD_POST_DECODE_LITERALS -> CMD_POST_WRITE_1
+      CMD_POST_WRITE_2 [style=dashed]
+      CMD_POST_WRAP_COPY -> CMD_POST_WRITE_2
+
+      CMD_POST_WRITE_1 -> CMD_BEGIN:s [constraint="false"]
+      CMD_INNER_WRITE -> {CMD_INNER CMD_POST_DECODE_LITERALS}
+          [constraint="false"]
+      CMD_BEGIN:ne -> CMD_POST_DECODE_LITERALS [constraint="false"]
+      CMD_POST_WRAP_COPY -> CMD_BEGIN [constraint="false"]
+      CMD_POST_DECODE_LITERALS -> CMD_BEGIN:ne [constraint="false"]
+      CMD_POST_WRITE_2 -> CMD_POST_WRAP_COPY [constraint="false"]
+      {rank=same; CMD_BEGIN; CMD_INNER; CMD_POST_DECODE_LITERALS;
+          CMD_POST_WRAP_COPY}
+      {rank=same; CMD_INNER_WRITE; CMD_POST_WRITE_1; CMD_POST_WRITE_2}
+
+      {CMD_INNER CMD_POST_DECODE_LITERALS CMD_POST_WRAP_COPY} ->
+          _METABLOCK_DONE_PORT_ [style=invis]
+      {CMD_INNER_WRITE CMD_POST_WRITE_1} -> _METABLOCK_DONE_PORT_
+          [constraint="false" style=invis]
+    }
+
+    BEFORE_COMPRESSED_METABLOCK_HEADER:s -> HUFFMAN_CODE_0:n
+    HUFFMAN_CODE_0 -> HUFFMAN_CODE_1 -> HUFFMAN_CODE_2 -> HUFFMAN_CODE_3
+    HUFFMAN_CODE_0 -> METABLOCK_HEADER_2 -> CONTEXT_MODES -> CONTEXT_MAP_1
+    CONTEXT_MAP_1 -> CONTEXT_MAP_2 -> TREE_GROUP
+    TREE_GROUP -> BEFORE_COMPRESSED_METABLOCK_BODY:e
+    BEFORE_COMPRESSED_METABLOCK_BODY:s -> CMD_BEGIN:n
+
+    HUFFMAN_CODE_3:e -> HUFFMAN_CODE_0:ne [constraint="false"]
+    {rank=same; HUFFMAN_CODE_0; HUFFMAN_CODE_1; HUFFMAN_CODE_2; HUFFMAN_CODE_3}
+    {rank=same; METABLOCK_HEADER_2; CONTEXT_MODES; CONTEXT_MAP_1; CONTEXT_MAP_2;
+        TREE_GROUP}
+  }
+  METABLOCK_HEADER:e -> BEFORE_COMPRESSED_METABLOCK_HEADER:n
+
+  _METABLOCK_DONE_PORT_ -> METABLOCK_DONE:se
+      [constraint="false" ltail=cluster_command]
+
+  UNINITED [shape=Mdiamond];
+  DONE [shape=Msquare];
+}
+
+
+ */
+
 typedef enum {
   BROTLI_STATE_UNINITED,
   BROTLI_STATE_LARGE_WINDOW_BITS,
@@ -39,6 +128,7 @@ typedef enum {
   BROTLI_STATE_METABLOCK_DONE,
   BROTLI_STATE_COMMAND_POST_WRITE_1,
   BROTLI_STATE_COMMAND_POST_WRITE_2,
+  BROTLI_STATE_BEFORE_COMPRESSED_METABLOCK_HEADER,
   BROTLI_STATE_HUFFMAN_CODE_0,
   BROTLI_STATE_HUFFMAN_CODE_1,
   BROTLI_STATE_HUFFMAN_CODE_2,
@@ -46,6 +136,7 @@ typedef enum {
   BROTLI_STATE_CONTEXT_MAP_1,
   BROTLI_STATE_CONTEXT_MAP_2,
   BROTLI_STATE_TREE_GROUP,
+  BROTLI_STATE_BEFORE_COMPRESSED_METABLOCK_BODY,
   BROTLI_STATE_DONE
 } BrotliRunningState;
 
@@ -98,6 +189,50 @@ typedef enum {
   BROTLI_STATE_READ_BLOCK_LENGTH_SUFFIX
 } BrotliRunningReadBlockLengthState;
 
+typedef struct BrotliMetablockHeaderArena {
+  BrotliRunningTreeGroupState substate_tree_group;
+  BrotliRunningContextMapState substate_context_map;
+  BrotliRunningHuffmanState substate_huffman;
+
+  uint32_t sub_loop_counter;
+
+  uint32_t repeat_code_len;
+  uint32_t prev_code_len;
+
+  /* For ReadHuffmanCode. */
+  uint32_t symbol;
+  uint32_t repeat;
+  uint32_t space;
+
+  /* Huffman table for "histograms". */
+  HuffmanCode table[32];
+  /* List of heads of symbol chains. */
+  uint16_t* symbol_lists;
+  /* Storage from symbol_lists. */
+  uint16_t symbols_lists_array[BROTLI_HUFFMAN_MAX_CODE_LENGTH + 1 +
+                               BROTLI_NUM_COMMAND_SYMBOLS];
+  /* Tails of symbol chains. */
+  int next_symbol[32];
+  uint8_t code_length_code_lengths[BROTLI_CODE_LENGTH_CODES];
+  /* Population counts for the code lengths. */
+  uint16_t code_length_histo[16];
+
+  /* For HuffmanTreeGroupDecode. */
+  int htree_index;
+  HuffmanCode* next;
+
+  /* For DecodeContextMap. */
+  uint32_t context_index;
+  uint32_t max_run_length_prefix;
+  uint32_t code;
+  HuffmanCode context_map_table[BROTLI_HUFFMAN_MAX_SIZE_272];
+} BrotliMetablockHeaderArena;
+
+typedef struct BrotliMetablockBodyArena {
+  uint8_t dist_extra_bits[544];
+  uint32_t dist_offset[544];
+} BrotliMetablockBodyArena;
+
 struct BrotliDecoderStateStruct {
   BrotliRunningState state;
 
@@ -110,7 +245,8 @@ struct BrotliDecoderStateStruct {
   brotli_free_func free_func;
   void* memory_manager_opaque;
 
-  /* Temporary storage for remaining input. */
+  /* Temporary storage for remaining input. Brotli stream format is designed in
+     a way, that 64 bits are enough to make progress in decoding. */
   union {
     uint64_t u64;
     uint8_t u8[8];
@@ -125,7 +261,6 @@ struct BrotliDecoderStateStruct {
   int dist_rb_idx;
   int dist_rb[4];
   int error_code;
-  uint32_t sub_loop_counter;
   uint8_t* ringbuffer;
   uint8_t* ringbuffer_end;
   HuffmanCode* htree_command;
@@ -153,13 +288,10 @@ struct BrotliDecoderStateStruct {
   uint32_t block_type_rb[6];
   uint32_t distance_postfix_bits;
   uint32_t num_direct_distance_codes;
-  int distance_postfix_mask;
   uint32_t num_dist_htrees;
   uint8_t* dist_context_map;
   HuffmanCode* literal_htree;
   uint8_t dist_htree_index;
-  uint32_t repeat_code_len;
-  uint32_t prev_code_len;
 
   int copy_length;
   int distance_code;
@@ -168,33 +300,6 @@ struct BrotliDecoderStateStruct {
   size_t rb_roundtrips;  /* how many times we went around the ring-buffer */
   size_t partial_pos_out;  /* how much output to the user in total */
 
-  /* For ReadHuffmanCode. */
-  uint32_t symbol;
-  uint32_t repeat;
-  uint32_t space;
-
-  HuffmanCode table[32];
-  /* List of heads of symbol chains. */
-  uint16_t* symbol_lists;
-  /* Storage from symbol_lists. */
-  uint16_t symbols_lists_array[BROTLI_HUFFMAN_MAX_CODE_LENGTH + 1 +
-                               BROTLI_NUM_COMMAND_SYMBOLS];
-  /* Tails of symbol chains. */
-  int next_symbol[32];
-  uint8_t code_length_code_lengths[BROTLI_CODE_LENGTH_CODES];
-  /* Population counts for the code lengths. */
-  uint16_t code_length_histo[16];
-
-  /* For HuffmanTreeGroupDecode. */
-  int htree_index;
-  HuffmanCode* next;
-
-  /* For DecodeContextMap. */
-  uint32_t context_index;
-  uint32_t max_run_length_prefix;
-  uint32_t code;
-  HuffmanCode context_map_table[BROTLI_HUFFMAN_MAX_SIZE_272];
-
   /* For InverseMoveToFrontTransform. */
   uint32_t mtf_upper_bound;
   uint32_t mtf[64 + 1];
@@ -203,10 +308,7 @@ struct BrotliDecoderStateStruct {
 
   /* States inside function calls. */
   BrotliRunningMetablockHeaderState substate_metablock_header;
-  BrotliRunningTreeGroupState substate_tree_group;
-  BrotliRunningContextMapState substate_context_map;
   BrotliRunningUncompressedState substate_uncompressed;
-  BrotliRunningHuffmanState substate_huffman;
   BrotliRunningDecodeUint8State substate_decode_uint8;
   BrotliRunningReadBlockLengthState substate_read_block_length;
 
@@ -229,6 +331,11 @@ struct BrotliDecoderStateStruct {
   const BrotliTransforms* transforms;
 
   uint32_t trivial_literal_contexts[8];  /* 256 bits */
+
+  union {
+    BrotliMetablockHeaderArena header;
+    BrotliMetablockBodyArena body;
+  } arena;
 };
 
 typedef struct BrotliDecoderStateStruct BrotliDecoderStateInternal;
@@ -241,8 +348,8 @@ BROTLI_INTERNAL void BrotliDecoderStateMetablockBegin(BrotliDecoderState* s);
 BROTLI_INTERNAL void BrotliDecoderStateCleanupAfterMetablock(
     BrotliDecoderState* s);
 BROTLI_INTERNAL BROTLI_BOOL BrotliDecoderHuffmanTreeGroupInit(
-    BrotliDecoderState* s, HuffmanTreeGroup* group, uint32_t alphabet_size,
-    uint32_t max_symbol, uint32_t ntrees);
+    BrotliDecoderState* s, HuffmanTreeGroup* group, uint32_t alphabet_size_max,
+    uint32_t alphabet_size_limit, uint32_t ntrees);
 
 #define BROTLI_DECODER_ALLOC(S, L) S->alloc_func(S->memory_manager_opaque, L)
 

data/vendor/brotli/c/enc/backward_references.c

@@ -9,6 +9,7 @@
 #include "./backward_references.h"
 
 #include "../common/constants.h"
+#include "../common/context.h"
 #include "../common/dictionary.h"
 #include "../common/platform.h"
 #include <brotli/types.h>
@@ -119,17 +120,17 @@ static BROTLI_INLINE size_t ComputeDistanceCode(size_t distance,
 #undef CAT
 #undef EXPAND_CAT
 
-void BrotliCreateBackwardReferences(
-    size_t num_bytes, size_t position, const uint8_t* ringbuffer,
-    size_t ringbuffer_mask, const BrotliEncoderParams* params,
-    HasherHandle hasher, int* dist_cache, size_t* last_insert_len,
+void BrotliCreateBackwardReferences(size_t num_bytes,
+    size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
+    ContextLut literal_context_lut, const BrotliEncoderParams* params,
+    Hasher* hasher, int* dist_cache, size_t* last_insert_len,
     Command* commands, size_t* num_commands, size_t* num_literals) {
   switch (params->hasher.type) {
 #define CASE_(N)                                                  \
     case N:                                                       \
-      CreateBackwardReferencesNH ## N(                            \
-          num_bytes, position, ringbuffer,                        \
-          ringbuffer_mask, params, hasher, dist_cache,            \
+      CreateBackwardReferencesNH ## N(num_bytes,                  \
+          position, ringbuffer, ringbuffer_mask,                  \
+          literal_context_lut, params, hasher, dist_cache,        \
           last_insert_len, commands, num_commands, num_literals); \
       return;
     FOR_GENERIC_HASHERS(CASE_)

data/vendor/brotli/c/enc/backward_references.h

@@ -10,6 +10,7 @@
 #define BROTLI_ENC_BACKWARD_REFERENCES_H_
 
 #include "../common/constants.h"
+#include "../common/context.h"
 #include "../common/dictionary.h"
 #include "../common/platform.h"
 #include <brotli/types.h>
@@ -25,10 +26,10 @@ extern "C" {
    initially the total amount of commands output by previous
    CreateBackwardReferences calls, and must be incremented by the amount written
    by this call. */
-BROTLI_INTERNAL void BrotliCreateBackwardReferences(
-    size_t num_bytes, size_t position, const uint8_t* ringbuffer,
-    size_t ringbuffer_mask, const BrotliEncoderParams* params,
-    HasherHandle hasher, int* dist_cache, size_t* last_insert_len,
+BROTLI_INTERNAL void BrotliCreateBackwardReferences(size_t num_bytes,
+    size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
+    ContextLut literal_context_lut, const BrotliEncoderParams* params,
+    Hasher* hasher, int* dist_cache, size_t* last_insert_len,
     Command* commands, size_t* num_commands, size_t* num_literals);
 
 #if defined(__cplusplus) || defined(c_plusplus)

data/vendor/brotli/c/enc/backward_references_hq.c

@@ -11,6 +11,7 @@
 #include <string.h>  /* memcpy, memset */
 
 #include "../common/constants.h"
+#include "../common/context.h"
 #include "../common/platform.h"
 #include <brotli/types.h>
 #include "./command.h"
@@ -26,6 +27,7 @@
 extern "C" {
 #endif
 
+/* BrotliCalculateDistanceCodeLimit(BROTLI_MAX_ALLOWED_DISTANCE, 3, 120). */
 #define BROTLI_MAX_EFFECTIVE_DISTANCE_ALPHABET_SIZE 544
 
 static const float kInfinity = 1.7e38f;  /* ~= 2 ^ 127 */
@@ -86,14 +88,10 @@ typedef struct ZopfliCostModel {
 static void InitZopfliCostModel(
     MemoryManager* m, ZopfliCostModel* self, const BrotliDistanceParams* dist,
     size_t num_bytes) {
-  uint32_t distance_histogram_size = dist->alphabet_size;
-  if (distance_histogram_size > BROTLI_MAX_EFFECTIVE_DISTANCE_ALPHABET_SIZE) {
-    distance_histogram_size = BROTLI_MAX_EFFECTIVE_DISTANCE_ALPHABET_SIZE;
-  }
   self->num_bytes_ = num_bytes;
   self->literal_costs_ = BROTLI_ALLOC(m, float, num_bytes + 2);
-  self->cost_dist_ = BROTLI_ALLOC(m, float, dist->alphabet_size);
-  self->distance_histogram_size = distance_histogram_size;
+  self->cost_dist_ = BROTLI_ALLOC(m, float, dist->alphabet_size_limit);
+  self->distance_histogram_size = dist->alphabet_size_limit;
   if (BROTLI_IS_OOM(m)) return;
 }
 
@@ -408,9 +406,12 @@ static size_t UpdateNodes(
     const int* starting_dist_cache, const size_t num_matches,
     const BackwardMatch* matches, const ZopfliCostModel* model,
     StartPosQueue* queue, ZopfliNode* nodes) {
+  const size_t stream_offset = params->stream_offset;
   const size_t cur_ix = block_start + pos;
   const size_t cur_ix_masked = cur_ix & ringbuffer_mask;
   const size_t max_distance = BROTLI_MIN(size_t, cur_ix, max_backward_limit);
+  const size_t dictionary_start = BROTLI_MIN(size_t,
+      cur_ix + stream_offset, max_backward_limit);
   const size_t max_len = num_bytes - pos;
   const size_t max_zopfli_len = MaxZopfliLen(params);
   const size_t max_iters = MaxZopfliCandidates(params);
@@ -419,8 +420,8 @@ static size_t UpdateNodes(
   size_t k;
   size_t gap = 0;
 
-  EvaluateNode(block_start, pos, max_backward_limit, gap, starting_dist_cache,
-      model, queue, nodes);
+  EvaluateNode(block_start + stream_offset, pos, max_backward_limit, gap,
+      starting_dist_cache, model, queue, nodes);
 
   {
     const PosData* posdata = StartPosQueueAt(queue, 0);
@@ -453,7 +454,7 @@ static size_t UpdateNodes(
       if (cur_ix_masked + best_len > ringbuffer_mask) {
         break;
       }
-      if (BROTLI_PREDICT_FALSE(backward > max_distance + gap)) {
+      if (BROTLI_PREDICT_FALSE(backward > dictionary_start + gap)) {
         /* Word dictionary -> ignore. */
         continue;
       }
@@ -472,6 +473,8 @@ static size_t UpdateNodes(
                                        &ringbuffer[cur_ix_masked],
                                        max_len);
       } else {
+        /* "Gray" area. It is addressable by decoder, but this encoder
+           instance does not have that data -> should not touch it. */
         continue;
       }
       {
@@ -506,7 +509,7 @@ static size_t UpdateNodes(
         BackwardMatch match = matches[j];
         size_t dist = match.distance;
         BROTLI_BOOL is_dictionary_match =
-            TO_BROTLI_BOOL(dist > max_distance + gap);
+            TO_BROTLI_BOOL(dist > dictionary_start + gap);
         /* We already tried all possible last distance matches, so we can use
            normal distance code here. */
         size_t dist_code = dist + BROTLI_NUM_DISTANCE_SHORT_CODES - 1;
@@ -569,6 +572,7 @@ void BrotliZopfliCreateCommands(const size_t num_bytes,
     const size_t block_start, const ZopfliNode* nodes, int* dist_cache,
     size_t* last_insert_len, const BrotliEncoderParams* params,
     Command* commands, size_t* num_literals) {
+  const size_t stream_offset = params->stream_offset;
   const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin);
   size_t pos = 0;
   uint32_t offset = nodes[0].u.next;
@@ -587,9 +591,10 @@ void BrotliZopfliCreateCommands(const size_t num_bytes,
     {
       size_t distance = ZopfliNodeCopyDistance(next);
       size_t len_code = ZopfliNodeLengthCode(next);
-      size_t max_distance =
-          BROTLI_MIN(size_t, block_start + pos, max_backward_limit);
-      BROTLI_BOOL is_dictionary = TO_BROTLI_BOOL(distance > max_distance + gap);
+      size_t dictionary_start = BROTLI_MIN(size_t,
+          block_start + pos + stream_offset, max_backward_limit);
+      BROTLI_BOOL is_dictionary =
+          TO_BROTLI_BOOL(distance > dictionary_start + gap);
       size_t dist_code = ZopfliNodeDistanceCode(next);
       InitCommand(&commands[i], &params->dist, insert_length,
           copy_length, (int)len_code - (int)copy_length, dist_code);
@@ -613,6 +618,7 @@ static size_t ZopfliIterate(size_t num_bytes, size_t position,
     const BrotliEncoderParams* params, const size_t gap, const int* dist_cache,
     const ZopfliCostModel* model, const uint32_t* num_matches,
     const BackwardMatch* matches, ZopfliNode* nodes) {
+  const size_t stream_offset = params->stream_offset;
   const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin);
   const size_t max_zopfli_len = MaxZopfliLen(params);
   StartPosQueue queue;
@@ -637,7 +643,7 @@ static size_t ZopfliIterate(size_t num_bytes, size_t position,
       while (skip) {
         i++;
         if (i + 3 >= num_bytes) break;
-        EvaluateNode(position, i, max_backward_limit, gap,
+        EvaluateNode(position + stream_offset, i, max_backward_limit, gap,
             dist_cache, model, &queue, nodes);
         cur_match_pos += num_matches[i];
         skip--;
@@ -650,8 +656,9 @@ static size_t ZopfliIterate(size_t num_bytes, size_t position,
 /* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */
 size_t BrotliZopfliComputeShortestPath(MemoryManager* m, size_t num_bytes,
     size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
-    const BrotliEncoderParams* params,
-    const int* dist_cache, HasherHandle hasher, ZopfliNode* nodes) {
+    ContextLut literal_context_lut, const BrotliEncoderParams* params,
+    const int* dist_cache, Hasher* hasher, ZopfliNode* nodes) {
+  const size_t stream_offset = params->stream_offset;
   const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin);
   const size_t max_zopfli_len = MaxZopfliLen(params);
   ZopfliCostModel model;
@@ -662,6 +669,7 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m, size_t num_bytes,
   size_t i;
   size_t gap = 0;
   size_t lz_matches_offset = 0;
+  BROTLI_UNUSED(literal_context_lut);
   nodes[0].length = 0;
   nodes[0].u.cost = 0;
   InitZopfliCostModel(m, &model, &params->dist, num_bytes);
@@ -672,12 +680,14 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m, size_t num_bytes,
   for (i = 0; i + HashTypeLengthH10() - 1 < num_bytes; i++) {
     const size_t pos = position + i;
     const size_t max_distance = BROTLI_MIN(size_t, pos, max_backward_limit);
+    const size_t dictionary_start = BROTLI_MIN(size_t,
+        pos + stream_offset, max_backward_limit);
     size_t skip;
     size_t num_matches;
-    num_matches = FindAllMatchesH10(hasher,
+    num_matches = FindAllMatchesH10(&hasher->privat._H10,
         &params->dictionary,
         ringbuffer, ringbuffer_mask, pos, num_bytes - i, max_distance,
-        gap, params, &matches[lz_matches_offset]);
+        dictionary_start + gap, params, &matches[lz_matches_offset]);
     if (num_matches > 0 &&
         BackwardMatchLength(&matches[num_matches - 1]) > max_zopfli_len) {
       matches[0] = matches[num_matches - 1];
@@ -692,13 +702,14 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m, size_t num_bytes,
     }
     if (skip > 1) {
       /* Add the tail of the copy to the hasher. */
-      StoreRangeH10(hasher, ringbuffer, ringbuffer_mask, pos + 1, BROTLI_MIN(
+      StoreRangeH10(&hasher->privat._H10,
+          ringbuffer, ringbuffer_mask, pos + 1, BROTLI_MIN(
           size_t, pos + skip, store_end));
       skip--;
       while (skip) {
         i++;
         if (i + HashTypeLengthH10() - 1 >= num_bytes) break;
-        EvaluateNode(position, i, max_backward_limit, gap,
+        EvaluateNode(position + stream_offset, i, max_backward_limit, gap,
             dist_cache, &model, &queue, nodes);
         skip--;
       }
@@ -710,15 +721,14 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m, size_t num_bytes,
 
 void BrotliCreateZopfliBackwardReferences(MemoryManager* m, size_t num_bytes,
     size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
-    const BrotliEncoderParams* params,
-    HasherHandle hasher, int* dist_cache, size_t* last_insert_len,
+    ContextLut literal_context_lut, const BrotliEncoderParams* params,
+    Hasher* hasher, int* dist_cache, size_t* last_insert_len,
     Command* commands, size_t* num_commands, size_t* num_literals) {
-  ZopfliNode* nodes;
-  nodes = BROTLI_ALLOC(m, ZopfliNode, num_bytes + 1);
-  if (BROTLI_IS_OOM(m)) return;
+  ZopfliNode* nodes = BROTLI_ALLOC(m, ZopfliNode, num_bytes + 1);
+  if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(nodes)) return;
   BrotliInitZopfliNodes(nodes, num_bytes + 1);
   *num_commands += BrotliZopfliComputeShortestPath(m, num_bytes,
-      position, ringbuffer, ringbuffer_mask, params,
+      position, ringbuffer, ringbuffer_mask, literal_context_lut, params,
       dist_cache, hasher, nodes);
   if (BROTLI_IS_OOM(m)) return;
   BrotliZopfliCreateCommands(num_bytes, position, nodes, dist_cache,
@@ -728,9 +738,10 @@ void BrotliCreateZopfliBackwardReferences(MemoryManager* m, size_t num_bytes,
 
 void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m, size_t num_bytes,
     size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
-    const BrotliEncoderParams* params,
-    HasherHandle hasher, int* dist_cache, size_t* last_insert_len,
+    ContextLut literal_context_lut, const BrotliEncoderParams* params,
+    Hasher* hasher, int* dist_cache, size_t* last_insert_len,
     Command* commands, size_t* num_commands, size_t* num_literals) {
+  const size_t stream_offset = params->stream_offset;
   const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin);
   uint32_t* num_matches = BROTLI_ALLOC(m, uint32_t, num_bytes);
   size_t matches_size = 4 * num_bytes;
@@ -747,10 +758,16 @@ void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m, size_t num_bytes,
   BackwardMatch* matches = BROTLI_ALLOC(m, BackwardMatch, matches_size);
   size_t gap = 0;
   size_t shadow_matches = 0;
-  if (BROTLI_IS_OOM(m)) return;
+  BROTLI_UNUSED(literal_context_lut);
+  if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(num_matches) ||
+      BROTLI_IS_NULL(matches)) {
+    return;
+  }
   for (i = 0; i + HashTypeLengthH10() - 1 < num_bytes; ++i) {
     const size_t pos = position + i;
     size_t max_distance = BROTLI_MIN(size_t, pos, max_backward_limit);
+    size_t dictionary_start = BROTLI_MIN(size_t,
+        pos + stream_offset, max_backward_limit);
     size_t max_length = num_bytes - i;
     size_t num_found_matches;
     size_t cur_match_end;
@@ -759,10 +776,10 @@ void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m, size_t num_bytes,
     BROTLI_ENSURE_CAPACITY(m, BackwardMatch, matches, matches_size,
         cur_match_pos + MAX_NUM_MATCHES_H10 + shadow_matches);
     if (BROTLI_IS_OOM(m)) return;
-    num_found_matches = FindAllMatchesH10(hasher,
+    num_found_matches = FindAllMatchesH10(&hasher->privat._H10,
         &params->dictionary,
         ringbuffer, ringbuffer_mask, pos, max_length,
-        max_distance, gap, params,
+        max_distance, dictionary_start + gap, params,
         &matches[cur_match_pos + shadow_matches]);
     cur_match_end = cur_match_pos + num_found_matches;
     for (j = cur_match_pos; j + 1 < cur_match_end; ++j) {
@@ -777,7 +794,8 @@ void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m, size_t num_bytes,
         matches[cur_match_pos++] = matches[cur_match_end - 1];
         num_matches[i] = 1;
         /* Add the tail of the copy to the hasher. */
-        StoreRangeH10(hasher, ringbuffer, ringbuffer_mask, pos + 1,
+        StoreRangeH10(&hasher->privat._H10,
+                      ringbuffer, ringbuffer_mask, pos + 1,
                       BROTLI_MIN(size_t, pos + match_len, store_end));
         memset(&num_matches[i + 1], 0, skip * sizeof(num_matches[0]));
         i += skip;
@@ -791,7 +809,7 @@ void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m, size_t num_bytes,
   memcpy(orig_dist_cache, dist_cache, 4 * sizeof(dist_cache[0]));
   orig_num_commands = *num_commands;
   nodes = BROTLI_ALLOC(m, ZopfliNode, num_bytes + 1);
-  if (BROTLI_IS_OOM(m)) return;
+  if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(nodes)) return;
   InitZopfliCostModel(m, &model, &params->dist, num_bytes);
   if (BROTLI_IS_OOM(m)) return;
   for (i = 0; i < 2; i++) {