brotli 0.1.1 → 0.1.2

data/vendor/brotli/enc/brotli_bit_stream.cc

@@ -11,70 +11,86 @@
 #include "./brotli_bit_stream.h"
 
 #include <algorithm>
+#include <cstdlib>  /* free, malloc */
+#include <cstring>
 #include <limits>
 #include <vector>
 
 #include "./bit_cost.h"
 #include "./context.h"
 #include "./entropy_encode.h"
+#include "./entropy_encode_static.h"
 #include "./fast_log.h"
 #include "./prefix.h"
 #include "./write_bits.h"
+
 namespace brotli {
 
-// returns false if fail
+namespace {
+
+static const size_t kMaxHuffmanTreeSize = 2 * kNumCommandPrefixes + 1;
+// Context map alphabet has 256 context id symbols plus max 16 rle symbols.
+static const size_t kContextMapAlphabetSize = 256 + 16;
+// Block type alphabet has 256 block id symbols plus 2 special symbols.
+static const size_t kBlockTypeAlphabetSize = 256 + 2;
+
 // nibblesbits represents the 2 bits to encode MNIBBLES (0-3)
-bool EncodeMlen(size_t length, int* bits, int* numbits, int* nibblesbits) {
-  if (length > (1 << 24)) {
-    return false;
-  }
+// REQUIRES: length > 0
+// REQUIRES: length <= (1 << 24)
+void EncodeMlen(size_t length, uint64_t* bits,
+                size_t* numbits, uint64_t* nibblesbits) {
+  assert(length > 0);
+  assert(length <= (1 << 24));
   length--;  // MLEN - 1 is encoded
-  int lg = length == 0 ? 1 : Log2Floor(static_cast<uint32_t>(length)) + 1;
+  size_t lg = length == 0 ? 1 : Log2FloorNonZero(
+      static_cast<uint32_t>(length)) + 1;
   assert(lg <= 24);
-  int mnibbles = (lg < 16 ? 16 : (lg + 3)) / 4;
+  size_t mnibbles = (lg < 16 ? 16 : (lg + 3)) / 4;
   *nibblesbits = mnibbles - 4;
   *numbits = mnibbles * 4;
-  *bits = static_cast<int>(length);
-  return true;
+  *bits = length;
 }
 
-void StoreVarLenUint8(int n, int* storage_ix, uint8_t* storage) {
+static inline void StoreCommandExtra(
+    const Command& cmd, size_t* storage_ix, uint8_t* storage) {
+  uint32_t copylen_code = cmd.copy_len_code();
+  uint16_t inscode = GetInsertLengthCode(cmd.insert_len_);
+  uint16_t copycode = GetCopyLengthCode(copylen_code);
+  uint32_t insnumextra = GetInsertExtra(inscode);
+  uint64_t insextraval = cmd.insert_len_ - GetInsertBase(inscode);
+  uint64_t copyextraval = copylen_code - GetCopyBase(copycode);
+  uint64_t bits = (copyextraval << insnumextra) | insextraval;
+  WriteBits(insnumextra + GetCopyExtra(copycode), bits, storage_ix, storage);
+}
+
+}  // namespace
+
+void StoreVarLenUint8(size_t n, size_t* storage_ix, uint8_t* storage) {
   if (n == 0) {
     WriteBits(1, 0, storage_ix, storage);
   } else {
     WriteBits(1, 1, storage_ix, storage);
-    int nbits = Log2Floor(n);
+    size_t nbits = Log2FloorNonZero(n);
     WriteBits(3, nbits, storage_ix, storage);
     WriteBits(nbits, n - (1 << nbits), storage_ix, storage);
   }
 }
 
-bool StoreCompressedMetaBlockHeader(bool final_block,
+void StoreCompressedMetaBlockHeader(bool final_block,
                                     size_t length,
-                                    int* storage_ix,
+                                    size_t* storage_ix,
                                     uint8_t* storage) {
   // Write ISLAST bit.
   WriteBits(1, final_block, storage_ix, storage);
   // Write ISEMPTY bit.
   if (final_block) {
-    WriteBits(1, length == 0, storage_ix, storage);
-    if (length == 0) {
-      return true;
-    }
-  }
-
-  if (length == 0) {
-    // Only the last meta-block can be empty.
-    return false;
-  }
-
-  int lenbits;
-  int nlenbits;
-  int nibblesbits;
-  if (!EncodeMlen(length, &lenbits, &nlenbits, &nibblesbits)) {
-    return false;
+    WriteBits(1, 0, storage_ix, storage);
   }
 
+  uint64_t lenbits;
+  size_t nlenbits;
+  uint64_t nibblesbits;
+  EncodeMlen(length, &lenbits, &nlenbits, &nibblesbits);
   WriteBits(2, nibblesbits, storage_ix, storage);
   WriteBits(nlenbits, lenbits, storage_ix, storage);
 
@@ -82,31 +98,27 @@ bool StoreCompressedMetaBlockHeader(bool final_block,
     // Write ISUNCOMPRESSED bit.
     WriteBits(1, 0, storage_ix, storage);
   }
-  return true;
 }
 
-bool StoreUncompressedMetaBlockHeader(size_t length,
-                                      int* storage_ix,
+void StoreUncompressedMetaBlockHeader(size_t length,
+                                      size_t* storage_ix,
                                       uint8_t* storage) {
   // Write ISLAST bit. Uncompressed block cannot be the last one, so set to 0.
   WriteBits(1, 0, storage_ix, storage);
-  int lenbits;
-  int nlenbits;
-  int nibblesbits;
-  if (!EncodeMlen(length, &lenbits, &nlenbits, &nibblesbits)) {
-    return false;
-  }
+  uint64_t lenbits;
+  size_t nlenbits;
+  uint64_t nibblesbits;
+  EncodeMlen(length, &lenbits, &nlenbits, &nibblesbits);
   WriteBits(2, nibblesbits, storage_ix, storage);
   WriteBits(nlenbits, lenbits, storage_ix, storage);
   // Write ISUNCOMPRESSED bit.
   WriteBits(1, 1, storage_ix, storage);
-  return true;
 }
 
 void StoreHuffmanTreeOfHuffmanTreeToBitMask(
     const int num_codes,
     const uint8_t *code_length_bitdepth,
-    int *storage_ix,
+    size_t *storage_ix,
     uint8_t *storage) {
   static const uint8_t kStorageOrder[kCodeLengthCodes] = {
     1, 2, 3, 4, 0, 5, 17, 6, 16, 7, 8, 9, 10, 11, 12, 13, 14, 15
@@ -129,7 +141,7 @@ void StoreHuffmanTreeOfHuffmanTreeToBitMask(
   };
 
   // Throw away trailing zeros:
-  int codes_to_store = kCodeLengthCodes;
+  size_t codes_to_store = kCodeLengthCodes;
   if (num_codes > 1) {
     for (; codes_to_store > 0; --codes_to_store) {
       if (code_length_bitdepth[kStorageOrder[codes_to_store - 1]] != 0) {
@@ -137,7 +149,7 @@ void StoreHuffmanTreeOfHuffmanTreeToBitMask(
       }
     }
   }
-  int skip_some = 0;  // skips none.
+  size_t skip_some = 0;  // skips none.
   if (code_length_bitdepth[kStorageOrder[0]] == 0 &&
       code_length_bitdepth[kStorageOrder[1]] == 0) {
     skip_some = 2;  // skips two.
@@ -146,22 +158,23 @@ void StoreHuffmanTreeOfHuffmanTreeToBitMask(
     }
   }
   WriteBits(2, skip_some, storage_ix, storage);
-  for (int i = skip_some; i < codes_to_store; ++i) {
-    uint8_t l = code_length_bitdepth[kStorageOrder[i]];
+  for (size_t i = skip_some; i < codes_to_store; ++i) {
+    size_t l = code_length_bitdepth[kStorageOrder[i]];
     WriteBits(kHuffmanBitLengthHuffmanCodeBitLengths[l],
               kHuffmanBitLengthHuffmanCodeSymbols[l], storage_ix, storage);
   }
 }
 
-void StoreHuffmanTreeToBitMask(
-    const std::vector<uint8_t> &huffman_tree,
-    const std::vector<uint8_t> &huffman_tree_extra_bits,
-    const uint8_t *code_length_bitdepth,
-    const std::vector<uint16_t> &code_length_bitdepth_symbols,
-    int * __restrict storage_ix,
+static void StoreHuffmanTreeToBitMask(
+    const size_t huffman_tree_size,
+    const uint8_t* huffman_tree,
+    const uint8_t* huffman_tree_extra_bits,
+    const uint8_t* code_length_bitdepth,
+    const uint16_t* code_length_bitdepth_symbols,
+    size_t * __restrict storage_ix,
     uint8_t * __restrict storage) {
-  for (size_t i = 0; i < huffman_tree.size(); ++i) {
-    int ix = huffman_tree[i];
+  for (size_t i = 0; i < huffman_tree_size; ++i) {
+    size_t ix = huffman_tree[i];
     WriteBits(code_length_bitdepth[ix], code_length_bitdepth_symbols[ix],
               storage_ix, storage);
     // Extra bits
@@ -176,18 +189,18 @@ void StoreHuffmanTreeToBitMask(
   }
 }
 
-void StoreSimpleHuffmanTree(const uint8_t* depths,
-                            int symbols[4],
-                            int num_symbols,
-                            int max_bits,
-                            int *storage_ix, uint8_t *storage) {
+static void StoreSimpleHuffmanTree(const uint8_t* depths,
+                                   size_t symbols[4],
+                                   size_t num_symbols,
+                                   size_t max_bits,
+                                   size_t *storage_ix, uint8_t *storage) {
   // value of 1 indicates a simple Huffman code
   WriteBits(2, 1, storage_ix, storage);
   WriteBits(2, num_symbols - 1, storage_ix, storage);  // NSYM - 1
 
   // Sort
-  for (int i = 0; i < num_symbols; i++) {
-    for (int j = i + 1; j < num_symbols; j++) {
+  for (size_t i = 0; i < num_symbols; i++) {
+    for (size_t j = i + 1; j < num_symbols; j++) {
       if (depths[symbols[j]] < depths[symbols[i]]) {
         std::swap(symbols[j], symbols[i]);
       }
@@ -213,19 +226,22 @@ void StoreSimpleHuffmanTree(const uint8_t* depths,
 
 // num = alphabet size
 // depths = symbol depths
-void StoreHuffmanTree(const uint8_t* depths, int num,
-                      int *storage_ix, uint8_t *storage) {
+void StoreHuffmanTree(const uint8_t* depths, size_t num,
+                      HuffmanTree* tree,
+                      size_t *storage_ix, uint8_t *storage) {
   // Write the Huffman tree into the brotli-representation.
-  std::vector<uint8_t> huffman_tree;
-  std::vector<uint8_t> huffman_tree_extra_bits;
-  // TODO: Consider allocating these from stack.
-  huffman_tree.reserve(256);
-  huffman_tree_extra_bits.reserve(256);
-  WriteHuffmanTree(depths, num, &huffman_tree, &huffman_tree_extra_bits);
+  // The command alphabet is the largest, so this allocation will fit all
+  // alphabets.
+  assert(num <= kNumCommandPrefixes);
+  uint8_t huffman_tree[kNumCommandPrefixes];
+  uint8_t huffman_tree_extra_bits[kNumCommandPrefixes];
+  size_t huffman_tree_size = 0;
+  WriteHuffmanTree(depths, num, &huffman_tree_size, huffman_tree,
+                   huffman_tree_extra_bits);
 
   // Calculate the statistics of the Huffman tree in brotli-representation.
-  int huffman_tree_histogram[kCodeLengthCodes] = { 0 };
-  for (size_t i = 0; i < huffman_tree.size(); ++i) {
+  uint32_t huffman_tree_histogram[kCodeLengthCodes] = { 0 };
+  for (size_t i = 0; i < huffman_tree_size; ++i) {
     ++huffman_tree_histogram[huffman_tree[i]];
   }
 
@@ -245,11 +261,10 @@ void StoreHuffmanTree(const uint8_t* depths, int num,
 
   // Calculate another Huffman tree to use for compressing both the
   // earlier Huffman tree with.
-  // TODO: Consider allocating these from stack.
   uint8_t code_length_bitdepth[kCodeLengthCodes] = { 0 };
-  std::vector<uint16_t> code_length_bitdepth_symbols(kCodeLengthCodes);
+  uint16_t code_length_bitdepth_symbols[kCodeLengthCodes] = { 0 };
   CreateHuffmanTree(&huffman_tree_histogram[0], kCodeLengthCodes,
-                    5, &code_length_bitdepth[0]);
+                    5, tree, &code_length_bitdepth[0]);
   ConvertBitDepthsToSymbols(code_length_bitdepth, kCodeLengthCodes,
                             &code_length_bitdepth_symbols[0]);
 
@@ -262,23 +277,24 @@ void StoreHuffmanTree(const uint8_t* depths, int num,
   }
 
   // Store the real huffman tree now.
-  StoreHuffmanTreeToBitMask(huffman_tree,
+  StoreHuffmanTreeToBitMask(huffman_tree_size,
+                            huffman_tree,
                             huffman_tree_extra_bits,
                             &code_length_bitdepth[0],
                             code_length_bitdepth_symbols,
                             storage_ix, storage);
 }
 
-
-void BuildAndStoreHuffmanTree(const int *histogram,
-                              const int length,
+void BuildAndStoreHuffmanTree(const uint32_t *histogram,
+                              const size_t length,
+                              HuffmanTree* tree,
                               uint8_t* depth,
                               uint16_t* bits,
-                              int* storage_ix,
+                              size_t* storage_ix,
                               uint8_t* storage) {
-  int count = 0;
-  int s4[4] = { 0 };
-  for (int i = 0; i < length; i++) {
+  size_t count = 0;
+  size_t s4[4] = { 0 };
+  for (size_t i = 0; i < length; i++) {
     if (histogram[i]) {
       if (count < 4) {
         s4[count] = i;
@@ -289,8 +305,8 @@ void BuildAndStoreHuffmanTree(const int *histogram,
     }
   }
 
-  int max_bits_counter = length - 1;
-  int max_bits = 0;
+  size_t max_bits_counter = length - 1;
+  size_t max_bits = 0;
   while (max_bits_counter) {
     max_bits_counter >>= 1;
     ++max_bits;
@@ -302,179 +318,365 @@ void BuildAndStoreHuffmanTree(const int *histogram,
     return;
   }
 
-  CreateHuffmanTree(histogram, length, 15, depth);
+  CreateHuffmanTree(histogram, length, 15, tree, depth);
   ConvertBitDepthsToSymbols(depth, length, bits);
 
   if (count <= 4) {
     StoreSimpleHuffmanTree(depth, s4, count, max_bits, storage_ix, storage);
   } else {
-    StoreHuffmanTree(depth, length, storage_ix, storage);
+    StoreHuffmanTree(depth, length, tree, storage_ix, storage);
+  }
+}
+
+static inline bool SortHuffmanTree(const HuffmanTree& v0,
+                                   const HuffmanTree& v1) {
+  return v0.total_count_ < v1.total_count_;
+}
+
+void BuildAndStoreHuffmanTreeFast(const uint32_t *histogram,
+                                  const size_t histogram_total,
+                                  const size_t max_bits,
+                                  uint8_t* depth,
+                                  uint16_t* bits,
+                                  size_t* storage_ix,
+                                  uint8_t* storage) {
+  size_t count = 0;
+  size_t symbols[4] = { 0 };
+  size_t length = 0;
+  size_t total = histogram_total;
+  while (total != 0) {
+    if (histogram[length]) {
+      if (count < 4) {
+        symbols[count] = length;
+      }
+      ++count;
+      total -= histogram[length];
+    }
+    ++length;
+  }
+
+  if (count <= 1) {
+    WriteBits(4, 1, storage_ix, storage);
+    WriteBits(max_bits, symbols[0], storage_ix, storage);
+    return;
+  }
+
+  const size_t max_tree_size = 2 * length + 1;
+  HuffmanTree* const tree =
+      static_cast<HuffmanTree*>(malloc(max_tree_size * sizeof(HuffmanTree)));
+  for (uint32_t count_limit = 1; ; count_limit *= 2) {
+    HuffmanTree* node = tree;
+    for (size_t i = length; i != 0;) {
+      --i;
+      if (histogram[i]) {
+        if (PREDICT_TRUE(histogram[i] >= count_limit)) {
+          *node = HuffmanTree(histogram[i], -1, static_cast<int16_t>(i));
+        } else {
+          *node = HuffmanTree(count_limit, -1, static_cast<int16_t>(i));
+        }
+        ++node;
+      }
+    }
+    const int n = static_cast<int>(node - tree);
+    std::sort(tree, node, SortHuffmanTree);
+    // The nodes are:
+    // [0, n): the sorted leaf nodes that we start with.
+    // [n]: we add a sentinel here.
+    // [n + 1, 2n): new parent nodes are added here, starting from
+    //              (n+1). These are naturally in ascending order.
+    // [2n]: we add a sentinel at the end as well.
+    // There will be (2n+1) elements at the end.
+    const HuffmanTree sentinel(std::numeric_limits<int>::max(), -1, -1);
+    *node++ = sentinel;
+    *node++ = sentinel;
+
+    int i = 0;      // Points to the next leaf node.
+    int j = n + 1;  // Points to the next non-leaf node.
+    for (int k = n - 1; k > 0; --k) {
+      int left, right;
+      if (tree[i].total_count_ <= tree[j].total_count_) {
+        left = i;
+        ++i;
+      } else {
+        left = j;
+        ++j;
+      }
+      if (tree[i].total_count_ <= tree[j].total_count_) {
+        right = i;
+        ++i;
+      } else {
+        right = j;
+        ++j;
+      }
+      // The sentinel node becomes the parent node.
+      node[-1].total_count_ =
+          tree[left].total_count_ + tree[right].total_count_;
+      node[-1].index_left_ = static_cast<int16_t>(left);
+      node[-1].index_right_or_value_ = static_cast<int16_t>(right);
+      // Add back the last sentinel node.
+      *node++ = sentinel;
+    }
+    SetDepth(tree[2 * n - 1], &tree[0], depth, 0);
+    // We need to pack the Huffman tree in 14 bits.
+    // If this was not successful, add fake entities to the lowest values
+    // and retry.
+    if (PREDICT_TRUE(*std::max_element(&depth[0], &depth[length]) <= 14)) {
+      break;
+    }
+  }
+  free(tree);
+  ConvertBitDepthsToSymbols(depth, length, bits);
+  if (count <= 4) {
+    // value of 1 indicates a simple Huffman code
+    WriteBits(2, 1, storage_ix, storage);
+    WriteBits(2, count - 1, storage_ix, storage);  // NSYM - 1
+
+    // Sort
+    for (size_t i = 0; i < count; i++) {
+      for (size_t j = i + 1; j < count; j++) {
+        if (depth[symbols[j]] < depth[symbols[i]]) {
+          std::swap(symbols[j], symbols[i]);
+        }
+      }
+    }
+
+    if (count == 2) {
+      WriteBits(max_bits, symbols[0], storage_ix, storage);
+      WriteBits(max_bits, symbols[1], storage_ix, storage);
+    } else if (count == 3) {
+      WriteBits(max_bits, symbols[0], storage_ix, storage);
+      WriteBits(max_bits, symbols[1], storage_ix, storage);
+      WriteBits(max_bits, symbols[2], storage_ix, storage);
+    } else {
+      WriteBits(max_bits, symbols[0], storage_ix, storage);
+      WriteBits(max_bits, symbols[1], storage_ix, storage);
+      WriteBits(max_bits, symbols[2], storage_ix, storage);
+      WriteBits(max_bits, symbols[3], storage_ix, storage);
+      // tree-select
+      WriteBits(1, depth[symbols[0]] == 1 ? 1 : 0, storage_ix, storage);
+    }
+  } else {
+    // Complex Huffman Tree
+    StoreStaticCodeLengthCode(storage_ix, storage);
+
+    // Actual rle coding.
+    uint8_t previous_value = 8;
+    for (size_t i = 0; i < length;) {
+      const uint8_t value = depth[i];
+      size_t reps = 1;
+      for (size_t k = i + 1; k < length && depth[k] == value; ++k) {
+        ++reps;
+      }
+      i += reps;
+      if (value == 0) {
+        WriteBits(kZeroRepsDepth[reps], kZeroRepsBits[reps],
+                  storage_ix, storage);
+      } else {
+        if (previous_value != value) {
+          WriteBits(kCodeLengthDepth[value], kCodeLengthBits[value],
+                    storage_ix, storage);
+          --reps;
+        }
+        if (reps < 3) {
+          while (reps != 0) {
+            reps--;
+            WriteBits(kCodeLengthDepth[value], kCodeLengthBits[value],
+                      storage_ix, storage);
+          }
+        } else {
+          reps -= 3;
+          WriteBits(kNonZeroRepsDepth[reps], kNonZeroRepsBits[reps],
+                    storage_ix, storage);
+        }
+        previous_value = value;
+      }
+    }
   }
 }
 
-int IndexOf(const std::vector<int>& v, int value) {
-  for (int i = 0; i < static_cast<int>(v.size()); ++i) {
+static size_t IndexOf(const uint8_t* v, size_t v_size, uint8_t value) {
+  size_t i = 0;
+  for (; i < v_size; ++i) {
     if (v[i] == value) return i;
   }
-  return -1;
+  return i;
 }
 
-void MoveToFront(std::vector<int>* v, int index) {
-  int value = (*v)[index];
-  for (int i = index; i > 0; --i) {
-    (*v)[i] = (*v)[i - 1];
+static void MoveToFront(uint8_t* v, size_t index) {
+  uint8_t value = v[index];
+  for (size_t i = index; i != 0; --i) {
+    v[i] = v[i - 1];
   }
-  (*v)[0] = value;
+  v[0] = value;
 }
 
-std::vector<int> MoveToFrontTransform(const std::vector<int>& v) {
-  if (v.empty()) return v;
-  std::vector<int> mtf(*std::max_element(v.begin(), v.end()) + 1);
-  for (int i = 0; i < static_cast<int>(mtf.size()); ++i) mtf[i] = i;
-  std::vector<int> result(v.size());
-  for (size_t i = 0; i < v.size(); ++i) {
-    int index = IndexOf(mtf, v[i]);
-    assert(index >= 0);
-    result[i] = index;
-    MoveToFront(&mtf, index);
-  }
-  return result;
+static void MoveToFrontTransform(const uint32_t* __restrict v_in,
+                                 const size_t v_size,
+                                 uint32_t* v_out) {
+  if (v_size == 0) {
+    return;
+  }
+  uint32_t max_value = *std::max_element(v_in, v_in + v_size);
+  assert(max_value < 256u);
+  uint8_t mtf[256];
+  size_t mtf_size = max_value + 1;
+  for (uint32_t i = 0; i <= max_value; ++i) {
+    mtf[i] = static_cast<uint8_t>(i);
+  }
+  for (size_t i = 0; i < v_size; ++i) {
+    size_t index = IndexOf(mtf, mtf_size, static_cast<uint8_t>(v_in[i]));
+    assert(index < mtf_size);
+    v_out[i] = static_cast<uint32_t>(index);
+    MoveToFront(mtf, index);
+  }
 }
 
-// Finds runs of zeros in v_in and replaces them with a prefix code of the run
-// length plus extra bits in *v_out and *extra_bits. Non-zero values in v_in are
-// shifted by *max_length_prefix. Will not create prefix codes bigger than the
-// initial value of *max_run_length_prefix. The prefix code of run length L is
-// simply Log2Floor(L) and the number of extra bits is the same as the prefix
-// code.
-void RunLengthCodeZeros(const std::vector<int>& v_in,
-                        int* max_run_length_prefix,
-                        std::vector<int>* v_out,
-                        std::vector<int>* extra_bits) {
-  int max_reps = 0;
-  for (size_t i = 0; i < v_in.size();) {
-    for (; i < v_in.size() && v_in[i] != 0; ++i) ;
-    int reps = 0;
-    for (; i < v_in.size() && v_in[i] == 0; ++i) {
+// Finds runs of zeros in v[0..in_size) and replaces them with a prefix code of
+// the run length plus extra bits (lower 9 bits is the prefix code and the rest
+// are the extra bits). Non-zero values in v[] are shifted by
+// *max_length_prefix. Will not create prefix codes bigger than the initial
+// value of *max_run_length_prefix. The prefix code of run length L is simply
+// Log2Floor(L) and the number of extra bits is the same as the prefix code.
+static void RunLengthCodeZeros(const size_t in_size,
+                               uint32_t* __restrict v,
+                               size_t* __restrict out_size,
+                               uint32_t* __restrict max_run_length_prefix) {
+  uint32_t max_reps = 0;
+  for (size_t i = 0; i < in_size;) {
+    for (; i < in_size && v[i] != 0; ++i) ;
+    uint32_t reps = 0;
+    for (; i < in_size && v[i] == 0; ++i) {
       ++reps;
     }
     max_reps = std::max(reps, max_reps);
   }
-  int max_prefix = max_reps > 0 ? Log2Floor(max_reps) : 0;
-  *max_run_length_prefix = std::min(max_prefix, *max_run_length_prefix);
-  for (size_t i = 0; i < v_in.size();) {
-    if (v_in[i] != 0) {
-      v_out->push_back(v_in[i] + *max_run_length_prefix);
-      extra_bits->push_back(0);
+  uint32_t max_prefix = max_reps > 0 ? Log2FloorNonZero(max_reps) : 0;
+  max_prefix = std::min(max_prefix, *max_run_length_prefix);
+  *max_run_length_prefix = max_prefix;
+  *out_size = 0;
+  for (size_t i = 0; i < in_size;) {
+    assert(*out_size <= i);
+    if (v[i] != 0) {
+      v[*out_size] = v[i] + *max_run_length_prefix;
       ++i;
+      ++(*out_size);
     } else {
-      int reps = 1;
-      for (size_t k = i + 1; k < v_in.size() && v_in[k] == 0; ++k) {
+      uint32_t reps = 1;
+      for (size_t k = i + 1; k < in_size && v[k] == 0; ++k) {
         ++reps;
       }
       i += reps;
-      while (reps) {
-        if (reps < (2 << *max_run_length_prefix)) {
-          int run_length_prefix = Log2Floor(reps);
-          v_out->push_back(run_length_prefix);
-          extra_bits->push_back(reps - (1 << run_length_prefix));
+      while (reps != 0) {
+        if (reps < (2u << max_prefix)) {
+          uint32_t run_length_prefix = Log2FloorNonZero(reps);
+          const uint32_t extra_bits = reps - (1u << run_length_prefix);
+          v[*out_size] = run_length_prefix + (extra_bits << 9);
+          ++(*out_size);
           break;
         } else {
-          v_out->push_back(*max_run_length_prefix);
-          extra_bits->push_back((1 << *max_run_length_prefix) - 1);
-          reps -= (2 << *max_run_length_prefix) - 1;
+          const uint32_t extra_bits = (1u << max_prefix) - 1u;
+          v[*out_size] = max_prefix + (extra_bits << 9);
+          reps -= (2u << max_prefix) - 1u;
+          ++(*out_size);
         }
       }
     }
   }
 }
 
-void EncodeContextMap(const std::vector<int>& context_map,
-                      int num_clusters,
-                      int* storage_ix, uint8_t* storage) {
+void EncodeContextMap(const std::vector<uint32_t>& context_map,
+                      size_t num_clusters,
+                      HuffmanTree* tree,
+                      size_t* storage_ix, uint8_t* storage) {
   StoreVarLenUint8(num_clusters - 1, storage_ix, storage);
 
   if (num_clusters == 1) {
     return;
   }
 
-  std::vector<int> transformed_symbols = MoveToFrontTransform(context_map);
-  std::vector<int> rle_symbols;
-  std::vector<int> extra_bits;
-  int max_run_length_prefix = 6;
-  RunLengthCodeZeros(transformed_symbols, &max_run_length_prefix,
-                     &rle_symbols, &extra_bits);
-  HistogramContextMap symbol_histogram;
-  for (size_t i = 0; i < rle_symbols.size(); ++i) {
-    symbol_histogram.Add(rle_symbols[i]);
+  uint32_t* rle_symbols = new uint32_t[context_map.size()];
+  MoveToFrontTransform(&context_map[0], context_map.size(), rle_symbols);
+  uint32_t max_run_length_prefix = 6;
+  size_t num_rle_symbols = 0;
+  RunLengthCodeZeros(context_map.size(), rle_symbols,
+                     &num_rle_symbols, &max_run_length_prefix);
+  uint32_t histogram[kContextMapAlphabetSize];
+  memset(histogram, 0, sizeof(histogram));
+  static const int kSymbolBits = 9;
+  static const uint32_t kSymbolMask = (1u << kSymbolBits) - 1u;
+  for (size_t i = 0; i < num_rle_symbols; ++i) {
+    ++histogram[rle_symbols[i] & kSymbolMask];
   }
   bool use_rle = max_run_length_prefix > 0;
   WriteBits(1, use_rle, storage_ix, storage);
   if (use_rle) {
     WriteBits(4, max_run_length_prefix - 1, storage_ix, storage);
   }
-  EntropyCodeContextMap symbol_code;
-  memset(symbol_code.depth_, 0, sizeof(symbol_code.depth_));
-  memset(symbol_code.bits_, 0, sizeof(symbol_code.bits_));
-  BuildAndStoreHuffmanTree(symbol_histogram.data_,
-                           num_clusters + max_run_length_prefix,
-                           symbol_code.depth_, symbol_code.bits_,
-                           storage_ix, storage);
-  for (size_t i = 0; i < rle_symbols.size(); ++i) {
-    WriteBits(symbol_code.depth_[rle_symbols[i]],
-              symbol_code.bits_[rle_symbols[i]],
-              storage_ix, storage);
-    if (rle_symbols[i] > 0 && rle_symbols[i] <= max_run_length_prefix) {
-      WriteBits(rle_symbols[i], extra_bits[i], storage_ix, storage);
+  uint8_t depths[kContextMapAlphabetSize];
+  uint16_t bits[kContextMapAlphabetSize];
+  memset(depths, 0, sizeof(depths));
+  memset(bits, 0, sizeof(bits));
+  BuildAndStoreHuffmanTree(histogram, num_clusters + max_run_length_prefix,
+                           tree, depths, bits, storage_ix, storage);
+  for (size_t i = 0; i < num_rle_symbols; ++i) {
+    const uint32_t rle_symbol = rle_symbols[i] & kSymbolMask;
+    const uint32_t extra_bits_val = rle_symbols[i] >> kSymbolBits;
+    WriteBits(depths[rle_symbol], bits[rle_symbol], storage_ix, storage);
+    if (rle_symbol > 0 && rle_symbol <= max_run_length_prefix) {
+      WriteBits(rle_symbol, extra_bits_val, storage_ix, storage);
     }
   }
   WriteBits(1, 1, storage_ix, storage);  // use move-to-front
+  delete[] rle_symbols;
 }
 
 void StoreBlockSwitch(const BlockSplitCode& code,
-                      const int block_ix,
-                      int* storage_ix,
+                      const size_t block_ix,
+                      size_t* storage_ix,
                       uint8_t* storage) {
   if (block_ix > 0) {
-    int typecode = code.type_code[block_ix];
+    size_t typecode = code.type_code[block_ix];
     WriteBits(code.type_depths[typecode], code.type_bits[typecode],
               storage_ix, storage);
   }
-  int lencode = code.length_prefix[block_ix];
+  size_t lencode = code.length_prefix[block_ix];
   WriteBits(code.length_depths[lencode], code.length_bits[lencode],
             storage_ix, storage);
   WriteBits(code.length_nextra[block_ix], code.length_extra[block_ix],
             storage_ix, storage);
 }
 
-void BuildAndStoreBlockSplitCode(const std::vector<int>& types,
-                                 const std::vector<int>& lengths,
-                                 const int num_types,
-                                 BlockSplitCode* code,
-                                 int* storage_ix,
-                                 uint8_t* storage) {
-  const int num_blocks = static_cast<int>(types.size());
-  std::vector<int> type_histo(num_types + 2);
-  std::vector<int> length_histo(26);
-  int last_type = 1;
-  int second_last_type = 0;
+static void BuildAndStoreBlockSplitCode(const std::vector<uint8_t>& types,
+                                        const std::vector<uint32_t>& lengths,
+                                        const size_t num_types,
+                                        HuffmanTree* tree,
+                                        BlockSplitCode* code,
+                                        size_t* storage_ix,
+                                        uint8_t* storage) {
+  const size_t num_blocks = types.size();
+  uint32_t type_histo[kBlockTypeAlphabetSize];
+  uint32_t length_histo[kNumBlockLenPrefixes];
+  memset(type_histo, 0, (num_types + 2) * sizeof(type_histo[0]));
+  memset(length_histo, 0, sizeof(length_histo));
+  size_t last_type = 1;
+  size_t second_last_type = 0;
   code->type_code.resize(num_blocks);
   code->length_prefix.resize(num_blocks);
   code->length_nextra.resize(num_blocks);
   code->length_extra.resize(num_blocks);
   code->type_depths.resize(num_types + 2);
   code->type_bits.resize(num_types + 2);
-  code->length_depths.resize(26);
-  code->length_bits.resize(26);
-  for (int i = 0; i < num_blocks; ++i) {
-    int type = types[i];
-    int type_code = (type == last_type + 1 ? 1 :
+  memset(code->length_depths, 0, sizeof(code->length_depths));
+  memset(code->length_bits, 0, sizeof(code->length_bits));
+  for (size_t i = 0; i < num_blocks; ++i) {
+    size_t type = types[i];
+    size_t type_code = (type == last_type + 1 ? 1 :
                      type == second_last_type ? 0 :
                      type + 2);
     second_last_type = last_type;
     last_type = type;
-    code->type_code[i] = type_code;
-    if (i > 0) ++type_histo[type_code];
+    code->type_code[i] = static_cast<uint32_t>(type_code);
+    if (i != 0) ++type_histo[type_code];
     GetBlockLengthPrefixCode(lengths[i],
                              &code->length_prefix[i],
                              &code->length_nextra[i],
@@ -483,41 +685,45 @@ void BuildAndStoreBlockSplitCode(const std::vector<int>& types,
   }
   StoreVarLenUint8(num_types - 1, storage_ix, storage);
   if (num_types > 1) {
-    BuildAndStoreHuffmanTree(&type_histo[0], num_types + 2,
+    BuildAndStoreHuffmanTree(&type_histo[0], num_types + 2, tree,
                              &code->type_depths[0], &code->type_bits[0],
                              storage_ix, storage);
-    BuildAndStoreHuffmanTree(&length_histo[0], 26,
+    BuildAndStoreHuffmanTree(&length_histo[0], kNumBlockLenPrefixes, tree,
                              &code->length_depths[0], &code->length_bits[0],
                              storage_ix, storage);
     StoreBlockSwitch(*code, 0, storage_ix, storage);
   }
 }
 
-void StoreTrivialContextMap(int num_types,
-                            int context_bits,
-                            int* storage_ix,
+void StoreTrivialContextMap(size_t num_types,
+                            size_t context_bits,
+                            HuffmanTree* tree,
+                            size_t* storage_ix,
                             uint8_t* storage) {
   StoreVarLenUint8(num_types - 1, storage_ix, storage);
   if (num_types > 1) {
-    int repeat_code = context_bits - 1;
-    int repeat_bits = (1 << repeat_code) - 1;
-    int alphabet_size = num_types + repeat_code;
-    std::vector<int> histogram(alphabet_size);
-    std::vector<uint8_t> depths(alphabet_size);
-    std::vector<uint16_t> bits(alphabet_size);
+    size_t repeat_code = context_bits - 1u;
+    size_t repeat_bits = (1u << repeat_code) - 1u;
+    size_t alphabet_size = num_types + repeat_code;
+    uint32_t histogram[kContextMapAlphabetSize];
+    uint8_t depths[kContextMapAlphabetSize];
+    uint16_t bits[kContextMapAlphabetSize];
+    memset(histogram, 0, alphabet_size * sizeof(histogram[0]));
+    memset(depths, 0, alphabet_size * sizeof(depths[0]));
+    memset(bits, 0, alphabet_size * sizeof(bits[0]));
     // Write RLEMAX.
     WriteBits(1, 1, storage_ix, storage);
     WriteBits(4, repeat_code - 1, storage_ix, storage);
-    histogram[repeat_code] = num_types;
+    histogram[repeat_code] = static_cast<uint32_t>(num_types);
     histogram[0] = 1;
-    for (int i = context_bits; i < alphabet_size; ++i) {
+    for (size_t i = context_bits; i < alphabet_size; ++i) {
       histogram[i] = 1;
     }
-    BuildAndStoreHuffmanTree(&histogram[0], alphabet_size,
+    BuildAndStoreHuffmanTree(&histogram[0], alphabet_size, tree,
                              &depths[0], &bits[0],
                              storage_ix, storage);
-    for (int i = 0; i < num_types; ++i) {
-      int code = (i == 0 ? 0 : i + context_bits - 1);
+    for (size_t i = 0; i < num_types; ++i) {
+      size_t code = (i == 0 ? 0 : i + context_bits - 1);
       WriteBits(depths[code], bits[code], storage_ix, storage);
       WriteBits(depths[repeat_code], bits[repeat_code], storage_ix, storage);
       WriteBits(repeat_code, repeat_bits, storage_ix, storage);
@@ -530,10 +736,10 @@ void StoreTrivialContextMap(int num_types,
 // Manages the encoding of one block category (literal, command or distance).
 class BlockEncoder {
  public:
-  BlockEncoder(int alphabet_size,
-               int num_block_types,
-               const std::vector<int>& block_types,
-               const std::vector<int>& block_lengths)
+  BlockEncoder(size_t alphabet_size,
+               size_t num_block_types,
+               const std::vector<uint8_t>& block_types,
+               const std::vector<uint32_t>& block_lengths)
       : alphabet_size_(alphabet_size),
         num_block_types_(num_block_types),
         block_types_(block_types),
@@ -544,10 +750,12 @@ class BlockEncoder {
 
   // Creates entropy codes of block lengths and block types and stores them
   // to the bit stream.
-  void BuildAndStoreBlockSwitchEntropyCodes(int* storage_ix, uint8_t* storage) {
+  void BuildAndStoreBlockSwitchEntropyCodes(HuffmanTree* tree,
+                                            size_t* storage_ix,
+                                            uint8_t* storage) {
     BuildAndStoreBlockSplitCode(
         block_types_, block_lengths_, num_block_types_,
-        &block_split_code_, storage_ix, storage);
+        tree, &block_split_code_, storage_ix, storage);
   }
 
   // Creates entropy codes for all block types and stores them to the bit
@@ -555,12 +763,14 @@ class BlockEncoder {
   template<int kSize>
   void BuildAndStoreEntropyCodes(
       const std::vector<Histogram<kSize> >& histograms,
-      int* storage_ix, uint8_t* storage) {
+      HuffmanTree* tree,
+      size_t* storage_ix, uint8_t* storage) {
     depths_.resize(histograms.size() * alphabet_size_);
     bits_.resize(histograms.size() * alphabet_size_);
     for (size_t i = 0; i < histograms.size(); ++i) {
       size_t ix = i * alphabet_size_;
       BuildAndStoreHuffmanTree(&histograms[i].data_[0], alphabet_size_,
+                               tree,
                                &depths_[ix], &bits_[ix],
                                storage_ix, storage);
     }
@@ -568,7 +778,7 @@ class BlockEncoder {
 
   // Stores the next symbol with the entropy code of the current block type.
   // Updates the block type and block length at block boundaries.
-  void StoreSymbol(int symbol, int* storage_ix, uint8_t* storage) {
+  void StoreSymbol(size_t symbol, size_t* storage_ix, uint8_t* storage) {
     if (block_len_ == 0) {
       ++block_ix_;
       block_len_ = block_lengths_[block_ix_];
@@ -576,7 +786,7 @@ class BlockEncoder {
       StoreBlockSwitch(block_split_code_, block_ix_, storage_ix, storage);
     }
     --block_len_;
-    int ix = entropy_ix_ + symbol;
+    size_t ix = entropy_ix_ + symbol;
     WriteBits(depths_[ix], bits_[ix], storage_ix, storage);
   }
 
@@ -584,68 +794,63 @@ class BlockEncoder {
   // context value.
   // Updates the block type and block length at block boundaries.
   template<int kContextBits>
-  void StoreSymbolWithContext(int symbol, int context,
-                              const std::vector<int>& context_map,
-                              int* storage_ix, uint8_t* storage) {
+  void StoreSymbolWithContext(size_t symbol, size_t context,
+                              const std::vector<uint32_t>& context_map,
+                              size_t* storage_ix, uint8_t* storage) {
     if (block_len_ == 0) {
       ++block_ix_;
       block_len_ = block_lengths_[block_ix_];
-      entropy_ix_ = block_types_[block_ix_] << kContextBits;
+      size_t block_type = block_types_[block_ix_];
+      entropy_ix_ = block_type << kContextBits;
       StoreBlockSwitch(block_split_code_, block_ix_, storage_ix, storage);
     }
     --block_len_;
-    int histo_ix = context_map[entropy_ix_ + context];
-    int ix = histo_ix * alphabet_size_ + symbol;
+    size_t histo_ix = context_map[entropy_ix_ + context];
+    size_t ix = histo_ix * alphabet_size_ + symbol;
     WriteBits(depths_[ix], bits_[ix], storage_ix, storage);
   }
 
  private:
-  const int alphabet_size_;
-  const int num_block_types_;
-  const std::vector<int>& block_types_;
-  const std::vector<int>& block_lengths_;
+  const size_t alphabet_size_;
+  const size_t num_block_types_;
+  const std::vector<uint8_t>& block_types_;
+  const std::vector<uint32_t>& block_lengths_;
   BlockSplitCode block_split_code_;
-  int block_ix_;
-  int block_len_;
-  int entropy_ix_;
+  size_t block_ix_;
+  size_t block_len_;
+  size_t entropy_ix_;
   std::vector<uint8_t> depths_;
   std::vector<uint16_t> bits_;
 };
 
-void JumpToByteBoundary(int* storage_ix, uint8_t* storage) {
-  *storage_ix = (*storage_ix + 7) & ~7;
+static void JumpToByteBoundary(size_t* storage_ix, uint8_t* storage) {
+  *storage_ix = (*storage_ix + 7u) & ~7u;
   storage[*storage_ix >> 3] = 0;
 }
 
-bool StoreMetaBlock(const uint8_t* input,
+void StoreMetaBlock(const uint8_t* input,
                     size_t start_pos,
                     size_t length,
                     size_t mask,
                     uint8_t prev_byte,
                     uint8_t prev_byte2,
                     bool is_last,
-                    int num_direct_distance_codes,
-                    int distance_postfix_bits,
-                    int literal_context_mode,
+                    uint32_t num_direct_distance_codes,
+                    uint32_t distance_postfix_bits,
+                    ContextType literal_context_mode,
                     const brotli::Command *commands,
                     size_t n_commands,
                     const MetaBlockSplit& mb,
-                    int *storage_ix,
+                    size_t *storage_ix,
                     uint8_t *storage) {
-  if (!StoreCompressedMetaBlockHeader(is_last, length, storage_ix, storage)) {
-    return false;
-  }
-
-  if (length == 0) {
-    // Only the last meta-block can be empty, so jump to next byte.
-    JumpToByteBoundary(storage_ix, storage);
-    return true;
-  }
+  StoreCompressedMetaBlockHeader(is_last, length, storage_ix, storage);
 
-  int num_distance_codes =
+  size_t num_distance_codes =
       kNumDistanceShortCodes + num_direct_distance_codes +
-      (48 << distance_postfix_bits);
+      (48u << distance_postfix_bits);
 
+  HuffmanTree* tree = static_cast<HuffmanTree*>(
+      malloc(kMaxHuffmanTreeSize * sizeof(HuffmanTree)));
   BlockEncoder literal_enc(256,
                            mb.literal_split.num_types,
                            mb.literal_split.types,
@@ -659,59 +864,57 @@ bool StoreMetaBlock(const uint8_t* input,
                             mb.distance_split.types,
                             mb.distance_split.lengths);
 
-  literal_enc.BuildAndStoreBlockSwitchEntropyCodes(storage_ix, storage);
-  command_enc.BuildAndStoreBlockSwitchEntropyCodes(storage_ix, storage);
-  distance_enc.BuildAndStoreBlockSwitchEntropyCodes(storage_ix, storage);
+  literal_enc.BuildAndStoreBlockSwitchEntropyCodes(tree, storage_ix, storage);
+  command_enc.BuildAndStoreBlockSwitchEntropyCodes(tree, storage_ix, storage);
+  distance_enc.BuildAndStoreBlockSwitchEntropyCodes(tree, storage_ix, storage);
 
   WriteBits(2, distance_postfix_bits, storage_ix, storage);
   WriteBits(4, num_direct_distance_codes >> distance_postfix_bits,
             storage_ix, storage);
-  for (int i = 0; i < mb.literal_split.num_types; ++i) {
+  for (size_t i = 0; i < mb.literal_split.num_types; ++i) {
     WriteBits(2, literal_context_mode, storage_ix, storage);
   }
 
-  int num_literal_histograms = static_cast<int>(mb.literal_histograms.size());
+  size_t num_literal_histograms = mb.literal_histograms.size();
   if (mb.literal_context_map.empty()) {
-    StoreTrivialContextMap(num_literal_histograms, kLiteralContextBits,
+    StoreTrivialContextMap(num_literal_histograms, kLiteralContextBits, tree,
                            storage_ix, storage);
   } else {
-    EncodeContextMap(mb.literal_context_map, num_literal_histograms,
+    EncodeContextMap(mb.literal_context_map, num_literal_histograms, tree,
                      storage_ix, storage);
   }
 
-  int num_dist_histograms = static_cast<int>(mb.distance_histograms.size());
+  size_t num_dist_histograms = mb.distance_histograms.size();
   if (mb.distance_context_map.empty()) {
-    StoreTrivialContextMap(num_dist_histograms, kDistanceContextBits,
+    StoreTrivialContextMap(num_dist_histograms, kDistanceContextBits, tree,
                            storage_ix, storage);
   } else {
-    EncodeContextMap(mb.distance_context_map, num_dist_histograms,
+    EncodeContextMap(mb.distance_context_map, num_dist_histograms, tree,
                      storage_ix, storage);
   }
 
-  literal_enc.BuildAndStoreEntropyCodes(mb.literal_histograms,
+  literal_enc.BuildAndStoreEntropyCodes(mb.literal_histograms, tree,
                                         storage_ix, storage);
-  command_enc.BuildAndStoreEntropyCodes(mb.command_histograms,
+  command_enc.BuildAndStoreEntropyCodes(mb.command_histograms, tree,
                                         storage_ix, storage);
-  distance_enc.BuildAndStoreEntropyCodes(mb.distance_histograms,
+  distance_enc.BuildAndStoreEntropyCodes(mb.distance_histograms, tree,
                                          storage_ix, storage);
+  free(tree);
 
   size_t pos = start_pos;
   for (size_t i = 0; i < n_commands; ++i) {
     const Command cmd = commands[i];
-    int cmd_code = cmd.cmd_prefix_;
-    int lennumextra = static_cast<int>(cmd.cmd_extra_ >> 48);
-    uint64_t lenextra = cmd.cmd_extra_ & 0xffffffffffffUL;
+    size_t cmd_code = cmd.cmd_prefix_;
     command_enc.StoreSymbol(cmd_code, storage_ix, storage);
-    WriteBits(lennumextra, lenextra, storage_ix, storage);
+    StoreCommandExtra(cmd, storage_ix, storage);
     if (mb.literal_context_map.empty()) {
-      for (int j = 0; j < cmd.insert_len_; j++) {
+      for (size_t j = cmd.insert_len_; j != 0; --j) {
         literal_enc.StoreSymbol(input[pos & mask], storage_ix, storage);
         ++pos;
       }
     } else {
-      for (int j = 0; j < cmd.insert_len_; ++j) {
-        int context = Context(prev_byte, prev_byte2,
-                              literal_context_mode);
+      for (size_t j = cmd.insert_len_; j != 0; --j) {
+        size_t context = Context(prev_byte, prev_byte2, literal_context_mode);
         uint8_t literal = input[pos & mask];
         literal_enc.StoreSymbolWithContext<kLiteralContextBits>(
             literal, context, mb.literal_context_map, storage_ix, storage);
@@ -720,18 +923,18 @@ bool StoreMetaBlock(const uint8_t* input,
         ++pos;
       }
     }
-    pos += cmd.copy_len_;
-    if (cmd.copy_len_ > 0) {
+    pos += cmd.copy_len();
+    if (cmd.copy_len()) {
       prev_byte2 = input[(pos - 2) & mask];
       prev_byte = input[(pos - 1) & mask];
       if (cmd.cmd_prefix_ >= 128) {
-        int dist_code = cmd.dist_prefix_;
-        int distnumextra = cmd.dist_extra_ >> 24;
-        int distextra = cmd.dist_extra_ & 0xffffff;
+        size_t dist_code = cmd.dist_prefix_;
+        uint32_t distnumextra = cmd.dist_extra_ >> 24;
+        uint64_t distextra = cmd.dist_extra_ & 0xffffff;
         if (mb.distance_context_map.empty()) {
           distance_enc.StoreSymbol(dist_code, storage_ix, storage);
         } else {
-          int context = cmd.DistanceContext();
+          size_t context = cmd.DistanceContext();
           distance_enc.StoreSymbolWithContext<kDistanceContextBits>(
               dist_code, context, mb.distance_context_map, storage_ix, storage);
         }
@@ -742,45 +945,84 @@ bool StoreMetaBlock(const uint8_t* input,
   if (is_last) {
     JumpToByteBoundary(storage_ix, storage);
   }
-  return true;
 }
 
-bool StoreMetaBlockTrivial(const uint8_t* input,
+static void BuildHistograms(const uint8_t* input,
+                            size_t start_pos,
+                            size_t mask,
+                            const brotli::Command *commands,
+                            size_t n_commands,
+                            HistogramLiteral* lit_histo,
+                            HistogramCommand* cmd_histo,
+                            HistogramDistance* dist_histo) {
+  size_t pos = start_pos;
+  for (size_t i = 0; i < n_commands; ++i) {
+    const Command cmd = commands[i];
+    cmd_histo->Add(cmd.cmd_prefix_);
+    for (size_t j = cmd.insert_len_; j != 0; --j) {
+      lit_histo->Add(input[pos & mask]);
+      ++pos;
+    }
+    pos += cmd.copy_len();
+    if (cmd.copy_len() && cmd.cmd_prefix_ >= 128) {
+      dist_histo->Add(cmd.dist_prefix_);
+    }
+  }
+}
+
+static void StoreDataWithHuffmanCodes(const uint8_t* input,
+                                      size_t start_pos,
+                                      size_t mask,
+                                      const brotli::Command *commands,
+                                      size_t n_commands,
+                                      const uint8_t* lit_depth,
+                                      const uint16_t* lit_bits,
+                                      const uint8_t* cmd_depth,
+                                      const uint16_t* cmd_bits,
+                                      const uint8_t* dist_depth,
+                                      const uint16_t* dist_bits,
+                                      size_t* storage_ix,
+                                      uint8_t* storage) {
+  size_t pos = start_pos;
+  for (size_t i = 0; i < n_commands; ++i) {
+    const Command cmd = commands[i];
+    const size_t cmd_code = cmd.cmd_prefix_;
+    WriteBits(cmd_depth[cmd_code], cmd_bits[cmd_code], storage_ix, storage);
+    StoreCommandExtra(cmd, storage_ix, storage);
+    for (size_t j = cmd.insert_len_; j != 0; --j) {
+      const uint8_t literal = input[pos & mask];
+      WriteBits(lit_depth[literal], lit_bits[literal], storage_ix, storage);
+      ++pos;
+    }
+    pos += cmd.copy_len();
+    if (cmd.copy_len() && cmd.cmd_prefix_ >= 128) {
+      const size_t dist_code = cmd.dist_prefix_;
+      const uint32_t distnumextra = cmd.dist_extra_ >> 24;
+      const uint32_t distextra = cmd.dist_extra_ & 0xffffff;
+      WriteBits(dist_depth[dist_code], dist_bits[dist_code],
+                storage_ix, storage);
+      WriteBits(distnumextra, distextra, storage_ix, storage);
+    }
+  }
+}
+
+void StoreMetaBlockTrivial(const uint8_t* input,
                            size_t start_pos,
                            size_t length,
                            size_t mask,
                            bool is_last,
                            const brotli::Command *commands,
                            size_t n_commands,
-                           int *storage_ix,
+                           size_t *storage_ix,
                            uint8_t *storage) {
-  if (!StoreCompressedMetaBlockHeader(is_last, length, storage_ix, storage)) {
-    return false;
-  }
-
-  if (length == 0) {
-    // Only the last meta-block can be empty, so jump to next byte.
-    JumpToByteBoundary(storage_ix, storage);
-    return true;
-  }
+  StoreCompressedMetaBlockHeader(is_last, length, storage_ix, storage);
 
   HistogramLiteral lit_histo;
   HistogramCommand cmd_histo;
   HistogramDistance dist_histo;
 
-  size_t pos = start_pos;
-  for (size_t i = 0; i < n_commands; ++i) {
-    const Command cmd = commands[i];
-    cmd_histo.Add(cmd.cmd_prefix_);
-    for (int j = 0; j < cmd.insert_len_; ++j) {
-      lit_histo.Add(input[pos & mask]);
-      ++pos;
-    }
-    pos += cmd.copy_len_;
-    if (cmd.copy_len_ > 0 && cmd.cmd_prefix_ >= 128) {
-      dist_histo.Add(cmd.dist_prefix_);
-    }
-  }
+  BuildHistograms(input, start_pos, mask, commands, n_commands,
+                  &lit_histo, &cmd_histo, &dist_histo);
 
   WriteBits(13, 0, storage_ix, storage);
 
@@ -791,68 +1033,126 @@ bool StoreMetaBlockTrivial(const uint8_t* input,
   std::vector<uint8_t> dist_depth(64);
   std::vector<uint16_t> dist_bits(64);
 
-  BuildAndStoreHuffmanTree(&lit_histo.data_[0], 256,
+  HuffmanTree* tree = static_cast<HuffmanTree*>(
+      malloc(kMaxHuffmanTreeSize * sizeof(HuffmanTree)));
+  BuildAndStoreHuffmanTree(&lit_histo.data_[0], 256, tree,
                            &lit_depth[0], &lit_bits[0],
                            storage_ix, storage);
-  BuildAndStoreHuffmanTree(&cmd_histo.data_[0], kNumCommandPrefixes,
+  BuildAndStoreHuffmanTree(&cmd_histo.data_[0], kNumCommandPrefixes, tree,
                            &cmd_depth[0], &cmd_bits[0],
                            storage_ix, storage);
-  BuildAndStoreHuffmanTree(&dist_histo.data_[0], 64,
+  BuildAndStoreHuffmanTree(&dist_histo.data_[0], 64, tree,
                            &dist_depth[0], &dist_bits[0],
                            storage_ix, storage);
+  free(tree);
+  StoreDataWithHuffmanCodes(input, start_pos, mask, commands,
+                            n_commands, &lit_depth[0], &lit_bits[0],
+                            &cmd_depth[0], &cmd_bits[0],
+                            &dist_depth[0], &dist_bits[0],
+                            storage_ix, storage);
+  if (is_last) {
+    JumpToByteBoundary(storage_ix, storage);
+  }
+}
 
-  pos = start_pos;
-  for (size_t i = 0; i < n_commands; ++i) {
-    const Command cmd = commands[i];
-    const int cmd_code = cmd.cmd_prefix_;
-    const int lennumextra = static_cast<int>(cmd.cmd_extra_ >> 48);
-    const uint64_t lenextra = cmd.cmd_extra_ & 0xffffffffffffUL;
-    WriteBits(cmd_depth[cmd_code], cmd_bits[cmd_code], storage_ix, storage);
-    WriteBits(lennumextra, lenextra, storage_ix, storage);
-    for (int j = 0; j < cmd.insert_len_; j++) {
-      const uint8_t literal = input[pos & mask];
-      WriteBits(lit_depth[literal], lit_bits[literal], storage_ix, storage);
-      ++pos;
-    }
-    pos += cmd.copy_len_;
-    if (cmd.copy_len_ > 0 && cmd.cmd_prefix_ >= 128) {
-      const int dist_code = cmd.dist_prefix_;
-      const int distnumextra = cmd.dist_extra_ >> 24;
-      const int distextra = cmd.dist_extra_ & 0xffffff;
-      WriteBits(dist_depth[dist_code], dist_bits[dist_code],
-                storage_ix, storage);
-      WriteBits(distnumextra, distextra, storage_ix, storage);
+void StoreMetaBlockFast(const uint8_t* input,
+                        size_t start_pos,
+                        size_t length,
+                        size_t mask,
+                        bool is_last,
+                        const brotli::Command *commands,
+                        size_t n_commands,
+                        size_t *storage_ix,
+                        uint8_t *storage) {
+  StoreCompressedMetaBlockHeader(is_last, length, storage_ix, storage);
+
+  WriteBits(13, 0, storage_ix, storage);
+
+  if (n_commands <= 128) {
+    uint32_t histogram[256] = { 0 };
+    size_t pos = start_pos;
+    size_t num_literals = 0;
+    for (size_t i = 0; i < n_commands; ++i) {
+      const Command cmd = commands[i];
+      for (size_t j = cmd.insert_len_; j != 0; --j) {
+        ++histogram[input[pos & mask]];
+        ++pos;
+      }
+      num_literals += cmd.insert_len_;
+      pos += cmd.copy_len();
     }
+    uint8_t lit_depth[256] = { 0 };
+    uint16_t lit_bits[256] = { 0 };
+    BuildAndStoreHuffmanTreeFast(histogram, num_literals,
+                                 /* max_bits = */ 8,
+                                 lit_depth, lit_bits,
+                                 storage_ix, storage);
+    StoreStaticCommandHuffmanTree(storage_ix, storage);
+    StoreStaticDistanceHuffmanTree(storage_ix, storage);
+    StoreDataWithHuffmanCodes(input, start_pos, mask, commands,
+                              n_commands, &lit_depth[0], &lit_bits[0],
+                              kStaticCommandCodeDepth,
+                              kStaticCommandCodeBits,
+                              kStaticDistanceCodeDepth,
+                              kStaticDistanceCodeBits,
+                              storage_ix, storage);
+  } else {
+    HistogramLiteral lit_histo;
+    HistogramCommand cmd_histo;
+    HistogramDistance dist_histo;
+    BuildHistograms(input, start_pos, mask, commands, n_commands,
+                    &lit_histo, &cmd_histo, &dist_histo);
+    std::vector<uint8_t> lit_depth(256);
+    std::vector<uint16_t> lit_bits(256);
+    std::vector<uint8_t> cmd_depth(kNumCommandPrefixes);
+    std::vector<uint16_t> cmd_bits(kNumCommandPrefixes);
+    std::vector<uint8_t> dist_depth(64);
+    std::vector<uint16_t> dist_bits(64);
+    BuildAndStoreHuffmanTreeFast(&lit_histo.data_[0], lit_histo.total_count_,
+                                 /* max_bits = */ 8,
+                                 &lit_depth[0], &lit_bits[0],
+                                 storage_ix, storage);
+    BuildAndStoreHuffmanTreeFast(&cmd_histo.data_[0], cmd_histo.total_count_,
+                                 /* max_bits = */ 10,
+                                 &cmd_depth[0], &cmd_bits[0],
+                                 storage_ix, storage);
+    BuildAndStoreHuffmanTreeFast(&dist_histo.data_[0], dist_histo.total_count_,
+                                 /* max_bits = */ 6,
+                                 &dist_depth[0], &dist_bits[0],
+                                 storage_ix, storage);
+    StoreDataWithHuffmanCodes(input, start_pos, mask, commands,
+                              n_commands, &lit_depth[0], &lit_bits[0],
+                              &cmd_depth[0], &cmd_bits[0],
+                              &dist_depth[0], &dist_bits[0],
+                              storage_ix, storage);
   }
+
   if (is_last) {
     JumpToByteBoundary(storage_ix, storage);
   }
-  return true;
 }
 
 // This is for storing uncompressed blocks (simple raw storage of
 // bytes-as-bytes).
-bool StoreUncompressedMetaBlock(bool final_block,
+void StoreUncompressedMetaBlock(bool final_block,
                                 const uint8_t * __restrict input,
                                 size_t position, size_t mask,
                                 size_t len,
-                                int * __restrict storage_ix,
+                                size_t * __restrict storage_ix,
                                 uint8_t * __restrict storage) {
-  if (!brotli::StoreUncompressedMetaBlockHeader(len, storage_ix, storage)) {
-    return false;
-  }
+  StoreUncompressedMetaBlockHeader(len, storage_ix, storage);
   JumpToByteBoundary(storage_ix, storage);
 
   size_t masked_pos = position & mask;
   if (masked_pos + len > mask + 1) {
     size_t len1 = mask + 1 - masked_pos;
     memcpy(&storage[*storage_ix >> 3], &input[masked_pos], len1);
-    *storage_ix += static_cast<int>(len1 << 3);
+    *storage_ix += len1 << 3;
     len -= len1;
     masked_pos = 0;
   }
   memcpy(&storage[*storage_ix >> 3], &input[masked_pos], len);
-  *storage_ix += static_cast<int>(len << 3);
+  *storage_ix += len << 3;
 
   // We need to clear the next 4 bytes to continue to be
   // compatible with WriteBits.
@@ -865,10 +1165,9 @@ bool StoreUncompressedMetaBlock(bool final_block,
     brotli::WriteBits(1, 1, storage_ix, storage);  // isempty
     JumpToByteBoundary(storage_ix, storage);
   }
-  return true;
 }
 
-void StoreSyncMetaBlock(int * __restrict storage_ix,
+void StoreSyncMetaBlock(size_t * __restrict storage_ix,
                         uint8_t * __restrict storage) {
   // Empty metadata meta-block bit pattern:
   //   1 bit:  is_last (0)