brotli 0.1.1 → 0.1.2

data/vendor/brotli/enc/brotli_bit_stream.h

@@ -18,6 +18,7 @@
 
 #include <vector>
 
+#include "./entropy_encode.h"
 #include "./metablock.h"
 #include "./types.h"
 
@@ -27,113 +28,151 @@ namespace brotli {
 // position for the current storage.
 
 // Stores a number between 0 and 255.
-void StoreVarLenUint8(int n, int* storage_ix, uint8_t* storage);
+void StoreVarLenUint8(size_t n, size_t* storage_ix, uint8_t* storage);
 
 // Stores the compressed meta-block header.
-bool StoreCompressedMetaBlockHeader(bool final_block,
+// REQUIRES: length > 0
+// REQUIRES: length <= (1 << 24)
+void StoreCompressedMetaBlockHeader(bool final_block,
                                     size_t length,
-                                    int* storage_ix,
+                                    size_t* storage_ix,
                                     uint8_t* storage);
 
 // Stores the uncompressed meta-block header.
-bool StoreUncompressedMetaBlockHeader(size_t length,
-                                      int* storage_ix,
+// REQUIRES: length > 0
+// REQUIRES: length <= (1 << 24)
+void StoreUncompressedMetaBlockHeader(size_t length,
+                                      size_t* storage_ix,
                                       uint8_t* storage);
 
 // Stores a context map where the histogram type is always the block type.
-void StoreTrivialContextMap(int num_types,
-                            int context_bits,
-                            int* storage_ix,
+void StoreTrivialContextMap(size_t num_types,
+                            size_t context_bits,
+                            HuffmanTree* tree,
+                            size_t* storage_ix,
                             uint8_t* storage);
 
 void StoreHuffmanTreeOfHuffmanTreeToBitMask(
     const int num_codes,
     const uint8_t *code_length_bitdepth,
-    int *storage_ix,
+    size_t *storage_ix,
     uint8_t *storage);
 
+void StoreHuffmanTree(const uint8_t* depths, size_t num, HuffmanTree* tree,
+                      size_t *storage_ix, uint8_t *storage);
+
 // Builds a Huffman tree from histogram[0:length] into depth[0:length] and
 // bits[0:length] and stores the encoded tree to the bit stream.
-void BuildAndStoreHuffmanTree(const int *histogram,
-                              const int length,
+void BuildAndStoreHuffmanTree(const uint32_t *histogram,
+                              const size_t length,
+                              HuffmanTree* tree,
                               uint8_t* depth,
                               uint16_t* bits,
-                              int* storage_ix,
+                              size_t* storage_ix,
                               uint8_t* storage);
 
+void BuildAndStoreHuffmanTreeFast(const uint32_t *histogram,
+                                  const size_t histogram_total,
+                                  const size_t max_bits,
+                                  uint8_t* depth,
+                                  uint16_t* bits,
+                                  size_t* storage_ix,
+                                  uint8_t* storage);
+
 // Encodes the given context map to the bit stream. The number of different
 // histogram ids is given by num_clusters.
-void EncodeContextMap(const std::vector<int>& context_map,
-                      int num_clusters,
-                      int* storage_ix, uint8_t* storage);
+void EncodeContextMap(const std::vector<uint32_t>& context_map,
+                      size_t num_clusters,
+                      HuffmanTree* tree,
+                      size_t* storage_ix, uint8_t* storage);
 
 // Data structure that stores everything that is needed to encode each block
 // switch command.
 struct BlockSplitCode {
-  std::vector<int> type_code;
-  std::vector<int> length_prefix;
-  std::vector<int> length_nextra;
-  std::vector<int> length_extra;
+  std::vector<uint32_t> type_code;
+  std::vector<uint32_t> length_prefix;
+  std::vector<uint32_t> length_nextra;
+  std::vector<uint32_t> length_extra;
   std::vector<uint8_t> type_depths;
   std::vector<uint16_t> type_bits;
-  std::vector<uint8_t> length_depths;
-  std::vector<uint16_t> length_bits;
+  uint8_t length_depths[kNumBlockLenPrefixes];
+  uint16_t length_bits[kNumBlockLenPrefixes];
 };
 
 // Builds a BlockSplitCode data structure from the block split given by the
 // vector of block types and block lengths and stores it to the bit stream.
-void BuildAndStoreBlockSplitCode(const std::vector<int>& types,
-                                 const std::vector<int>& lengths,
-                                 const int num_types,
+void BuildAndStoreBlockSplitCode(const std::vector<uint8_t>& types,
+                                 const std::vector<uint32_t>& lengths,
+                                 const size_t num_types,
                                  BlockSplitCode* code,
-                                 int* storage_ix,
+                                 size_t* storage_ix,
                                  uint8_t* storage);
 
 // Stores the block switch command with index block_ix to the bit stream.
 void StoreBlockSwitch(const BlockSplitCode& code,
-                      const int block_ix,
-                      int* storage_ix,
+                      const size_t block_ix,
+                      size_t* storage_ix,
                       uint8_t* storage);
 
-bool StoreMetaBlock(const uint8_t* input,
+// REQUIRES: length > 0
+// REQUIRES: length <= (1 << 24)
+void StoreMetaBlock(const uint8_t* input,
                     size_t start_pos,
                     size_t length,
                     size_t mask,
                     uint8_t prev_byte,
                     uint8_t prev_byte2,
                     bool final_block,
-                    int num_direct_distance_codes,
-                    int distance_postfix_bits,
-                    int literal_context_mode,
+                    uint32_t num_direct_distance_codes,
+                    uint32_t distance_postfix_bits,
+                    ContextType literal_context_mode,
                     const brotli::Command *commands,
                     size_t n_commands,
                     const MetaBlockSplit& mb,
-                    int *storage_ix,
+                    size_t *storage_ix,
                     uint8_t *storage);
 
 // Stores the meta-block without doing any block splitting, just collects
 // one histogram per block category and uses that for entropy coding.
-bool StoreMetaBlockTrivial(const uint8_t* input,
+// REQUIRES: length > 0
+// REQUIRES: length <= (1 << 24)
+void StoreMetaBlockTrivial(const uint8_t* input,
                            size_t start_pos,
                            size_t length,
                            size_t mask,
                            bool is_last,
                            const brotli::Command *commands,
                            size_t n_commands,
-                           int *storage_ix,
+                           size_t *storage_ix,
                            uint8_t *storage);
 
+// Same as above, but uses static prefix codes for histograms with a only a few
+// symbols, and uses static code length prefix codes for all other histograms.
+// REQUIRES: length > 0
+// REQUIRES: length <= (1 << 24)
+void StoreMetaBlockFast(const uint8_t* input,
+                        size_t start_pos,
+                        size_t length,
+                        size_t mask,
+                        bool is_last,
+                        const brotli::Command *commands,
+                        size_t n_commands,
+                        size_t *storage_ix,
+                        uint8_t *storage);
+
 // This is for storing uncompressed blocks (simple raw storage of
 // bytes-as-bytes).
-bool StoreUncompressedMetaBlock(bool final_block,
+// REQUIRES: length > 0
+// REQUIRES: length <= (1 << 24)
+void StoreUncompressedMetaBlock(bool final_block,
                                 const uint8_t* input,
                                 size_t position, size_t mask,
                                 size_t len,
-                                int* storage_ix,
+                                size_t* storage_ix,
                                 uint8_t* storage);
 
 // Stores an empty metadata meta-block and syncs to a byte boundary.
-void StoreSyncMetaBlock(int* storage_ix, uint8_t* storage);
+void StoreSyncMetaBlock(size_t* storage_ix, uint8_t* storage);
 
 }  // namespace brotli
 

data/vendor/brotli/enc/cluster.h

@@ -10,11 +10,7 @@
 #define BROTLI_ENC_CLUSTER_H_
 
 #include <math.h>
-#include <stdio.h>
 #include <algorithm>
-#include <complex>
-#include <map>
-#include <set>
 #include <utility>
 #include <vector>
 
@@ -28,41 +24,41 @@
 namespace brotli {
 
 struct HistogramPair {
-  int idx1;
-  int idx2;
-  bool valid;
+  uint32_t idx1;
+  uint32_t idx2;
   double cost_combo;
   double cost_diff;
 };
 
-struct HistogramPairComparator {
-  bool operator()(const HistogramPair& p1, const HistogramPair& p2) const {
-    if (p1.cost_diff != p2.cost_diff) {
-      return p1.cost_diff > p2.cost_diff;
-    }
-    return abs(p1.idx1 - p1.idx2) > abs(p2.idx1 - p2.idx2);
+inline bool operator<(const HistogramPair& p1, const HistogramPair& p2) {
+  if (p1.cost_diff != p2.cost_diff) {
+    return p1.cost_diff > p2.cost_diff;
   }
-};
+  return (p1.idx2 - p1.idx1) > (p2.idx2 - p2.idx1);
+}
 
 // Returns entropy reduction of the context map when we combine two clusters.
-inline double ClusterCostDiff(int size_a, int size_b) {
-  int size_c = size_a + size_b;
-  return size_a * FastLog2(size_a) + size_b * FastLog2(size_b) -
-      size_c * FastLog2(size_c);
+inline double ClusterCostDiff(size_t size_a, size_t size_b) {
+  size_t size_c = size_a + size_b;
+  return static_cast<double>(size_a) * FastLog2(size_a) +
+      static_cast<double>(size_b) * FastLog2(size_b) -
+      static_cast<double>(size_c) * FastLog2(size_c);
 }
 
 // Computes the bit cost reduction by combining out[idx1] and out[idx2] and if
-// it is below a threshold, stores the pair (idx1, idx2) in the *pairs heap.
+// it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue.
 template<typename HistogramType>
-void CompareAndPushToHeap(const HistogramType* out,
-                          const int* cluster_size,
-                          int idx1, int idx2,
-                          std::vector<HistogramPair>* pairs) {
+void CompareAndPushToQueue(const HistogramType* out,
+                           const uint32_t* cluster_size,
+                           uint32_t idx1, uint32_t idx2,
+                           size_t max_num_pairs,
+                           HistogramPair* pairs,
+                           size_t* num_pairs) {
   if (idx1 == idx2) {
     return;
   }
   if (idx2 < idx1) {
-    int t = idx2;
+    uint32_t t = idx2;
     idx2 = idx1;
     idx1 = t;
   }
@@ -70,7 +66,6 @@ void CompareAndPushToHeap(const HistogramType* out,
   HistogramPair p;
   p.idx1 = idx1;
   p.idx2 = idx2;
-  p.valid = true;
   p.cost_diff = 0.5 * ClusterCostDiff(cluster_size[idx1], cluster_size[idx2]);
   p.cost_diff -= out[idx1].bit_cost_;
   p.cost_diff -= out[idx2].bit_cost_;
@@ -82,8 +77,8 @@ void CompareAndPushToHeap(const HistogramType* out,
     p.cost_combo = out[idx1].bit_cost_;
     store_pair = true;
   } else {
-    double threshold = pairs->empty() ? 1e99 :
-        std::max(0.0, (*pairs)[0].cost_diff);
+    double threshold = *num_pairs == 0 ? 1e99 :
+        std::max(0.0, pairs[0].cost_diff);
     HistogramType combo = out[idx1];
     combo.AddHistogram(out[idx2]);
     double cost_combo = PopulationCost(combo);
@@ -94,81 +89,96 @@ void CompareAndPushToHeap(const HistogramType* out,
   }
   if (store_pair) {
     p.cost_diff += p.cost_combo;
-    pairs->push_back(p);
-    std::push_heap(pairs->begin(), pairs->end(), HistogramPairComparator());
+    if (*num_pairs > 0 && pairs[0] < p) {
+      // Replace the top of the queue if needed.
+      if (*num_pairs < max_num_pairs) {
+        pairs[*num_pairs] = pairs[0];
+        ++(*num_pairs);
+      }
+      pairs[0] = p;
+    } else if (*num_pairs < max_num_pairs) {
+      pairs[*num_pairs] = p;
+      ++(*num_pairs);
+    }
   }
 }
 
 template<typename HistogramType>
-void HistogramCombine(HistogramType* out,
-                      int* cluster_size,
-                      int* symbols,
-                      int symbols_size,
-                      size_t max_clusters) {
+size_t HistogramCombine(HistogramType* out,
+                        uint32_t* cluster_size,
+                        uint32_t* symbols,
+                        uint32_t* clusters,
+                        HistogramPair* pairs,
+                        size_t num_clusters,
+                        size_t symbols_size,
+                        size_t max_clusters,
+                        size_t max_num_pairs) {
   double cost_diff_threshold = 0.0;
   size_t min_cluster_size = 1;
-  std::set<int> all_symbols;
-  std::vector<int> clusters;
-  for (int i = 0; i < symbols_size; ++i) {
-    if (all_symbols.find(symbols[i]) == all_symbols.end()) {
-      all_symbols.insert(symbols[i]);
-      if (!clusters.empty()) {
-        BROTLI_DCHECK(clusters.back() < symbols[i]);
-      }
-      clusters.push_back(symbols[i]);
-    }
-  }
 
-  // We maintain a heap of histogram pairs, ordered by the bit cost reduction.
-  std::vector<HistogramPair> pairs;
-  for (size_t idx1 = 0; idx1 < clusters.size(); ++idx1) {
-    for (size_t idx2 = idx1 + 1; idx2 < clusters.size(); ++idx2) {
-      CompareAndPushToHeap(out, cluster_size, clusters[idx1], clusters[idx2],
-                           &pairs);
+  // We maintain a vector of histogram pairs, with the property that the pair
+  // with the maximum bit cost reduction is the first.
+  size_t num_pairs = 0;
+  for (size_t idx1 = 0; idx1 < num_clusters; ++idx1) {
+    for (size_t idx2 = idx1 + 1; idx2 < num_clusters; ++idx2) {
+      CompareAndPushToQueue(out, cluster_size, clusters[idx1], clusters[idx2],
+                            max_num_pairs, &pairs[0], &num_pairs);
     }
   }
 
-  while (clusters.size() > min_cluster_size) {
+  while (num_clusters > min_cluster_size) {
     if (pairs[0].cost_diff >= cost_diff_threshold) {
       cost_diff_threshold = 1e99;
       min_cluster_size = max_clusters;
       continue;
     }
     // Take the best pair from the top of heap.
-    int best_idx1 = pairs[0].idx1;
-    int best_idx2 = pairs[0].idx2;
+    uint32_t best_idx1 = pairs[0].idx1;
+    uint32_t best_idx2 = pairs[0].idx2;
     out[best_idx1].AddHistogram(out[best_idx2]);
     out[best_idx1].bit_cost_ = pairs[0].cost_combo;
     cluster_size[best_idx1] += cluster_size[best_idx2];
-    for (int i = 0; i < symbols_size; ++i) {
+    for (size_t i = 0; i < symbols_size; ++i) {
       if (symbols[i] == best_idx2) {
         symbols[i] = best_idx1;
       }
     }
-    for (size_t i = 0; i + 1 < clusters.size(); ++i) {
-      if (clusters[i] >= best_idx2) {
-        clusters[i] = clusters[i + 1];
+    for (size_t i = 0; i < num_clusters; ++i) {
+      if (clusters[i] == best_idx2) {
+        memmove(&clusters[i], &clusters[i + 1],
+                (num_clusters - i - 1) * sizeof(clusters[0]));
+        break;
       }
     }
-    clusters.pop_back();
-    // Invalidate pairs intersecting the just combined best pair.
-    for (size_t i = 0; i < pairs.size(); ++i) {
+    --num_clusters;
+    // Remove pairs intersecting the just combined best pair.
+    size_t copy_to_idx = 0;
+    for (size_t i = 0; i < num_pairs; ++i) {
       HistogramPair& p = pairs[i];
       if (p.idx1 == best_idx1 || p.idx2 == best_idx1 ||
           p.idx1 == best_idx2 || p.idx2 == best_idx2) {
-        p.valid = false;
+        // Remove invalid pair from the queue.
+        continue;
       }
+      if (pairs[0] < p) {
+        // Replace the top of the queue if needed.
+        HistogramPair front = pairs[0];
+        pairs[0] = p;
+        pairs[copy_to_idx] = front;
+      } else {
+        pairs[copy_to_idx] = p;
+      }
+      ++copy_to_idx;
     }
-    // Pop invalid pairs from the top of the heap.
-    while (!pairs.empty() && !pairs[0].valid) {
-      std::pop_heap(pairs.begin(), pairs.end(), HistogramPairComparator());
-      pairs.pop_back();
-    }
+    num_pairs = copy_to_idx;
+
     // Push new pairs formed with the combined histogram to the heap.
-    for (size_t i = 0; i < clusters.size(); ++i) {
-      CompareAndPushToHeap(out, cluster_size, best_idx1, clusters[i], &pairs);
+    for (size_t i = 0; i < num_clusters; ++i) {
+      CompareAndPushToQueue(out, cluster_size, best_idx1, clusters[i],
+                            max_num_pairs, &pairs[0], &num_pairs);
     }
   }
+  return num_clusters;
 }
 
 // -----------------------------------------------------------------------------
@@ -187,58 +197,69 @@ double HistogramBitCostDistance(const HistogramType& histogram,
 }
 
 // Find the best 'out' histogram for each of the 'in' histograms.
+// When called, clusters[0..num_clusters) contains the unique values from
+// symbols[0..in_size), but this property is not preserved in this function.
 // Note: we assume that out[]->bit_cost_ is already up-to-date.
 template<typename HistogramType>
-void HistogramRemap(const HistogramType* in, int in_size,
-                    HistogramType* out, int* symbols) {
-  std::set<int> all_symbols;
-  for (int i = 0; i < in_size; ++i) {
-    all_symbols.insert(symbols[i]);
-  }
-  for (int i = 0; i < in_size; ++i) {
-    int best_out = i == 0 ? symbols[0] : symbols[i - 1];
+void HistogramRemap(const HistogramType* in, size_t in_size,
+                    const uint32_t* clusters, size_t num_clusters,
+                    HistogramType* out, uint32_t* symbols) {
+  for (size_t i = 0; i < in_size; ++i) {
+    uint32_t best_out = i == 0 ? symbols[0] : symbols[i - 1];
     double best_bits = HistogramBitCostDistance(in[i], out[best_out]);
-    for (std::set<int>::const_iterator k = all_symbols.begin();
-         k != all_symbols.end(); ++k) {
-      const double cur_bits = HistogramBitCostDistance(in[i], out[*k]);
+    for (size_t j = 0; j < num_clusters; ++j) {
+      const double cur_bits = HistogramBitCostDistance(in[i], out[clusters[j]]);
       if (cur_bits < best_bits) {
         best_bits = cur_bits;
-        best_out = *k;
+        best_out = clusters[j];
       }
     }
     symbols[i] = best_out;
   }
 
-
   // Recompute each out based on raw and symbols.
-  for (std::set<int>::const_iterator k = all_symbols.begin();
-       k != all_symbols.end(); ++k) {
-    out[*k].Clear();
+  for (size_t j = 0; j < num_clusters; ++j) {
+    out[clusters[j]].Clear();
   }
-  for (int i = 0; i < in_size; ++i) {
+  for (size_t i = 0; i < in_size; ++i) {
     out[symbols[i]].AddHistogram(in[i]);
   }
 }
 
-// Reorder histograms in *out so that the new symbols in *symbols come in
-// increasing order.
+// Reorders elements of the out[0..length) array and changes values in
+// symbols[0..length) array in the following way:
+//   * when called, symbols[] contains indexes into out[], and has N unique
+//     values (possibly N < length)
+//   * on return, symbols'[i] = f(symbols[i]) and
+//                out'[symbols'[i]] = out[symbols[i]], for each 0 <= i < length,
+//     where f is a bijection between the range of symbols[] and [0..N), and
+//     the first occurrences of values in symbols'[i] come in consecutive
+//     increasing order.
+// Returns N, the number of unique values in symbols[].
 template<typename HistogramType>
-void HistogramReindex(std::vector<HistogramType>* out,
-                      std::vector<int>* symbols) {
-  std::vector<HistogramType> tmp(*out);
-  std::map<int, int> new_index;
-  int next_index = 0;
-  for (size_t i = 0; i < symbols->size(); ++i) {
-    if (new_index.find((*symbols)[i]) == new_index.end()) {
-      new_index[(*symbols)[i]] = next_index;
-      (*out)[next_index] = tmp[(*symbols)[i]];
+size_t HistogramReindex(HistogramType* out, uint32_t* symbols, size_t length) {
+  static const uint32_t kInvalidIndex = std::numeric_limits<uint32_t>::max();
+  std::vector<uint32_t> new_index(length, kInvalidIndex);
+  uint32_t next_index = 0;
+  for (size_t i = 0; i < length; ++i) {
+    if (new_index[symbols[i]] == kInvalidIndex) {
+      new_index[symbols[i]] = next_index;
+      ++next_index;
+    }
+  }
+  std::vector<HistogramType> tmp(next_index);
+  next_index = 0;
+  for (size_t i = 0; i < length; ++i) {
+    if (new_index[symbols[i]] == next_index) {
+      tmp[next_index] = out[symbols[i]];
       ++next_index;
     }
+    symbols[i] = new_index[symbols[i]];
   }
-  out->resize(next_index);
-  for (size_t i = 0; i < symbols->size(); ++i) {
-    (*symbols)[i] = new_index[(*symbols)[i]];
+  for (size_t i = 0; i < next_index; ++i) {
+    out[i] = tmp[i];
   }
+  return next_index;
 }
 
 // Clusters similar histograms in 'in' together, the selected histograms are
@@ -246,44 +267,64 @@ void HistogramReindex(std::vector<HistogramType>* out,
 // indicate which of the 'out' histograms is the best approximation.
 template<typename HistogramType>
 void ClusterHistograms(const std::vector<HistogramType>& in,
-                       int num_contexts, int num_blocks,
+                       size_t num_contexts, size_t num_blocks,
                        size_t max_histograms,
                        std::vector<HistogramType>* out,
-                       std::vector<int>* histogram_symbols) {
-  const int in_size = num_contexts * num_blocks;
-  BROTLI_DCHECK(in_size == in.size());
-  std::vector<int> cluster_size(in_size, 1);
+                       std::vector<uint32_t>* histogram_symbols) {
+  const size_t in_size = num_contexts * num_blocks;
+  assert(in_size == in.size());
+  std::vector<uint32_t> cluster_size(in_size, 1);
+  std::vector<uint32_t> clusters(in_size);
+  size_t num_clusters = 0;
   out->resize(in_size);
   histogram_symbols->resize(in_size);
-  for (int i = 0; i < in_size; ++i) {
+  for (size_t i = 0; i < in_size; ++i) {
     (*out)[i] = in[i];
     (*out)[i].bit_cost_ = PopulationCost(in[i]);
-    (*histogram_symbols)[i] = i;
+    (*histogram_symbols)[i] = static_cast<uint32_t>(i);
   }
 
+  const size_t max_input_histograms = 64;
+  // For the first pass of clustering, we allow all pairs.
+  size_t max_num_pairs = max_input_histograms * max_input_histograms / 2;
+  std::vector<HistogramPair> pairs(max_num_pairs + 1);
 
-  const int max_input_histograms = 64;
-  for (int i = 0; i < in_size; i += max_input_histograms) {
-    int num_to_combine = std::min(in_size - i, max_input_histograms);
-    HistogramCombine(&(*out)[0], &cluster_size[0],
-                     &(*histogram_symbols)[i], num_to_combine,
-                     max_histograms);
+  for (size_t i = 0; i < in_size; i += max_input_histograms) {
+    size_t num_to_combine = std::min(in_size - i, max_input_histograms);
+    for (size_t j = 0; j < num_to_combine; ++j) {
+      clusters[num_clusters + j] = static_cast<uint32_t>(i + j);
+    }
+    size_t num_new_clusters =
+        HistogramCombine(&(*out)[0], &cluster_size[0],
+                         &(*histogram_symbols)[i],
+                         &clusters[num_clusters], &pairs[0],
+                         num_to_combine, num_to_combine,
+                         max_histograms, max_num_pairs);
+    num_clusters += num_new_clusters;
   }
 
+  // For the second pass, we limit the total number of histogram pairs.
+  // After this limit is reached, we only keep searching for the best pair.
+  max_num_pairs =
+      std::min(64 * num_clusters, (num_clusters / 2) * num_clusters);
+  pairs.resize(max_num_pairs + 1);
+
   // Collapse similar histograms.
-  HistogramCombine(&(*out)[0], &cluster_size[0],
-                   &(*histogram_symbols)[0], in_size,
-                   max_histograms);
+  num_clusters = HistogramCombine(&(*out)[0], &cluster_size[0],
+                                  &(*histogram_symbols)[0], &clusters[0],
+                                  &pairs[0], num_clusters, in_size,
+                                  max_histograms, max_num_pairs);
 
   // Find the optimal map from original histograms to the final ones.
-  HistogramRemap(&in[0], in_size, &(*out)[0], &(*histogram_symbols)[0]);
+  HistogramRemap(&in[0], in_size, &clusters[0], num_clusters,
+                 &(*out)[0], &(*histogram_symbols)[0]);
 
   // Convert the context map to a canonical form.
-  HistogramReindex(out, histogram_symbols);
-
+  size_t num_histograms =
+      HistogramReindex(&(*out)[0], &(*histogram_symbols)[0], in_size);
+  out->resize(num_histograms);
 }
 
-
 }  // namespace brotli
 
 #endif  // BROTLI_ENC_CLUSTER_H_