extbrotli 0.0.1.PROTOTYPE

data/contrib/brotli/enc/brotli_bit_stream.h

@@ -0,0 +1,149 @@
+// Copyright 2014 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Functions to convert brotli-related data structures into the
+// brotli bit stream. The functions here operate under
+// assumption that there is enough space in the storage, i.e., there are
+// no out-of-range checks anywhere.
+//
+// These functions do bit addressing into a byte array. The byte array
+// is called "storage" and the index to the bit is called storage_ix
+// in function arguments.
+
+#ifndef BROTLI_ENC_BROTLI_BIT_STREAM_H_
+#define BROTLI_ENC_BROTLI_BIT_STREAM_H_
+
+#include <stddef.h>
+#include <stdint.h>
+#include <vector>
+
+#include "./metablock.h"
+
+namespace brotli {
+
+// All Store functions here will use a storage_ix, which is always the bit
+// position for the current storage.
+
+// Stores a number between 0 and 255.
+void StoreVarLenUint8(int n, int* storage_ix, uint8_t* storage);
+
+// Stores the compressed meta-block header.
+bool StoreCompressedMetaBlockHeader(bool final_block,
+                                    size_t length,
+                                    int* storage_ix,
+                                    uint8_t* storage);
+
+// Stores the uncompressed meta-block header.
+bool StoreUncompressedMetaBlockHeader(size_t length,
+                                      int* storage_ix,
+                                      uint8_t* storage);
+
+// Stores a context map where the histogram type is always the block type.
+void StoreTrivialContextMap(int num_types,
+                            int context_bits,
+                            int* storage_ix,
+                            uint8_t* storage);
+
+void StoreHuffmanTreeOfHuffmanTreeToBitMask(
+    const int num_codes,
+    const uint8_t *code_length_bitdepth,
+    int *storage_ix,
+    uint8_t *storage);
+
+// Builds a Huffman tree from histogram[0:length] into depth[0:length] and
+// bits[0:length] and stores the encoded tree to the bit stream.
+void BuildAndStoreHuffmanTree(const int *histogram,
+                              const int length,
+                              uint8_t* depth,
+                              uint16_t* bits,
+                              int* storage_ix,
+                              uint8_t* storage);
+
+// Encodes the given context map to the bit stream. The number of different
+// histogram ids is given by num_clusters.
+void EncodeContextMap(const std::vector<int>& context_map,
+                      int num_clusters,
+                      int* storage_ix, uint8_t* storage);
+
+// Data structure that stores everything that is needed to encode each block
+// switch command.
+struct BlockSplitCode {
+  std::vector<int> type_code;
+  std::vector<int> length_prefix;
+  std::vector<int> length_nextra;
+  std::vector<int> length_extra;
+  std::vector<uint8_t> type_depths;
+  std::vector<uint16_t> type_bits;
+  std::vector<uint8_t> length_depths;
+  std::vector<uint16_t> length_bits;
+};
+
+// Builds a BlockSplitCode data structure from the block split given by the
+// vector of block types and block lengths and stores it to the bit stream.
+void BuildAndStoreBlockSplitCode(const std::vector<int>& types,
+                                 const std::vector<int>& lengths,
+                                 const int num_types,
+                                 BlockSplitCode* code,
+                                 int* storage_ix,
+                                 uint8_t* storage);
+
+// Stores the block switch command with index block_ix to the bit stream.
+void StoreBlockSwitch(const BlockSplitCode& code,
+                      const int block_ix,
+                      int* storage_ix,
+                      uint8_t* storage);
+
+bool StoreMetaBlock(const uint8_t* input,
+                    size_t start_pos,
+                    size_t length,
+                    size_t mask,
+                    uint8_t prev_byte,
+                    uint8_t prev_byte2,
+                    bool final_block,
+                    int num_direct_distance_codes,
+                    int distance_postfix_bits,
+                    int literal_context_mode,
+                    const brotli::Command *commands,
+                    size_t n_commands,
+                    const MetaBlockSplit& mb,
+                    int *storage_ix,
+                    uint8_t *storage);
+
+// Stores the meta-block without doing any block splitting, just collects
+// one histogram per block category and uses that for entropy coding.
+bool StoreMetaBlockTrivial(const uint8_t* input,
+                           size_t start_pos,
+                           size_t length,
+                           size_t mask,
+                           bool is_last,
+                           const brotli::Command *commands,
+                           size_t n_commands,
+                           int *storage_ix,
+                           uint8_t *storage);
+
+// This is for storing uncompressed blocks (simple raw storage of
+// bytes-as-bytes).
+bool StoreUncompressedMetaBlock(bool final_block,
+                                const uint8_t* input,
+                                size_t position, size_t mask,
+                                size_t len,
+                                int* storage_ix,
+                                uint8_t* storage);
+
+// Stores an empty metadata meta-block and syncs to a byte boundary.
+void StoreSyncMetaBlock(int* storage_ix, uint8_t* storage);
+
+}  // namespace brotli
+
+#endif  // BROTLI_ENC_BROTLI_BIT_STREAM_H_

data/contrib/brotli/enc/cluster.h

@@ -0,0 +1,290 @@
+// Copyright 2013 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Functions for clustering similar histograms together.
+
+#ifndef BROTLI_ENC_CLUSTER_H_
+#define BROTLI_ENC_CLUSTER_H_
+
+#include <math.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <algorithm>
+#include <complex>
+#include <map>
+#include <set>
+#include <utility>
+#include <vector>
+
+#include "./bit_cost.h"
+#include "./entropy_encode.h"
+#include "./fast_log.h"
+#include "./histogram.h"
+
+namespace brotli {
+
+struct HistogramPair {
+  int idx1;
+  int idx2;
+  bool valid;
+  double cost_combo;
+  double cost_diff;
+};
+
+struct HistogramPairComparator {
+  bool operator()(const HistogramPair& p1, const HistogramPair& p2) const {
+    if (p1.cost_diff != p2.cost_diff) {
+      return p1.cost_diff > p2.cost_diff;
+    }
+    return abs(p1.idx1 - p1.idx2) > abs(p2.idx1 - p2.idx2);
+  }
+};
+
+// Returns entropy reduction of the context map when we combine two clusters.
+inline double ClusterCostDiff(int size_a, int size_b) {
+  int size_c = size_a + size_b;
+  return size_a * FastLog2(size_a) + size_b * FastLog2(size_b) -
+      size_c * FastLog2(size_c);
+}
+
+// Computes the bit cost reduction by combining out[idx1] and out[idx2] and if
+// it is below a threshold, stores the pair (idx1, idx2) in the *pairs heap.
+template<typename HistogramType>
+void CompareAndPushToHeap(const HistogramType* out,
+                          const int* cluster_size,
+                          int idx1, int idx2,
+                          std::vector<HistogramPair>* pairs) {
+  if (idx1 == idx2) {
+    return;
+  }
+  if (idx2 < idx1) {
+    int t = idx2;
+    idx2 = idx1;
+    idx1 = t;
+  }
+  bool store_pair = false;
+  HistogramPair p;
+  p.idx1 = idx1;
+  p.idx2 = idx2;
+  p.valid = true;
+  p.cost_diff = 0.5 * ClusterCostDiff(cluster_size[idx1], cluster_size[idx2]);
+  p.cost_diff -= out[idx1].bit_cost_;
+  p.cost_diff -= out[idx2].bit_cost_;
+
+  if (out[idx1].total_count_ == 0) {
+    p.cost_combo = out[idx2].bit_cost_;
+    store_pair = true;
+  } else if (out[idx2].total_count_ == 0) {
+    p.cost_combo = out[idx1].bit_cost_;
+    store_pair = true;
+  } else {
+    double threshold = pairs->empty() ? 1e99 :
+        std::max(0.0, (*pairs)[0].cost_diff);
+    HistogramType combo = out[idx1];
+    combo.AddHistogram(out[idx2]);
+    double cost_combo = PopulationCost(combo);
+    if (cost_combo < threshold - p.cost_diff) {
+      p.cost_combo = cost_combo;
+      store_pair = true;
+    }
+  }
+  if (store_pair) {
+    p.cost_diff += p.cost_combo;
+    pairs->push_back(p);
+    std::push_heap(pairs->begin(), pairs->end(), HistogramPairComparator());
+  }
+}
+
+template<typename HistogramType>
+void HistogramCombine(HistogramType* out,
+                      int* cluster_size,
+                      int* symbols,
+                      int symbols_size,
+                      int max_clusters) {
+  double cost_diff_threshold = 0.0;
+  int min_cluster_size = 1;
+  std::set<int> all_symbols;
+  std::vector<int> clusters;
+  for (int i = 0; i < symbols_size; ++i) {
+    if (all_symbols.find(symbols[i]) == all_symbols.end()) {
+      all_symbols.insert(symbols[i]);
+      clusters.push_back(symbols[i]);
+    }
+  }
+
+  // We maintain a heap of histogram pairs, ordered by the bit cost reduction.
+  std::vector<HistogramPair> pairs;
+  for (int idx1 = 0; idx1 < clusters.size(); ++idx1) {
+    for (int idx2 = idx1 + 1; idx2 < clusters.size(); ++idx2) {
+      CompareAndPushToHeap(out, cluster_size, clusters[idx1], clusters[idx2],
+                           &pairs);
+    }
+  }
+
+  while (clusters.size() > min_cluster_size) {
+    if (pairs[0].cost_diff >= cost_diff_threshold) {
+      cost_diff_threshold = 1e99;
+      min_cluster_size = max_clusters;
+      continue;
+    }
+    // Take the best pair from the top of heap.
+    int best_idx1 = pairs[0].idx1;
+    int best_idx2 = pairs[0].idx2;
+    out[best_idx1].AddHistogram(out[best_idx2]);
+    out[best_idx1].bit_cost_ = pairs[0].cost_combo;
+    cluster_size[best_idx1] += cluster_size[best_idx2];
+    for (int i = 0; i < symbols_size; ++i) {
+      if (symbols[i] == best_idx2) {
+        symbols[i] = best_idx1;
+      }
+    }
+    for (int i = 0; i + 1 < clusters.size(); ++i) {
+      if (clusters[i] >= best_idx2) {
+        clusters[i] = clusters[i + 1];
+      }
+    }
+    clusters.pop_back();
+    // Invalidate pairs intersecting the just combined best pair.
+    for (int i = 0; i < pairs.size(); ++i) {
+      HistogramPair& p = pairs[i];
+      if (p.idx1 == best_idx1 || p.idx2 == best_idx1 ||
+          p.idx1 == best_idx2 || p.idx2 == best_idx2) {
+        p.valid = false;
+      }
+    }
+    // Pop invalid pairs from the top of the heap.
+    while (!pairs.empty() && !pairs[0].valid) {
+      std::pop_heap(pairs.begin(), pairs.end(), HistogramPairComparator());
+      pairs.pop_back();
+    }
+    // Push new pairs formed with the combined histogram to the heap.
+    for (int i = 0; i < clusters.size(); ++i) {
+      CompareAndPushToHeap(out, cluster_size, best_idx1, clusters[i], &pairs);
+    }
+  }
+}
+
+// -----------------------------------------------------------------------------
+// Histogram refinement
+
+// What is the bit cost of moving histogram from cur_symbol to candidate.
+template<typename HistogramType>
+double HistogramBitCostDistance(const HistogramType& histogram,
+                                const HistogramType& candidate) {
+  if (histogram.total_count_ == 0) {
+    return 0.0;
+  }
+  HistogramType tmp = histogram;
+  tmp.AddHistogram(candidate);
+  return PopulationCost(tmp) - candidate.bit_cost_;
+}
+
+// Find the best 'out' histogram for each of the 'in' histograms.
+// Note: we assume that out[]->bit_cost_ is already up-to-date.
+template<typename HistogramType>
+void HistogramRemap(const HistogramType* in, int in_size,
+                    HistogramType* out, int* symbols) {
+  std::set<int> all_symbols;
+  for (int i = 0; i < in_size; ++i) {
+    all_symbols.insert(symbols[i]);
+  }
+  for (int i = 0; i < in_size; ++i) {
+    int best_out = i == 0 ? symbols[0] : symbols[i - 1];
+    double best_bits = HistogramBitCostDistance(in[i], out[best_out]);
+    for (std::set<int>::const_iterator k = all_symbols.begin();
+         k != all_symbols.end(); ++k) {
+      const double cur_bits = HistogramBitCostDistance(in[i], out[*k]);
+      if (cur_bits < best_bits) {
+        best_bits = cur_bits;
+        best_out = *k;
+      }
+    }
+    symbols[i] = best_out;
+  }
+
+
+  // Recompute each out based on raw and symbols.
+  for (std::set<int>::const_iterator k = all_symbols.begin();
+       k != all_symbols.end(); ++k) {
+    out[*k].Clear();
+  }
+  for (int i = 0; i < in_size; ++i) {
+    out[symbols[i]].AddHistogram(in[i]);
+  }
+}
+
+// Reorder histograms in *out so that the new symbols in *symbols come in
+// increasing order.
+template<typename HistogramType>
+void HistogramReindex(std::vector<HistogramType>* out,
+                      std::vector<int>* symbols) {
+  std::vector<HistogramType> tmp(*out);
+  std::map<int, int> new_index;
+  int next_index = 0;
+  for (int i = 0; i < symbols->size(); ++i) {
+    if (new_index.find((*symbols)[i]) == new_index.end()) {
+      new_index[(*symbols)[i]] = next_index;
+      (*out)[next_index] = tmp[(*symbols)[i]];
+      ++next_index;
+    }
+  }
+  out->resize(next_index);
+  for (int i = 0; i < symbols->size(); ++i) {
+    (*symbols)[i] = new_index[(*symbols)[i]];
+  }
+}
+
+// Clusters similar histograms in 'in' together, the selected histograms are
+// placed in 'out', and for each index in 'in', *histogram_symbols will
+// indicate which of the 'out' histograms is the best approximation.
+template<typename HistogramType>
+void ClusterHistograms(const std::vector<HistogramType>& in,
+                       int num_contexts, int num_blocks,
+                       int max_histograms,
+                       std::vector<HistogramType>* out,
+                       std::vector<int>* histogram_symbols) {
+  const int in_size = num_contexts * num_blocks;
+  std::vector<int> cluster_size(in_size, 1);
+  out->resize(in_size);
+  histogram_symbols->resize(in_size);
+  for (int i = 0; i < in_size; ++i) {
+    (*out)[i] = in[i];
+    (*out)[i].bit_cost_ = PopulationCost(in[i]);
+    (*histogram_symbols)[i] = i;
+  }
+
+  const int max_input_histograms = 64;
+  for (int i = 0; i < in_size; i += max_input_histograms) {
+    int num_to_combine = std::min(in_size - i, max_input_histograms);
+    HistogramCombine(&(*out)[0], &cluster_size[0],
+                     &(*histogram_symbols)[i], num_to_combine,
+                     max_histograms);
+  }
+
+  // Collapse similar histograms.
+  HistogramCombine(&(*out)[0], &cluster_size[0],
+                   &(*histogram_symbols)[0], in_size,
+                   max_histograms);
+
+  // Find the optimal map from original histograms to the final ones.
+  HistogramRemap(&in[0], in_size, &(*out)[0], &(*histogram_symbols)[0]);
+
+  // Convert the context map to a canonical form.
+  HistogramReindex(out, histogram_symbols);
+}
+
+
+}  // namespace brotli
+
+#endif  // BROTLI_ENC_CLUSTER_H_

data/contrib/brotli/enc/command.h

@@ -0,0 +1,140 @@
+// Copyright 2013 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// This class models a sequence of literals and a backward reference copy.
+
+#ifndef BROTLI_ENC_COMMAND_H_
+#define BROTLI_ENC_COMMAND_H_
+
+#include <stdint.h>
+#include "./fast_log.h"
+#include "./prefix.h"
+
+namespace brotli {
+
+static int insbase[] =   { 0, 1, 2, 3, 4, 5, 6, 8, 10, 14, 18, 26, 34, 50, 66,
+    98, 130, 194, 322, 578, 1090, 2114, 6210, 22594 };
+static int insextra[] =  { 0, 0, 0, 0, 0, 0, 1, 1,  2,  2,  3,  3,  4,  4,  5,
+    5,   6,   7,   8,   9,   10,   12,   14,    24 };
+static int copybase[] =  { 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 18, 22, 30, 38,
+    54,  70, 102, 134, 198, 326,   582, 1094,  2118 };
+static int copyextra[] = { 0, 0, 0, 0, 0, 0, 0, 0,  1,  1,  2,  2,  3,  3,  4,
+    4,   5,   5,   6,   7,   8,     9,   10,    24 };
+
+static inline int GetInsertLengthCode(int insertlen) {
+  if (insertlen < 6) {
+    return insertlen;
+  } else if (insertlen < 130) {
+    insertlen -= 2;
+    int nbits = Log2FloorNonZero(insertlen) - 1;
+    return (nbits << 1) + (insertlen >> nbits) + 2;
+  } else if (insertlen < 2114) {
+    return Log2FloorNonZero(insertlen - 66) + 10;
+  } else if (insertlen < 6210) {
+    return 21;
+  } else if (insertlen < 22594) {
+    return 22;
+  } else {
+    return 23;
+  }
+}
+
+static inline int GetCopyLengthCode(int copylen) {
+  if (copylen < 10) {
+    return copylen - 2;
+  } else if (copylen < 134) {
+    copylen -= 6;
+    int nbits = Log2FloorNonZero(copylen) - 1;
+    return (nbits << 1) + (copylen >> nbits) + 4;
+  } else if (copylen < 2118) {
+    return Log2FloorNonZero(copylen - 70) + 12;
+  } else {
+    return 23;
+  }
+}
+
+static inline int CombineLengthCodes(
+    int inscode, int copycode, int distancecode) {
+  int bits64 = (copycode & 0x7u) | ((inscode & 0x7u) << 3);
+  if (distancecode == 0 && inscode < 8 && copycode < 16) {
+    return (copycode < 8) ? bits64 : (bits64 | 64);
+  } else {
+    // "To convert an insert-and-copy length code to an insert length code and
+    // a copy length code, the following table can be used"
+    static const int cells[9] = { 2, 3, 6, 4, 5, 8, 7, 9, 10 };
+    return (cells[(copycode >> 3) + 3 * (inscode >> 3)] << 6) | bits64;
+  }
+}
+
+static inline void GetLengthCode(int insertlen, int copylen, int distancecode,
+                                 uint16_t* code, uint64_t* extra) {
+  int inscode = GetInsertLengthCode(insertlen);
+  int copycode = GetCopyLengthCode(copylen);
+  uint64_t insnumextra = insextra[inscode];
+  uint64_t numextra = insnumextra + copyextra[copycode];
+  uint64_t insextraval = insertlen - insbase[inscode];
+  uint64_t copyextraval = copylen - copybase[copycode];
+  *code = CombineLengthCodes(inscode, copycode, distancecode);
+  *extra = (numextra << 48) | (copyextraval << insnumextra) | insextraval;
+}
+
+struct Command {
+  Command() {}
+
+  // distance_code is e.g. 0 for same-as-last short code, or 16 for offset 1.
+  Command(int insertlen, int copylen, int copylen_code, int distance_code)
+      : insert_len_(insertlen), copy_len_(copylen) {
+    // The distance prefix and extra bits are stored in this Command as if
+    // npostfix and ndirect were 0, they are only recomputed later after the
+    // clustering if needed.
+    PrefixEncodeCopyDistance(distance_code, 0, 0, &dist_prefix_, &dist_extra_);
+    GetLengthCode(insertlen, copylen_code, dist_prefix_,
+                  &cmd_prefix_, &cmd_extra_);
+  }
+
+  Command(int insertlen)
+      : insert_len_(insertlen), copy_len_(0), dist_prefix_(16), dist_extra_(0) {
+    GetLengthCode(insertlen, 4, dist_prefix_, &cmd_prefix_, &cmd_extra_);
+  }
+
+  int DistanceCode() const {
+    if (dist_prefix_ < 16) {
+      return dist_prefix_;
+    }
+    int nbits = dist_extra_ >> 24;
+    int extra = dist_extra_ & 0xffffff;
+    int prefix = dist_prefix_ - 12 - 2 * nbits;
+    return (prefix << nbits) + extra + 12;
+  }
+
+  int DistanceContext() const {
+    int r = cmd_prefix_ >> 6;
+    int c = cmd_prefix_ & 7;
+    if ((r == 0 || r == 2 || r == 4 || r == 7) && (c <= 2)) {
+      return c;
+    }
+    return 3;
+  }
+
+  int insert_len_;
+  int copy_len_;
+  uint16_t cmd_prefix_;
+  uint16_t dist_prefix_;
+  uint64_t cmd_extra_;
+  uint32_t dist_extra_;
+};
+
+}  // namespace brotli
+
+#endif  // BROTLI_ENC_COMMAND_H_