brotli 0.1.3 → 0.1.4

data/vendor/brotli/enc/hash_longest_match_quickly_inc.h

@@ -0,0 +1,230 @@
+/* NOLINT(build/header_guard) */
+/* Copyright 2010 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* template parameters: FN, BUCKET_BITS, BUCKET_SWEEP, USE_DICTIONARY */
+
+#define HashLongestMatchQuickly HASHER()
+
+#define BUCKET_SIZE (1 << BUCKET_BITS)
+
+#define HASH_MAP_SIZE (4 << BUCKET_BITS)
+
+static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 8; }
+static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 8; }
+
+/* HashBytes is the function that chooses the bucket to place
+   the address in. The HashLongestMatch and HashLongestMatchQuickly
+   classes have separate, different implementations of hashing. */
+static uint32_t FN(HashBytes)(const uint8_t *data) {
+  /* Computing a hash based on 5 bytes works much better for
+     qualities 1 and 3, where the next hash value is likely to replace */
+  uint64_t h = (BROTLI_UNALIGNED_LOAD64(data) << 24) * kHashMul32;
+  /* The higher bits contain more mixture from the multiplication,
+     so we take our results from there. */
+  return (uint32_t)(h >> (64 - BUCKET_BITS));
+}
+
+/* A (forgetful) hash table to the data seen by the compressor, to
+   help create backward references to previous data.
+
+   This is a hash map of fixed size (BUCKET_SIZE). Starting from the
+   given index, BUCKET_SWEEP buckets are used to store values of a key. */
+typedef struct HashLongestMatchQuickly {
+  uint32_t buckets_[BUCKET_SIZE + BUCKET_SWEEP];
+  /* True if buckets_ array needs to be initialized. */
+  BROTLI_BOOL is_dirty_;
+  DictionarySearchStatictics dict_search_stats_;
+} HashLongestMatchQuickly;
+
+static void FN(Reset)(HashLongestMatchQuickly* self) {
+  self->is_dirty_ = BROTLI_TRUE;
+  DictionarySearchStaticticsReset(&self->dict_search_stats_);
+}
+
+static void FN(InitEmpty)(HashLongestMatchQuickly* self) {
+  if (self->is_dirty_) {
+    /* It is not strictly necessary to fill this buffer here, but
+       not filling will make the results of the compression stochastic
+       (but correct). This is because random data would cause the
+       system to find accidentally good backward references here and there. */
+    memset(&self->buckets_[0], 0, sizeof(self->buckets_));
+    self->is_dirty_ = BROTLI_FALSE;
+  }
+}
+
+static void FN(InitForData)(HashLongestMatchQuickly* self, const uint8_t* data,
+    size_t num) {
+  size_t i;
+  for (i = 0; i < num; ++i) {
+    const uint32_t key = FN(HashBytes)(&data[i]);
+    memset(&self->buckets_[key], 0, BUCKET_SWEEP * sizeof(self->buckets_[0]));
+  }
+  if (num != 0) {
+    self->is_dirty_ = BROTLI_FALSE;
+  }
+}
+
+static void FN(Init)(
+    MemoryManager* m, HashLongestMatchQuickly* self, const uint8_t* data,
+    const BrotliEncoderParams* params, size_t position, size_t bytes,
+    BROTLI_BOOL is_last) {
+  /* Choose which init method is faster.
+     Init() is about 100 times faster than InitForData(). */
+  const size_t kMaxBytesForPartialHashInit = HASH_MAP_SIZE >> 7;
+  BROTLI_UNUSED(m);
+  BROTLI_UNUSED(params);
+  if (position == 0 && is_last && bytes <= kMaxBytesForPartialHashInit) {
+    FN(InitForData)(self, data, bytes);
+  } else {
+    FN(InitEmpty)(self);
+  }
+}
+
+/* Look at 5 bytes at &data[ix & mask].
+   Compute a hash from these, and store the value somewhere within
+   [ix .. ix+3]. */
+static BROTLI_INLINE void FN(Store)(HashLongestMatchQuickly* self,
+    const uint8_t *data, const size_t mask, const size_t ix) {
+  const uint32_t key = FN(HashBytes)(&data[ix & mask]);
+  /* Wiggle the value with the bucket sweep range. */
+  const uint32_t off = (ix >> 3) % BUCKET_SWEEP;
+  self->buckets_[key + off] = (uint32_t)ix;
+}
+
+static BROTLI_INLINE void FN(StoreRange)(HashLongestMatchQuickly* self,
+    const uint8_t *data, const size_t mask, const size_t ix_start,
+    const size_t ix_end) {
+  size_t i;
+  for (i = ix_start; i < ix_end; ++i) {
+    FN(Store)(self, data, mask, i);
+  }
+}
+
+static BROTLI_INLINE void FN(StitchToPreviousBlock)(
+    HashLongestMatchQuickly* self, size_t num_bytes, size_t position,
+    const uint8_t* ringbuffer, size_t ringbuffer_mask) {
+  if (num_bytes >= FN(HashTypeLength)() - 1 && position >= 3) {
+    /* Prepare the hashes for three last bytes of the last write.
+       These could not be calculated before, since they require knowledge
+       of both the previous and the current block. */
+    FN(Store)(self, ringbuffer, ringbuffer_mask, position - 3);
+    FN(Store)(self, ringbuffer, ringbuffer_mask, position - 2);
+    FN(Store)(self, ringbuffer, ringbuffer_mask, position - 1);
+  }
+}
+
+/* Find a longest backward match of &data[cur_ix & ring_buffer_mask]
+   up to the length of max_length and stores the position cur_ix in the
+   hash table.
+
+   Does not look for matches longer than max_length.
+   Does not look for matches further away than max_backward.
+   Writes the best match into |out|.
+   Returns true if match is found, otherwise false. */
+static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(
+    HashLongestMatchQuickly* self, const uint8_t* BROTLI_RESTRICT data,
+    const size_t ring_buffer_mask, const int* BROTLI_RESTRICT distance_cache,
+    const size_t cur_ix, const size_t max_length, const size_t max_backward,
+    HasherSearchResult* BROTLI_RESTRICT out) {
+  const size_t best_len_in = out->len;
+  const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
+  const uint32_t key = FN(HashBytes)(&data[cur_ix_masked]);
+  int compare_char = data[cur_ix_masked + best_len_in];
+  score_t best_score = out->score;
+  size_t best_len = best_len_in;
+  size_t cached_backward = (size_t)distance_cache[0];
+  size_t prev_ix = cur_ix - cached_backward;
+  BROTLI_BOOL is_match_found = BROTLI_FALSE;
+  out->len_x_code = 0;
+  if (prev_ix < cur_ix) {
+    prev_ix &= (uint32_t)ring_buffer_mask;
+    if (compare_char == data[prev_ix + best_len]) {
+      size_t len = FindMatchLengthWithLimit(&data[prev_ix],
+                                            &data[cur_ix_masked],
+                                            max_length);
+      if (len >= 4) {
+        best_score = BackwardReferenceScoreUsingLastDistance(len, 0);
+        best_len = len;
+        out->len = len;
+        out->distance = cached_backward;
+        out->score = best_score;
+        compare_char = data[cur_ix_masked + best_len];
+        if (BUCKET_SWEEP == 1) {
+          self->buckets_[key] = (uint32_t)cur_ix;
+          return BROTLI_TRUE;
+        } else {
+          is_match_found = BROTLI_TRUE;
+        }
+      }
+    }
+  }
+  if (BUCKET_SWEEP == 1) {
+    size_t backward;
+    size_t len;
+    /* Only one to look for, don't bother to prepare for a loop. */
+    prev_ix = self->buckets_[key];
+    self->buckets_[key] = (uint32_t)cur_ix;
+    backward = cur_ix - prev_ix;
+    prev_ix &= (uint32_t)ring_buffer_mask;
+    if (compare_char != data[prev_ix + best_len_in]) {
+      return BROTLI_FALSE;
+    }
+    if (PREDICT_FALSE(backward == 0 || backward > max_backward)) {
+      return BROTLI_FALSE;
+    }
+    len = FindMatchLengthWithLimit(&data[prev_ix],
+                                   &data[cur_ix_masked],
+                                   max_length);
+    if (len >= 4) {
+      out->len = len;
+      out->distance = backward;
+      out->score = BackwardReferenceScore(len, backward);
+      return BROTLI_TRUE;
+    }
+  } else {
+    uint32_t *bucket = self->buckets_ + key;
+    int i;
+    prev_ix = *bucket++;
+    for (i = 0; i < BUCKET_SWEEP; ++i, prev_ix = *bucket++) {
+      const size_t backward = cur_ix - prev_ix;
+      size_t len;
+      prev_ix &= (uint32_t)ring_buffer_mask;
+      if (compare_char != data[prev_ix + best_len]) {
+        continue;
+      }
+      if (PREDICT_FALSE(backward == 0 || backward > max_backward)) {
+        continue;
+      }
+      len = FindMatchLengthWithLimit(&data[prev_ix],
+                                     &data[cur_ix_masked],
+                                     max_length);
+      if (len >= 4) {
+        const score_t score = BackwardReferenceScore(len, backward);
+        if (best_score < score) {
+          best_score = score;
+          best_len = len;
+          out->len = best_len;
+          out->distance = backward;
+          out->score = score;
+          compare_char = data[cur_ix_masked + best_len];
+          is_match_found = BROTLI_TRUE;
+        }
+      }
+    }
+  }
+  if (USE_DICTIONARY && !is_match_found) {
+    is_match_found = SearchInStaticDictionary(&self->dict_search_stats_,
+        &data[cur_ix_masked], max_length, max_backward, out, BROTLI_TRUE);
+  }
+  self->buckets_[key + ((cur_ix >> 3) % BUCKET_SWEEP)] = (uint32_t)cur_ix;
+  return is_match_found;
+}
+
+#undef HASH_MAP_SIZE
+#undef BUCKET_SIZE
+
+#undef HashLongestMatchQuickly

data/vendor/brotli/enc/histogram.c

@@ -0,0 +1,95 @@
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Build per-context histograms of literals, commands and distance codes. */
+
+#include "./histogram.h"
+
+#include "./block_splitter.h"
+#include "./command.h"
+#include "./context.h"
+
+#if defined(__cplusplus) || defined(c_plusplus)
+extern "C" {
+#endif
+
+typedef struct BlockSplitIterator {
+  const BlockSplit* split_;  /* Not owned. */
+  size_t idx_;
+  size_t type_;
+  size_t length_;
+} BlockSplitIterator;
+
+static void InitBlockSplitIterator(BlockSplitIterator* self,
+    const BlockSplit* split) {
+  self->split_ = split;
+  self->idx_ = 0;
+  self->type_ = 0;
+  self->length_ = split->lengths ? split->lengths[0] : 0;
+}
+
+static void BlockSplitIteratorNext(BlockSplitIterator* self) {
+  if (self->length_ == 0) {
+    ++self->idx_;
+    self->type_ = self->split_->types[self->idx_];
+    self->length_ = self->split_->lengths[self->idx_];
+  }
+  --self->length_;
+}
+
+void BrotliBuildHistogramsWithContext(
+    const Command* cmds, const size_t num_commands,
+    const BlockSplit* literal_split, const BlockSplit* insert_and_copy_split,
+    const BlockSplit* dist_split, const uint8_t* ringbuffer, size_t start_pos,
+    size_t mask, uint8_t prev_byte, uint8_t prev_byte2,
+    const ContextType* context_modes, HistogramLiteral* literal_histograms,
+    HistogramCommand* insert_and_copy_histograms,
+    HistogramDistance* copy_dist_histograms) {
+  size_t pos = start_pos;
+  BlockSplitIterator literal_it;
+  BlockSplitIterator insert_and_copy_it;
+  BlockSplitIterator dist_it;
+  size_t i;
+
+  InitBlockSplitIterator(&literal_it, literal_split);
+  InitBlockSplitIterator(&insert_and_copy_it, insert_and_copy_split);
+  InitBlockSplitIterator(&dist_it, dist_split);
+  for (i = 0; i < num_commands; ++i) {
+    const Command* cmd = &cmds[i];
+    size_t j;
+    BlockSplitIteratorNext(&insert_and_copy_it);
+    HistogramAddCommand(&insert_and_copy_histograms[insert_and_copy_it.type_],
+        cmd->cmd_prefix_);
+    for (j = cmd->insert_len_; j != 0; --j) {
+      size_t context;
+      BlockSplitIteratorNext(&literal_it);
+      context = (literal_it.type_ << BROTLI_LITERAL_CONTEXT_BITS) +
+          Context(prev_byte, prev_byte2, context_modes[literal_it.type_]);
+      HistogramAddLiteral(&literal_histograms[context],
+          ringbuffer[pos & mask]);
+      prev_byte2 = prev_byte;
+      prev_byte = ringbuffer[pos & mask];
+      ++pos;
+    }
+    pos += CommandCopyLen(cmd);
+    if (CommandCopyLen(cmd)) {
+      prev_byte2 = ringbuffer[(pos - 2) & mask];
+      prev_byte = ringbuffer[(pos - 1) & mask];
+      if (cmd->cmd_prefix_ >= 128) {
+        size_t context;
+        BlockSplitIteratorNext(&dist_it);
+        context = (dist_it.type_ << BROTLI_DISTANCE_CONTEXT_BITS) +
+            CommandDistanceContext(cmd);
+        HistogramAddDistance(&copy_dist_histograms[context],
+            cmd->dist_prefix_);
+      }
+    }
+  }
+}
+
+#if defined(__cplusplus) || defined(c_plusplus)
+}  /* extern "C" */
+#endif

data/vendor/brotli/enc/histogram.h

@@ -4,91 +4,57 @@
    See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
 */
 
-// Models the histograms of literals, commands and distance codes.
+/* Models the histograms of literals, commands and distance codes. */
 
 #ifndef BROTLI_ENC_HISTOGRAM_H_
 #define BROTLI_ENC_HISTOGRAM_H_
 
-#include <cstring>
-#include <limits>
-#include <vector>
-#include "./context.h"
-#include "./command.h"
-#include "./fast_log.h"
-#include "./prefix.h"
-#include "./types.h"
-
-namespace brotli {
-
-struct BlockSplit;
-
-// A simple container for histograms of data in blocks.
-template<int kDataSize>
-struct Histogram {
-  Histogram(void) {
-    Clear();
-  }
-  void Clear(void) {
-    memset(data_, 0, sizeof(data_));
-    total_count_ = 0;
-    bit_cost_ = std::numeric_limits<double>::infinity();
-  }
-  void Add(size_t val) {
-    ++data_[val];
-    ++total_count_;
-  }
-  void Remove(size_t val) {
-    --data_[val];
-    --total_count_;
-  }
-  template<typename DataType>
-  void Add(const DataType *p, size_t n) {
-    total_count_ += n;
-    n += 1;
-    while(--n) ++data_[*p++];
-  }
-  void AddHistogram(const Histogram& v) {
-    total_count_ += v.total_count_;
-    for (size_t i = 0; i < kDataSize; ++i) {
-      data_[i] += v.data_[i];
-    }
-  }
-
-  uint32_t data_[kDataSize];
-  size_t total_count_;
-  double bit_cost_;
-};
+#include <string.h>  /* memset */
 
-// Literal histogram.
-typedef Histogram<256> HistogramLiteral;
-// Prefix histograms.
-typedef Histogram<kNumCommandPrefixes> HistogramCommand;
-typedef Histogram<kNumDistancePrefixes> HistogramDistance;
-typedef Histogram<kNumBlockLenPrefixes> HistogramBlockLength;
-// Context map histogram, 256 Huffman tree indexes + 16 run length codes.
-typedef Histogram<272> HistogramContextMap;
-// Block type histogram, 256 block types + 2 special symbols.
-typedef Histogram<258> HistogramBlockType;
-
-static const size_t kLiteralContextBits = 6;
-static const size_t kDistanceContextBits = 2;
-
-void BuildHistograms(
-    const Command* cmds,
-    const size_t num_commands,
-    const BlockSplit& literal_split,
-    const BlockSplit& insert_and_copy_split,
-    const BlockSplit& dist_split,
-    const uint8_t* ringbuffer,
-    size_t pos,
-    size_t mask,
-    uint8_t prev_byte,
-    uint8_t prev_byte2,
-    const std::vector<ContextType>& context_modes,
-    std::vector<HistogramLiteral>* literal_histograms,
-    std::vector<HistogramCommand>* insert_and_copy_histograms,
-    std::vector<HistogramDistance>* copy_dist_histograms);
-
-}  // namespace brotli
-
-#endif  // BROTLI_ENC_HISTOGRAM_H_
+#include "../common/constants.h"
+#include "../common/types.h"
+#include "./block_splitter.h"
+#include "./command.h"
+#include "./context.h"
+#include "./port.h"
+
+#if defined(__cplusplus) || defined(c_plusplus)
+extern "C" {
+#endif
+
+#define FN(X) X ## Literal
+#define DATA_SIZE BROTLI_NUM_LITERAL_SYMBOLS
+#define DataType uint8_t
+#include "./histogram_inc.h"  /* NOLINT(build/include) */
+#undef DataType
+#undef DATA_SIZE
+#undef FN
+
+#define FN(X) X ## Command
+#define DataType uint16_t
+#define DATA_SIZE BROTLI_NUM_COMMAND_SYMBOLS
+#include "./histogram_inc.h"  /* NOLINT(build/include) */
+#undef DATA_SIZE
+#undef FN
+
+#define FN(X) X ## Distance
+#define DATA_SIZE BROTLI_NUM_DISTANCE_SYMBOLS
+#include "./histogram_inc.h"  /* NOLINT(build/include) */
+#undef DataType
+#undef DATA_SIZE
+#undef FN
+
+BROTLI_INTERNAL void BrotliBuildHistogramsWithContext(
+    const Command* cmds, const size_t num_commands,
+    const BlockSplit* literal_split, const BlockSplit* insert_and_copy_split,
+    const BlockSplit* dist_split, const uint8_t* ringbuffer, size_t pos,
+    size_t mask, uint8_t prev_byte, uint8_t prev_byte2,
+    const ContextType* context_modes, HistogramLiteral* literal_histograms,
+    HistogramCommand* insert_and_copy_histograms,
+    HistogramDistance* copy_dist_histograms);
+
+#if defined(__cplusplus) || defined(c_plusplus)
+}  /* extern "C" */
+#endif
+
+#endif  /* BROTLI_ENC_HISTOGRAM_H_ */

data/vendor/brotli/enc/histogram_inc.h

@@ -0,0 +1,51 @@
+/* NOLINT(build/header_guard) */
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* template parameters: Histogram, DATA_SIZE, DataType */
+
+/* A simple container for histograms of data in blocks. */
+
+typedef struct FN(Histogram) {
+  uint32_t data_[DATA_SIZE];
+  size_t total_count_;
+  double bit_cost_;
+} FN(Histogram);
+
+static BROTLI_INLINE void FN(HistogramClear)(FN(Histogram)* self) {
+  memset(self->data_, 0, sizeof(self->data_));
+  self->total_count_ = 0;
+  self->bit_cost_ = HUGE_VAL;
+}
+
+static BROTLI_INLINE void FN(ClearHistograms)(
+    FN(Histogram)* array, size_t length) {
+  size_t i;
+  for (i = 0; i < length; ++i) FN(HistogramClear)(array + i);
+}
+
+static BROTLI_INLINE void FN(HistogramAdd)(FN(Histogram)* self, size_t val) {
+  ++self->data_[val];
+  ++self->total_count_;
+}
+
+static BROTLI_INLINE void FN(HistogramAddVector)(FN(Histogram)* self,
+    const DataType *p, size_t n) {
+  self->total_count_ += n;
+  n += 1;
+  while (--n) ++self->data_[*p++];
+}
+
+static BROTLI_INLINE void FN(HistogramAddHistogram)(FN(Histogram)* self,
+    const FN(Histogram)* v) {
+  size_t i;
+  self->total_count_ += v->total_count_;
+  for (i = 0; i < DATA_SIZE; ++i) {
+    self->data_[i] += v->data_[i];
+  }
+}
+
+static BROTLI_INLINE size_t FN(HistogramDataSize)(void) { return DATA_SIZE; }

data/vendor/brotli/enc/literal_cost.c

@@ -0,0 +1,178 @@
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Literal cost model to allow backward reference replacement to be efficient.
+*/
+
+#include "./literal_cost.h"
+
+#include "../common/types.h"
+#include "./fast_log.h"
+#include "./port.h"
+#include "./utf8_util.h"
+
+#if defined(__cplusplus) || defined(c_plusplus)
+extern "C" {
+#endif
+
+static size_t UTF8Position(size_t last, size_t c, size_t clamp) {
+  if (c < 128) {
+    return 0;  /* Next one is the 'Byte 1' again. */
+  } else if (c >= 192) {  /* Next one is the 'Byte 2' of utf-8 encoding. */
+    return BROTLI_MIN(size_t, 1, clamp);
+  } else {
+    /* Let's decide over the last byte if this ends the sequence. */
+    if (last < 0xe0) {
+      return 0;  /* Completed two or three byte coding. */
+    } else {  /* Next one is the 'Byte 3' of utf-8 encoding. */
+      return BROTLI_MIN(size_t, 2, clamp);
+    }
+  }
+}
+
+static size_t DecideMultiByteStatsLevel(size_t pos, size_t len, size_t mask,
+                                        const uint8_t *data) {
+  size_t counts[3] = { 0 };
+  size_t max_utf8 = 1;  /* should be 2, but 1 compresses better. */
+  size_t last_c = 0;
+  size_t utf8_pos = 0;
+  size_t i;
+  for (i = 0; i < len; ++i) {
+    size_t c = data[(pos + i) & mask];
+    utf8_pos = UTF8Position(last_c, c, 2);
+    ++counts[utf8_pos];
+    last_c = c;
+  }
+  if (counts[2] < 500) {
+    max_utf8 = 1;
+  }
+  if (counts[1] + counts[2] < 25) {
+    max_utf8 = 0;
+  }
+  return max_utf8;
+}
+
+static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
+                                            const uint8_t *data, float *cost) {
+  /* max_utf8 is 0 (normal ascii single byte modeling),
+     1 (for 2-byte utf-8 modeling), or 2 (for 3-byte utf-8 modeling). */
+  const size_t max_utf8 = DecideMultiByteStatsLevel(pos, len, mask, data);
+  size_t histogram[3][256] = { { 0 } };
+  size_t window_half = 495;
+  size_t in_window = BROTLI_MIN(size_t, window_half, len);
+  size_t in_window_utf8[3] = { 0 };
+
+
+  size_t i;
+  {  /* Bootstrap histograms. */
+    size_t last_c = 0;
+    size_t utf8_pos = 0;
+    for (i = 0; i < in_window; ++i) {
+      size_t c = data[(pos + i) & mask];
+      ++histogram[utf8_pos][c];
+      ++in_window_utf8[utf8_pos];
+      utf8_pos = UTF8Position(last_c, c, max_utf8);
+      last_c = c;
+    }
+  }
+
+  /* Compute bit costs with sliding window. */
+  for (i = 0; i < len; ++i) {
+    if (i >= window_half) {
+      /* Remove a byte in the past. */
+      size_t c =
+          i < window_half + 1 ? 0 : data[(pos + i - window_half - 1) & mask];
+      size_t last_c =
+          i < window_half + 2 ? 0 : data[(pos + i - window_half - 2) & mask];
+      size_t utf8_pos2 = UTF8Position(last_c, c, max_utf8);
+      --histogram[utf8_pos2][data[(pos + i - window_half) & mask]];
+      --in_window_utf8[utf8_pos2];
+    }
+    if (i + window_half < len) {
+      /* Add a byte in the future. */
+      size_t c = data[(pos + i + window_half - 1) & mask];
+      size_t last_c = data[(pos + i + window_half - 2) & mask];
+      size_t utf8_pos2 = UTF8Position(last_c, c, max_utf8);
+      ++histogram[utf8_pos2][data[(pos + i + window_half) & mask]];
+      ++in_window_utf8[utf8_pos2];
+    }
+    {
+      size_t c = i < 1 ? 0 : data[(pos + i - 1) & mask];
+      size_t last_c = i < 2 ? 0 : data[(pos + i - 2) & mask];
+      size_t utf8_pos = UTF8Position(last_c, c, max_utf8);
+      size_t masked_pos = (pos + i) & mask;
+      size_t histo = histogram[utf8_pos][data[masked_pos]];
+      double lit_cost;
+      if (histo == 0) {
+        histo = 1;
+      }
+      lit_cost = FastLog2(in_window_utf8[utf8_pos]) - FastLog2(histo);
+      lit_cost += 0.02905;
+      if (lit_cost < 1.0) {
+        lit_cost *= 0.5;
+        lit_cost += 0.5;
+      }
+      /* Make the first bytes more expensive -- seems to help, not sure why.
+         Perhaps because the entropy source is changing its properties
+         rapidly in the beginning of the file, perhaps because the beginning
+         of the data is a statistical "anomaly". */
+      if (i < 2000) {
+        lit_cost += 0.7 - ((double)(2000 - i) / 2000.0 * 0.35);
+      }
+      cost[i] = (float)lit_cost;
+    }
+  }
+}
+
+void BrotliEstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask,
+                                       const uint8_t *data, float *cost) {
+  if (BrotliIsMostlyUTF8(data, pos, mask, len, kMinUTF8Ratio)) {
+    EstimateBitCostsForLiteralsUTF8(pos, len, mask, data, cost);
+    return;
+  } else {
+    size_t histogram[256] = { 0 };
+    size_t window_half = 2000;
+    size_t in_window = BROTLI_MIN(size_t, window_half, len);
+
+    /* Bootstrap histogram. */
+    size_t i;
+    for (i = 0; i < in_window; ++i) {
+      ++histogram[data[(pos + i) & mask]];
+    }
+
+    /* Compute bit costs with sliding window. */
+    for (i = 0; i < len; ++i) {
+      size_t histo;
+      if (i >= window_half) {
+        /* Remove a byte in the past. */
+        --histogram[data[(pos + i - window_half) & mask]];
+        --in_window;
+      }
+      if (i + window_half < len) {
+        /* Add a byte in the future. */
+        ++histogram[data[(pos + i + window_half) & mask]];
+        ++in_window;
+      }
+      histo = histogram[data[(pos + i) & mask]];
+      if (histo == 0) {
+        histo = 1;
+      }
+      {
+        double lit_cost = FastLog2(in_window) - FastLog2(histo);
+        lit_cost += 0.029;
+        if (lit_cost < 1.0) {
+          lit_cost *= 0.5;
+          lit_cost += 0.5;
+        }
+        cost[i] = (float)lit_cost;
+      }
+    }
+  }
+}
+
+#if defined(__cplusplus) || defined(c_plusplus)
+}  /* extern "C" */
+#endif