brotli 0.1.0 → 0.1.1

data/vendor/brotli/enc/encode_parallel.cc

@@ -1,17 +1,9 @@
-// Copyright 2013 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
 // Implementation of parallel Brotli compressor.
 
 #include "./encode_parallel.h"
@@ -31,84 +23,23 @@
 #include "./fast_log.h"
 #include "./hash.h"
 #include "./histogram.h"
-#include "./literal_cost.h"
 #include "./prefix.h"
+#include "./utf8_util.h"
 #include "./write_bits.h"
 
 namespace brotli {
 
 namespace {
 
-int ParseAsUTF8(int* symbol, const uint8_t* input, int size) {
-  // ASCII
-  if ((input[0] & 0x80) == 0) {
-    *symbol = input[0];
-    if (*symbol > 0) {
-      return 1;
-    }
-  }
-  // 2-byte UTF8
-  if (size > 1 &&
-      (input[0] & 0xe0) == 0xc0 &&
-      (input[1] & 0xc0) == 0x80) {
-    *symbol = (((input[0] & 0x1f) << 6) |
-               (input[1] & 0x3f));
-    if (*symbol > 0x7f) {
-      return 2;
-    }
-  }
-  // 3-byte UFT8
-  if (size > 2 &&
-      (input[0] & 0xf0) == 0xe0 &&
-      (input[1] & 0xc0) == 0x80 &&
-      (input[2] & 0xc0) == 0x80) {
-    *symbol = (((input[0] & 0x0f) << 12) |
-               ((input[1] & 0x3f) << 6) |
-               (input[2] & 0x3f));
-    if (*symbol > 0x7ff) {
-      return 3;
-    }
-  }
-  // 4-byte UFT8
-  if (size > 3 &&
-      (input[0] & 0xf8) == 0xf0 &&
-      (input[1] & 0xc0) == 0x80 &&
-      (input[2] & 0xc0) == 0x80 &&
-      (input[3] & 0xc0) == 0x80) {
-    *symbol = (((input[0] & 0x07) << 18) |
-               ((input[1] & 0x3f) << 12) |
-               ((input[2] & 0x3f) << 6) |
-               (input[3] & 0x3f));
-    if (*symbol > 0xffff && *symbol <= 0x10ffff) {
-      return 4;
-    }
-  }
-  // Not UTF8, emit a special symbol above the UTF8-code space
-  *symbol = 0x110000 | input[0];
-  return 1;
-}
-
-// Returns true if at least min_fraction of the data is UTF8-encoded.
-bool IsMostlyUTF8(const uint8_t* data, size_t length, double min_fraction) {
-  size_t size_utf8 = 0;
-  for (size_t pos = 0; pos < length; ) {
-    int symbol;
-    int bytes_read = ParseAsUTF8(&symbol, data + pos, length - pos);
-    pos += bytes_read;
-    if (symbol < 0x110000) size_utf8 += bytes_read;
-  }
-  return size_utf8 > min_fraction * length;
-}
-
-void RecomputeDistancePrefixes(std::vector<Command>* cmds,
+void RecomputeDistancePrefixes(Command* cmds, size_t num_commands,
                                int num_direct_distance_codes,
                                int distance_postfix_bits) {
   if (num_direct_distance_codes == 0 &&
       distance_postfix_bits == 0) {
     return;
   }
-  for (int i = 0; i < cmds->size(); ++i) {
-    Command* cmd = &(*cmds)[i];
+  for (size_t i = 0; i < num_commands; ++i) {
+    Command* cmd = &cmds[i];
     if (cmd->copy_len_ > 0 && cmd->cmd_prefix_ >= 128) {
       PrefixEncodeCopyDistance(cmd->DistanceCode(),
                                num_direct_distance_codes,
@@ -151,50 +82,44 @@ bool WriteMetaBlockParallel(const BrotliParams& params,
 
   // Decide about UTF8 mode.
   static const double kMinUTF8Ratio = 0.75;
-  bool utf8_mode = IsMostlyUTF8(&input[input_pos], input_size, kMinUTF8Ratio);
-
-  // Compute literal costs. The 4 bytes at the end are there to cover for an
-  // over-read past the end of input, but not past the mask, in
-  // CreateBackwardReferences.
-  std::vector<float> literal_cost(prefix_size + input_size + 4);
-  if (utf8_mode) {
-    EstimateBitCostsForLiteralsUTF8(input_pos, input_size, mask, mask,
-                                    &input[0], &literal_cost[0]);
-  } else {
-    EstimateBitCostsForLiterals(input_pos, input_size, mask, mask,
-                                &input[0], &literal_cost[0]);
-  }
+  bool utf8_mode = IsMostlyUTF8(&input[0], input_pos, mask, input_size,
+                                kMinUTF8Ratio);
 
   // Initialize hashers.
   int hash_type = std::min(9, params.quality);
-  std::unique_ptr<Hashers> hashers(new Hashers());
+  Hashers* hashers = new Hashers();
   hashers->Init(hash_type);
 
   // Compute backward references.
   int last_insert_len = 0;
-  int num_commands = 0;
+  size_t num_commands = 0;
   int num_literals = 0;
   int max_backward_distance = (1 << params.lgwin) - 16;
   int dist_cache[4] = { -4, -4, -4, -4 };
-  std::vector<Command> commands((input_size + 1) >> 1);
+  Command* commands = static_cast<Command*>(
+      malloc(sizeof(Command) * ((input_size + 1) >> 1)));
+  if (commands == 0) {
+    delete hashers;
+    return false;
+  }
   CreateBackwardReferences(
       input_size, input_pos,
       &input[0], mask,
-      &literal_cost[0], mask,
       max_backward_distance,
       params.quality,
-      hashers.get(),
+      hashers,
       hash_type,
       dist_cache,
       &last_insert_len,
-      &commands[0],
+      commands,
       &num_commands,
       &num_literals);
-  commands.resize(num_commands);
+  delete hashers;
   if (last_insert_len > 0) {
-    commands.push_back(Command(last_insert_len));
+    commands[num_commands++] = Command(last_insert_len);
     num_literals += last_insert_len;
   }
+  assert(num_commands != 0);
 
   // Build the meta-block.
   MetaBlockSplit mb;
@@ -202,17 +127,17 @@ bool WriteMetaBlockParallel(const BrotliParams& params,
       params.mode == BrotliParams::MODE_FONT ? 12 : 0;
   int distance_postfix_bits = params.mode == BrotliParams::MODE_FONT ? 1 : 0;
   int literal_context_mode = utf8_mode ? CONTEXT_UTF8 : CONTEXT_SIGNED;
-  RecomputeDistancePrefixes(&commands,
+  RecomputeDistancePrefixes(commands, num_commands,
                             num_direct_distance_codes,
                             distance_postfix_bits);
   if (params.quality <= 9) {
     BuildMetaBlockGreedy(&input[0], input_pos, mask,
-                         commands.data(), commands.size(),
+                         commands, num_commands,
                          &mb);
   } else {
     BuildMetaBlock(&input[0], input_pos, mask,
                    prev_byte, prev_byte2,
-                   commands.data(), commands.size(),
+                   commands, num_commands,
                    literal_context_mode,
                    &mb);
   }
@@ -234,7 +159,7 @@ bool WriteMetaBlockParallel(const BrotliParams& params,
       first_byte_bits = 4;
     }
   }
-  storage[0] = first_byte;
+  storage[0] = static_cast<uint8_t>(first_byte);
   int storage_ix = first_byte_bits;
 
   // Store the meta-block to the temporary output.
@@ -244,11 +169,13 @@ bool WriteMetaBlockParallel(const BrotliParams& params,
                       num_direct_distance_codes,
                       distance_postfix_bits,
                       literal_context_mode,
-                      commands.data(), commands.size(),
+                      commands, num_commands,
                       mb,
                       &storage_ix, &storage[0])) {
+    free(commands);
     return false;
   }
+  free(commands);
 
   // If this is not the last meta-block, store an empty metadata
   // meta-block so that the meta-block will end at a byte boundary.
@@ -260,7 +187,7 @@ bool WriteMetaBlockParallel(const BrotliParams& params,
   // meta-block.
   size_t output_size = storage_ix >> 3;
   if (input_size + 4 < output_size) {
-    storage[0] = first_byte;
+    storage[0] = static_cast<uint8_t>(first_byte);
     storage_ix = first_byte_bits;
     if (!StoreUncompressedMetaBlock(is_last, &input[0], input_pos, mask,
                                     input_size,
@@ -318,7 +245,7 @@ int BrotliCompressBufferParallel(BrotliParams params,
   // Compress block-by-block independently.
   for (size_t pos = 0; pos < input_size; ) {
     size_t input_block_size = std::min(max_input_block_size, input_size - pos);
-    size_t out_size = 1.2 * input_block_size + 1024;
+    size_t out_size = input_block_size + (input_block_size >> 3) + 1024;
     std::vector<uint8_t> out(out_size);
     if (!WriteMetaBlockParallel(params,
                                 input_block_size,
@@ -338,7 +265,7 @@ int BrotliCompressBufferParallel(BrotliParams params,
 
   // Piece together the output.
   size_t out_pos = 0;
-  for (int i = 0; i < compressed_pieces.size(); ++i) {
+  for (size_t i = 0; i < compressed_pieces.size(); ++i) {
     const std::vector<uint8_t>& out = compressed_pieces[i];
     if (out_pos + out.size() > *encoded_size) {
       return false;

data/vendor/brotli/enc/encode_parallel.h

@@ -1,17 +1,9 @@
-// Copyright 2013 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
 // API for parallel Brotli compression
 // Note that this is only a proof of concept currently and not part of the
 // final API yet.
@@ -19,10 +11,9 @@
 #ifndef BROTLI_ENC_ENCODE_PARALLEL_H_
 #define BROTLI_ENC_ENCODE_PARALLEL_H_
 
-#include <stddef.h>
-#include <stdint.h>
 
 #include "./encode.h"
+#include "./types.h"
 
 namespace brotli {
 

data/vendor/brotli/enc/entropy_encode.cc

@@ -1,35 +1,27 @@
-// Copyright 2010 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
+/* Copyright 2010 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
 // Entropy encoding (Huffman) utilities.
 
 #include "./entropy_encode.h"
 
-#include <stdint.h>
 #include <algorithm>
 #include <limits>
 #include <vector>
 #include <cstdlib>
 
 #include "./histogram.h"
+#include "./port.h"
+#include "./types.h"
 
 namespace brotli {
 
 namespace {
 
 struct HuffmanTree {
-  HuffmanTree();
   HuffmanTree(int count, int16_t left, int16_t right)
       : total_count_(count),
         index_left_(left),
@@ -40,8 +32,6 @@ struct HuffmanTree {
   int16_t index_right_or_value_;
 };
 
-HuffmanTree::HuffmanTree() {}
-
 // Sort the root nodes, least popular first.
 bool SortHuffmanTree(const HuffmanTree &v0, const HuffmanTree &v1) {
   return v0.total_count_ < v1.total_count_;
@@ -50,7 +40,7 @@ bool SortHuffmanTree(const HuffmanTree &v0, const HuffmanTree &v1) {
 void SetDepth(const HuffmanTree &p,
               HuffmanTree *pool,
               uint8_t *depth,
-              int level) {
+              uint8_t level) {
   if (p.index_left_ >= 0) {
     ++level;
     SetDepth(pool[p.index_left_], pool, depth, level);
@@ -92,11 +82,11 @@ void CreateHuffmanTree(const int *data,
     for (int i = length - 1; i >= 0; --i) {
       if (data[i]) {
         const int count = std::max(data[i], count_limit);
-        tree.push_back(HuffmanTree(count, -1, i));
+        tree.push_back(HuffmanTree(count, -1, static_cast<int16_t>(i)));
       }
     }
 
-    const int n = tree.size();
+    const int n = static_cast<int>(tree.size());
     if (n == 1) {
       depth[tree[0].index_right_or_value_] = 1;      // Only one element.
       break;
@@ -135,15 +125,16 @@ void CreateHuffmanTree(const int *data,
       }
 
       // The sentinel node becomes the parent node.
-      int j_end = tree.size() - 1;
+      int j_end = static_cast<int>(tree.size()) - 1;
       tree[j_end].total_count_ =
           tree[left].total_count_ + tree[right].total_count_;
-      tree[j_end].index_left_ = left;
-      tree[j_end].index_right_or_value_ = right;
+      tree[j_end].index_left_ = static_cast<int16_t>(left);
+      tree[j_end].index_right_or_value_ = static_cast<int16_t>(right);
 
       // Add back the last sentinel node.
       tree.push_back(sentinel);
     }
+    BROTLI_DCHECK(tree.size() == 2 * n + 1);
     SetDepth(tree[2 * n - 1], &tree[0], depth, 0);
 
     // We need to pack the Huffman tree in tree_limit bits.
@@ -158,7 +149,7 @@ void CreateHuffmanTree(const int *data,
 void Reverse(std::vector<uint8_t>* v, int start, int end) {
   --end;
   while (start < end) {
-    int tmp = (*v)[start];
+    uint8_t tmp = (*v)[start];
     (*v)[start] = (*v)[end];
     (*v)[end] = tmp;
     ++start;
@@ -167,8 +158,8 @@ void Reverse(std::vector<uint8_t>* v, int start, int end) {
 }
 
 void WriteHuffmanTreeRepetitions(
-    const int previous_value,
-    const int value,
+    const uint8_t previous_value,
+    const uint8_t value,
     int repetitions,
     std::vector<uint8_t> *tree,
     std::vector<uint8_t> *extra_bits_data) {
@@ -189,15 +180,15 @@ void WriteHuffmanTreeRepetitions(
     }
   } else {
     repetitions -= 3;
-    int start = tree->size();
+    int start = static_cast<int>(tree->size());
     while (repetitions >= 0) {
       tree->push_back(16);
       extra_bits_data->push_back(repetitions & 0x3);
       repetitions >>= 2;
       --repetitions;
     }
-    Reverse(tree, start, tree->size());
-    Reverse(extra_bits_data, start, tree->size());
+    Reverse(tree, start, static_cast<int>(tree->size()));
+    Reverse(extra_bits_data, start, static_cast<int>(tree->size()));
   }
 }
 
@@ -217,15 +208,15 @@ void WriteHuffmanTreeRepetitionsZeros(
     }
   } else {
     repetitions -= 3;
-    int start = tree->size();
+    int start = static_cast<int>(tree->size());
     while (repetitions >= 0) {
       tree->push_back(17);
       extra_bits_data->push_back(repetitions & 0x7);
       repetitions >>= 3;
       --repetitions;
     }
-    Reverse(tree, start, tree->size());
-    Reverse(extra_bits_data, start, tree->size());
+    Reverse(tree, start, static_cast<int>(tree->size()));
+    Reverse(extra_bits_data, start, static_cast<int>(tree->size()));
   }
 }
 
@@ -374,10 +365,10 @@ static void DecideOverRleUse(const uint8_t* depth, const int length,
   int total_reps_non_zero = 0;
   int count_reps_zero = 0;
   int count_reps_non_zero = 0;
-  for (uint32_t i = 0; i < length;) {
+  for (int i = 0; i < length;) {
     const int value = depth[i];
     int reps = 1;
-    for (uint32_t k = i + 1; k < length && depth[k] == value; ++k) {
+    for (int k = i + 1; k < length && depth[k] == value; ++k) {
       ++reps;
     }
     if (reps >= 3 && value == 0) {
@@ -400,11 +391,11 @@ void WriteHuffmanTree(const uint8_t* depth,
                       uint32_t length,
                       std::vector<uint8_t> *tree,
                       std::vector<uint8_t> *extra_bits_data) {
-  int previous_value = 8;
+  uint8_t previous_value = 8;
 
   // Throw away trailing zeros.
-  int new_length = length;
-  for (int i = 0; i < length; ++i) {
+  uint32_t new_length = length;
+  for (uint32_t i = 0; i < length; ++i) {
     if (depth[length - i - 1] == 0) {
       --new_length;
     } else {
@@ -424,7 +415,7 @@ void WriteHuffmanTree(const uint8_t* depth,
 
   // Actual rle coding.
   for (uint32_t i = 0; i < new_length;) {
-    const int value = depth[i];
+    const uint8_t value = depth[i];
     int reps = 1;
     if ((value != 0 && use_rle_for_non_zero) ||
         (value == 0 && use_rle_for_zero)) {
@@ -453,11 +444,11 @@ uint16_t ReverseBits(int num_bits, uint16_t bits) {
   size_t retval = kLut[bits & 0xf];
   for (int i = 4; i < num_bits; i += 4) {
     retval <<= 4;
-    bits >>= 4;
+    bits = static_cast<uint16_t>(bits >> 4);
     retval |= kLut[bits & 0xf];
   }
   retval >>= (-num_bits & 0x3);
-  return retval;
+  return static_cast<uint16_t>(retval);
 }
 
 }  // namespace
@@ -479,7 +470,7 @@ void ConvertBitDepthsToSymbols(const uint8_t *depth, int len, uint16_t *bits) {
     int code = 0;
     for (int bits = 1; bits < kMaxBits; ++bits) {
       code = (code + bl_count[bits - 1]) << 1;
-      next_code[bits] = code;
+      next_code[bits] = static_cast<uint16_t>(code);
     }
   }
   for (int i = 0; i < len; ++i) {

data/vendor/brotli/enc/entropy_encode.h

@@ -1,27 +1,19 @@
-// Copyright 2010 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
+/* Copyright 2010 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
 // Entropy encoding (Huffman) utilities.
 
 #ifndef BROTLI_ENC_ENTROPY_ENCODE_H_
 #define BROTLI_ENC_ENTROPY_ENCODE_H_
 
-#include <stdint.h>
 #include <string.h>
 #include <vector>
 #include "./histogram.h"
 #include "./prefix.h"
+#include "./types.h"
 
 namespace brotli {
 
@@ -47,7 +39,7 @@ void CreateHuffmanTree(const int *data,
 // counts contains the population counts.
 int OptimizeHuffmanCountsForRle(int length, int* counts);
 
-// Write a huffman tree from bit depths into the bitstream representation
+// Write a Huffman tree from bit depths into the bitstream representation
 // of a Huffman tree. The generated Huffman tree is to be compressed once
 // more using a Huffman tree
 void WriteHuffmanTree(const uint8_t* depth,

data/vendor/brotli/enc/fast_log.h

@@ -1,17 +1,9 @@
-// Copyright 2013 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
 // Utilities for fast computation of logarithms.
 
 #ifndef BROTLI_ENC_FAST_LOG_H_
@@ -19,7 +11,8 @@
 
 #include <assert.h>
 #include <math.h>
-#include <stdint.h>
+
+#include "./types.h"
 
 namespace brotli {
 

data/vendor/brotli/enc/find_match_length.h

@@ -1,27 +1,17 @@
-// Copyright 2010 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
+/* Copyright 2010 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
 // Function to find maximal matching prefixes of strings.
 
 #ifndef BROTLI_ENC_FIND_MATCH_LENGTH_H_
 #define BROTLI_ENC_FIND_MATCH_LENGTH_H_
 
-#include <stdint.h>
-
-#include <stddef.h>
 
 #include "./port.h"
+#include "./types.h"
 
 namespace brotli {