brotli 0.1.1 → 0.1.2

data/vendor/brotli/enc/encode.h

@@ -26,7 +26,7 @@ static const int kMinInputBlockBits = 16;
 static const int kMaxInputBlockBits = 24;
 
 struct BrotliParams {
-  BrotliParams()
+  BrotliParams(void)
       : mode(MODE_GENERIC),
         quality(11),
         lgwin(22),
@@ -68,10 +68,10 @@ struct BrotliParams {
 class BrotliCompressor {
  public:
   explicit BrotliCompressor(BrotliParams params);
-  ~BrotliCompressor();
+  ~BrotliCompressor(void);
 
   // The maximum input size that can be processed at once.
-  size_t input_block_size() const { return size_t(1) << params_.lgblock; }
+  size_t input_block_size(void) const { return size_t(1) << params_.lgblock; }
 
   // Encodes the data in input_buffer as a meta-block and writes it to
   // encoded_buffer (*encoded_size should be set to the size of
@@ -115,9 +115,9 @@ class BrotliCompressor {
   // the new output meta-block, or to zero if no new output meta-block was
   // created (in this case the processed input data is buffered internally).
   // If *out_size is positive, *output points to the start of the output data.
-  // Returns false if the size of the input data is larger than
-  // input_block_size() or if there was an error during writing the output.
   // If is_last or force_flush is true, an output meta-block is always created.
+  // Returns false if the size of the input data is larger than
+  // input_block_size().
   bool WriteBrotliData(const bool is_last, const bool force_flush,
                        size_t* out_size, uint8_t** output);
 
@@ -129,28 +129,29 @@ class BrotliCompressor {
   void BrotliSetCustomDictionary(size_t size, const uint8_t* dict);
 
   // No-op, but we keep it here for API backward-compatibility.
-  void WriteStreamHeader() {}
+  void WriteStreamHeader(void) {}
 
  private:
   uint8_t* GetBrotliStorage(size_t size);
 
-  bool WriteMetaBlockInternal(const bool is_last,
-                              size_t* out_size,
-                              uint8_t** output);
+  // Allocates and clears a hash table using memory in "*this",
+  // stores the number of buckets in "*table_size" and returns a pointer to
+  // the base of the hash table.
+  int* GetHashTable(int quality,
+                    size_t input_size, size_t* table_size);
 
   BrotliParams params_;
-  int max_backward_distance_;
   Hashers* hashers_;
   int hash_type_;
-  size_t input_pos_;
+  uint64_t input_pos_;
   RingBuffer* ringbuffer_;
   size_t cmd_alloc_size_;
   Command* commands_;
   size_t num_commands_;
-  int num_literals_;
-  int last_insert_len_;
-  size_t last_flush_pos_;
-  size_t last_processed_pos_;
+  size_t num_literals_;
+  size_t last_insert_len_;
+  uint64_t last_flush_pos_;
+  uint64_t last_processed_pos_;
   int dist_cache_[4];
   int saved_dist_cache_[4];
   uint8_t last_byte_;
@@ -159,6 +160,28 @@ class BrotliCompressor {
   uint8_t prev_byte2_;
   size_t storage_size_;
   uint8_t* storage_;
+  // Hash table for quality 0 mode.
+  int small_table_[1 << 10];  // 2KB
+  int* large_table_;          // Allocated only when needed
+  // Command and distance prefix codes (each 64 symbols, stored back-to-back)
+  // used for the next block in quality 0. The command prefix code is over a
+  // smaller alphabet with the following 64 symbols:
+  //    0 - 15: insert length code 0, copy length code 0 - 15, same distance
+  //   16 - 39: insert length code 0, copy length code 0 - 23
+  //   40 - 63: insert length code 0 - 23, copy length code 0
+  // Note that symbols 16 and 40 represent the same code in the full alphabet,
+  // but we do not use either of them in quality 0.
+  uint8_t cmd_depths_[128];
+  uint16_t cmd_bits_[128];
+  // The compressed form of the command and distance prefix codes for the next
+  // block in quality 0.
+  uint8_t cmd_code_[512];
+  size_t cmd_code_numbits_;
+  // Command and literal buffers for quality 1.
+  uint32_t* command_buf_;
+  uint8_t* literal_buf_;
+  
+  int is_last_block_emitted_;
 };
 
 // Compresses the data in input_buffer into encoded_buffer, and sets

data/vendor/brotli/enc/encode_parallel.cc

@@ -32,15 +32,15 @@ namespace brotli {
 namespace {
 
 void RecomputeDistancePrefixes(Command* cmds, size_t num_commands,
-                               int num_direct_distance_codes,
-                               int distance_postfix_bits) {
+                               uint32_t num_direct_distance_codes,
+                               uint32_t distance_postfix_bits) {
   if (num_direct_distance_codes == 0 &&
       distance_postfix_bits == 0) {
     return;
   }
   for (size_t i = 0; i < num_commands; ++i) {
     Command* cmd = &cmds[i];
-    if (cmd->copy_len_ > 0 && cmd->cmd_prefix_ >= 128) {
+    if (cmd->copy_len() && cmd->cmd_prefix_ >= 128) {
       PrefixEncodeCopyDistance(cmd->DistanceCode(),
                                num_direct_distance_codes,
                                distance_postfix_bits,
@@ -51,21 +51,20 @@ void RecomputeDistancePrefixes(Command* cmds, size_t num_commands,
 }
 
 bool WriteMetaBlockParallel(const BrotliParams& params,
-                            const size_t block_size,
+                            const uint32_t input_size,
                             const uint8_t* input_buffer,
-                            const size_t prefix_size,
+                            const uint32_t prefix_size,
                             const uint8_t* prefix_buffer,
                             const bool is_first,
                             const bool is_last,
                             size_t* encoded_size,
                             uint8_t* encoded_buffer) {
-  if (block_size == 0) {
+  if (input_size == 0) {
     return false;
   }
-  const size_t input_size = block_size;
 
   // Copy prefix + next input block into a continuous area.
-  size_t input_pos = prefix_size;
+  uint32_t input_pos = prefix_size;
   // CreateBackwardReferences reads up to 3 bytes past the end of input if the
   // mask points past the end of input.
   // FindMatchLengthWithLimit could do another 8 bytes look-forward.
@@ -75,7 +74,7 @@ bool WriteMetaBlockParallel(const BrotliParams& params,
   // Since we don't have a ringbuffer, masking is a no-op.
   // We use one less bit than the full range because some of the code uses
   // mask + 1 as the size of the ringbuffer.
-  const size_t mask = std::numeric_limits<size_t>::max() >> 1;
+  const uint32_t mask = std::numeric_limits<uint32_t>::max() >> 1;
 
   uint8_t prev_byte = input_pos > 0 ? input[(input_pos - 1) & mask] : 0;
   uint8_t prev_byte2 = input_pos > 1 ? input[(input_pos - 2) & mask] : 0;
@@ -86,15 +85,14 @@ bool WriteMetaBlockParallel(const BrotliParams& params,
                                 kMinUTF8Ratio);
 
   // Initialize hashers.
-  int hash_type = std::min(9, params.quality);
+  int hash_type = std::min(10, params.quality);
   Hashers* hashers = new Hashers();
   hashers->Init(hash_type);
 
   // Compute backward references.
-  int last_insert_len = 0;
+  size_t last_insert_len = 0;
   size_t num_commands = 0;
-  int num_literals = 0;
-  int max_backward_distance = (1 << params.lgwin) - 16;
+  size_t num_literals = 0;
   int dist_cache[4] = { -4, -4, -4, -4 };
   Command* commands = static_cast<Command*>(
       malloc(sizeof(Command) * ((input_size + 1) >> 1)));
@@ -103,10 +101,10 @@ bool WriteMetaBlockParallel(const BrotliParams& params,
     return false;
   }
   CreateBackwardReferences(
-      input_size, input_pos,
+      input_size, input_pos, is_last,
       &input[0], mask,
-      max_backward_distance,
       params.quality,
+      params.lgwin,
       hashers,
       hash_type,
       dist_cache,
@@ -123,10 +121,11 @@ bool WriteMetaBlockParallel(const BrotliParams& params,
 
   // Build the meta-block.
   MetaBlockSplit mb;
-  int num_direct_distance_codes =
+  uint32_t num_direct_distance_codes =
       params.mode == BrotliParams::MODE_FONT ? 12 : 0;
-  int distance_postfix_bits = params.mode == BrotliParams::MODE_FONT ? 1 : 0;
-  int literal_context_mode = utf8_mode ? CONTEXT_UTF8 : CONTEXT_SIGNED;
+  uint32_t distance_postfix_bits =
+      params.mode == BrotliParams::MODE_FONT ? 1 : 0;
+  ContextType literal_context_mode = utf8_mode ? CONTEXT_UTF8 : CONTEXT_SIGNED;
   RecomputeDistancePrefixes(commands, num_commands,
                             num_direct_distance_codes,
                             distance_postfix_bits);
@@ -145,8 +144,8 @@ bool WriteMetaBlockParallel(const BrotliParams& params,
   // Set up the temporary output storage.
   const size_t max_out_size = 2 * input_size + 500;
   std::vector<uint8_t> storage(max_out_size);
-  int first_byte = 0;
-  int first_byte_bits = 0;
+  uint8_t first_byte = 0;
+  size_t first_byte_bits = 0;
   if (is_first) {
     if (params.lgwin == 16) {
       first_byte = 0;
@@ -155,26 +154,23 @@ bool WriteMetaBlockParallel(const BrotliParams& params,
       first_byte = 1;
       first_byte_bits = 7;
     } else {
-      first_byte = ((params.lgwin - 17) << 1) | 1;
+      first_byte = static_cast<uint8_t>(((params.lgwin - 17) << 1) | 1);
       first_byte_bits = 4;
     }
   }
   storage[0] = static_cast<uint8_t>(first_byte);
-  int storage_ix = first_byte_bits;
+  size_t storage_ix = first_byte_bits;
 
   // Store the meta-block to the temporary output.
-  if (!StoreMetaBlock(&input[0], input_pos, input_size, mask,
-                      prev_byte, prev_byte2,
-                      is_last,
-                      num_direct_distance_codes,
-                      distance_postfix_bits,
-                      literal_context_mode,
-                      commands, num_commands,
-                      mb,
-                      &storage_ix, &storage[0])) {
-    free(commands);
-    return false;
-  }
+  StoreMetaBlock(&input[0], input_pos, input_size, mask,
+                 prev_byte, prev_byte2,
+                 is_last,
+                 num_direct_distance_codes,
+                 distance_postfix_bits,
+                 literal_context_mode,
+                 commands, num_commands,
+                 mb,
+                 &storage_ix, &storage[0]);
   free(commands);
 
   // If this is not the last meta-block, store an empty metadata
@@ -189,11 +185,9 @@ bool WriteMetaBlockParallel(const BrotliParams& params,
   if (input_size + 4 < output_size) {
     storage[0] = static_cast<uint8_t>(first_byte);
     storage_ix = first_byte_bits;
-    if (!StoreUncompressedMetaBlock(is_last, &input[0], input_pos, mask,
-                                    input_size,
-                                    &storage_ix, &storage[0])) {
-      return false;
-    }
+    StoreUncompressedMetaBlock(is_last, &input[0], input_pos, mask,
+                               input_size,
+                               &storage_ix, &storage[0]);
     output_size = storage_ix >> 3;
   }
 
@@ -239,19 +233,23 @@ int BrotliCompressBufferParallel(BrotliParams params,
     params.lgblock = kMaxInputBlockBits;
   }
   size_t max_input_block_size = 1 << params.lgblock;
+  size_t max_prefix_size = 1u << params.lgwin;
 
   std::vector<std::vector<uint8_t> > compressed_pieces;
 
   // Compress block-by-block independently.
   for (size_t pos = 0; pos < input_size; ) {
-    size_t input_block_size = std::min(max_input_block_size, input_size - pos);
+    uint32_t input_block_size =
+        static_cast<uint32_t>(std::min(max_input_block_size, input_size - pos));
+    uint32_t prefix_size =
+        static_cast<uint32_t>(std::min(max_prefix_size, pos));
     size_t out_size = input_block_size + (input_block_size >> 3) + 1024;
     std::vector<uint8_t> out(out_size);
     if (!WriteMetaBlockParallel(params,
                                 input_block_size,
                                 &input_buffer[pos],
-                                pos,
-                                input_buffer,
+                                prefix_size,
+                                &input_buffer[pos - prefix_size],
                                 pos == 0,
                                 pos + input_block_size == input_size,
                                 &out_size,