brotli 0.4.0 → 0.5.0

data/vendor/brotli/c/enc/encoder_dict.h

@@ -7,15 +7,56 @@
 #ifndef BROTLI_ENC_ENCODER_DICT_H_
 #define BROTLI_ENC_ENCODER_DICT_H_
 
+#include <brotli/shared_dictionary.h>
+#include <brotli/types.h>
+
 #include "../common/dictionary.h"
 #include "../common/platform.h"
-#include <brotli/types.h>
-#include "./static_dict_lut.h"
+#include "compound_dictionary.h"
+#include "memory.h"
+#include "static_dict_lut.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
 #endif
 
+/*
+Dictionary hierarchy for Encoder:
+-SharedEncoderDictionary
+--CompoundDictionary
+---PreparedDictionary [up to 15x]
+   = prefix dictionary with precomputed hashes
+--ContextualEncoderDictionary
+---BrotliEncoderDictionary [up to 64x]
+   = for each context, precomputed static dictionary with words + transforms
+
+Dictionary hiearchy from common: similar, but without precomputed hashes
+-BrotliSharedDictionary
+--BrotliDictionary [up to 64x]
+--BrotliTransforms [up to 64x]
+--const uint8_t* prefix [up to 15x]: compound dictionaries
+*/
+
+typedef struct BrotliTrieNode {
+  uint8_t single;  /* if 1, sub is a single node for c instead of 256 */
+  uint8_t c;
+  uint8_t len_;  /* untransformed length */
+  uint32_t idx_;  /* word index + num words * transform index */
+  uint32_t sub;  /* index of sub node(s) in the pool */
+} BrotliTrieNode;
+
+typedef struct BrotliTrie {
+  BrotliTrieNode* pool;
+  size_t pool_capacity;
+  size_t pool_size;
+  BrotliTrieNode root;
+} BrotliTrie;
+
+#if defined(BROTLI_EXPERIMENTAL)
+BROTLI_INTERNAL const BrotliTrieNode* BrotliTrieSub(const BrotliTrie* trie,
+    const BrotliTrieNode* node, uint8_t c);
+#endif  /* BROTLI_EXPERIMENTAL */
+
 /* Dictionary data (words and transforms) for 1 possible context */
 typedef struct BrotliEncoderDictionary {
   const BrotliDictionary* words;
@@ -32,9 +73,82 @@ typedef struct BrotliEncoderDictionary {
   /* from static_dict_lut.h, for slow encoder */
   const uint16_t* buckets;
   const DictWord* dict_words;
+  /* Heavy version, for use by slow encoder when there are custom transforms.
+     Contains every possible transformed dictionary word in a trie. It encodes
+     about as fast as the non-heavy encoder but consumes a lot of memory and
+     takes time to build. */
+  BrotliTrie trie;
+  BROTLI_BOOL has_words_heavy;
+
+  /* Reference to other dictionaries. */
+  const struct ContextualEncoderDictionary* parent;
+
+  /* Allocated memory, used only when not using the Brotli defaults */
+  uint16_t* hash_table_data_words_;
+  uint8_t* hash_table_data_lengths_;
+  size_t buckets_alloc_size_;
+  uint16_t* buckets_data_;
+  size_t dict_words_alloc_size_;
+  DictWord* dict_words_data_;
+  BrotliDictionary* words_instance_;
 } BrotliEncoderDictionary;
 
-BROTLI_INTERNAL void BrotliInitEncoderDictionary(BrotliEncoderDictionary* dict);
+/* Dictionary data for all 64 contexts */
+typedef struct ContextualEncoderDictionary {
+  BROTLI_BOOL context_based;
+  uint8_t num_dictionaries;
+  uint8_t context_map[SHARED_BROTLI_NUM_DICTIONARY_CONTEXTS];
+  const BrotliEncoderDictionary* dict[SHARED_BROTLI_NUM_DICTIONARY_CONTEXTS];
+
+  /* If num_instances_ is 1, instance_ is used, else dynamic allocation with
+     instances_ is used. */
+  size_t num_instances_;
+  BrotliEncoderDictionary instance_;
+  BrotliEncoderDictionary* instances_;
+} ContextualEncoderDictionary;
+
+typedef struct SharedEncoderDictionary {
+  /* Magic value to distinguish this struct from PreparedDictionary for
+     certain external usages. */
+  uint32_t magic;
+
+  /* LZ77 prefix, compound dictionary */
+  CompoundDictionary compound;
+
+  /* Custom static dictionary (optionally context-based) */
+  ContextualEncoderDictionary contextual;
+
+  /* The maximum quality the dictionary was computed for */
+  int max_quality;
+} SharedEncoderDictionary;
+
+typedef struct ManagedDictionary {
+  uint32_t magic;
+  MemoryManager memory_manager_;
+  uint32_t* dictionary;
+} ManagedDictionary;
+
+/* Initializes to the brotli built-in dictionary */
+BROTLI_INTERNAL void BrotliInitSharedEncoderDictionary(
+    SharedEncoderDictionary* dict);
+
+#if defined(BROTLI_EXPERIMENTAL)
+/* Initializes to shared dictionary that will be parsed from
+   encoded_dict. Requires that you keep the encoded_dict buffer
+   around, parts of data will point to it. */
+BROTLI_INTERNAL BROTLI_BOOL BrotliInitCustomSharedEncoderDictionary(
+    MemoryManager* m, const uint8_t* encoded_dict, size_t size,
+    int quality, SharedEncoderDictionary* dict);
+#endif  /* BROTLI_EXPERIMENTAL */
+
+BROTLI_INTERNAL void BrotliCleanupSharedEncoderDictionary(
+    MemoryManager* m, SharedEncoderDictionary* dict);
+
+BROTLI_INTERNAL ManagedDictionary* BrotliCreateManagedDictionary(
+    brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque);
+
+BROTLI_INTERNAL void BrotliDestroyManagedDictionary(
+    ManagedDictionary* dictionary);
 
 #if defined(__cplusplus) || defined(c_plusplus)
 }  /* extern "C" */

data/vendor/brotli/c/enc/entropy_encode.c

@@ -6,13 +6,14 @@
 
 /* Entropy encoding (Huffman) utilities. */
 
-#include "./entropy_encode.h"
+#include "entropy_encode.h"
 
 #include <string.h>  /* memset */
 
+#include <brotli/types.h>
+
 #include "../common/constants.h"
 #include "../common/platform.h"
-#include <brotli/types.h>
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {

data/vendor/brotli/c/enc/entropy_encode.h

@@ -9,9 +9,10 @@
 #ifndef BROTLI_ENC_ENTROPY_ENCODE_H_
 #define BROTLI_ENC_ENTROPY_ENCODE_H_
 
-#include "../common/platform.h"
 #include <brotli/types.h>
 
+#include "../common/platform.h"
+
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
 #endif

data/vendor/brotli/c/enc/entropy_encode_static.h

@@ -9,10 +9,11 @@
 #ifndef BROTLI_ENC_ENTROPY_ENCODE_STATIC_H_
 #define BROTLI_ENC_ENTROPY_ENCODE_STATIC_H_
 
+#include <brotli/types.h>
+
 #include "../common/constants.h"
 #include "../common/platform.h"
-#include <brotli/types.h>
-#include "./write_bits.h"
+#include "write_bits.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
@@ -76,6 +77,7 @@ static const uint8_t kStaticDistanceCodeDepth[64] = {
   6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
 };
 
+/* GENERATED CODE START */
 static const uint32_t kCodeLengthBits[18] = {
   0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 15, 31, 0, 11, 7,
 };
@@ -531,6 +533,7 @@ static BROTLI_INLINE void StoreStaticDistanceHuffmanTree(
     size_t* storage_ix, uint8_t* storage) {
   BrotliWriteBits(28, 0x0369DC03u, storage_ix, storage);
 }
+/* GENERATED CODE END */
 
 #if defined(__cplusplus) || defined(c_plusplus)
 }  /* extern "C" */

data/vendor/brotli/c/enc/fast_log.c

@@ -4,7 +4,7 @@
    See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
 */
 
-#include "./fast_log.h"
+#include "fast_log.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {

data/vendor/brotli/c/enc/fast_log.h

@@ -11,9 +11,10 @@
 
 #include <math.h>
 
-#include "../common/platform.h"
 #include <brotli/types.h>
 
+#include "../common/platform.h"
+
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
 #endif

data/vendor/brotli/c/enc/find_match_length.h

@@ -9,9 +9,10 @@
 #ifndef BROTLI_ENC_FIND_MATCH_LENGTH_H_
 #define BROTLI_ENC_FIND_MATCH_LENGTH_H_
 
-#include "../common/platform.h"
 #include <brotli/types.h>
 
+#include "../common/platform.h"
+
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
 #endif
@@ -21,31 +22,23 @@ extern "C" {
 static BROTLI_INLINE size_t FindMatchLengthWithLimit(const uint8_t* s1,
                                                      const uint8_t* s2,
                                                      size_t limit) {
-  size_t matched = 0;
-  size_t limit2 = (limit >> 3) + 1;  /* + 1 is for pre-decrement in while */
-  while (BROTLI_PREDICT_TRUE(--limit2)) {
-    if (BROTLI_PREDICT_FALSE(BROTLI_UNALIGNED_LOAD64LE(s2) ==
-                      BROTLI_UNALIGNED_LOAD64LE(s1 + matched))) {
-      s2 += 8;
-      matched += 8;
-    } else {
-      uint64_t x = BROTLI_UNALIGNED_LOAD64LE(s2) ^
-          BROTLI_UNALIGNED_LOAD64LE(s1 + matched);
+  const uint8_t *s1_orig = s1;
+  for (; limit >= 8; limit -= 8) {
+    uint64_t x = BROTLI_UNALIGNED_LOAD64LE(s2) ^
+                 BROTLI_UNALIGNED_LOAD64LE(s1);
+    s2 += 8;
+    if (x != 0) {
       size_t matching_bits = (size_t)BROTLI_TZCNT64(x);
-      matched += matching_bits >> 3;
-      return matched;
+      return (size_t)(s1 - s1_orig) + (matching_bits >> 3);
     }
+    s1 += 8;
   }
-  limit = (limit & 7) + 1;  /* + 1 is for pre-decrement in while */
-  while (--limit) {
-    if (BROTLI_PREDICT_TRUE(s1[matched] == *s2)) {
-      ++s2;
-      ++matched;
-    } else {
-      return matched;
-    }
+  while (limit && *s1 == *s2) {
+    limit--;
+    ++s2;
+    ++s1;
   }
-  return matched;
+  return (size_t)(s1 - s1_orig);
 }
 #else
 static BROTLI_INLINE size_t FindMatchLengthWithLimit(const uint8_t* s1,