brotli 0.2.3 → 0.5.0

data/vendor/brotli/c/enc/histogram.h

@@ -11,12 +11,13 @@
 
 #include <string.h>  /* memset */
 
+#include <brotli/types.h>
+
 #include "../common/constants.h"
 #include "../common/context.h"
 #include "../common/platform.h"
-#include <brotli/types.h>
-#include "./block_splitter.h"
-#include "./command.h"
+#include "block_splitter.h"
+#include "command.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
@@ -28,7 +29,7 @@ extern "C" {
 #define FN(X) X ## Literal
 #define DATA_SIZE BROTLI_NUM_LITERAL_SYMBOLS
 #define DataType uint8_t
-#include "./histogram_inc.h"  /* NOLINT(build/include) */
+#include "histogram_inc.h"  /* NOLINT(build/include) */
 #undef DataType
 #undef DATA_SIZE
 #undef FN
@@ -36,13 +37,13 @@ extern "C" {
 #define FN(X) X ## Command
 #define DataType uint16_t
 #define DATA_SIZE BROTLI_NUM_COMMAND_SYMBOLS
-#include "./histogram_inc.h"  /* NOLINT(build/include) */
+#include "histogram_inc.h"  /* NOLINT(build/include) */
 #undef DATA_SIZE
 #undef FN
 
 #define FN(X) X ## Distance
 #define DATA_SIZE BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS
-#include "./histogram_inc.h"  /* NOLINT(build/include) */
+#include "histogram_inc.h"  /* NOLINT(build/include) */
 #undef DataType
 #undef DATA_SIZE
 #undef FN

data/vendor/brotli/c/enc/literal_cost.c

@@ -7,12 +7,15 @@
 /* Literal cost model to allow backward reference replacement to be efficient.
 */
 
-#include "./literal_cost.h"
+#include "literal_cost.h"
+
+#include <string.h>  /* memset */
 
-#include "../common/platform.h"
 #include <brotli/types.h>
-#include "./fast_log.h"
-#include "./utf8_util.h"
+
+#include "../common/platform.h"
+#include "fast_log.h"
+#include "utf8_util.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
@@ -54,22 +57,23 @@ static size_t DecideMultiByteStatsLevel(size_t pos, size_t len, size_t mask,
 }
 
 static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
-                                            const uint8_t* data, float* cost) {
+                                            const uint8_t* data,
+                                            size_t* histogram, float* cost) {
   /* max_utf8 is 0 (normal ASCII single byte modeling),
      1 (for 2-byte UTF-8 modeling), or 2 (for 3-byte UTF-8 modeling). */
   const size_t max_utf8 = DecideMultiByteStatsLevel(pos, len, mask, data);
-  size_t histogram[3][256] = { { 0 } };
   size_t window_half = 495;
   size_t in_window = BROTLI_MIN(size_t, window_half, len);
   size_t in_window_utf8[3] = { 0 };
-
   size_t i;
+  memset(histogram, 0, 3 * 256 * sizeof(histogram[0]));
+
   {  /* Bootstrap histograms. */
     size_t last_c = 0;
     size_t utf8_pos = 0;
     for (i = 0; i < in_window; ++i) {
       size_t c = data[(pos + i) & mask];
-      ++histogram[utf8_pos][c];
+      ++histogram[256 * utf8_pos + c];
       ++in_window_utf8[utf8_pos];
       utf8_pos = UTF8Position(last_c, c, max_utf8);
       last_c = c;
@@ -85,7 +89,7 @@ static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
       size_t last_c =
           i < window_half + 2 ? 0 : data[(pos + i - window_half - 2) & mask];
       size_t utf8_pos2 = UTF8Position(last_c, c, max_utf8);
-      --histogram[utf8_pos2][data[(pos + i - window_half) & mask]];
+      --histogram[256 * utf8_pos2 + data[(pos + i - window_half) & mask]];
       --in_window_utf8[utf8_pos2];
     }
     if (i + window_half < len) {
@@ -93,7 +97,7 @@ static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
       size_t c = data[(pos + i + window_half - 1) & mask];
       size_t last_c = data[(pos + i + window_half - 2) & mask];
       size_t utf8_pos2 = UTF8Position(last_c, c, max_utf8);
-      ++histogram[utf8_pos2][data[(pos + i + window_half) & mask]];
+      ++histogram[256 * utf8_pos2 + data[(pos + i + window_half) & mask]];
       ++in_window_utf8[utf8_pos2];
     }
     {
@@ -101,7 +105,7 @@ static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
       size_t last_c = i < 2 ? 0 : data[(pos + i - 2) & mask];
       size_t utf8_pos = UTF8Position(last_c, c, max_utf8);
       size_t masked_pos = (pos + i) & mask;
-      size_t histo = histogram[utf8_pos][data[masked_pos]];
+      size_t histo = histogram[256 * utf8_pos + data[masked_pos]];
       double lit_cost;
       if (histo == 0) {
         histo = 1;
@@ -125,17 +129,18 @@ static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
 }
 
 void BrotliEstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask,
-                                       const uint8_t* data, float* cost) {
+                                       const uint8_t* data,
+                                       size_t* histogram, float* cost) {
   if (BrotliIsMostlyUTF8(data, pos, mask, len, kMinUTF8Ratio)) {
-    EstimateBitCostsForLiteralsUTF8(pos, len, mask, data, cost);
+    EstimateBitCostsForLiteralsUTF8(pos, len, mask, data, histogram, cost);
     return;
   } else {
-    size_t histogram[256] = { 0 };
     size_t window_half = 2000;
     size_t in_window = BROTLI_MIN(size_t, window_half, len);
+    size_t i;
+    memset(histogram, 0, 256 * sizeof(histogram[0]));
 
     /* Bootstrap histogram. */
-    size_t i;
     for (i = 0; i < in_window; ++i) {
       ++histogram[data[(pos + i) & mask]];
     }

data/vendor/brotli/c/enc/literal_cost.h

@@ -10,9 +10,10 @@
 #ifndef BROTLI_ENC_LITERAL_COST_H_
 #define BROTLI_ENC_LITERAL_COST_H_
 
-#include "../common/platform.h"
 #include <brotli/types.h>
 
+#include "../common/platform.h"
+
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
 #endif
@@ -21,7 +22,8 @@ extern "C" {
    ring-buffer (data, mask) will take entropy coded and writes these estimates
    to the cost[0..len) array. */
 BROTLI_INTERNAL void BrotliEstimateBitCostsForLiterals(
-    size_t pos, size_t len, size_t mask, const uint8_t* data, float* cost);
+    size_t pos, size_t len, size_t mask, const uint8_t* data, size_t* histogram,
+    float* cost);
 
 #if defined(__cplusplus) || defined(c_plusplus)
 }  /* extern "C" */

data/vendor/brotli/c/enc/memory.c

@@ -7,21 +7,22 @@
 /* Algorithms for distributing the literals and commands of a metablock between
    block types and contexts. */
 
-#include "./memory.h"
+#include "memory.h"
 
 #include <stdlib.h>  /* exit, free, malloc */
 #include <string.h>  /* memcpy */
 
-#include "../common/platform.h"
 #include <brotli/types.h>
 
+#include "../common/platform.h"
+
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
 #endif
 
-#define MAX_PERM_ALLOCATED 128
-#define MAX_NEW_ALLOCATED 64
-#define MAX_NEW_FREED 64
+#define MAX_NEW_ALLOCATED (BROTLI_ENCODER_MEMORY_MANAGER_SLOTS >> 2)
+#define MAX_NEW_FREED (BROTLI_ENCODER_MEMORY_MANAGER_SLOTS >> 2)
+#define MAX_PERM_ALLOCATED (BROTLI_ENCODER_MEMORY_MANAGER_SLOTS >> 1)
 
 #define PERM_ALLOCATED_OFFSET 0
 #define NEW_ALLOCATED_OFFSET MAX_PERM_ALLOCATED
@@ -67,6 +68,7 @@ void BrotliWipeOutMemoryManager(MemoryManager* m) {
 
 static void SortPointers(void** items, const size_t n) {
   /* Shell sort. */
+  /* TODO(eustas): fine-tune for "many slots" case */
   static const size_t gaps[] = {23, 10, 4, 1};
   int g = 0;
   for (; g < 4; ++g) {
@@ -165,6 +167,28 @@ void BrotliWipeOutMemoryManager(MemoryManager* m) {
 
 #endif  /* BROTLI_ENCODER_EXIT_ON_OOM */
 
+void* BrotliBootstrapAlloc(size_t size,
+    brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque) {
+  if (!alloc_func && !free_func) {
+    return malloc(size);
+  } else if (alloc_func && free_func) {
+    return alloc_func(opaque, size);
+  }
+  return NULL;
+}
+
+void BrotliBootstrapFree(void* address, MemoryManager* m) {
+  if (!address) {
+    /* Should not happen! */
+    return;
+  } else {
+    /* Copy values, as those would be freed. */
+    brotli_free_func free_func = m->free_func;
+    void* opaque = m->opaque;
+    free_func(opaque, address);
+  }
+}
+
 #if defined(__cplusplus) || defined(c_plusplus)
 }  /* extern "C" */
 #endif

data/vendor/brotli/c/enc/memory.h

@@ -11,9 +11,10 @@
 
 #include <string.h>  /* memcpy */
 
-#include "../common/platform.h"
 #include <brotli/types.h>
 
+#include "../common/platform.h"
+
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
 #endif
@@ -23,6 +24,16 @@ extern "C" {
 #define BROTLI_ENCODER_EXIT_ON_OOM
 #endif
 
+#if !defined(BROTLI_ENCODER_EXIT_ON_OOM)
+#if defined(BROTLI_EXPERIMENTAL)
+#define BROTLI_ENCODER_MEMORY_MANAGER_SLOTS (48*1024)
+#else  /* BROTLI_EXPERIMENTAL */
+#define BROTLI_ENCODER_MEMORY_MANAGER_SLOTS 256
+#endif  /* BROTLI_EXPERIMENTAL */
+#else /* BROTLI_ENCODER_EXIT_ON_OOM */
+#define BROTLI_ENCODER_MEMORY_MANAGER_SLOTS 0
+#endif  /* BROTLI_ENCODER_EXIT_ON_OOM */
+
 typedef struct MemoryManager {
   brotli_alloc_func alloc_func;
   brotli_free_func free_func;
@@ -32,7 +43,7 @@ typedef struct MemoryManager {
   size_t perm_allocated;
   size_t new_allocated;
   size_t new_freed;
-  void* pointers[256];
+  void* pointers[BROTLI_ENCODER_MEMORY_MANAGER_SLOTS];
 #endif  /* BROTLI_ENCODER_EXIT_ON_OOM */
 } MemoryManager;
 
@@ -56,6 +67,18 @@ BROTLI_INTERNAL void BrotliFree(MemoryManager* m, void* p);
 #define BROTLI_IS_OOM(M) (!!(M)->is_oom)
 #endif  /* BROTLI_ENCODER_EXIT_ON_OOM */
 
+/*
+BROTLI_IS_NULL is a fake check, BROTLI_IS_OOM does the heavy lifting.
+The only purpose of it is to explain static analyzers the state of things.
+NB: use ONLY together with BROTLI_IS_OOM
+    AND ONLY for allocations in the current scope.
+ */
+#if defined(__clang_analyzer__) && !defined(BROTLI_ENCODER_EXIT_ON_OOM)
+#define BROTLI_IS_NULL(A) ((A) == nullptr)
+#else  /* defined(__clang_analyzer__) */
+#define BROTLI_IS_NULL(A) (!!0)
+#endif  /* defined(__clang_analyzer__) */
+
 BROTLI_INTERNAL void BrotliWipeOutMemoryManager(MemoryManager* m);
 
 /*
@@ -66,18 +89,18 @@ A: array
 C: capacity
 R: requested size
 */
-#define BROTLI_ENSURE_CAPACITY(M, T, A, C, R) {  \
-  if (C < (R)) {                                 \
-    size_t _new_size = (C == 0) ? (R) : C;       \
-    T* new_array;                                \
-    while (_new_size < (R)) _new_size *= 2;      \
-    new_array = BROTLI_ALLOC((M), T, _new_size); \
-    if (!BROTLI_IS_OOM(M) && C != 0)             \
-      memcpy(new_array, A, C * sizeof(T));       \
-    BROTLI_FREE((M), A);                         \
-    A = new_array;                               \
-    C = _new_size;                               \
-  }                                              \
+#define BROTLI_ENSURE_CAPACITY(M, T, A, C, R) {                    \
+  if (C < (R)) {                                                   \
+    size_t _new_size = (C == 0) ? (R) : C;                         \
+    T* new_array;                                                  \
+    while (_new_size < (R)) _new_size *= 2;                        \
+    new_array = BROTLI_ALLOC((M), T, _new_size);                   \
+    if (!BROTLI_IS_OOM(M) && !BROTLI_IS_NULL(new_array) && C != 0) \
+      memcpy(new_array, A, C * sizeof(T));                         \
+    BROTLI_FREE((M), A);                                           \
+    A = new_array;                                                 \
+    C = _new_size;                                                 \
+  }                                                                \
 }
 
 /*
@@ -95,6 +118,12 @@ V: value to append
   A[(S) - 1] = (V);                                       \
 }
 
+/* "Bootstrap" allocations are not tracked by memory manager; should be used
+   only to allocate MemoryManager itself (or structure containing it). */
+BROTLI_INTERNAL void* BrotliBootstrapAlloc(size_t size,
+    brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque);
+BROTLI_INTERNAL void BrotliBootstrapFree(void* address, MemoryManager* m);
+
 #if defined(__cplusplus) || defined(c_plusplus)
 }  /* extern "C" */
 #endif