duckdb 1.0.1-dev22.0 → 1.0.1-dev27.0

package/src/duckdb/third_party/brotli/enc/literal_cost.cpp

@@ -0,0 +1,176 @@
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Literal cost model to allow backward reference replacement to be efficient.
+*/
+
+#include "literal_cost.h"
+
+#include <string.h>  /* memset */
+
+#include <brotli/types.h>
+
+#include "../common/brotli_platform.h"
+#include "fast_log.h"
+#include "utf8_util.h"
+
+using namespace duckdb_brotli;
+
+static size_t UTF8Position(size_t last, size_t c, size_t clamp) {
+  if (c < 128) {
+    return 0;  /* Next one is the 'Byte 1' again. */
+  } else if (c >= 192) {  /* Next one is the 'Byte 2' of utf-8 encoding. */
+    return BROTLI_MIN(size_t, 1, clamp);
+  } else {
+    /* Let's decide over the last byte if this ends the sequence. */
+    if (last < 0xE0) {
+      return 0;  /* Completed two or three byte coding. */
+    } else {  /* Next one is the 'Byte 3' of utf-8 encoding. */
+      return BROTLI_MIN(size_t, 2, clamp);
+    }
+  }
+}
+
+static size_t DecideMultiByteStatsLevel(size_t pos, size_t len, size_t mask,
+                                        const uint8_t* data) {
+  size_t counts[3] = { 0 };
+  size_t max_utf8 = 1;  /* should be 2, but 1 compresses better. */
+  size_t last_c = 0;
+  size_t i;
+  for (i = 0; i < len; ++i) {
+    size_t c = data[(pos + i) & mask];
+    ++counts[UTF8Position(last_c, c, 2)];
+    last_c = c;
+  }
+  if (counts[2] < 500) {
+    max_utf8 = 1;
+  }
+  if (counts[1] + counts[2] < 25) {
+    max_utf8 = 0;
+  }
+  return max_utf8;
+}
+
+static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
+                                            const uint8_t* data,
+                                            size_t* histogram, float* cost) {
+  /* max_utf8 is 0 (normal ASCII single byte modeling),
+     1 (for 2-byte UTF-8 modeling), or 2 (for 3-byte UTF-8 modeling). */
+  const size_t max_utf8 = DecideMultiByteStatsLevel(pos, len, mask, data);
+  size_t window_half = 495;
+  size_t in_window = BROTLI_MIN(size_t, window_half, len);
+  size_t in_window_utf8[3] = { 0 };
+  size_t i;
+  memset(histogram, 0, 3 * 256 * sizeof(histogram[0]));
+
+  {  /* Bootstrap histograms. */
+    size_t last_c = 0;
+    size_t utf8_pos = 0;
+    for (i = 0; i < in_window; ++i) {
+      size_t c = data[(pos + i) & mask];
+      ++histogram[256 * utf8_pos + c];
+      ++in_window_utf8[utf8_pos];
+      utf8_pos = UTF8Position(last_c, c, max_utf8);
+      last_c = c;
+    }
+  }
+
+  /* Compute bit costs with sliding window. */
+  for (i = 0; i < len; ++i) {
+    if (i >= window_half) {
+      /* Remove a byte in the past. */
+      size_t c =
+          i < window_half + 1 ? 0 : data[(pos + i - window_half - 1) & mask];
+      size_t last_c =
+          i < window_half + 2 ? 0 : data[(pos + i - window_half - 2) & mask];
+      size_t utf8_pos2 = UTF8Position(last_c, c, max_utf8);
+      --histogram[256 * utf8_pos2 + data[(pos + i - window_half) & mask]];
+      --in_window_utf8[utf8_pos2];
+    }
+    if (i + window_half < len) {
+      /* Add a byte in the future. */
+      size_t c = data[(pos + i + window_half - 1) & mask];
+      size_t last_c = data[(pos + i + window_half - 2) & mask];
+      size_t utf8_pos2 = UTF8Position(last_c, c, max_utf8);
+      ++histogram[256 * utf8_pos2 + data[(pos + i + window_half) & mask]];
+      ++in_window_utf8[utf8_pos2];
+    }
+    {
+      size_t c = i < 1 ? 0 : data[(pos + i - 1) & mask];
+      size_t last_c = i < 2 ? 0 : data[(pos + i - 2) & mask];
+      size_t utf8_pos = UTF8Position(last_c, c, max_utf8);
+      size_t masked_pos = (pos + i) & mask;
+      size_t histo = histogram[256 * utf8_pos + data[masked_pos]];
+      double lit_cost;
+      if (histo == 0) {
+        histo = 1;
+      }
+      lit_cost = FastLog2(in_window_utf8[utf8_pos]) - FastLog2(histo);
+      lit_cost += 0.02905;
+      if (lit_cost < 1.0) {
+        lit_cost *= 0.5;
+        lit_cost += 0.5;
+      }
+      /* Make the first bytes more expensive -- seems to help, not sure why.
+         Perhaps because the entropy source is changing its properties
+         rapidly in the beginning of the file, perhaps because the beginning
+         of the data is a statistical "anomaly". */
+      if (i < 2000) {
+        lit_cost += 0.7 - ((double)(2000 - i) / 2000.0 * 0.35);
+      }
+      cost[i] = (float)lit_cost;
+    }
+  }
+}
+
+void duckdb_brotli::BrotliEstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask,
+                                       const uint8_t* data,
+                                       size_t* histogram, float* cost) {
+  if (BrotliIsMostlyUTF8(data, pos, mask, len, kMinUTF8Ratio)) {
+    EstimateBitCostsForLiteralsUTF8(pos, len, mask, data, histogram, cost);
+    return;
+  } else {
+    size_t window_half = 2000;
+    size_t in_window = BROTLI_MIN(size_t, window_half, len);
+    size_t i;
+    memset(histogram, 0, 256 * sizeof(histogram[0]));
+
+    /* Bootstrap histogram. */
+    for (i = 0; i < in_window; ++i) {
+      ++histogram[data[(pos + i) & mask]];
+    }
+
+    /* Compute bit costs with sliding window. */
+    for (i = 0; i < len; ++i) {
+      size_t histo;
+      if (i >= window_half) {
+        /* Remove a byte in the past. */
+        --histogram[data[(pos + i - window_half) & mask]];
+        --in_window;
+      }
+      if (i + window_half < len) {
+        /* Add a byte in the future. */
+        ++histogram[data[(pos + i + window_half) & mask]];
+        ++in_window;
+      }
+      histo = histogram[data[(pos + i) & mask]];
+      if (histo == 0) {
+        histo = 1;
+      }
+      {
+        double lit_cost = FastLog2(in_window) - FastLog2(histo);
+        lit_cost += 0.029;
+        if (lit_cost < 1.0) {
+          lit_cost *= 0.5;
+          lit_cost += 0.5;
+        }
+        cost[i] = (float)lit_cost;
+      }
+    }
+  }
+}
+
+

package/src/duckdb/third_party/brotli/enc/literal_cost.h

@@ -0,0 +1,28 @@
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Literal cost model to allow backward reference replacement to be efficient.
+*/
+
+#ifndef BROTLI_ENC_LITERAL_COST_H_
+#define BROTLI_ENC_LITERAL_COST_H_
+
+#include <brotli/types.h>
+
+#include "../common/brotli_platform.h"
+
+namespace duckdb_brotli {
+
+/* Estimates how many bits the literals in the interval [pos, pos + len) in the
+   ring-buffer (data, mask) will take entropy coded and writes these estimates
+   to the cost[0..len) array. */
+BROTLI_INTERNAL void BrotliEstimateBitCostsForLiterals(
+    size_t pos, size_t len, size_t mask, const uint8_t* data, size_t* histogram,
+    float* cost);
+
+}
+
+#endif  /* BROTLI_ENC_LITERAL_COST_H_ */

package/src/duckdb/third_party/brotli/enc/memory.cpp

@@ -0,0 +1,190 @@
+/* Copyright 2015 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Algorithms for distributing the literals and commands of a metablock between
+   block types and contexts. */
+
+#include "memory.h"
+
+#include <stdlib.h>  /* exit, free, malloc */
+#include <string.h>  /* memcpy */
+
+#include <brotli/types.h>
+
+#include "../common/brotli_platform.h"
+
+using namespace duckdb_brotli;
+
+#define MAX_NEW_ALLOCATED (BROTLI_ENCODER_MEMORY_MANAGER_SLOTS >> 2)
+#define MAX_NEW_FREED (BROTLI_ENCODER_MEMORY_MANAGER_SLOTS >> 2)
+#define MAX_PERM_ALLOCATED (BROTLI_ENCODER_MEMORY_MANAGER_SLOTS >> 1)
+
+#define PERM_ALLOCATED_OFFSET 0
+#define NEW_ALLOCATED_OFFSET MAX_PERM_ALLOCATED
+#define NEW_FREED_OFFSET (MAX_PERM_ALLOCATED + MAX_NEW_ALLOCATED)
+
+void duckdb_brotli::BrotliInitMemoryManager(
+    MemoryManager* m, brotli_alloc_func alloc_func, brotli_free_func free_func,
+    void* opaque) {
+  if (!alloc_func) {
+    m->alloc_func = duckdb_brotli::BrotliDefaultAllocFunc;
+    m->free_func =  duckdb_brotli::BrotliDefaultFreeFunc;
+    m->opaque = 0;
+  } else {
+    m->alloc_func = alloc_func;
+    m->free_func = free_func;
+    m->opaque = opaque;
+  }
+#if !defined(BROTLI_ENCODER_EXIT_ON_OOM)
+  m->is_oom = BROTLI_FALSE;
+  m->perm_allocated = 0;
+  m->new_allocated = 0;
+  m->new_freed = 0;
+#endif  /* BROTLI_ENCODER_EXIT_ON_OOM */
+}
+
+#if defined(BROTLI_ENCODER_EXIT_ON_OOM)
+
+void* duckdb_brotli::BrotliAllocate(MemoryManager* m, size_t n) {
+  void* result = m->alloc_func(m->opaque, n);
+  if (!result) exit(EXIT_FAILURE);
+  return result;
+}
+
+void duckdb_brotli::BrotliFree(MemoryManager* m, void* p) {
+  m->free_func(m->opaque, p);
+}
+
+void duckdb_brotli::BrotliWipeOutMemoryManager(MemoryManager* m) {
+  BROTLI_UNUSED(m);
+}
+
+#else  /* BROTLI_ENCODER_EXIT_ON_OOM */
+
+static void SortPointers(void** items, const size_t n) {
+  /* Shell sort. */
+  /* TODO(eustas): fine-tune for "many slots" case */
+  static const size_t gaps[] = {23, 10, 4, 1};
+  int g = 0;
+  for (; g < 4; ++g) {
+    size_t gap = gaps[g];
+    size_t i;
+    for (i = gap; i < n; ++i) {
+      size_t j = i;
+      void* tmp = items[i];
+      for (; j >= gap && tmp < items[j - gap]; j -= gap) {
+        items[j] = items[j - gap];
+      }
+      items[j] = tmp;
+    }
+  }
+}
+
+static size_t Annihilate(void** a, size_t a_len, void** b, size_t b_len) {
+  size_t a_read_index = 0;
+  size_t b_read_index = 0;
+  size_t a_write_index = 0;
+  size_t b_write_index = 0;
+  size_t annihilated = 0;
+  while (a_read_index < a_len && b_read_index < b_len) {
+    if (a[a_read_index] == b[b_read_index]) {
+      a_read_index++;
+      b_read_index++;
+      annihilated++;
+    } else if (a[a_read_index] < b[b_read_index]) {
+      a[a_write_index++] = a[a_read_index++];
+    } else {
+      b[b_write_index++] = b[b_read_index++];
+    }
+  }
+  while (a_read_index < a_len) a[a_write_index++] = a[a_read_index++];
+  while (b_read_index < b_len) b[b_write_index++] = b[b_read_index++];
+  return annihilated;
+}
+
+static void CollectGarbagePointers(MemoryManager* m) {
+  size_t annihilated;
+  SortPointers(m->pointers + NEW_ALLOCATED_OFFSET, m->new_allocated);
+  SortPointers(m->pointers + NEW_FREED_OFFSET, m->new_freed);
+  annihilated = Annihilate(
+      m->pointers + NEW_ALLOCATED_OFFSET, m->new_allocated,
+      m->pointers + NEW_FREED_OFFSET, m->new_freed);
+  m->new_allocated -= annihilated;
+  m->new_freed -= annihilated;
+
+  if (m->new_freed != 0) {
+    annihilated = Annihilate(
+        m->pointers + PERM_ALLOCATED_OFFSET, m->perm_allocated,
+        m->pointers + NEW_FREED_OFFSET, m->new_freed);
+    m->perm_allocated -= annihilated;
+    m->new_freed -= annihilated;
+    BROTLI_DCHECK(m->new_freed == 0);
+  }
+
+  if (m->new_allocated != 0) {
+    BROTLI_DCHECK(m->perm_allocated + m->new_allocated <= MAX_PERM_ALLOCATED);
+    memcpy(m->pointers + PERM_ALLOCATED_OFFSET + m->perm_allocated,
+           m->pointers + NEW_ALLOCATED_OFFSET,
+           sizeof(void*) * m->new_allocated);
+    m->perm_allocated += m->new_allocated;
+    m->new_allocated = 0;
+    SortPointers(m->pointers + PERM_ALLOCATED_OFFSET, m->perm_allocated);
+  }
+}
+
+void* BrotliAllocate(MemoryManager* m, size_t n) {
+  void* result = m->alloc_func(m->opaque, n);
+  if (!result) {
+    m->is_oom = BROTLI_TRUE;
+    return NULL;
+  }
+  if (m->new_allocated == MAX_NEW_ALLOCATED) CollectGarbagePointers(m);
+  m->pointers[NEW_ALLOCATED_OFFSET + (m->new_allocated++)] = result;
+  return result;
+}
+
+void BrotliFree(MemoryManager* m, void* p) {
+  if (!p) return;
+  m->free_func(m->opaque, p);
+  if (m->new_freed == MAX_NEW_FREED) CollectGarbagePointers(m);
+  m->pointers[NEW_FREED_OFFSET + (m->new_freed++)] = p;
+}
+
+void BrotliWipeOutMemoryManager(MemoryManager* m) {
+  size_t i;
+  CollectGarbagePointers(m);
+  /* Now all unfreed pointers are in perm-allocated list. */
+  for (i = 0; i < m->perm_allocated; ++i) {
+    m->free_func(m->opaque, m->pointers[PERM_ALLOCATED_OFFSET + i]);
+  }
+  m->perm_allocated = 0;
+}
+
+#endif  /* BROTLI_ENCODER_EXIT_ON_OOM */
+
+void* duckdb_brotli::BrotliBootstrapAlloc(size_t size,
+    brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque) {
+  if (!alloc_func && !free_func) {
+    return malloc(size);
+  } else if (alloc_func && free_func) {
+    return alloc_func(opaque, size);
+  }
+  return NULL;
+}
+
+void duckdb_brotli::BrotliBootstrapFree(void* address, MemoryManager* m) {
+  if (!address) {
+    /* Should not happen! */
+    return;
+  } else {
+    /* Copy values, as those would be freed. */
+    brotli_free_func free_func = m->free_func;
+    void* opaque = m->opaque;
+    free_func(opaque, address);
+  }
+}
+
+

package/src/duckdb/third_party/brotli/enc/memory.h

@@ -0,0 +1,127 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Macros for memory management. */
+
+#ifndef BROTLI_ENC_MEMORY_H_
+#define BROTLI_ENC_MEMORY_H_
+
+#include <string.h>  /* memcpy */
+
+#include <brotli/types.h>
+
+#include "../common/brotli_platform.h"
+
+namespace duckdb_brotli {
+
+#if !defined(BROTLI_ENCODER_CLEANUP_ON_OOM) && \
+    !defined(BROTLI_ENCODER_EXIT_ON_OOM)
+#define BROTLI_ENCODER_EXIT_ON_OOM
+#endif
+
+#if !defined(BROTLI_ENCODER_EXIT_ON_OOM)
+#if defined(BROTLI_EXPERIMENTAL)
+#define BROTLI_ENCODER_MEMORY_MANAGER_SLOTS (48*1024)
+#else  /* BROTLI_EXPERIMENTAL */
+#define BROTLI_ENCODER_MEMORY_MANAGER_SLOTS 256
+#endif  /* BROTLI_EXPERIMENTAL */
+#else /* BROTLI_ENCODER_EXIT_ON_OOM */
+#define BROTLI_ENCODER_MEMORY_MANAGER_SLOTS 0
+#endif  /* BROTLI_ENCODER_EXIT_ON_OOM */
+
+typedef struct MemoryManager {
+  brotli_alloc_func alloc_func;
+  brotli_free_func free_func;
+  void* opaque;
+#if !defined(BROTLI_ENCODER_EXIT_ON_OOM)
+  BROTLI_BOOL is_oom;
+  size_t perm_allocated;
+  size_t new_allocated;
+  size_t new_freed;
+  void* pointers[BROTLI_ENCODER_MEMORY_MANAGER_SLOTS];
+#endif  /* BROTLI_ENCODER_EXIT_ON_OOM */
+} MemoryManager;
+
+BROTLI_INTERNAL void BrotliInitMemoryManager(
+    MemoryManager* m, brotli_alloc_func alloc_func, brotli_free_func free_func,
+    void* opaque);
+
+BROTLI_INTERNAL void* BrotliAllocate(MemoryManager* m, size_t n);
+#define BROTLI_ALLOC(M, T, N)                               \
+  ((N) > 0 ? ((T*)BrotliAllocate((M), (N) * sizeof(T))) : NULL)
+
+BROTLI_INTERNAL void BrotliFree(MemoryManager* m, void* p);
+#define BROTLI_FREE(M, P) { \
+  BrotliFree((M), (P));     \
+  P = NULL;                 \
+}
+
+#if defined(BROTLI_ENCODER_EXIT_ON_OOM)
+#define BROTLI_IS_OOM(M) (!!0)
+#else  /* BROTLI_ENCODER_EXIT_ON_OOM */
+#define BROTLI_IS_OOM(M) (!!(M)->is_oom)
+#endif  /* BROTLI_ENCODER_EXIT_ON_OOM */
+
+/*
+BROTLI_IS_NULL is a fake check, BROTLI_IS_OOM does the heavy lifting.
+The only purpose of it is to explain static analyzers the state of things.
+NB: use ONLY together with BROTLI_IS_OOM
+    AND ONLY for allocations in the current scope.
+ */
+#if defined(__clang_analyzer__) && !defined(BROTLI_ENCODER_EXIT_ON_OOM)
+#define BROTLI_IS_NULL(A) ((A) == nullptr)
+#else  /* defined(__clang_analyzer__) */
+#define BROTLI_IS_NULL(A) (!!0)
+#endif  /* defined(__clang_analyzer__) */
+
+BROTLI_INTERNAL void BrotliWipeOutMemoryManager(MemoryManager* m);
+
+/*
+Dynamically grows array capacity to at least the requested size
+M: MemoryManager
+T: data type
+A: array
+C: capacity
+R: requested size
+*/
+#define BROTLI_ENSURE_CAPACITY(M, T, A, C, R) {                    \
+  if (C < (R)) {                                                   \
+    size_t _new_size = (C == 0) ? (R) : C;                         \
+    T* new_array;                                                  \
+    while (_new_size < (R)) _new_size *= 2;                        \
+    new_array = BROTLI_ALLOC((M), T, _new_size);                   \
+    if (!BROTLI_IS_OOM(M) && !BROTLI_IS_NULL(new_array) && C != 0) \
+      memcpy(new_array, A, C * sizeof(T));                         \
+    BROTLI_FREE((M), A);                                           \
+    A = new_array;                                                 \
+    C = _new_size;                                                 \
+  }                                                                \
+}
+
+/*
+Appends value and dynamically grows array capacity when needed
+M: MemoryManager
+T: data type
+A: array
+C: array capacity
+S: array size
+V: value to append
+*/
+#define BROTLI_ENSURE_CAPACITY_APPEND(M, T, A, C, S, V) { \
+  (S)++;                                                  \
+  BROTLI_ENSURE_CAPACITY(M, T, A, C, S);                  \
+  A[(S) - 1] = (V);                                       \
+}
+
+/* "Bootstrap" allocations are not tracked by memory manager; should be used
+   only to allocate MemoryManager itself (or structure containing it). */
+BROTLI_INTERNAL void* BrotliBootstrapAlloc(size_t size,
+    brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque);
+BROTLI_INTERNAL void BrotliBootstrapFree(void* address, MemoryManager* m);
+
+}
+
+#endif  /* BROTLI_ENC_MEMORY_H_ */