multi_compress 0.1.0

data/ext/multi_compress/vendor/brotli/c/enc/command.h

@@ -0,0 +1,191 @@
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* This class models a sequence of literals and a backward reference copy. */
+
+#ifndef BROTLI_ENC_COMMAND_H_
+#define BROTLI_ENC_COMMAND_H_
+
+#include <brotli/types.h>
+
+#include "../common/constants.h"
+#include "../common/platform.h"
+#include "fast_log.h"
+#include "params.h"
+#include "prefix.h"
+
+#if defined(__cplusplus) || defined(c_plusplus)
+extern "C" {
+#endif
+
+BROTLI_INTERNAL extern const uint32_t
+    kBrotliInsBase[BROTLI_NUM_INS_COPY_CODES];
+BROTLI_INTERNAL extern const uint32_t
+    kBrotliInsExtra[BROTLI_NUM_INS_COPY_CODES];
+BROTLI_INTERNAL extern const uint32_t
+    kBrotliCopyBase[BROTLI_NUM_INS_COPY_CODES];
+BROTLI_INTERNAL extern const uint32_t
+    kBrotliCopyExtra[BROTLI_NUM_INS_COPY_CODES];
+
+static BROTLI_INLINE uint16_t GetInsertLengthCode(size_t insertlen) {
+  if (insertlen < 6) {
+    return (uint16_t)insertlen;
+  } else if (insertlen < 130) {
+    uint32_t nbits = Log2FloorNonZero(insertlen - 2) - 1u;
+    return (uint16_t)((nbits << 1) + ((insertlen - 2) >> nbits) + 2);
+  } else if (insertlen < 2114) {
+    return (uint16_t)(Log2FloorNonZero(insertlen - 66) + 10);
+  } else if (insertlen < 6210) {
+    return 21u;
+  } else if (insertlen < 22594) {
+    return 22u;
+  } else {
+    return 23u;
+  }
+}
+
+static BROTLI_INLINE uint16_t GetCopyLengthCode(size_t copylen) {
+  if (copylen < 10) {
+    return (uint16_t)(copylen - 2);
+  } else if (copylen < 134) {
+    uint32_t nbits = Log2FloorNonZero(copylen - 6) - 1u;
+    return (uint16_t)((nbits << 1) + ((copylen - 6) >> nbits) + 4);
+  } else if (copylen < 2118) {
+    return (uint16_t)(Log2FloorNonZero(copylen - 70) + 12);
+  } else {
+    return 23u;
+  }
+}
+
+static BROTLI_INLINE uint16_t CombineLengthCodes(
+    uint16_t inscode, uint16_t copycode, BROTLI_BOOL use_last_distance) {
+  uint16_t bits64 =
+      (uint16_t)((copycode & 0x7u) | ((inscode & 0x7u) << 3u));
+  if (use_last_distance && inscode < 8u && copycode < 16u) {
+    return (copycode < 8u) ? bits64 : (bits64 | 64u);
+  } else {
+    /* Specification: 5 Encoding of ... (last table) */
+    /* offset = 2 * index, where index is in range [0..8] */
+    uint32_t offset = 2u * ((copycode >> 3u) + 3u * (inscode >> 3u));
+    /* All values in specification are K * 64,
+       where   K = [2, 3, 6, 4, 5, 8, 7, 9, 10],
+           i + 1 = [1, 2, 3, 4, 5, 6, 7, 8,  9],
+       K - i - 1 = [1, 1, 3, 0, 0, 2, 0, 1,  2] = D.
+       All values in D require only 2 bits to encode.
+       Magic constant is shifted 6 bits left, to avoid final multiplication. */
+    offset = (offset << 5u) + 0x40u + ((0x520D40u >> offset) & 0xC0u);
+    return (uint16_t)(offset | bits64);
+  }
+}
+
+static BROTLI_INLINE void GetLengthCode(size_t insertlen, size_t copylen,
+                                        BROTLI_BOOL use_last_distance,
+                                        uint16_t* code) {
+  uint16_t inscode = GetInsertLengthCode(insertlen);
+  uint16_t copycode = GetCopyLengthCode(copylen);
+  *code = CombineLengthCodes(inscode, copycode, use_last_distance);
+}
+
+static BROTLI_INLINE uint32_t GetInsertBase(uint16_t inscode) {
+  return kBrotliInsBase[inscode];
+}
+
+static BROTLI_INLINE uint32_t GetInsertExtra(uint16_t inscode) {
+  return kBrotliInsExtra[inscode];
+}
+
+static BROTLI_INLINE uint32_t GetCopyBase(uint16_t copycode) {
+  return kBrotliCopyBase[copycode];
+}
+
+static BROTLI_INLINE uint32_t GetCopyExtra(uint16_t copycode) {
+  return kBrotliCopyExtra[copycode];
+}
+
+typedef struct Command {
+  uint32_t insert_len_;
+  /* Stores copy_len in low 25 bits and copy_code - copy_len in high 7 bit. */
+  uint32_t copy_len_;
+  /* Stores distance extra bits. */
+  uint32_t dist_extra_;
+  uint16_t cmd_prefix_;
+  /* Stores distance code in low 10 bits
+     and number of extra bits in high 6 bits. */
+  uint16_t dist_prefix_;
+} Command;
+
+/* distance_code is e.g. 0 for same-as-last short code, or 16 for offset 1. */
+static BROTLI_INLINE void InitCommand(Command* self,
+    const BrotliDistanceParams* dist, size_t insertlen,
+    size_t copylen, int copylen_code_delta, size_t distance_code) {
+  /* Don't rely on signed int representation, use honest casts. */
+  uint32_t delta = (uint8_t)((int8_t)copylen_code_delta);
+  self->insert_len_ = (uint32_t)insertlen;
+  self->copy_len_ = (uint32_t)(copylen | (delta << 25));
+  /* The distance prefix and extra bits are stored in this Command as if
+     npostfix and ndirect were 0, they are only recomputed later after the
+     clustering if needed. */
+  PrefixEncodeCopyDistance(
+      distance_code, dist->num_direct_distance_codes,
+      dist->distance_postfix_bits, &self->dist_prefix_, &self->dist_extra_);
+  GetLengthCode(
+      insertlen, (size_t)((int)copylen + copylen_code_delta),
+      TO_BROTLI_BOOL((self->dist_prefix_ & 0x3FF) == 0), &self->cmd_prefix_);
+}
+
+static BROTLI_INLINE void InitInsertCommand(Command* self, size_t insertlen) {
+  self->insert_len_ = (uint32_t)insertlen;
+  self->copy_len_ = 4 << 25;
+  self->dist_extra_ = 0;
+  self->dist_prefix_ = BROTLI_NUM_DISTANCE_SHORT_CODES;
+  GetLengthCode(insertlen, 4, BROTLI_FALSE, &self->cmd_prefix_);
+}
+
+static BROTLI_INLINE uint32_t CommandRestoreDistanceCode(
+    const Command* self, const BrotliDistanceParams* dist) {
+  if ((self->dist_prefix_ & 0x3FFu) <
+      BROTLI_NUM_DISTANCE_SHORT_CODES + dist->num_direct_distance_codes) {
+    return self->dist_prefix_ & 0x3FFu;
+  } else {
+    uint32_t dcode = self->dist_prefix_ & 0x3FFu;
+    uint32_t nbits = self->dist_prefix_ >> 10;
+    uint32_t extra = self->dist_extra_;
+    uint32_t postfix_mask = (1U << dist->distance_postfix_bits) - 1U;
+    uint32_t hcode = (dcode - dist->num_direct_distance_codes -
+        BROTLI_NUM_DISTANCE_SHORT_CODES) >>
+        dist->distance_postfix_bits;
+    uint32_t lcode = (dcode - dist->num_direct_distance_codes -
+        BROTLI_NUM_DISTANCE_SHORT_CODES) & postfix_mask;
+    uint32_t offset = ((2U + (hcode & 1U)) << nbits) - 4U;
+    return ((offset + extra) << dist->distance_postfix_bits) + lcode +
+        dist->num_direct_distance_codes + BROTLI_NUM_DISTANCE_SHORT_CODES;
+  }
+}
+
+static BROTLI_INLINE uint32_t CommandDistanceContext(const Command* self) {
+  uint32_t r = self->cmd_prefix_ >> 6;
+  uint32_t c = self->cmd_prefix_ & 7;
+  if ((r == 0 || r == 2 || r == 4 || r == 7) && (c <= 2)) {
+    return c;
+  }
+  return 3;
+}
+
+static BROTLI_INLINE uint32_t CommandCopyLen(const Command* self) {
+  return self->copy_len_ & 0x1FFFFFF;
+}
+
+static BROTLI_INLINE uint32_t CommandCopyLenCode(const Command* self) {
+  uint32_t modifier = self->copy_len_ >> 25;
+  int32_t delta = (int8_t)((uint8_t)(modifier | ((modifier & 0x40) << 1)));
+  return (uint32_t)((int32_t)(self->copy_len_ & 0x1FFFFFF) + delta);
+}
+
+#if defined(__cplusplus) || defined(c_plusplus)
+}  /* extern "C" */
+#endif
+
+#endif  /* BROTLI_ENC_COMMAND_H_ */

data/ext/multi_compress/vendor/brotli/c/enc/compound_dictionary.c

@@ -0,0 +1,207 @@
+/* Copyright 2017 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+#include "compound_dictionary.h"
+
+#include <brotli/types.h>
+
+#include "../common/platform.h"
+#include "memory.h"
+#include "quality.h"
+
+static PreparedDictionary* CreatePreparedDictionaryWithParams(MemoryManager* m,
+    const uint8_t* source, size_t source_size, uint32_t bucket_bits,
+    uint32_t slot_bits, uint32_t hash_bits, uint16_t bucket_limit) {
+  /* Step 1: create "bloated" hasher. */
+  uint32_t num_slots = 1u << slot_bits;
+  uint32_t num_buckets = 1u << bucket_bits;
+  uint32_t hash_shift = 64u - bucket_bits;
+  uint64_t hash_mask = (~((uint64_t)0U)) >> (64 - hash_bits);
+  uint32_t slot_mask = num_slots - 1;
+  size_t alloc_size = (sizeof(uint32_t) << slot_bits) +
+      (sizeof(uint32_t) << slot_bits) +
+      (sizeof(uint16_t) << bucket_bits) +
+      (sizeof(uint32_t) << bucket_bits) +
+      (sizeof(uint32_t) * source_size);
+  uint8_t* flat = NULL;
+  PreparedDictionary* result = NULL;
+  uint16_t* num = NULL;
+  uint32_t* bucket_heads = NULL;
+  uint32_t* next_bucket = NULL;
+  uint32_t* slot_offsets = NULL;
+  uint16_t* heads = NULL;
+  uint32_t* items = NULL;
+  uint8_t** source_ref = NULL;
+  uint32_t i;
+  uint32_t* slot_size = NULL;
+  uint32_t* slot_limit = NULL;
+  uint32_t total_items = 0;
+  if (slot_bits > 16) return NULL;
+  if (slot_bits > bucket_bits) return NULL;
+  if (bucket_bits - slot_bits >= 16) return NULL;
+
+  flat = BROTLI_ALLOC(m, uint8_t, alloc_size);
+  if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(flat)) return NULL;
+
+  slot_size = (uint32_t*)flat;
+  slot_limit = (uint32_t*)(&slot_size[num_slots]);
+  num = (uint16_t*)(&slot_limit[num_slots]);
+  bucket_heads = (uint32_t*)(&num[num_buckets]);
+  next_bucket = (uint32_t*)(&bucket_heads[num_buckets]);
+  memset(num, 0, num_buckets * sizeof(num[0]));
+
+  /* TODO(eustas): apply custom "store" order. */
+  for (i = 0; i + 7 < source_size; ++i) {
+    const uint64_t h = (BROTLI_UNALIGNED_LOAD64LE(&source[i]) & hash_mask) *
+        kPreparedDictionaryHashMul64Long;
+    const uint32_t key = (uint32_t)(h >> hash_shift);
+    uint16_t count = num[key];
+    next_bucket[i] = (count == 0) ? ((uint32_t)(-1)) : bucket_heads[key];
+    bucket_heads[key] = i;
+    count++;
+    if (count > bucket_limit) count = bucket_limit;
+    num[key] = count;
+  }
+
+  /* Step 2: find slot limits. */
+  for (i = 0; i < num_slots; ++i) {
+    BROTLI_BOOL overflow = BROTLI_FALSE;
+    slot_limit[i] = bucket_limit;
+    while (BROTLI_TRUE) {
+      uint32_t limit = slot_limit[i];
+      size_t j;
+      uint32_t count = 0;
+      overflow = BROTLI_FALSE;
+      for (j = i; j < num_buckets; j += num_slots) {
+        uint32_t size = num[j];
+        /* Last chain may span behind 64K limit; overflow happens only if
+           we are about to use 0xFFFF+ as item offset. */
+        if (count >= 0xFFFF) {
+          overflow = BROTLI_TRUE;
+          break;
+        }
+        if (size > limit) size = limit;
+        count += size;
+      }
+      if (!overflow) {
+        slot_size[i] = count;
+        total_items += count;
+        break;
+      }
+      slot_limit[i]--;
+    }
+  }
+
+  /* Step 3: transfer data to "slim" hasher. */
+  alloc_size = sizeof(PreparedDictionary) + (sizeof(uint32_t) << slot_bits) +
+      (sizeof(uint16_t) << bucket_bits) + (sizeof(uint32_t) * total_items) +
+      sizeof(uint8_t*);
+
+  result = (PreparedDictionary*)BROTLI_ALLOC(m, uint8_t, alloc_size);
+  if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(result)) {
+    BROTLI_FREE(m, flat);
+    return NULL;
+  }
+  slot_offsets = (uint32_t*)(&result[1]);
+  heads = (uint16_t*)(&slot_offsets[num_slots]);
+  items = (uint32_t*)(&heads[num_buckets]);
+  source_ref = (uint8_t**)(&items[total_items]);
+
+  result->magic = kLeanPreparedDictionaryMagic;
+  result->num_items = total_items;
+  result->source_size = (uint32_t)source_size;
+  result->hash_bits = hash_bits;
+  result->bucket_bits = bucket_bits;
+  result->slot_bits = slot_bits;
+  BROTLI_UNALIGNED_STORE_PTR(source_ref, source);
+
+  total_items = 0;
+  for (i = 0; i < num_slots; ++i) {
+    slot_offsets[i] = total_items;
+    total_items += slot_size[i];
+    slot_size[i] = 0;
+  }
+  for (i = 0; i < num_buckets; ++i) {
+    uint32_t slot = i & slot_mask;
+    uint32_t count = num[i];
+    uint32_t pos;
+    size_t j;
+    size_t cursor = slot_size[slot];
+    if (count > slot_limit[slot]) count = slot_limit[slot];
+    if (count == 0) {
+      heads[i] = 0xFFFF;
+      continue;
+    }
+    heads[i] = (uint16_t)cursor;
+    cursor += slot_offsets[slot];
+    slot_size[slot] += count;
+    pos = bucket_heads[i];
+    for (j = 0; j < count; j++) {
+      items[cursor++] = pos;
+      pos = next_bucket[pos];
+    }
+    items[cursor - 1] |= 0x80000000;
+  }
+
+  BROTLI_FREE(m, flat);
+  return result;
+}
+
+PreparedDictionary* CreatePreparedDictionary(MemoryManager* m,
+    const uint8_t* source, size_t source_size) {
+  uint32_t bucket_bits = 17;
+  uint32_t slot_bits = 7;
+  uint32_t hash_bits = 40;
+  uint16_t bucket_limit = 32;
+  size_t volume = 16u << bucket_bits;
+  /* Tune parameters to fit dictionary size. */
+  while (volume < source_size && bucket_bits < 22) {
+    bucket_bits++;
+    slot_bits++;
+    volume <<= 1;
+  }
+  return CreatePreparedDictionaryWithParams(m,
+      source, source_size, bucket_bits, slot_bits, hash_bits, bucket_limit);
+}
+
+void DestroyPreparedDictionary(MemoryManager* m,
+    PreparedDictionary* dictionary) {
+  if (!dictionary) return;
+  BROTLI_FREE(m, dictionary);
+}
+
+BROTLI_BOOL AttachPreparedDictionary(
+    CompoundDictionary* compound, const PreparedDictionary* dictionary) {
+  size_t length = 0;
+  size_t index = 0;
+
+  if (compound->num_chunks == SHARED_BROTLI_MAX_COMPOUND_DICTS) {
+    return BROTLI_FALSE;
+  }
+
+  if (!dictionary) return BROTLI_FALSE;
+
+  length = dictionary->source_size;
+  index = compound->num_chunks;
+  compound->total_size += length;
+  compound->chunks[index] = dictionary;
+  compound->chunk_offsets[index + 1] = compound->total_size;
+  {
+    uint32_t* slot_offsets = (uint32_t*)(&dictionary[1]);
+    uint16_t* heads = (uint16_t*)(&slot_offsets[1u << dictionary->slot_bits]);
+    uint32_t* items = (uint32_t*)(&heads[1u << dictionary->bucket_bits]);
+    const void* tail = (void*)&items[dictionary->num_items];
+    if (dictionary->magic == kPreparedDictionaryMagic) {
+      compound->chunk_source[index] = (const uint8_t*)tail;
+    } else {
+      /* dictionary->magic == kLeanPreparedDictionaryMagic */
+      compound->chunk_source[index] =
+          (const uint8_t*)BROTLI_UNALIGNED_LOAD_PTR((const uint8_t**)tail);
+    }
+  }
+  compound->num_chunks++;
+  return BROTLI_TRUE;
+}

data/ext/multi_compress/vendor/brotli/c/enc/compound_dictionary.h

@@ -0,0 +1,74 @@
+/* Copyright 2017 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+#ifndef BROTLI_ENC_PREPARED_DICTIONARY_H_
+#define BROTLI_ENC_PREPARED_DICTIONARY_H_
+
+#include <brotli/shared_dictionary.h>
+#include <brotli/types.h>
+
+#include "../common/platform.h"
+#include "../common/constants.h"
+#include "memory.h"
+
+/* "Fat" prepared dictionary, could be cooked outside of C implementation,
+ * e.g. on Java side. LZ77 data is copied inside PreparedDictionary struct. */
+static const uint32_t kPreparedDictionaryMagic = 0xDEBCEDE0;
+
+static const uint32_t kSharedDictionaryMagic = 0xDEBCEDE1;
+
+static const uint32_t kManagedDictionaryMagic = 0xDEBCEDE2;
+
+/* "Lean" prepared dictionary. LZ77 data is referenced. It is the responsibility
+ * of caller of "prepare dictionary" to keep the LZ77 data while prepared
+ * dictionary is in use. */
+static const uint32_t kLeanPreparedDictionaryMagic = 0xDEBCEDE3;
+
+static const uint64_t kPreparedDictionaryHashMul64Long =
+    BROTLI_MAKE_UINT64_T(0x1FE35A7Bu, 0xD3579BD3u);
+
+typedef struct PreparedDictionary {
+  uint32_t magic;
+  uint32_t num_items;
+  uint32_t source_size;
+  uint32_t hash_bits;
+  uint32_t bucket_bits;
+  uint32_t slot_bits;
+
+  /* --- Dynamic size members --- */
+
+  /* uint32_t slot_offsets[1 << slot_bits]; */
+  /* uint16_t heads[1 << bucket_bits]; */
+  /* uint32_t items[variable]; */
+
+  /* [maybe] uint8_t* source_ref, depending on magic. */
+  /* [maybe] uint8_t source[source_size], depending on magic. */
+} PreparedDictionary;
+
+BROTLI_INTERNAL PreparedDictionary* CreatePreparedDictionary(MemoryManager* m,
+    const uint8_t* source, size_t source_size);
+
+BROTLI_INTERNAL void DestroyPreparedDictionary(MemoryManager* m,
+    PreparedDictionary* dictionary);
+
+typedef struct CompoundDictionary {
+  /* LZ77 prefix, compound dictionary */
+  size_t num_chunks;
+  size_t total_size;
+  /* Client instances. */
+  const PreparedDictionary* chunks[SHARED_BROTLI_MAX_COMPOUND_DICTS + 1];
+  const uint8_t* chunk_source[SHARED_BROTLI_MAX_COMPOUND_DICTS + 1];
+  size_t chunk_offsets[SHARED_BROTLI_MAX_COMPOUND_DICTS + 1];
+
+  size_t num_prepared_instances_;
+  /* Owned instances. */
+  PreparedDictionary* prepared_instances_[SHARED_BROTLI_MAX_COMPOUND_DICTS + 1];
+} CompoundDictionary;
+
+BROTLI_INTERNAL BROTLI_BOOL AttachPreparedDictionary(
+    CompoundDictionary* compound, const PreparedDictionary* dictionary);
+
+#endif /* BROTLI_ENC_PREPARED_DICTIONARY */