zstd-native-ruby 1.0.0

data/vendor/zstd/contrib/pzstd/test/RoundTrip.h

@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+#pragma once
+
+#include "Options.h"
+#include "Pzstd.h"
+#include "utils/ScopeGuard.h"
+
+#include <cstdio>
+#include <string>
+#include <cstdint>
+#include <memory>
+
+namespace pzstd {
+
+inline bool check(std::string source, std::string decompressed) {
+  std::unique_ptr<std::uint8_t[]> sBuf(new std::uint8_t[1024]);
+  std::unique_ptr<std::uint8_t[]> dBuf(new std::uint8_t[1024]);
+
+  auto sFd = std::fopen(source.c_str(), "rb");
+  auto dFd = std::fopen(decompressed.c_str(), "rb");
+  auto guard = makeScopeGuard([&] {
+    std::fclose(sFd);
+    std::fclose(dFd);
+  });
+
+  size_t sRead, dRead;
+
+  do {
+    sRead = std::fread(sBuf.get(), 1, 1024, sFd);
+    dRead = std::fread(dBuf.get(), 1, 1024, dFd);
+    if (std::ferror(sFd) || std::ferror(dFd)) {
+      return false;
+    }
+    if (sRead != dRead) {
+      return false;
+    }
+
+    for (size_t i = 0; i < sRead; ++i) {
+      if (sBuf.get()[i] != dBuf.get()[i]) {
+        return false;
+      }
+    }
+  } while (sRead == 1024);
+  if (!std::feof(sFd) || !std::feof(dFd)) {
+    return false;
+  }
+  return true;
+}
+
+inline bool roundTrip(Options& options) {
+  if (options.inputFiles.size() != 1) {
+    return false;
+  }
+  std::string source = options.inputFiles.front();
+  std::string compressedFile = std::tmpnam(nullptr);
+  std::string decompressedFile = std::tmpnam(nullptr);
+  auto guard = makeScopeGuard([&] {
+    std::remove(compressedFile.c_str());
+    std::remove(decompressedFile.c_str());
+  });
+
+  {
+    options.outputFile = compressedFile;
+    options.decompress = false;
+    if (pzstdMain(options) != 0) {
+      return false;
+    }
+  }
+  {
+    options.decompress = true;
+    options.inputFiles.front() = compressedFile;
+    options.outputFile = decompressedFile;
+    if (pzstdMain(options) != 0) {
+      return false;
+    }
+  }
+  return check(source, decompressedFile);
+}
+}

data/vendor/zstd/contrib/pzstd/test/RoundTripTest.cpp

@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+#include "datagen.h"
+#include "Options.h"
+#include "test/RoundTrip.h"
+#include "utils/ScopeGuard.h"
+
+#include <cstddef>
+#include <cstdio>
+#include <cstdlib>
+#include <memory>
+#include <random>
+
+using namespace std;
+using namespace pzstd;
+
+namespace {
+string
+writeData(size_t size, double matchProba, double litProba, unsigned seed) {
+  std::unique_ptr<uint8_t[]> buf(new uint8_t[size]);
+  RDG_genBuffer(buf.get(), size, matchProba, litProba, seed);
+  string file = tmpnam(nullptr);
+  auto fd = std::fopen(file.c_str(), "wb");
+  auto guard = makeScopeGuard([&] { std::fclose(fd); });
+  auto bytesWritten = std::fwrite(buf.get(), 1, size, fd);
+  if (bytesWritten != size) {
+    std::abort();
+  }
+  return file;
+}
+
+template <typename Generator>
+string generateInputFile(Generator& gen) {
+  // Use inputs ranging from 1 Byte to 2^16 Bytes
+  std::uniform_int_distribution<size_t> size{1, 1 << 16};
+  std::uniform_real_distribution<> prob{0, 1};
+  return writeData(size(gen), prob(gen), prob(gen), gen());
+}
+
+template <typename Generator>
+Options generateOptions(Generator& gen, const string& inputFile) {
+  Options options;
+  options.inputFiles = {inputFile};
+  options.overwrite = true;
+
+  std::uniform_int_distribution<unsigned> numThreads{1, 32};
+  std::uniform_int_distribution<unsigned> compressionLevel{1, 10};
+
+  options.numThreads = numThreads(gen);
+  options.compressionLevel = compressionLevel(gen);
+
+  return options;
+}
+}
+
+int main() {
+  std::mt19937 gen(std::random_device{}());
+
+  auto newlineGuard = makeScopeGuard([] { std::fprintf(stderr, "\n"); });
+  for (unsigned i = 0; i < 10000; ++i) {
+    if (i % 100 == 0) {
+      std::fprintf(stderr, "Progress: %u%%\r", i / 100);
+    }
+    auto inputFile = generateInputFile(gen);
+    auto inputGuard = makeScopeGuard([&] { std::remove(inputFile.c_str()); });
+    for (unsigned i = 0; i < 10; ++i) {
+      auto options = generateOptions(gen, inputFile);
+      if (!roundTrip(options)) {
+        std::fprintf(stderr, "numThreads: %u\n", options.numThreads);
+        std::fprintf(stderr, "level: %u\n", options.compressionLevel);
+        std::fprintf(stderr, "decompress? %u\n", (unsigned)options.decompress);
+        std::fprintf(stderr, "file: %s\n", inputFile.c_str());
+        return 1;
+      }
+    }
+  }
+  return 0;
+}

data/vendor/zstd/contrib/pzstd/utils/BUCK

@@ -0,0 +1,75 @@
+cxx_library(
+    name='buffer',
+    visibility=['PUBLIC'],
+    header_namespace='utils',
+    exported_headers=['Buffer.h'],
+    deps=[':range'],
+)
+
+cxx_library(
+    name='file_system',
+    visibility=['PUBLIC'],
+    header_namespace='utils',
+    exported_headers=['FileSystem.h'],
+    deps=[':range'],
+)
+
+cxx_library(
+    name='likely',
+    visibility=['PUBLIC'],
+    header_namespace='utils',
+    exported_headers=['Likely.h'],
+)
+
+cxx_library(
+    name='range',
+    visibility=['PUBLIC'],
+    header_namespace='utils',
+    exported_headers=['Range.h'],
+    deps=[':likely'],
+)
+
+cxx_library(
+    name='resource_pool',
+    visibility=['PUBLIC'],
+    header_namespace='utils',
+    exported_headers=['ResourcePool.h'],
+)
+
+cxx_library(
+    name='scope_guard',
+    visibility=['PUBLIC'],
+    header_namespace='utils',
+    exported_headers=['ScopeGuard.h'],
+)
+
+cxx_library(
+    name='thread_pool',
+    visibility=['PUBLIC'],
+    header_namespace='utils',
+    exported_headers=['ThreadPool.h'],
+    deps=[':work_queue'],
+)
+
+cxx_library(
+    name='work_queue',
+    visibility=['PUBLIC'],
+    header_namespace='utils',
+    exported_headers=['WorkQueue.h'],
+    deps=[':buffer'],
+)
+
+cxx_library(
+    name='utils',
+    visibility=['PUBLIC'],
+    deps=[
+        ':buffer',
+        ':file_system',
+        ':likely',
+        ':range',
+        ':resource_pool',
+        ':scope_guard',
+        ':thread_pool',
+        ':work_queue',
+    ],
+)

data/vendor/zstd/contrib/pzstd/utils/Buffer.h

@@ -0,0 +1,99 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+#pragma once
+
+#include "utils/Range.h"
+
+#include <array>
+#include <cstddef>
+#include <memory>
+
+namespace pzstd {
+
+/**
+ * A `Buffer` has a pointer to a shared buffer, and a range of the buffer that
+ * it owns.
+ * The idea is that you can allocate one buffer, and write chunks into it
+ * and break off those chunks.
+ * The underlying buffer is reference counted, and will be destroyed when all
+ * `Buffer`s that reference it are destroyed.
+ */
+class Buffer {
+  std::shared_ptr<unsigned char> buffer_;
+  MutableByteRange range_;
+
+  static void delete_buffer(unsigned char* buffer) {
+    delete[] buffer;
+  }
+
+ public:
+  /// Construct an empty buffer that owns no data.
+  explicit Buffer() {}
+
+  /// Construct a `Buffer` that owns a new underlying buffer of size `size`.
+  explicit Buffer(std::size_t size)
+      : buffer_(new unsigned char[size], delete_buffer),
+        range_(buffer_.get(), buffer_.get() + size) {}
+
+  explicit Buffer(std::shared_ptr<unsigned char> buffer, MutableByteRange data)
+      : buffer_(buffer), range_(data) {}
+
+  Buffer(Buffer&&) = default;
+  Buffer& operator=(Buffer&&) = default;
+
+  /**
+   * Splits the data into two pieces: [begin, begin + n), [begin + n, end).
+   * Their data both points into the same underlying buffer.
+   * Modifies the original `Buffer` to point to only [begin + n, end).
+   *
+   * @param n  The offset to split at.
+   * @returns  A buffer that owns the data [begin, begin + n).
+   */
+  Buffer splitAt(std::size_t n) {
+    auto firstPiece = range_.subpiece(0, n);
+    range_.advance(n);
+    return Buffer(buffer_, firstPiece);
+  }
+
+  /// Modifies the buffer to point to the range [begin + n, end).
+  void advance(std::size_t n) {
+    range_.advance(n);
+  }
+
+  /// Modifies the buffer to point to the range [begin, end - n).
+  void subtract(std::size_t n) {
+    range_.subtract(n);
+  }
+
+  /// Returns a read only `Range` pointing to the `Buffer`s data.
+  ByteRange range() const {
+    return range_;
+  }
+  /// Returns a mutable `Range` pointing to the `Buffer`s data.
+  MutableByteRange range() {
+    return range_;
+  }
+
+  const unsigned char* data() const {
+    return range_.data();
+  }
+
+  unsigned char* data() {
+    return range_.data();
+  }
+
+  std::size_t size() const {
+    return range_.size();
+  }
+
+  bool empty() const {
+    return range_.empty();
+  }
+};
+}

data/vendor/zstd/contrib/pzstd/utils/FileSystem.h

@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+#pragma once
+
+#include "utils/Portability.h"
+#include "utils/Range.h"
+
+#include <sys/stat.h>
+#include <cerrno>
+#include <cstdint>
+#include <limits>
+#include <system_error>
+
+// A small subset of `std::filesystem`.
+// `std::filesystem` should be a drop in replacement.
+// See https://en.cppreference.com/w/cpp/filesystem for documentation.
+
+namespace pzstd {
+
+// using file_status = ... causes gcc to emit a false positive warning
+#if defined(_MSC_VER)
+typedef struct ::_stat64 file_status;
+#else
+typedef struct ::stat file_status;
+#endif
+
+/// https://en.cppreference.com/w/cpp/filesystem/status
+inline file_status status(StringPiece path, std::error_code& ec) noexcept {
+  file_status status;
+#if defined(_MSC_VER)
+  const auto error = ::_stat64(path.data(), &status);
+#else
+  const auto error = ::stat(path.data(), &status);
+#endif
+  if (error) {
+    ec.assign(errno, std::generic_category());
+  } else {
+    ec.clear();
+  }
+  return status;
+}
+
+/// https://en.cppreference.com/w/cpp/filesystem/is_regular_file
+inline bool is_regular_file(file_status status) noexcept {
+#if defined(S_ISREG)
+  return S_ISREG(status.st_mode);
+#elif !defined(S_ISREG) && defined(S_IFMT) && defined(S_IFREG)
+  return (status.st_mode & S_IFMT) == S_IFREG;
+#else
+  static_assert(false, "No POSIX stat() support.");
+#endif
+}
+
+/// https://en.cppreference.com/w/cpp/filesystem/is_regular_file
+inline bool is_regular_file(StringPiece path, std::error_code& ec) noexcept {
+  return is_regular_file(status(path, ec));
+}
+
+/// https://en.cppreference.com/w/cpp/filesystem/is_directory
+inline bool is_directory(file_status status) noexcept {
+#if defined(S_ISDIR)
+  return S_ISDIR(status.st_mode);
+#elif !defined(S_ISDIR) && defined(S_IFMT) && defined(S_IFDIR)
+  return (status.st_mode & S_IFMT) == S_IFDIR;
+#else
+  static_assert(false, "NO POSIX stat() support.");
+#endif
+}
+
+/// https://en.cppreference.com/w/cpp/filesystem/is_directory
+inline bool is_directory(StringPiece path, std::error_code& ec) noexcept {
+  return is_directory(status(path, ec));
+}
+
+/// https://en.cppreference.com/w/cpp/filesystem/file_size
+inline std::uintmax_t file_size(
+    StringPiece path,
+    std::error_code& ec) noexcept {
+  auto stat = status(path, ec);
+  if (ec) {
+    return std::numeric_limits<uintmax_t>::max();
+  }
+  if (!is_regular_file(stat)) {
+    ec.assign(ENOTSUP, std::generic_category());
+    return std::numeric_limits<uintmax_t>::max();
+  }
+  ec.clear();
+  return stat.st_size;
+}
+}

data/vendor/zstd/contrib/pzstd/utils/Likely.h

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+ 
+/**
+ * Compiler hints to indicate the fast path of an "if" branch: whether
+ * the if condition is likely to be true or false.
+ *
+ * @author Tudor Bosman (tudorb@fb.com)
+ */
+
+#pragma once
+
+#undef LIKELY
+#undef UNLIKELY
+
+#if defined(__GNUC__) && __GNUC__ >= 4
+#define LIKELY(x) (__builtin_expect((x), 1))
+#define UNLIKELY(x) (__builtin_expect((x), 0))
+#else
+#define LIKELY(x) (x)
+#define UNLIKELY(x) (x)
+#endif

data/vendor/zstd/contrib/pzstd/utils/Portability.h

@@ -0,0 +1,16 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+
+#pragma once
+
+#include <algorithm>
+
+// Required for windows, which defines min/max, but we want the std:: version.
+#undef min
+#undef max

data/vendor/zstd/contrib/pzstd/utils/Range.h

@@ -0,0 +1,133 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+
+/**
+ * A subset of `folly/Range.h`.
+ * All code copied verbatim modulo formatting
+ */
+#pragma once
+
+#include "utils/Likely.h"
+#include "utils/Portability.h"
+
+#include <algorithm>
+#include <cstddef>
+#include <cstring>
+#include <stdexcept>
+#include <string>
+#include <type_traits>
+
+namespace pzstd {
+
+namespace detail {
+/*
+ *Use IsCharPointer<T>::type to enable const char* or char*.
+ *Use IsCharPointer<T>::const_type to enable only const char*.
+*/
+template <class T>
+struct IsCharPointer {};
+
+template <>
+struct IsCharPointer<char*> {
+  typedef int type;
+};
+
+template <>
+struct IsCharPointer<const char*> {
+  typedef int const_type;
+  typedef int type;
+};
+
+} // namespace detail
+
+template <typename Iter>
+class Range {
+  Iter b_;
+  Iter e_;
+
+ public:
+  using size_type = std::size_t;
+  using iterator = Iter;
+  using const_iterator = Iter;
+  using value_type = typename std::remove_reference<
+      typename std::iterator_traits<Iter>::reference>::type;
+  using reference = typename std::iterator_traits<Iter>::reference;
+
+  constexpr Range() : b_(), e_() {}
+  constexpr Range(Iter begin, Iter end) : b_(begin), e_(end) {}
+
+  constexpr Range(Iter begin, size_type size) : b_(begin), e_(begin + size) {}
+
+  template <class T = Iter, typename detail::IsCharPointer<T>::type = 0>
+  /* implicit */ Range(Iter str) : b_(str), e_(str + std::strlen(str)) {}
+
+  template <class T = Iter, typename detail::IsCharPointer<T>::const_type = 0>
+  /* implicit */ Range(const std::string& str)
+      : b_(str.data()), e_(b_ + str.size()) {}
+
+  // Allow implicit conversion from Range<From> to Range<To> if From is
+  // implicitly convertible to To.
+  template <
+      class OtherIter,
+      typename std::enable_if<
+          (!std::is_same<Iter, OtherIter>::value &&
+           std::is_convertible<OtherIter, Iter>::value),
+          int>::type = 0>
+  constexpr /* implicit */ Range(const Range<OtherIter>& other)
+      : b_(other.begin()), e_(other.end()) {}
+
+  Range(const Range&) = default;
+  Range(Range&&) = default;
+
+  Range& operator=(const Range&) = default;
+  Range& operator=(Range&&) = default;
+
+  constexpr size_type size() const {
+    return e_ - b_;
+  }
+  bool empty() const {
+    return b_ == e_;
+  }
+  Iter data() const {
+    return b_;
+  }
+  Iter begin() const {
+    return b_;
+  }
+  Iter end() const {
+    return e_;
+  }
+
+  void advance(size_type n) {
+    if (UNLIKELY(n > size())) {
+      throw std::out_of_range("index out of range");
+    }
+    b_ += n;
+  }
+
+  void subtract(size_type n) {
+    if (UNLIKELY(n > size())) {
+      throw std::out_of_range("index out of range");
+    }
+    e_ -= n;
+  }
+
+  Range subpiece(size_type first, size_type length = std::string::npos) const {
+    if (UNLIKELY(first > size())) {
+      throw std::out_of_range("index out of range");
+    }
+
+    return Range(b_ + first, std::min(length, size() - first));
+  }
+};
+
+using ByteRange = Range<const unsigned char*>;
+using MutableByteRange = Range<unsigned char*>;
+using StringPiece = Range<const char*>;
+}