zstd-native-ruby 1.0.0

data/vendor/zstd/contrib/pzstd/Pzstd.h

@@ -0,0 +1,153 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+#pragma once
+
+#include "ErrorHolder.h"
+#include "Logging.h"
+#include "Options.h"
+#include "utils/Buffer.h"
+#include "utils/Range.h"
+#include "utils/ResourcePool.h"
+#include "utils/ThreadPool.h"
+#include "utils/WorkQueue.h"
+#define ZSTD_STATIC_LINKING_ONLY
+#define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, pzstd itself is deprecated
+                                         * and uses deprecated functions
+                                         */
+#include "zstd.h"
+#undef ZSTD_STATIC_LINKING_ONLY
+
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+
+namespace pzstd {
+/**
+ * Runs pzstd with `options` and returns the number of bytes written.
+ * An error occurred if `errorHandler.hasError()`.
+ *
+ * @param options      The pzstd options to use for (de)compression
+ * @returns            0 upon success and non-zero on failure.
+ */
+int pzstdMain(const Options& options);
+
+class SharedState {
+ public:
+  SharedState(const Options& options) : log(options.verbosity) {
+    if (!options.decompress) {
+      auto parameters = options.determineParameters();
+      cStreamPool.reset(new ResourcePool<ZSTD_CStream>{
+          [this, parameters]() -> ZSTD_CStream* {
+            this->log(kLogVerbose, "%s\n", "Creating new ZSTD_CStream");
+            auto zcs = ZSTD_createCStream();
+            if (zcs) {
+              auto err = ZSTD_initCStream_advanced(
+                  zcs, nullptr, 0, parameters, 0);
+              if (ZSTD_isError(err)) {
+                ZSTD_freeCStream(zcs);
+                return nullptr;
+              }
+            }
+            return zcs;
+          },
+          [](ZSTD_CStream *zcs) {
+            ZSTD_freeCStream(zcs);
+          }});
+    } else {
+      dStreamPool.reset(new ResourcePool<ZSTD_DStream>{
+          [this]() -> ZSTD_DStream* {
+            this->log(kLogVerbose, "%s\n", "Creating new ZSTD_DStream");
+            auto zds = ZSTD_createDStream();
+            if (zds) {
+              auto err = ZSTD_initDStream(zds);
+              if (ZSTD_isError(err)) {
+                ZSTD_freeDStream(zds);
+                return nullptr;
+              }
+            }
+            return zds;
+          },
+          [](ZSTD_DStream *zds) {
+            ZSTD_freeDStream(zds);
+          }});
+    }
+  }
+
+  ~SharedState() {
+    // The resource pools have references to this, so destroy them first.
+    cStreamPool.reset();
+    dStreamPool.reset();
+  }
+
+  Logger log;
+  ErrorHolder errorHolder;
+  std::unique_ptr<ResourcePool<ZSTD_CStream>> cStreamPool;
+  std::unique_ptr<ResourcePool<ZSTD_DStream>> dStreamPool;
+};
+
+/**
+ * Streams input from `fd`, breaks input up into chunks, and compresses each
+ * chunk independently.  Output of each chunk gets streamed to a queue, and
+ * the output queues get put into `chunks` in order.
+ *
+ * @param state        The shared state
+ * @param chunks       Each compression jobs output queue gets `pushed()` here
+ *                      as soon as it is available
+ * @param executor     The thread pool to run compression jobs in
+ * @param fd           The input file descriptor
+ * @param size         The size of the input file if known, 0 otherwise
+ * @param numThreads   The number of threads in the thread pool
+ * @param parameters   The zstd parameters to use for compression
+ * @returns            The number of bytes read from the file
+ */
+std::uint64_t asyncCompressChunks(
+    SharedState& state,
+    WorkQueue<std::shared_ptr<BufferWorkQueue>>& chunks,
+    ThreadPool& executor,
+    FILE* fd,
+    std::uintmax_t size,
+    std::size_t numThreads,
+    ZSTD_parameters parameters);
+
+/**
+ * Streams input from `fd`.  If pzstd headers are available it breaks the input
+ * up into independent frames.  It sends each frame to an independent
+ * decompression job.  Output of each frame gets streamed to a queue, and
+ * the output queues get put into `frames` in order.
+ *
+ * @param state        The shared state
+ * @param frames       Each decompression jobs output queue gets `pushed()` here
+ *                      as soon as it is available
+ * @param executor     The thread pool to run compression jobs in
+ * @param fd           The input file descriptor
+ * @returns            The number of bytes read from the file
+ */
+std::uint64_t asyncDecompressFrames(
+    SharedState& state,
+    WorkQueue<std::shared_ptr<BufferWorkQueue>>& frames,
+    ThreadPool& executor,
+    FILE* fd);
+
+/**
+ * Streams input in from each queue in `outs` in order, and writes the data to
+ * `outputFd`.
+ *
+ * @param state        The shared state
+ * @param outs         A queue of output queues, one for each
+ *                      (de)compression job.
+ * @param outputFd     The file descriptor to write to
+ * @param decompress   Are we decompressing?
+ * @returns            The number of bytes written
+ */
+std::uint64_t writeFile(
+    SharedState& state,
+    WorkQueue<std::shared_ptr<BufferWorkQueue>>& outs,
+    FILE* outputFd,
+    bool decompress);
+}

data/vendor/zstd/contrib/pzstd/README.md

@@ -0,0 +1,56 @@
+# Parallel Zstandard (PZstandard)
+
+Parallel Zstandard is a Pigz-like tool for Zstandard.
+It provides Zstandard format compatible compression and decompression that is able to utilize multiple cores.
+It breaks the input up into equal sized chunks and compresses each chunk independently into a Zstandard frame.
+It then concatenates the frames together to produce the final compressed output.
+Pzstandard will write a 12 byte header for each frame that is a skippable frame in the Zstandard format, which tells PZstandard the size of the next compressed frame.
+PZstandard supports parallel decompression of files compressed with PZstandard.
+When decompressing files compressed with Zstandard, PZstandard does IO in one thread, and decompression in another.
+
+## Usage
+
+PZstandard supports the same command line interface as Zstandard, but also provides the `-p` option to specify the number of threads.
+Dictionary mode is not currently supported.
+
+Basic usage
+
+    pzstd input-file -o output-file -p num-threads -#          # Compression
+    pzstd -d input-file -o output-file -p num-threads          # Decompression
+
+PZstandard also supports piping and fifo pipes
+
+    cat input-file | pzstd -p num-threads -# -c > /dev/null
+
+For more options
+
+    pzstd --help
+
+PZstandard tries to pick a smart default number of threads if not specified (displayed in `pzstd --help`).
+If this number is not suitable, during compilation you can define `PZSTD_NUM_THREADS` to the number of threads you prefer.
+
+## Benchmarks
+
+As a reference, PZstandard and Pigz were compared on an Intel Core i7 @ 3.1 GHz, each using 4 threads, with the [Silesia compression corpus](https://sun.aei.polsl.pl//~sdeor/index.php?page=silesia).
+
+Compression Speed vs Ratio with 4 Threads | Decompression Speed with 4 Threads
+------------------------------------------|-----------------------------------
+![Compression Speed vs Ratio](images/Cspeed.png "Compression Speed vs Ratio") | ![Decompression Speed](images/Dspeed.png "Decompression Speed")
+
+The test procedure was to run each of the following commands 2 times for each compression level, and take the minimum time.
+
+    time pzstd -# -p 4    -c silesia.tar     > silesia.tar.zst
+    time pzstd -d -p 4    -c silesia.tar.zst > /dev/null
+
+    time pigz  -# -p 4 -k -c silesia.tar     > silesia.tar.gz
+    time pigz  -d -p 4 -k -c silesia.tar.gz  > /dev/null
+
+PZstandard was tested using compression levels 1-19, and Pigz was tested using compression levels 1-9.
+Pigz cannot do parallel decompression, it simply does each of reading, decompression, and writing on separate threads.
+
+## Tests
+
+Tests require that you have [gtest](https://github.com/google/googletest) installed.
+Set `GTEST_INC` and `GTEST_LIB` in `Makefile` to specify the location of the gtest headers and libraries.
+Alternatively, run `make googletest`, which will clone googletest and build it.
+Run `make tests && make check` to run tests.

data/vendor/zstd/contrib/pzstd/SkippableFrame.cpp

@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+#include "SkippableFrame.h"
+#include "mem.h"
+#include "utils/Range.h"
+
+#include <cstdio>
+
+using namespace pzstd;
+
+SkippableFrame::SkippableFrame(std::uint32_t size) : frameSize_(size) {
+  MEM_writeLE32(data_.data(), kSkippableFrameMagicNumber);
+  MEM_writeLE32(data_.data() + 4, kFrameContentsSize);
+  MEM_writeLE32(data_.data() + 8, frameSize_);
+}
+
+/* static */ std::size_t SkippableFrame::tryRead(ByteRange bytes) {
+  if (bytes.size() < SkippableFrame::kSize ||
+      MEM_readLE32(bytes.begin()) != kSkippableFrameMagicNumber ||
+      MEM_readLE32(bytes.begin() + 4) != kFrameContentsSize) {
+    return 0;
+  }
+  return MEM_readLE32(bytes.begin() + 8);
+}

data/vendor/zstd/contrib/pzstd/SkippableFrame.h

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+#pragma once
+
+#include "utils/Range.h"
+
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+
+namespace pzstd {
+/**
+ * We put a skippable frame before each frame.
+ * It contains a skippable frame magic number, the size of the skippable frame,
+ * and the size of the next frame.
+ * Each skippable frame is exactly 12 bytes in little endian format.
+ * The first 8 bytes are for compatibility with the ZSTD format.
+ * If we have N threads, the output will look like
+ *
+ * [0x184D2A50|4|size1] [frame1 of size size1]
+ * [0x184D2A50|4|size2] [frame2 of size size2]
+ * ...
+ * [0x184D2A50|4|sizeN] [frameN of size sizeN]
+ *
+ * Each sizeX is 4 bytes.
+ *
+ * These skippable frames should allow us to skip through the compressed file
+ * and only load at most N pages.
+ */
+class SkippableFrame {
+ public:
+  static constexpr std::size_t kSize = 12;
+
+ private:
+  std::uint32_t frameSize_;
+  std::array<std::uint8_t, kSize> data_;
+  static constexpr std::uint32_t kSkippableFrameMagicNumber = 0x184D2A50;
+  // Could be improved if the size fits in less bytes
+  static constexpr std::uint32_t kFrameContentsSize = kSize - 8;
+
+ public:
+   // Write the skippable frame to data_ in LE format.
+  explicit SkippableFrame(std::uint32_t size);
+
+  // Read the skippable frame from bytes in LE format.
+  static std::size_t tryRead(ByteRange bytes);
+
+  ByteRange data() const {
+    return {data_.data(), data_.size()};
+  }
+
+  // Size of the next frame.
+  std::size_t frameSize() const {
+    return frameSize_;
+  }
+};
+}

data/vendor/zstd/contrib/pzstd/main.cpp

@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+#include "ErrorHolder.h"
+#include "Options.h"
+#include "Pzstd.h"
+
+using namespace pzstd;
+
+int main(int argc, const char** argv) {
+  Options options;
+  switch (options.parse(argc, argv)) {
+  case Options::Status::Failure:
+    return 1;
+  case Options::Status::Message:
+    return 0;
+  default:
+    break;
+  }
+
+  return pzstdMain(options);
+}

data/vendor/zstd/contrib/pzstd/test/BUCK

@@ -0,0 +1,37 @@
+cxx_test(
+    name='options_test',
+    srcs=['OptionsTest.cpp'],
+    deps=['//contrib/pzstd:options'],
+)
+
+cxx_test(
+    name='pzstd_test',
+    srcs=['PzstdTest.cpp'],
+    deps=[
+        ':round_trip',
+        '//contrib/pzstd:libpzstd',
+        '//contrib/pzstd/utils:scope_guard',
+        '//programs:datagen',
+    ],
+)
+
+cxx_binary(
+    name='round_trip_test',
+    srcs=['RoundTripTest.cpp'],
+    deps=[
+        ':round_trip',
+        '//contrib/pzstd/utils:scope_guard',
+        '//programs:datagen',
+    ]
+)
+
+cxx_library(
+    name='round_trip',
+    header_namespace='test',
+    exported_headers=['RoundTrip.h'],
+    deps=[
+        '//contrib/pzstd:libpzstd',
+        '//contrib/pzstd:options',
+        '//contrib/pzstd/utils:scope_guard',
+    ]
+)