electrobun 0.0.1 → 0.0.3

package/src/bsdiff/zstd/contrib/pzstd/Pzstd.h

@@ -1,153 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- */
-#pragma once
-
-#include "ErrorHolder.h"
-#include "Logging.h"
-#include "Options.h"
-#include "utils/Buffer.h"
-#include "utils/Range.h"
-#include "utils/ResourcePool.h"
-#include "utils/ThreadPool.h"
-#include "utils/WorkQueue.h"
-#define ZSTD_STATIC_LINKING_ONLY
-#define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, pzstd itself is deprecated
-                                         * and uses deprecated functions
-                                         */
-#include "zstd.h"
-#undef ZSTD_STATIC_LINKING_ONLY
-
-#include <cstddef>
-#include <cstdint>
-#include <memory>
-
-namespace pzstd {
-/**
- * Runs pzstd with `options` and returns the number of bytes written.
- * An error occurred if `errorHandler.hasError()`.
- *
- * @param options      The pzstd options to use for (de)compression
- * @returns            0 upon success and non-zero on failure.
- */
-int pzstdMain(const Options& options);
-
-class SharedState {
- public:
-  SharedState(const Options& options) : log(options.verbosity) {
-    if (!options.decompress) {
-      auto parameters = options.determineParameters();
-      cStreamPool.reset(new ResourcePool<ZSTD_CStream>{
-          [this, parameters]() -> ZSTD_CStream* {
-            this->log(kLogVerbose, "%s\n", "Creating new ZSTD_CStream");
-            auto zcs = ZSTD_createCStream();
-            if (zcs) {
-              auto err = ZSTD_initCStream_advanced(
-                  zcs, nullptr, 0, parameters, 0);
-              if (ZSTD_isError(err)) {
-                ZSTD_freeCStream(zcs);
-                return nullptr;
-              }
-            }
-            return zcs;
-          },
-          [](ZSTD_CStream *zcs) {
-            ZSTD_freeCStream(zcs);
-          }});
-    } else {
-      dStreamPool.reset(new ResourcePool<ZSTD_DStream>{
-          [this]() -> ZSTD_DStream* {
-            this->log(kLogVerbose, "%s\n", "Creating new ZSTD_DStream");
-            auto zds = ZSTD_createDStream();
-            if (zds) {
-              auto err = ZSTD_initDStream(zds);
-              if (ZSTD_isError(err)) {
-                ZSTD_freeDStream(zds);
-                return nullptr;
-              }
-            }
-            return zds;
-          },
-          [](ZSTD_DStream *zds) {
-            ZSTD_freeDStream(zds);
-          }});
-    }
-  }
-
-  ~SharedState() {
-    // The resource pools have references to this, so destroy them first.
-    cStreamPool.reset();
-    dStreamPool.reset();
-  }
-
-  Logger log;
-  ErrorHolder errorHolder;
-  std::unique_ptr<ResourcePool<ZSTD_CStream>> cStreamPool;
-  std::unique_ptr<ResourcePool<ZSTD_DStream>> dStreamPool;
-};
-
-/**
- * Streams input from `fd`, breaks input up into chunks, and compresses each
- * chunk independently.  Output of each chunk gets streamed to a queue, and
- * the output queues get put into `chunks` in order.
- *
- * @param state        The shared state
- * @param chunks       Each compression jobs output queue gets `pushed()` here
- *                      as soon as it is available
- * @param executor     The thread pool to run compression jobs in
- * @param fd           The input file descriptor
- * @param size         The size of the input file if known, 0 otherwise
- * @param numThreads   The number of threads in the thread pool
- * @param parameters   The zstd parameters to use for compression
- * @returns            The number of bytes read from the file
- */
-std::uint64_t asyncCompressChunks(
-    SharedState& state,
-    WorkQueue<std::shared_ptr<BufferWorkQueue>>& chunks,
-    ThreadPool& executor,
-    FILE* fd,
-    std::uintmax_t size,
-    std::size_t numThreads,
-    ZSTD_parameters parameters);
-
-/**
- * Streams input from `fd`.  If pzstd headers are available it breaks the input
- * up into independent frames.  It sends each frame to an independent
- * decompression job.  Output of each frame gets streamed to a queue, and
- * the output queues get put into `frames` in order.
- *
- * @param state        The shared state
- * @param frames       Each decompression jobs output queue gets `pushed()` here
- *                      as soon as it is available
- * @param executor     The thread pool to run compression jobs in
- * @param fd           The input file descriptor
- * @returns            The number of bytes read from the file
- */
-std::uint64_t asyncDecompressFrames(
-    SharedState& state,
-    WorkQueue<std::shared_ptr<BufferWorkQueue>>& frames,
-    ThreadPool& executor,
-    FILE* fd);
-
-/**
- * Streams input in from each queue in `outs` in order, and writes the data to
- * `outputFd`.
- *
- * @param state        The shared state
- * @param outs         A queue of output queues, one for each
- *                      (de)compression job.
- * @param outputFd     The file descriptor to write to
- * @param decompress   Are we decompressing?
- * @returns            The number of bytes written
- */
-std::uint64_t writeFile(
-    SharedState& state,
-    WorkQueue<std::shared_ptr<BufferWorkQueue>>& outs,
-    FILE* outputFd,
-    bool decompress);
-}

package/src/bsdiff/zstd/contrib/pzstd/README.md

@@ -1,56 +0,0 @@
-# Parallel Zstandard (PZstandard)
-
-Parallel Zstandard is a Pigz-like tool for Zstandard.
-It provides Zstandard format compatible compression and decompression that is able to utilize multiple cores.
-It breaks the input up into equal sized chunks and compresses each chunk independently into a Zstandard frame.
-It then concatenates the frames together to produce the final compressed output.
-Pzstandard will write a 12 byte header for each frame that is a skippable frame in the Zstandard format, which tells PZstandard the size of the next compressed frame.
-PZstandard supports parallel decompression of files compressed with PZstandard.
-When decompressing files compressed with Zstandard, PZstandard does IO in one thread, and decompression in another.
-
-## Usage
-
-PZstandard supports the same command line interface as Zstandard, but also provides the `-p` option to specify the number of threads.
-Dictionary mode is not currently supported.
-
-Basic usage
-
-    pzstd input-file -o output-file -p num-threads -#          # Compression
-    pzstd -d input-file -o output-file -p num-threads          # Decompression
-
-PZstandard also supports piping and fifo pipes
-
-    cat input-file | pzstd -p num-threads -# -c > /dev/null
-
-For more options
-
-    pzstd --help
-
-PZstandard tries to pick a smart default number of threads if not specified (displayed in `pzstd --help`).
-If this number is not suitable, during compilation you can define `PZSTD_NUM_THREADS` to the number of threads you prefer.
-
-## Benchmarks
-
-As a reference, PZstandard and Pigz were compared on an Intel Core i7 @ 3.1 GHz, each using 4 threads, with the [Silesia compression corpus](https://sun.aei.polsl.pl//~sdeor/index.php?page=silesia).
-
-Compression Speed vs Ratio with 4 Threads | Decompression Speed with 4 Threads
-------------------------------------------|-----------------------------------
-![Compression Speed vs Ratio](images/Cspeed.png "Compression Speed vs Ratio") | ![Decompression Speed](images/Dspeed.png "Decompression Speed")
-
-The test procedure was to run each of the following commands 2 times for each compression level, and take the minimum time.
-
-    time pzstd -# -p 4    -c silesia.tar     > silesia.tar.zst
-    time pzstd -d -p 4    -c silesia.tar.zst > /dev/null
-
-    time pigz  -# -p 4 -k -c silesia.tar     > silesia.tar.gz
-    time pigz  -d -p 4 -k -c silesia.tar.gz  > /dev/null
-
-PZstandard was tested using compression levels 1-19, and Pigz was tested using compression levels 1-9.
-Pigz cannot do parallel decompression, it simply does each of reading, decompression, and writing on separate threads.
-
-## Tests
-
-Tests require that you have [gtest](https://github.com/google/googletest) installed.
-Set `GTEST_INC` and `GTEST_LIB` in `Makefile` to specify the location of the gtest headers and libraries.
-Alternatively, run `make googletest`, which will clone googletest and build it.
-Run `make tests && make check` to run tests.

package/src/bsdiff/zstd/contrib/pzstd/SkippableFrame.cpp

@@ -1,30 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- */
-#include "SkippableFrame.h"
-#include "mem.h"
-#include "utils/Range.h"
-
-#include <cstdio>
-
-using namespace pzstd;
-
-SkippableFrame::SkippableFrame(std::uint32_t size) : frameSize_(size) {
-  MEM_writeLE32(data_.data(), kSkippableFrameMagicNumber);
-  MEM_writeLE32(data_.data() + 4, kFrameContentsSize);
-  MEM_writeLE32(data_.data() + 8, frameSize_);
-}
-
-/* static */ std::size_t SkippableFrame::tryRead(ByteRange bytes) {
-  if (bytes.size() < SkippableFrame::kSize ||
-      MEM_readLE32(bytes.begin()) != kSkippableFrameMagicNumber ||
-      MEM_readLE32(bytes.begin() + 4) != kFrameContentsSize) {
-    return 0;
-  }
-  return MEM_readLE32(bytes.begin() + 8);
-}

package/src/bsdiff/zstd/contrib/pzstd/SkippableFrame.h

@@ -1,64 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- */
-#pragma once
-
-#include "utils/Range.h"
-
-#include <array>
-#include <cstddef>
-#include <cstdint>
-#include <cstdio>
-
-namespace pzstd {
-/**
- * We put a skippable frame before each frame.
- * It contains a skippable frame magic number, the size of the skippable frame,
- * and the size of the next frame.
- * Each skippable frame is exactly 12 bytes in little endian format.
- * The first 8 bytes are for compatibility with the ZSTD format.
- * If we have N threads, the output will look like
- *
- * [0x184D2A50|4|size1] [frame1 of size size1]
- * [0x184D2A50|4|size2] [frame2 of size size2]
- * ...
- * [0x184D2A50|4|sizeN] [frameN of size sizeN]
- *
- * Each sizeX is 4 bytes.
- *
- * These skippable frames should allow us to skip through the compressed file
- * and only load at most N pages.
- */
-class SkippableFrame {
- public:
-  static constexpr std::size_t kSize = 12;
-
- private:
-  std::uint32_t frameSize_;
-  std::array<std::uint8_t, kSize> data_;
-  static constexpr std::uint32_t kSkippableFrameMagicNumber = 0x184D2A50;
-  // Could be improved if the size fits in less bytes
-  static constexpr std::uint32_t kFrameContentsSize = kSize - 8;
-
- public:
-   // Write the skippable frame to data_ in LE format.
-  explicit SkippableFrame(std::uint32_t size);
-
-  // Read the skippable frame from bytes in LE format.
-  static std::size_t tryRead(ByteRange bytes);
-
-  ByteRange data() const {
-    return {data_.data(), data_.size()};
-  }
-
-  // Size of the next frame.
-  std::size_t frameSize() const {
-    return frameSize_;
-  }
-};
-}

package/src/bsdiff/zstd/contrib/pzstd/main.cpp

@@ -1,27 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- */
-#include "ErrorHolder.h"
-#include "Options.h"
-#include "Pzstd.h"
-
-using namespace pzstd;
-
-int main(int argc, const char** argv) {
-  Options options;
-  switch (options.parse(argc, argv)) {
-  case Options::Status::Failure:
-    return 1;
-  case Options::Status::Message:
-    return 0;
-  default:
-    break;
-  }
-
-  return pzstdMain(options);
-}

package/src/bsdiff/zstd/contrib/pzstd/test/BUCK

@@ -1,37 +0,0 @@
-cxx_test(
-    name='options_test',
-    srcs=['OptionsTest.cpp'],
-    deps=['//contrib/pzstd:options'],
-)
-
-cxx_test(
-    name='pzstd_test',
-    srcs=['PzstdTest.cpp'],
-    deps=[
-        ':round_trip',
-        '//contrib/pzstd:libpzstd',
-        '//contrib/pzstd/utils:scope_guard',
-        '//programs:datagen',
-    ],
-)
-
-cxx_binary(
-    name='round_trip_test',
-    srcs=['RoundTripTest.cpp'],
-    deps=[
-        ':round_trip',
-        '//contrib/pzstd/utils:scope_guard',
-        '//programs:datagen',
-    ]
-)
-
-cxx_library(
-    name='round_trip',
-    header_namespace='test',
-    exported_headers=['RoundTrip.h'],
-    deps=[
-        '//contrib/pzstd:libpzstd',
-        '//contrib/pzstd:options',
-        '//contrib/pzstd/utils:scope_guard',
-    ]
-)