zstd-native-ruby 1.0.0

data/vendor/zstd/contrib/pzstd/utils/test/ThreadPoolTest.cpp

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+#include "utils/ThreadPool.h"
+
+#include <gtest/gtest.h>
+#include <atomic>
+#include <iostream>
+#include <thread>
+#include <vector>
+
+using namespace pzstd;
+
+TEST(ThreadPool, Ordering) {
+  std::vector<int> results;
+
+  {
+    ThreadPool executor(1);
+    for (int i = 0; i < 10; ++i) {
+      executor.add([ &results, i ] { results.push_back(i); });
+    }
+  }
+
+  for (int i = 0; i < 10; ++i) {
+    EXPECT_EQ(i, results[i]);
+  }
+}
+
+TEST(ThreadPool, AllJobsFinished) {
+  std::atomic<unsigned> numFinished{0};
+  std::atomic<bool> start{false};
+  {
+    std::cerr << "Creating executor" << std::endl;
+    ThreadPool executor(5);
+    for (int i = 0; i < 10; ++i) {
+      executor.add([ &numFinished, &start ] {
+        while (!start.load()) {
+          std::this_thread::yield();
+        }
+        ++numFinished;
+      });
+    }
+    std::cerr << "Starting" << std::endl;
+    start.store(true);
+    std::cerr << "Finishing" << std::endl;
+  }
+  EXPECT_EQ(10, numFinished.load());
+}
+
+TEST(ThreadPool, AddJobWhileJoining) {
+  std::atomic<bool> done{false};
+  {
+    ThreadPool executor(1);
+    executor.add([&executor, &done] {
+      while (!done.load()) {
+        std::this_thread::yield();
+      }
+      // Sleep for a second to be sure that we are joining
+      std::this_thread::sleep_for(std::chrono::seconds(1));
+      executor.add([] {
+        EXPECT_TRUE(false);
+      });
+    });
+    done.store(true);
+  }
+}

data/vendor/zstd/contrib/pzstd/utils/test/WorkQueueTest.cpp

@@ -0,0 +1,282 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+#include "utils/Buffer.h"
+#include "utils/WorkQueue.h"
+
+#include <gtest/gtest.h>
+#include <iostream>
+#include <memory>
+#include <mutex>
+#include <thread>
+#include <vector>
+
+using namespace pzstd;
+
+namespace {
+struct Popper {
+  WorkQueue<int>* queue;
+  int* results;
+  std::mutex* mutex;
+
+  void operator()() {
+    int result;
+    while (queue->pop(result)) {
+      std::lock_guard<std::mutex> lock(*mutex);
+      results[result] = result;
+    }
+  }
+};
+}
+
+TEST(WorkQueue, SingleThreaded) {
+  WorkQueue<int> queue;
+  int result;
+
+  queue.push(5);
+  EXPECT_TRUE(queue.pop(result));
+  EXPECT_EQ(5, result);
+
+  queue.push(1);
+  queue.push(2);
+  EXPECT_TRUE(queue.pop(result));
+  EXPECT_EQ(1, result);
+  EXPECT_TRUE(queue.pop(result));
+  EXPECT_EQ(2, result);
+
+  queue.push(1);
+  queue.push(2);
+  queue.finish();
+  EXPECT_TRUE(queue.pop(result));
+  EXPECT_EQ(1, result);
+  EXPECT_TRUE(queue.pop(result));
+  EXPECT_EQ(2, result);
+  EXPECT_FALSE(queue.pop(result));
+
+  queue.waitUntilFinished();
+}
+
+TEST(WorkQueue, SPSC) {
+  WorkQueue<int> queue;
+  const int max = 100;
+
+  for (int i = 0; i < 10; ++i) {
+    queue.push(int{i});
+  }
+
+  std::thread thread([ &queue, max ] {
+    int result;
+    for (int i = 0;; ++i) {
+      if (!queue.pop(result)) {
+        EXPECT_EQ(i, max);
+        break;
+      }
+      EXPECT_EQ(i, result);
+    }
+  });
+
+  std::this_thread::yield();
+  for (int i = 10; i < max; ++i) {
+    queue.push(int{i});
+  }
+  queue.finish();
+
+  thread.join();
+}
+
+TEST(WorkQueue, SPMC) {
+  WorkQueue<int> queue;
+  std::vector<int> results(50, -1);
+  std::mutex mutex;
+  std::vector<std::thread> threads;
+  for (int i = 0; i < 5; ++i) {
+    threads.emplace_back(Popper{&queue, results.data(), &mutex});
+  }
+
+  for (int i = 0; i < 50; ++i) {
+    queue.push(int{i});
+  }
+  queue.finish();
+
+  for (auto& thread : threads) {
+    thread.join();
+  }
+
+  for (int i = 0; i < 50; ++i) {
+    EXPECT_EQ(i, results[i]);
+  }
+}
+
+TEST(WorkQueue, MPMC) {
+  WorkQueue<int> queue;
+  std::vector<int> results(100, -1);
+  std::mutex mutex;
+  std::vector<std::thread> popperThreads;
+  for (int i = 0; i < 4; ++i) {
+    popperThreads.emplace_back(Popper{&queue, results.data(), &mutex});
+  }
+
+  std::vector<std::thread> pusherThreads;
+  for (int i = 0; i < 2; ++i) {
+    auto min = i * 50;
+    auto max = (i + 1) * 50;
+    pusherThreads.emplace_back(
+        [ &queue, min, max ] {
+          for (int i = min; i < max; ++i) {
+            queue.push(int{i});
+          }
+        });
+  }
+
+  for (auto& thread : pusherThreads) {
+    thread.join();
+  }
+  queue.finish();
+
+  for (auto& thread : popperThreads) {
+    thread.join();
+  }
+
+  for (int i = 0; i < 100; ++i) {
+    EXPECT_EQ(i, results[i]);
+  }
+}
+
+TEST(WorkQueue, BoundedSizeWorks) {
+  WorkQueue<int> queue(1);
+  int result;
+  queue.push(5);
+  queue.pop(result);
+  queue.push(5);
+  queue.pop(result);
+  queue.push(5);
+  queue.finish();
+  queue.pop(result);
+  EXPECT_EQ(5, result);
+}
+
+TEST(WorkQueue, BoundedSizePushAfterFinish) {
+  WorkQueue<int> queue(1);
+  int result;
+  queue.push(5);
+  std::thread pusher([&queue] {
+    queue.push(6);
+  });
+  // Dirtily try and make sure that pusher has run.
+  std::this_thread::sleep_for(std::chrono::seconds(1));
+  queue.finish();
+  EXPECT_TRUE(queue.pop(result));
+  EXPECT_EQ(5, result);
+  EXPECT_FALSE(queue.pop(result));
+
+  pusher.join();
+}
+
+TEST(WorkQueue, SetMaxSize) {
+  WorkQueue<int> queue(2);
+  int result;
+  queue.push(5);
+  queue.push(6);
+  queue.setMaxSize(1);
+  std::thread pusher([&queue] {
+    queue.push(7);
+  });
+  // Dirtily try and make sure that pusher has run.
+  std::this_thread::sleep_for(std::chrono::seconds(1));
+  queue.finish();
+  EXPECT_TRUE(queue.pop(result));
+  EXPECT_EQ(5, result);
+  EXPECT_TRUE(queue.pop(result));
+  EXPECT_EQ(6, result);
+  EXPECT_FALSE(queue.pop(result));
+
+  pusher.join();
+}
+
+TEST(WorkQueue, BoundedSizeMPMC) {
+  WorkQueue<int> queue(10);
+  std::vector<int> results(200, -1);
+  std::mutex mutex;
+  std::cerr << "Creating popperThreads" << std::endl;
+  std::vector<std::thread> popperThreads;
+  for (int i = 0; i < 4; ++i) {
+    popperThreads.emplace_back(Popper{&queue, results.data(), &mutex});
+  }
+
+  std::cerr << "Creating pusherThreads" << std::endl;
+  std::vector<std::thread> pusherThreads;
+  for (int i = 0; i < 2; ++i) {
+    auto min = i * 100;
+    auto max = (i + 1) * 100;
+    pusherThreads.emplace_back(
+        [ &queue, min, max ] {
+          for (int i = min; i < max; ++i) {
+            queue.push(int{i});
+          }
+        });
+  }
+
+  std::cerr << "Joining pusherThreads" << std::endl;
+  for (auto& thread : pusherThreads) {
+    thread.join();
+  }
+  std::cerr << "Finishing queue" << std::endl;
+  queue.finish();
+
+  std::cerr << "Joining popperThreads" << std::endl;
+  for (auto& thread : popperThreads) {
+    thread.join();
+  }
+
+  std::cerr << "Inspecting results" << std::endl;
+  for (int i = 0; i < 200; ++i) {
+    EXPECT_EQ(i, results[i]);
+  }
+}
+
+TEST(WorkQueue, FailedPush) {
+  WorkQueue<std::unique_ptr<int>> queue;
+  std::unique_ptr<int> x(new int{5});
+  EXPECT_TRUE(queue.push(std::move(x)));
+  EXPECT_EQ(nullptr, x);
+  queue.finish();
+  x.reset(new int{6});
+  EXPECT_FALSE(queue.push(std::move(x)));
+  EXPECT_NE(nullptr, x);
+  EXPECT_EQ(6, *x);
+}
+
+TEST(BufferWorkQueue, SizeCalculatedCorrectly) {
+  {
+    BufferWorkQueue queue;
+    queue.finish();
+    EXPECT_EQ(0, queue.size());
+  }
+  {
+    BufferWorkQueue queue;
+    queue.push(Buffer(10));
+    queue.finish();
+    EXPECT_EQ(10, queue.size());
+  }
+  {
+    BufferWorkQueue queue;
+    queue.push(Buffer(10));
+    queue.push(Buffer(5));
+    queue.finish();
+    EXPECT_EQ(15, queue.size());
+  }
+  {
+    BufferWorkQueue queue;
+    queue.push(Buffer(10));
+    queue.push(Buffer(5));
+    queue.finish();
+    Buffer buffer;
+    queue.pop(buffer);
+    EXPECT_EQ(5, queue.size());
+  }
+}

data/vendor/zstd/contrib/recovery/Makefile

@@ -0,0 +1,35 @@
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# ################################################################
+
+.PHONY: all
+all: recover_directory
+
+ZSTDLIBDIR ?= ../../lib
+PROGRAMDIR ?= ../../programs
+
+CFLAGS     ?= -O3
+CFLAGS     += -I$(ZSTDLIBDIR) -I$(PROGRAMDIR)
+CFLAGS     += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow                 \
+              -Wstrict-aliasing=1 -Wswitch-enum                               \
+              -Wstrict-prototypes -Wundef                                     \
+              -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings      \
+              -Wredundant-decls -Wmissing-prototypes
+CFLAGS     += $(DEBUGFLAGS) $(MOREFLAGS)
+FLAGS       = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
+
+.PHONY: $(ZSTDLIBDIR)/libzstd.a
+$(ZSTDLIBDIR)/libzstd.a:
+	$(MAKE) -C $(ZSTDLIBDIR) libzstd.a
+
+recover_directory: recover_directory.c $(ZSTDLIBDIR)/libzstd.a $(PROGRAMDIR)/util.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
+
+.PHONY: clean
+clean:
+	rm -f recover_directory

data/vendor/zstd/contrib/recovery/recover_directory.c

@@ -0,0 +1,152 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#define ZSTD_STATIC_LINKING_ONLY
+#include "util.h"
+#include "zstd.h"
+
+#define CHECK(cond, ...)                                                       \
+  do {                                                                         \
+    if (!(cond)) {                                                             \
+      fprintf(stderr, "%s:%d CHECK(%s) failed: ", __FILE__, __LINE__, #cond);  \
+      fprintf(stderr, "" __VA_ARGS__);                                         \
+      fprintf(stderr, "\n");                                                   \
+      exit(1);                                                                 \
+    }                                                                          \
+  } while (0)
+
+static void usage(char const *program) {
+  fprintf(stderr, "USAGE: %s FILE.zst PREFIX\n", program);
+  fprintf(stderr, "FILE.zst: A zstd compressed file with multiple frames\n");
+  fprintf(stderr, "PREFIX:   The output prefix. Uncompressed files will be "
+                  "created named ${PREFIX}0 ${PREFIX}1...\n\n");
+  fprintf(stderr, "This program takes concatenated zstd frames and "
+                  "decompresses them into individual files.\n");
+  fprintf(stderr, "E.g. files created with a command like: zstd -r directory "
+                  "-o file.zst\n");
+}
+
+typedef struct {
+  char *data;
+  size_t size;
+  size_t frames;
+  size_t maxFrameSize;
+} ZstdFrames;
+
+static ZstdFrames readFile(char const *fileName) {
+  U64 const fileSize = UTIL_getFileSize(fileName);
+  CHECK(fileSize != UTIL_FILESIZE_UNKNOWN, "Unknown file size!");
+
+  char *const data = (char *)malloc(fileSize);
+  CHECK(data != NULL, "Allocation failed");
+
+  FILE *file = fopen(fileName, "rb");
+  CHECK(file != NULL, "fopen failed");
+
+  size_t const readSize = fread(data, 1, fileSize, file);
+  CHECK(readSize == fileSize, "fread failed");
+
+  fclose(file);
+  ZstdFrames frames;
+  frames.data = (char *)data;
+  frames.size = fileSize;
+  frames.frames = 0;
+
+  size_t index;
+  size_t maxFrameSize = 0;
+  for (index = 0; index < fileSize;) {
+    size_t const frameSize =
+        ZSTD_findFrameCompressedSize(data + index, fileSize - index);
+    CHECK(!ZSTD_isError(frameSize), "Bad zstd frame: %s",
+          ZSTD_getErrorName(frameSize));
+    if (frameSize > maxFrameSize)
+      maxFrameSize = frameSize;
+    frames.frames += 1;
+    index += frameSize;
+  }
+  CHECK(index == fileSize, "Zstd file corrupt!");
+  frames.maxFrameSize = maxFrameSize;
+
+  return frames;
+}
+
+static int computePadding(size_t numFrames) {
+  return snprintf(NULL, 0, "%u", (unsigned)numFrames);
+}
+
+int main(int argc, char **argv) {
+  if (argc != 3) {
+    usage(argv[0]);
+    exit(1);
+  }
+  char const *const zstdFile = argv[1];
+  char const *const prefix = argv[2];
+
+  ZstdFrames frames = readFile(zstdFile);
+
+  if (frames.frames <= 1) {
+    fprintf(
+        stderr,
+        "%s only has %u zstd frame. Simply use `zstd -d` to decompress it.\n",
+        zstdFile, (unsigned)frames.frames);
+    exit(1);
+  }
+
+  int const padding = computePadding(frames.frames - 1);
+
+  size_t const outFileNameSize = strlen(prefix) + padding + 1;
+  char* outFileName = malloc(outFileNameSize);
+  CHECK(outFileName != NULL, "Allocation failure");
+
+  size_t const bufferSize = 128 * 1024;
+  void *buffer = malloc(bufferSize);
+  CHECK(buffer != NULL, "Allocation failure");
+
+  ZSTD_DCtx* dctx = ZSTD_createDCtx();
+  CHECK(dctx != NULL, "Allocation failure");
+
+  fprintf(stderr, "Recovering %u files...\n", (unsigned)frames.frames);
+
+  size_t index;
+  size_t frame = 0;
+  for (index = 0; index < frames.size; ++frame) {
+    size_t const frameSize =
+        ZSTD_findFrameCompressedSize(frames.data + index, frames.size - index);
+
+    int const ret = snprintf(outFileName, outFileNameSize, "%s%0*u", prefix, padding, (unsigned)frame);
+    CHECK(ret >= 0 && (size_t)ret <= outFileNameSize, "snprintf failed!");
+
+    FILE* outFile = fopen(outFileName, "wb");
+    CHECK(outFile != NULL, "fopen failed");
+
+    ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
+    ZSTD_inBuffer in = {frames.data + index, frameSize, 0};
+    while (in.pos < in.size) {
+        ZSTD_outBuffer out = {buffer, bufferSize, 0};
+        CHECK(!ZSTD_isError(ZSTD_decompressStream(dctx, &out, &in)), "decompression failed");
+        size_t const writeSize = fwrite(out.dst, 1, out.pos, outFile);
+        CHECK(writeSize == out.pos, "fwrite failed");
+    }
+    fclose(outFile);
+    fprintf(stderr, "Recovered %s\n", outFileName);
+    index += frameSize;
+  }
+  fprintf(stderr, "Complete\n");
+
+  free(outFileName);
+  ZSTD_freeDCtx(dctx);
+  free(buffer);
+  free(frames.data);
+  return 0;
+}

data/vendor/zstd/contrib/seekable_format/README.md

@@ -0,0 +1,42 @@
+# Zstandard Seekable Format
+
+The seekable format splits compressed data into a series of independent "frames",
+each compressed individually,
+so that decompression of a section in the middle of an archive
+only requires zstd to decompress at most a frame's worth of extra data,
+instead of the entire archive.
+
+The frames are appended, so that the decompression of the entire payload
+still regenerates the original content, using any compliant zstd decoder.
+
+On top of that, the seekable format generates a jump table,
+which makes it possible to jump directly to the position of the relevant frame
+when requesting only a segment of the data.
+The jump table is simply ignored by zstd decoders unaware of the seekable format.
+
+The format is delivered with an API to create seekable archives
+and to retrieve arbitrary segments inside the archive.
+
+### Maximum Frame Size parameter
+
+When creating a seekable archive, the main parameter is the maximum frame size.
+
+At compression time, user can manually select the boundaries between segments,
+but they don't have to: long segments will be automatically split
+when larger than selected maximum frame size.
+
+Small frame sizes reduce decompression cost when requesting small segments,
+because the decoder will nonetheless have to decompress an entire frame
+to recover just a single byte from it.
+
+A good rule of thumb is to select a maximum frame size roughly equivalent
+to the access pattern when it's known.
+For example, if the application tends to request 4KB blocks,
+then it's a good idea to set a maximum frame size in the vicinity of 4 KB.
+
+But small frame sizes also reduce compression ratio,
+and increase the cost for the jump table,
+so there is a balance to find.
+
+In general, try to avoid really tiny frame sizes (<1 KB),
+which would have a large negative impact on compression ratio.

data/vendor/zstd/contrib/seekable_format/examples/Makefile

@@ -0,0 +1,55 @@
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# ################################################################
+
+# This Makefile presumes libzstd is built, using `make` in / or /lib/
+
+ZSTDLIB_PATH = ../../../lib
+ZSTDLIB_NAME = libzstd.a
+# Parallel tools only make sense against multi-threaded libzstd
+ZSTDLIB_TARGET = $(ZSTDLIB_NAME)-mt
+ZSTDLIB = $(ZSTDLIB_PATH)/$(ZSTDLIB_NAME)
+
+CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I../ -I../../../lib -I../../../lib/common
+
+CFLAGS ?= -O3
+CFLAGS += -g
+
+SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_decompress.c $(ZSTDLIB)
+
+.PHONY: default all clean test
+
+default: all
+
+all: seekable_compression seekable_decompression seekable_decompression_mem \
+	parallel_processing
+
+$(ZSTDLIB):
+	$(MAKE) -C $(ZSTDLIB_PATH) $(ZSTDLIB_TARGET)
+
+seekable_compression : seekable_compression.c $(SEEKABLE_OBJS)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+seekable_decompression : seekable_decompression.c $(SEEKABLE_OBJS)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+seekable_decompression_mem : seekable_decompression_mem.c $(SEEKABLE_OBJS)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+parallel_processing : parallel_processing.c $(SEEKABLE_OBJS)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ -pthread
+
+parallel_compression : parallel_compression.c $(SEEKABLE_OBJS)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ -pthread
+
+clean:
+	@rm -f core *.o tmp* result* *.zst \
+		seekable_compression seekable_decompression \
+		seekable_decompression_mem \
+		parallel_processing parallel_compression
+	@echo Cleaning completed