electrobun 0.0.1 → 0.0.3

package/src/bsdiff/zstd/contrib/pzstd/utils/test/ThreadPoolTest.cpp

@@ -1,71 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- */
-#include "utils/ThreadPool.h"
-
-#include <gtest/gtest.h>
-#include <atomic>
-#include <iostream>
-#include <thread>
-#include <vector>
-
-using namespace pzstd;
-
-TEST(ThreadPool, Ordering) {
-  std::vector<int> results;
-
-  {
-    ThreadPool executor(1);
-    for (int i = 0; i < 10; ++i) {
-      executor.add([ &results, i ] { results.push_back(i); });
-    }
-  }
-
-  for (int i = 0; i < 10; ++i) {
-    EXPECT_EQ(i, results[i]);
-  }
-}
-
-TEST(ThreadPool, AllJobsFinished) {
-  std::atomic<unsigned> numFinished{0};
-  std::atomic<bool> start{false};
-  {
-    std::cerr << "Creating executor" << std::endl;
-    ThreadPool executor(5);
-    for (int i = 0; i < 10; ++i) {
-      executor.add([ &numFinished, &start ] {
-        while (!start.load()) {
-          std::this_thread::yield();
-        }
-        ++numFinished;
-      });
-    }
-    std::cerr << "Starting" << std::endl;
-    start.store(true);
-    std::cerr << "Finishing" << std::endl;
-  }
-  EXPECT_EQ(10, numFinished.load());
-}
-
-TEST(ThreadPool, AddJobWhileJoining) {
-  std::atomic<bool> done{false};
-  {
-    ThreadPool executor(1);
-    executor.add([&executor, &done] {
-      while (!done.load()) {
-        std::this_thread::yield();
-      }
-      // Sleep for a second to be sure that we are joining
-      std::this_thread::sleep_for(std::chrono::seconds(1));
-      executor.add([] {
-        EXPECT_TRUE(false);
-      });
-    });
-    done.store(true);
-  }
-}

package/src/bsdiff/zstd/contrib/pzstd/utils/test/WorkQueueTest.cpp

@@ -1,282 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- */
-#include "utils/Buffer.h"
-#include "utils/WorkQueue.h"
-
-#include <gtest/gtest.h>
-#include <iostream>
-#include <memory>
-#include <mutex>
-#include <thread>
-#include <vector>
-
-using namespace pzstd;
-
-namespace {
-struct Popper {
-  WorkQueue<int>* queue;
-  int* results;
-  std::mutex* mutex;
-
-  void operator()() {
-    int result;
-    while (queue->pop(result)) {
-      std::lock_guard<std::mutex> lock(*mutex);
-      results[result] = result;
-    }
-  }
-};
-}
-
-TEST(WorkQueue, SingleThreaded) {
-  WorkQueue<int> queue;
-  int result;
-
-  queue.push(5);
-  EXPECT_TRUE(queue.pop(result));
-  EXPECT_EQ(5, result);
-
-  queue.push(1);
-  queue.push(2);
-  EXPECT_TRUE(queue.pop(result));
-  EXPECT_EQ(1, result);
-  EXPECT_TRUE(queue.pop(result));
-  EXPECT_EQ(2, result);
-
-  queue.push(1);
-  queue.push(2);
-  queue.finish();
-  EXPECT_TRUE(queue.pop(result));
-  EXPECT_EQ(1, result);
-  EXPECT_TRUE(queue.pop(result));
-  EXPECT_EQ(2, result);
-  EXPECT_FALSE(queue.pop(result));
-
-  queue.waitUntilFinished();
-}
-
-TEST(WorkQueue, SPSC) {
-  WorkQueue<int> queue;
-  const int max = 100;
-
-  for (int i = 0; i < 10; ++i) {
-    queue.push(int{i});
-  }
-
-  std::thread thread([ &queue, max ] {
-    int result;
-    for (int i = 0;; ++i) {
-      if (!queue.pop(result)) {
-        EXPECT_EQ(i, max);
-        break;
-      }
-      EXPECT_EQ(i, result);
-    }
-  });
-
-  std::this_thread::yield();
-  for (int i = 10; i < max; ++i) {
-    queue.push(int{i});
-  }
-  queue.finish();
-
-  thread.join();
-}
-
-TEST(WorkQueue, SPMC) {
-  WorkQueue<int> queue;
-  std::vector<int> results(50, -1);
-  std::mutex mutex;
-  std::vector<std::thread> threads;
-  for (int i = 0; i < 5; ++i) {
-    threads.emplace_back(Popper{&queue, results.data(), &mutex});
-  }
-
-  for (int i = 0; i < 50; ++i) {
-    queue.push(int{i});
-  }
-  queue.finish();
-
-  for (auto& thread : threads) {
-    thread.join();
-  }
-
-  for (int i = 0; i < 50; ++i) {
-    EXPECT_EQ(i, results[i]);
-  }
-}
-
-TEST(WorkQueue, MPMC) {
-  WorkQueue<int> queue;
-  std::vector<int> results(100, -1);
-  std::mutex mutex;
-  std::vector<std::thread> popperThreads;
-  for (int i = 0; i < 4; ++i) {
-    popperThreads.emplace_back(Popper{&queue, results.data(), &mutex});
-  }
-
-  std::vector<std::thread> pusherThreads;
-  for (int i = 0; i < 2; ++i) {
-    auto min = i * 50;
-    auto max = (i + 1) * 50;
-    pusherThreads.emplace_back(
-        [ &queue, min, max ] {
-          for (int i = min; i < max; ++i) {
-            queue.push(int{i});
-          }
-        });
-  }
-
-  for (auto& thread : pusherThreads) {
-    thread.join();
-  }
-  queue.finish();
-
-  for (auto& thread : popperThreads) {
-    thread.join();
-  }
-
-  for (int i = 0; i < 100; ++i) {
-    EXPECT_EQ(i, results[i]);
-  }
-}
-
-TEST(WorkQueue, BoundedSizeWorks) {
-  WorkQueue<int> queue(1);
-  int result;
-  queue.push(5);
-  queue.pop(result);
-  queue.push(5);
-  queue.pop(result);
-  queue.push(5);
-  queue.finish();
-  queue.pop(result);
-  EXPECT_EQ(5, result);
-}
-
-TEST(WorkQueue, BoundedSizePushAfterFinish) {
-  WorkQueue<int> queue(1);
-  int result;
-  queue.push(5);
-  std::thread pusher([&queue] {
-    queue.push(6);
-  });
-  // Dirtily try and make sure that pusher has run.
-  std::this_thread::sleep_for(std::chrono::seconds(1));
-  queue.finish();
-  EXPECT_TRUE(queue.pop(result));
-  EXPECT_EQ(5, result);
-  EXPECT_FALSE(queue.pop(result));
-
-  pusher.join();
-}
-
-TEST(WorkQueue, SetMaxSize) {
-  WorkQueue<int> queue(2);
-  int result;
-  queue.push(5);
-  queue.push(6);
-  queue.setMaxSize(1);
-  std::thread pusher([&queue] {
-    queue.push(7);
-  });
-  // Dirtily try and make sure that pusher has run.
-  std::this_thread::sleep_for(std::chrono::seconds(1));
-  queue.finish();
-  EXPECT_TRUE(queue.pop(result));
-  EXPECT_EQ(5, result);
-  EXPECT_TRUE(queue.pop(result));
-  EXPECT_EQ(6, result);
-  EXPECT_FALSE(queue.pop(result));
-
-  pusher.join();
-}
-
-TEST(WorkQueue, BoundedSizeMPMC) {
-  WorkQueue<int> queue(10);
-  std::vector<int> results(200, -1);
-  std::mutex mutex;
-  std::cerr << "Creating popperThreads" << std::endl;
-  std::vector<std::thread> popperThreads;
-  for (int i = 0; i < 4; ++i) {
-    popperThreads.emplace_back(Popper{&queue, results.data(), &mutex});
-  }
-
-  std::cerr << "Creating pusherThreads" << std::endl;
-  std::vector<std::thread> pusherThreads;
-  for (int i = 0; i < 2; ++i) {
-    auto min = i * 100;
-    auto max = (i + 1) * 100;
-    pusherThreads.emplace_back(
-        [ &queue, min, max ] {
-          for (int i = min; i < max; ++i) {
-            queue.push(int{i});
-          }
-        });
-  }
-
-  std::cerr << "Joining pusherThreads" << std::endl;
-  for (auto& thread : pusherThreads) {
-    thread.join();
-  }
-  std::cerr << "Finishing queue" << std::endl;
-  queue.finish();
-
-  std::cerr << "Joining popperThreads" << std::endl;
-  for (auto& thread : popperThreads) {
-    thread.join();
-  }
-
-  std::cerr << "Inspecting results" << std::endl;
-  for (int i = 0; i < 200; ++i) {
-    EXPECT_EQ(i, results[i]);
-  }
-}
-
-TEST(WorkQueue, FailedPush) {
-  WorkQueue<std::unique_ptr<int>> queue;
-  std::unique_ptr<int> x(new int{5});
-  EXPECT_TRUE(queue.push(std::move(x)));
-  EXPECT_EQ(nullptr, x);
-  queue.finish();
-  x.reset(new int{6});
-  EXPECT_FALSE(queue.push(std::move(x)));
-  EXPECT_NE(nullptr, x);
-  EXPECT_EQ(6, *x);
-}
-
-TEST(BufferWorkQueue, SizeCalculatedCorrectly) {
-  {
-    BufferWorkQueue queue;
-    queue.finish();
-    EXPECT_EQ(0, queue.size());
-  }
-  {
-    BufferWorkQueue queue;
-    queue.push(Buffer(10));
-    queue.finish();
-    EXPECT_EQ(10, queue.size());
-  }
-  {
-    BufferWorkQueue queue;
-    queue.push(Buffer(10));
-    queue.push(Buffer(5));
-    queue.finish();
-    EXPECT_EQ(15, queue.size());
-  }
-  {
-    BufferWorkQueue queue;
-    queue.push(Buffer(10));
-    queue.push(Buffer(5));
-    queue.finish();
-    Buffer buffer;
-    queue.pop(buffer);
-    EXPECT_EQ(5, queue.size());
-  }
-}

package/src/bsdiff/zstd/contrib/recovery/Makefile

@@ -1,35 +0,0 @@
-# ################################################################
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under both the BSD-style license (found in the
-# LICENSE file in the root directory of this source tree) and the GPLv2 (found
-# in the COPYING file in the root directory of this source tree).
-# ################################################################
-
-.PHONY: all
-all: recover_directory
-
-ZSTDLIBDIR ?= ../../lib
-PROGRAMDIR ?= ../../programs
-
-CFLAGS     ?= -O3
-CFLAGS     += -I$(ZSTDLIBDIR) -I$(PROGRAMDIR)
-CFLAGS     += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow                 \
-              -Wstrict-aliasing=1 -Wswitch-enum                               \
-              -Wstrict-prototypes -Wundef                                     \
-              -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings      \
-              -Wredundant-decls -Wmissing-prototypes
-CFLAGS     += $(DEBUGFLAGS) $(MOREFLAGS)
-FLAGS       = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
-
-.PHONY: $(ZSTDLIBDIR)/libzstd.a
-$(ZSTDLIBDIR)/libzstd.a:
-	$(MAKE) -C $(ZSTDLIBDIR) libzstd.a
-
-recover_directory: recover_directory.c $(ZSTDLIBDIR)/libzstd.a $(PROGRAMDIR)/util.c
-	$(CC) $(FLAGS) $^ -o $@$(EXT)
-
-.PHONY: clean
-clean:
-	rm -f recover_directory

package/src/bsdiff/zstd/contrib/recovery/recover_directory.c

@@ -1,152 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-#include <string.h>
-#include <stdio.h>
-#include <stdlib.h>
-
-#define ZSTD_STATIC_LINKING_ONLY
-#include "util.h"
-#include "zstd.h"
-
-#define CHECK(cond, ...)                                                       \
-  do {                                                                         \
-    if (!(cond)) {                                                             \
-      fprintf(stderr, "%s:%d CHECK(%s) failed: ", __FILE__, __LINE__, #cond);  \
-      fprintf(stderr, "" __VA_ARGS__);                                         \
-      fprintf(stderr, "\n");                                                   \
-      exit(1);                                                                 \
-    }                                                                          \
-  } while (0)
-
-static void usage(char const *program) {
-  fprintf(stderr, "USAGE: %s FILE.zst PREFIX\n", program);
-  fprintf(stderr, "FILE.zst: A zstd compressed file with multiple frames\n");
-  fprintf(stderr, "PREFIX:   The output prefix. Uncompressed files will be "
-                  "created named ${PREFIX}0 ${PREFIX}1...\n\n");
-  fprintf(stderr, "This program takes concatenated zstd frames and "
-                  "decompresses them into individual files.\n");
-  fprintf(stderr, "E.g. files created with a command like: zstd -r directory "
-                  "-o file.zst\n");
-}
-
-typedef struct {
-  char *data;
-  size_t size;
-  size_t frames;
-  size_t maxFrameSize;
-} ZstdFrames;
-
-static ZstdFrames readFile(char const *fileName) {
-  U64 const fileSize = UTIL_getFileSize(fileName);
-  CHECK(fileSize != UTIL_FILESIZE_UNKNOWN, "Unknown file size!");
-
-  char *const data = (char *)malloc(fileSize);
-  CHECK(data != NULL, "Allocation failed");
-
-  FILE *file = fopen(fileName, "rb");
-  CHECK(file != NULL, "fopen failed");
-
-  size_t const readSize = fread(data, 1, fileSize, file);
-  CHECK(readSize == fileSize, "fread failed");
-
-  fclose(file);
-  ZstdFrames frames;
-  frames.data = (char *)data;
-  frames.size = fileSize;
-  frames.frames = 0;
-
-  size_t index;
-  size_t maxFrameSize = 0;
-  for (index = 0; index < fileSize;) {
-    size_t const frameSize =
-        ZSTD_findFrameCompressedSize(data + index, fileSize - index);
-    CHECK(!ZSTD_isError(frameSize), "Bad zstd frame: %s",
-          ZSTD_getErrorName(frameSize));
-    if (frameSize > maxFrameSize)
-      maxFrameSize = frameSize;
-    frames.frames += 1;
-    index += frameSize;
-  }
-  CHECK(index == fileSize, "Zstd file corrupt!");
-  frames.maxFrameSize = maxFrameSize;
-
-  return frames;
-}
-
-static int computePadding(size_t numFrames) {
-  return snprintf(NULL, 0, "%u", (unsigned)numFrames);
-}
-
-int main(int argc, char **argv) {
-  if (argc != 3) {
-    usage(argv[0]);
-    exit(1);
-  }
-  char const *const zstdFile = argv[1];
-  char const *const prefix = argv[2];
-
-  ZstdFrames frames = readFile(zstdFile);
-
-  if (frames.frames <= 1) {
-    fprintf(
-        stderr,
-        "%s only has %u zstd frame. Simply use `zstd -d` to decompress it.\n",
-        zstdFile, (unsigned)frames.frames);
-    exit(1);
-  }
-
-  int const padding = computePadding(frames.frames - 1);
-
-  size_t const outFileNameSize = strlen(prefix) + padding + 1;
-  char* outFileName = malloc(outFileNameSize);
-  CHECK(outFileName != NULL, "Allocation failure");
-
-  size_t const bufferSize = 128 * 1024;
-  void *buffer = malloc(bufferSize);
-  CHECK(buffer != NULL, "Allocation failure");
-
-  ZSTD_DCtx* dctx = ZSTD_createDCtx();
-  CHECK(dctx != NULL, "Allocation failure");
-
-  fprintf(stderr, "Recovering %u files...\n", (unsigned)frames.frames);
-
-  size_t index;
-  size_t frame = 0;
-  for (index = 0; index < frames.size; ++frame) {
-    size_t const frameSize =
-        ZSTD_findFrameCompressedSize(frames.data + index, frames.size - index);
-
-    int const ret = snprintf(outFileName, outFileNameSize, "%s%0*u", prefix, padding, (unsigned)frame);
-    CHECK(ret >= 0 && (size_t)ret <= outFileNameSize, "snprintf failed!");
-
-    FILE* outFile = fopen(outFileName, "wb");
-    CHECK(outFile != NULL, "fopen failed");
-
-    ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
-    ZSTD_inBuffer in = {frames.data + index, frameSize, 0};
-    while (in.pos < in.size) {
-        ZSTD_outBuffer out = {buffer, bufferSize, 0};
-        CHECK(!ZSTD_isError(ZSTD_decompressStream(dctx, &out, &in)), "decompression failed");
-        size_t const writeSize = fwrite(out.dst, 1, out.pos, outFile);
-        CHECK(writeSize == out.pos, "fwrite failed");
-    }
-    fclose(outFile);
-    fprintf(stderr, "Recovered %s\n", outFileName);
-    index += frameSize;
-  }
-  fprintf(stderr, "Complete\n");
-
-  free(outFileName);
-  ZSTD_freeDCtx(dctx);
-  free(buffer);
-  free(frames.data);
-  return 0;
-}

package/src/bsdiff/zstd/contrib/seekable_format/README.md

@@ -1,42 +0,0 @@
-# Zstandard Seekable Format
-
-The seekable format splits compressed data into a series of independent "frames",
-each compressed individually,
-so that decompression of a section in the middle of an archive
-only requires zstd to decompress at most a frame's worth of extra data,
-instead of the entire archive.
-
-The frames are appended, so that the decompression of the entire payload
-still regenerates the original content, using any compliant zstd decoder.
-
-On top of that, the seekable format generates a jump table,
-which makes it possible to jump directly to the position of the relevant frame
-when requesting only a segment of the data.
-The jump table is simply ignored by zstd decoders unaware of the seekable format.
-
-The format is delivered with an API to create seekable archives
-and to retrieve arbitrary segments inside the archive.
-
-### Maximum Frame Size parameter
-
-When creating a seekable archive, the main parameter is the maximum frame size.
-
-At compression time, user can manually select the boundaries between segments,
-but they don't have to: long segments will be automatically split
-when larger than selected maximum frame size.
-
-Small frame sizes reduce decompression cost when requesting small segments,
-because the decoder will nonetheless have to decompress an entire frame
-to recover just a single byte from it.
-
-A good rule of thumb is to select a maximum frame size roughly equivalent
-to the access pattern when it's known.
-For example, if the application tends to request 4KB blocks,
-then it's a good idea to set a maximum frame size in the vicinity of 4 KB.
-
-But small frame sizes also reduce compression ratio,
-and increase the cost for the jump table,
-so there is a balance to find.
-
-In general, try to avoid really tiny frame sizes (<1 KB),
-which would have a large negative impact on compression ratio.

package/src/bsdiff/zstd/contrib/seekable_format/examples/Makefile

@@ -1,53 +0,0 @@
-# ################################################################
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under both the BSD-style license (found in the
-# LICENSE file in the root directory of this source tree) and the GPLv2 (found
-# in the COPYING file in the root directory of this source tree).
-# ################################################################
-
-# This Makefile presumes libzstd is built, using `make` in / or /lib/
-
-ZSTDLIB_PATH = ../../../lib
-ZSTDLIB_NAME = libzstd.a
-ZSTDLIB = $(ZSTDLIB_PATH)/$(ZSTDLIB_NAME)
-
-CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I../ -I../../../lib -I../../../lib/common
-
-CFLAGS ?= -O3
-CFLAGS += -g
-
-SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_decompress.c $(ZSTDLIB)
-
-.PHONY: default all clean test
-
-default: all
-
-all: seekable_compression seekable_decompression seekable_decompression_mem \
-	parallel_processing
-
-$(ZSTDLIB):
-	make -C $(ZSTDLIB_PATH) $(ZSTDLIB_NAME)
-
-seekable_compression : seekable_compression.c $(SEEKABLE_OBJS)
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
-
-seekable_decompression : seekable_decompression.c $(SEEKABLE_OBJS)
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
-
-seekable_decompression_mem : seekable_decompression_mem.c $(SEEKABLE_OBJS)
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
-
-parallel_processing : parallel_processing.c $(SEEKABLE_OBJS)
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ -pthread
-
-parallel_compression : parallel_compression.c $(SEEKABLE_OBJS)
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ -pthread
-
-clean:
-	@rm -f core *.o tmp* result* *.zst \
-		seekable_compression seekable_decompression \
-		seekable_decompression_mem \
-		parallel_processing parallel_compression
-	@echo Cleaning completed