electrobun 0.0.1 → 0.0.3

package/src/bsdiff/zstd/programs/fileio_asyncio.h

@@ -1,203 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
- /*
-  * FileIO AsyncIO exposes read/write IO pools that allow doing IO asynchronously.
-  * Current implementation relies on having one thread that reads and one that
-  * writes.
-  * Each IO pool supports up to `MAX_IO_JOBS` that can be enqueued for work, but
-  * are performed serially by the appropriate worker thread.
-  * Most systems exposes better primitives to perform asynchronous IO, such as
-  * io_uring on newer linux systems. The API is built in such a way that in the
-  * future we could replace the threads with better solutions when available.
-  */
-
-#ifndef ZSTD_FILEIO_ASYNCIO_H
-#define ZSTD_FILEIO_ASYNCIO_H
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-#include "../lib/common/mem.h"     /* U32, U64 */
-#include "fileio_types.h"
-#include "platform.h"
-#include "util.h"
-#include "../lib/common/pool.h"
-#include "../lib/common/threading.h"
-
-#define MAX_IO_JOBS          (10)
-
-typedef struct {
-    /* These struct fields should be set only on creation and not changed afterwards */
-    POOL_ctx* threadPool;
-    int threadPoolActive;
-    int totalIoJobs;
-    const FIO_prefs_t* prefs;
-    POOL_function poolFunction;
-
-    /* Controls the file we currently write to, make changes only by using provided utility functions */
-    FILE* file;
-
-    /* The jobs and availableJobsCount fields are accessed by both the main and worker threads and should
-     * only be mutated after locking the mutex */
-    ZSTD_pthread_mutex_t ioJobsMutex;
-    void* availableJobs[MAX_IO_JOBS];
-    int availableJobsCount;
-    size_t jobBufferSize;
-} IOPoolCtx_t;
-
-typedef struct {
-    IOPoolCtx_t base;
-
-    /* State regarding the currently read file */
-    int reachedEof;
-    U64 nextReadOffset;
-    U64 waitingOnOffset;
-
-    /* We may hold an IOJob object as needed if we actively expose its buffer. */
-    void *currentJobHeld;
-
-    /* Coalesce buffer is used to join two buffers in case where we need to read more bytes than left in
-     * the first of them. Shouldn't be accessed from outside ot utility functions. */
-    U8 *coalesceBuffer;
-
-    /* Read buffer can be used by consumer code, take care when copying this pointer aside as it might
-     * change when consuming / refilling buffer. */
-    U8 *srcBuffer;
-    size_t srcBufferLoaded;
-
-    /* We need to know what tasks completed so we can use their buffers when their time comes.
-     * Should only be accessed after locking base.ioJobsMutex . */
-    void* completedJobs[MAX_IO_JOBS];
-    int completedJobsCount;
-    ZSTD_pthread_cond_t jobCompletedCond;
-} ReadPoolCtx_t;
-
-typedef struct {
-    IOPoolCtx_t base;
-    unsigned storedSkips;
-} WritePoolCtx_t;
-
-typedef struct {
-    /* These fields are automatically set and shouldn't be changed by non WritePool code. */
-    void *ctx;
-    FILE* file;
-    void *buffer;
-    size_t bufferSize;
-
-    /* This field should be changed before a job is queued for execution and should contain the number
-     * of bytes to write from the buffer. */
-    size_t usedBufferSize;
-    U64 offset;
-} IOJob_t;
-
-/* AIO_supported:
- * Returns 1 if AsyncIO is supported on the system, 0 otherwise. */
-int AIO_supported(void);
-
-
-/* AIO_WritePool_releaseIoJob:
- * Releases an acquired job back to the pool. Doesn't execute the job. */
-void AIO_WritePool_releaseIoJob(IOJob_t *job);
-
-/* AIO_WritePool_acquireJob:
- * Returns an available write job to be used for a future write. */
-IOJob_t* AIO_WritePool_acquireJob(WritePoolCtx_t *ctx);
-
-/* AIO_WritePool_enqueueAndReacquireWriteJob:
- * Enqueues a write job for execution and acquires a new one.
- * After execution `job`'s pointed value would change to the newly acquired job.
- * Make sure to set `usedBufferSize` to the wanted length before call.
- * The queued job shouldn't be used directly after queueing it. */
-void AIO_WritePool_enqueueAndReacquireWriteJob(IOJob_t **job);
-
-/* AIO_WritePool_sparseWriteEnd:
- * Ends sparse writes to the current file.
- * Blocks on completion of all current write jobs before executing. */
-void AIO_WritePool_sparseWriteEnd(WritePoolCtx_t *ctx);
-
-/* AIO_WritePool_setFile:
- * Sets the destination file for future writes in the pool.
- * Requires completion of all queues write jobs and release of all otherwise acquired jobs.
- * Also requires ending of sparse write if a previous file was used in sparse mode. */
-void AIO_WritePool_setFile(WritePoolCtx_t *ctx, FILE* file);
-
-/* AIO_WritePool_getFile:
- * Returns the file the writePool is currently set to write to. */
-FILE* AIO_WritePool_getFile(const WritePoolCtx_t* ctx);
-
-/* AIO_WritePool_closeFile:
- * Ends sparse write and closes the writePool's current file and sets the file to NULL.
- * Requires completion of all queues write jobs and release of all otherwise acquired jobs.  */
-int AIO_WritePool_closeFile(WritePoolCtx_t *ctx);
-
-/* AIO_WritePool_create:
- * Allocates and sets and a new write pool including its included jobs.
- * bufferSize should be set to the maximal buffer we want to write to at a time. */
-WritePoolCtx_t* AIO_WritePool_create(const FIO_prefs_t* prefs, size_t bufferSize);
-
-/* AIO_WritePool_free:
- * Frees and releases a writePool and its resources. Closes destination file. */
-void AIO_WritePool_free(WritePoolCtx_t* ctx);
-
-/* AIO_WritePool_setAsync:
- * Allows (de)activating async mode, to be used when the expected overhead
- * of asyncio costs more than the expected gains. */
-void AIO_WritePool_setAsync(WritePoolCtx_t* ctx, int async);
-
-/* AIO_ReadPool_create:
- * Allocates and sets and a new readPool including its included jobs.
- * bufferSize should be set to the maximal buffer we want to read at a time, will also be used
- * as our basic read size. */
-ReadPoolCtx_t* AIO_ReadPool_create(const FIO_prefs_t* prefs, size_t bufferSize);
-
-/* AIO_ReadPool_free:
- * Frees and releases a readPool and its resources. Closes source file. */
-void AIO_ReadPool_free(ReadPoolCtx_t* ctx);
-
-/* AIO_ReadPool_setAsync:
- * Allows (de)activating async mode, to be used when the expected overhead
- * of asyncio costs more than the expected gains. */
-void AIO_ReadPool_setAsync(ReadPoolCtx_t* ctx, int async);
-
-/* AIO_ReadPool_consumeBytes:
- * Consumes byes from srcBuffer's beginning and updates srcBufferLoaded accordingly. */
-void AIO_ReadPool_consumeBytes(ReadPoolCtx_t *ctx, size_t n);
-
-/* AIO_ReadPool_fillBuffer:
- * Makes sure buffer has at least n bytes loaded (as long as n is not bigger than the initialized bufferSize).
- * Returns if srcBuffer has at least n bytes loaded or if we've reached the end of the file.
- * Return value is the number of bytes added to the buffer.
- * Note that srcBuffer might have up to 2 times bufferSize bytes. */
-size_t AIO_ReadPool_fillBuffer(ReadPoolCtx_t *ctx, size_t n);
-
-/* AIO_ReadPool_consumeAndRefill:
- * Consumes the current buffer and refills it with bufferSize bytes. */
-size_t AIO_ReadPool_consumeAndRefill(ReadPoolCtx_t *ctx);
-
-/* AIO_ReadPool_setFile:
- * Sets the source file for future read in the pool. Initiates reading immediately if file is not NULL.
- * Waits for all current enqueued tasks to complete if a previous file was set. */
-void AIO_ReadPool_setFile(ReadPoolCtx_t *ctx, FILE* file);
-
-/* AIO_ReadPool_getFile:
- * Returns the current file set for the read pool. */
-FILE* AIO_ReadPool_getFile(const ReadPoolCtx_t *ctx);
-
-/* AIO_ReadPool_closeFile:
- * Closes the current set file. Waits for all current enqueued tasks to complete and resets state. */
-int AIO_ReadPool_closeFile(ReadPoolCtx_t *ctx);
-
-#if defined (__cplusplus)
-}
-#endif
-
-#endif /* ZSTD_FILEIO_ASYNCIO_H */

package/src/bsdiff/zstd/programs/fileio_common.h

@@ -1,125 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-#ifndef ZSTD_FILEIO_COMMON_H
-#define ZSTD_FILEIO_COMMON_H
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-#include "../lib/common/mem.h"     /* U32, U64 */
-#include "fileio_types.h"
-#include "platform.h"
-#include "timefn.h"     /* UTIL_getTime, UTIL_clockSpanMicro */
-
-/*-*************************************
-*  Macros
-***************************************/
-#define KB *(1 <<10)
-#define MB *(1 <<20)
-#define GB *(1U<<30)
-#undef MAX
-#define MAX(a,b) ((a)>(b) ? (a) : (b))
-
-extern FIO_display_prefs_t g_display_prefs;
-
-#define DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
-#define DISPLAYOUT(...)      fprintf(stdout, __VA_ARGS__)
-#define DISPLAYLEVEL(l, ...) { if (g_display_prefs.displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
-
-extern UTIL_time_t g_displayClock;
-
-#define REFRESH_RATE  ((U64)(SEC_TO_MICRO / 6))
-#define READY_FOR_UPDATE() (UTIL_clockSpanMicro(g_displayClock) > REFRESH_RATE || g_display_prefs.displayLevel >= 4)
-#define DELAY_NEXT_UPDATE() { g_displayClock = UTIL_getTime(); }
-#define DISPLAYUPDATE(l, ...) {                              \
-        if (g_display_prefs.displayLevel>=l && (g_display_prefs.progressSetting != FIO_ps_never)) { \
-            if (READY_FOR_UPDATE()) { \
-                DELAY_NEXT_UPDATE();                         \
-                DISPLAY(__VA_ARGS__);                        \
-                if (g_display_prefs.displayLevel>=4) fflush(stderr);       \
-    }   }   }
-
-#define SHOULD_DISPLAY_SUMMARY() \
-    (g_display_prefs.displayLevel >= 2 || g_display_prefs.progressSetting == FIO_ps_always)
-#define SHOULD_DISPLAY_PROGRESS()                       \
-    (g_display_prefs.progressSetting != FIO_ps_never && SHOULD_DISPLAY_SUMMARY())
-#define DISPLAY_PROGRESS(...) { if (SHOULD_DISPLAY_PROGRESS()) { DISPLAYLEVEL(1, __VA_ARGS__); }}
-#define DISPLAYUPDATE_PROGRESS(...) { if (SHOULD_DISPLAY_PROGRESS()) { DISPLAYUPDATE(1, __VA_ARGS__); }}
-#define DISPLAY_SUMMARY(...) { if (SHOULD_DISPLAY_SUMMARY()) { DISPLAYLEVEL(1, __VA_ARGS__); } }
-
-#undef MIN  /* in case it would be already defined */
-#define MIN(a,b)    ((a) < (b) ? (a) : (b))
-
-
-#define EXM_THROW(error, ...)                                             \
-{                                                                         \
-    DISPLAYLEVEL(1, "zstd: ");                                            \
-    DISPLAYLEVEL(5, "Error defined at %s, line %i : \n", __FILE__, __LINE__); \
-    DISPLAYLEVEL(1, "error %i : ", error);                                \
-    DISPLAYLEVEL(1, __VA_ARGS__);                                         \
-    DISPLAYLEVEL(1, " \n");                                               \
-    exit(error);                                                          \
-}
-
-#define CHECK_V(v, f)                                \
-    v = f;                                           \
-    if (ZSTD_isError(v)) {                           \
-        DISPLAYLEVEL(5, "%s \n", #f);                \
-        EXM_THROW(11, "%s", ZSTD_getErrorName(v));   \
-    }
-#define CHECK(f) { size_t err; CHECK_V(err, f); }
-
-
-/* Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW */
-#if defined(_MSC_VER) && _MSC_VER >= 1400
-#   define LONG_SEEK _fseeki64
-#   define LONG_TELL _ftelli64
-#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */
-#  define LONG_SEEK fseeko
-#  define LONG_TELL ftello
-#elif defined(__MINGW32__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) && defined(__MSVCRT__)
-#   define LONG_SEEK fseeko64
-#   define LONG_TELL ftello64
-#elif defined(_WIN32) && !defined(__DJGPP__)
-#   include <windows.h>
-    static int LONG_SEEK(FILE* file, __int64 offset, int origin) {
-        LARGE_INTEGER off;
-        DWORD method;
-        off.QuadPart = offset;
-        if (origin == SEEK_END)
-            method = FILE_END;
-        else if (origin == SEEK_CUR)
-            method = FILE_CURRENT;
-        else
-            method = FILE_BEGIN;
-
-        if (SetFilePointerEx((HANDLE) _get_osfhandle(_fileno(file)), off, NULL, method))
-            return 0;
-        else
-            return -1;
-    }
-    static __int64 LONG_TELL(FILE* file) {
-        LARGE_INTEGER off, newOff;
-        off.QuadPart = 0;
-        newOff.QuadPart = 0;
-        SetFilePointerEx((HANDLE) _get_osfhandle(_fileno(file)), off, &newOff, FILE_CURRENT);
-        return newOff.QuadPart;
-    }
-#else
-#   define LONG_SEEK fseek
-#   define LONG_TELL ftell
-#endif
-
-#if defined (__cplusplus)
-}
-#endif
-#endif /* ZSTD_FILEIO_COMMON_H */

package/src/bsdiff/zstd/programs/fileio_types.h

@@ -1,86 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-#ifndef FILEIO_TYPES_HEADER
-#define FILEIO_TYPES_HEADER
-
-#define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_compressionParameters */
-#include "../lib/zstd.h"           /* ZSTD_* */
-
-/*-*************************************
-*  Parameters: FIO_prefs_t
-***************************************/
-
-typedef struct FIO_display_prefs_s FIO_display_prefs_t;
-
-typedef enum { FIO_ps_auto, FIO_ps_never, FIO_ps_always } FIO_progressSetting_e;
-
-struct FIO_display_prefs_s {
-    int displayLevel;   /* 0 : no display;  1: errors;  2: + result + interaction + warnings;  3: + progression;  4: + information */
-    FIO_progressSetting_e progressSetting;
-};
-
-
-typedef enum { FIO_zstdCompression, FIO_gzipCompression, FIO_xzCompression, FIO_lzmaCompression, FIO_lz4Compression } FIO_compressionType_t;
-
-typedef struct FIO_prefs_s {
-
-    /* Algorithm preferences */
-    FIO_compressionType_t compressionType;
-    int sparseFileSupport;   /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
-    int dictIDFlag;
-    int checksumFlag;
-    int blockSize;
-    int overlapLog;
-    int adaptiveMode;
-    int useRowMatchFinder;
-    int rsyncable;
-    int minAdaptLevel;
-    int maxAdaptLevel;
-    int ldmFlag;
-    int ldmHashLog;
-    int ldmMinMatch;
-    int ldmBucketSizeLog;
-    int ldmHashRateLog;
-    size_t streamSrcSize;
-    size_t targetCBlockSize;
-    int srcSizeHint;
-    int testMode;
-    ZSTD_paramSwitch_e literalCompressionMode;
-
-    /* IO preferences */
-    int removeSrcFile;
-    int overwrite;
-    int asyncIO;
-
-    /* Computation resources preferences */
-    unsigned memLimit;
-    int nbWorkers;
-
-    int excludeCompressedFiles;
-    int patchFromMode;
-    int contentSize;
-    int allowBlockDevices;
-    int passThrough;
-    ZSTD_paramSwitch_e mmapDict;
-} FIO_prefs_t;
-
-typedef enum {FIO_mallocDict, FIO_mmapDict} FIO_dictBufferType_t;
-
-typedef struct {
-    void* dictBuffer;
-    size_t dictBufferSize;
-    FIO_dictBufferType_t dictBufferType;
-#if defined(_MSC_VER) || defined(_WIN32)
-    HANDLE dictHandle;
-#endif
-} FIO_Dict_t;
-
-#endif /* FILEIO_TYPES_HEADER */