electrobun 0.0.1 → 0.0.3

package/src/bsdiff/zstd/programs/benchzstd.h

@@ -1,198 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
- /* benchzstd :
-  * benchmark Zstandard compression / decompression
-  * over a set of files or buffers
-  * and display progress result and final summary
-  */
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-#ifndef BENCH_ZSTD_H_3242387
-#define BENCH_ZSTD_H_3242387
-
-/* ===  Dependencies  === */
-#include <stddef.h>   /* size_t */
-#define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_compressionParameters */
-#include "../lib/zstd.h"     /* ZSTD_compressionParameters */
-
-
-/* ===  Constants  === */
-
-#define MB_UNIT 1000000
-
-
-/* ===  Benchmark functions  === */
-
-/* Creates a variant `typeName`, able to express "error or valid result".
- * Functions with return type `typeName`
- * must first check if result is valid, using BMK_isSuccessful_*(),
- * and only then can extract `baseType`.
- */
-#define VARIANT_ERROR_RESULT(baseType, variantName)  \
-                                             \
-typedef struct {                             \
-    baseType internal_never_use_directly;    \
-    int tag;                                 \
-} variantName
-
-
-typedef struct {
-    size_t cSize;
-    unsigned long long cSpeed;   /* bytes / sec */
-    unsigned long long dSpeed;
-    size_t cMem;                 /* memory usage during compression */
-} BMK_benchResult_t;
-
-VARIANT_ERROR_RESULT(BMK_benchResult_t, BMK_benchOutcome_t);
-
-/* check first if the return structure represents an error or a valid result */
-int BMK_isSuccessful_benchOutcome(BMK_benchOutcome_t outcome);
-
-/* extract result from variant type.
- * note : this function will abort() program execution if result is not valid
- *        check result validity first, by using BMK_isSuccessful_benchOutcome()
- */
-BMK_benchResult_t BMK_extract_benchResult(BMK_benchOutcome_t outcome);
-
-
-/*! BMK_benchFiles() -- called by zstdcli */
-/*  Loads files from fileNamesTable into memory,
- *  and an optional dictionary from dictFileName (can be NULL),
- *  then uses benchMem().
- *  fileNamesTable - name of files to benchmark.
- *  nbFiles - number of files (size of fileNamesTable), must be > 0.
- *  dictFileName - name of dictionary file to load.
- *  cLevel - compression level to benchmark, errors if invalid.
- *  compressionParams - advanced compression Parameters.
- *  displayLevel - what gets printed:
- *      0 : no display;
- *      1 : errors;
- *      2 : + result + interaction + warnings;
- *      3 : + information;
- *      4 : + debug
- * @return: 0 on success, !0 on error
- */
-int BMK_benchFiles(
-            const char* const * fileNamesTable, unsigned nbFiles,
-            const char* dictFileName,
-            int cLevel, const ZSTD_compressionParameters* compressionParams,
-            int displayLevel);
-
-
-typedef enum {
-    BMK_both = 0,
-    BMK_decodeOnly = 1,
-    BMK_compressOnly = 2
-} BMK_mode_t;
-
-typedef struct {
-    BMK_mode_t mode;        /* 0: all, 1: compress only 2: decode only */
-    unsigned nbSeconds;     /* default timing is in nbSeconds */
-    size_t blockSize;       /* Maximum size of each block*/
-    size_t targetCBlockSize;/* Approximative size of compressed blocks */
-    int nbWorkers;          /* multithreading */
-    unsigned realTime;      /* real time priority */
-    int additionalParam;    /* used by python speed benchmark */
-    int ldmFlag;            /* enables long distance matching */
-    int ldmMinMatch;        /* below: parameters for long distance matching, see zstd.1.md */
-    int ldmHashLog;
-    int ldmBucketSizeLog;
-    int ldmHashRateLog;
-    ZSTD_paramSwitch_e literalCompressionMode;
-    int useRowMatchFinder;  /* use row-based matchfinder if possible */
-} BMK_advancedParams_t;
-
-/* returns default parameters used by nonAdvanced functions */
-BMK_advancedParams_t BMK_initAdvancedParams(void);
-
-/*! BMK_benchFilesAdvanced():
- *  Same as BMK_benchFiles(),
- *  with more controls, provided through advancedParams_t structure */
-int BMK_benchFilesAdvanced(
-               const char* const * fileNamesTable, unsigned nbFiles,
-               const char* dictFileName,
-               int cLevel, const ZSTD_compressionParameters* compressionParams,
-               int displayLevel, const BMK_advancedParams_t* adv);
-
-/*! BMK_syntheticTest() -- called from zstdcli */
-/*  Generates a sample with datagen, using compressibility argument */
-/* @cLevel - compression level to benchmark, errors if invalid
- * @compressibility - determines compressibility of sample, range [0.0 - 1.0]
- *        if @compressibility < 0.0, uses the lorem ipsum generator
- * @compressionParams - basic compression Parameters
- * @displayLevel - see benchFiles
- * @adv - see advanced_Params_t
- * @return: 0 on success, !0 on error
- */
-int BMK_syntheticTest(int cLevel, double compressibility,
-                      const ZSTD_compressionParameters* compressionParams,
-                      int displayLevel, const BMK_advancedParams_t* adv);
-
-
-
-/* ===  Benchmark Zstandard in a memory-to-memory scenario  === */
-
-/** BMK_benchMem() -- core benchmarking function, called in paramgrill
- *  applies ZSTD_compress_generic() and ZSTD_decompress_generic() on data in srcBuffer
- *  with specific compression parameters provided by other arguments using benchFunction
- *  (cLevel, comprParams + adv in advanced Mode) */
-/*  srcBuffer - data source, expected to be valid compressed data if in Decode Only Mode
- *  srcSize - size of data in srcBuffer
- *  fileSizes - srcBuffer is considered cut into 1+ segments, to compress separately.
- *              note : sum(fileSizes) must be == srcSize.  (<== ensure it's properly checked)
- *  nbFiles - nb of segments
- *  cLevel - compression level
- *  comprParams - basic compression parameters
- *  dictBuffer - a dictionary if used, null otherwise
- *  dictBufferSize - size of dictBuffer, 0 otherwise
- *  displayLevel - see BMK_benchFiles
- *  displayName - name used by display
- * @return:
- *      a variant, which expresses either an error, or a valid result.
- *      Use BMK_isSuccessful_benchOutcome() to check if function was successful.
- *      If yes, extract the valid result with BMK_extract_benchResult(),
- *      it will contain :
- *          .cSpeed: compression speed in bytes per second,
- *          .dSpeed: decompression speed in bytes per second,
- *          .cSize : compressed size, in bytes
- *          .cMem  : memory budget required for the compression context
- */
-BMK_benchOutcome_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
-                        const size_t* fileSizes, unsigned nbFiles,
-                        int cLevel, const ZSTD_compressionParameters* comprParams,
-                        const void* dictBuffer, size_t dictBufferSize,
-                        int displayLevel, const char* displayName);
-
-
-/* BMK_benchMemAdvanced() : used by Paramgrill
- * same as BMK_benchMem() with following additional options :
- * dstBuffer - destination buffer to write compressed output in, NULL if none provided.
- * dstCapacity - capacity of destination buffer, give 0 if dstBuffer = NULL
- * adv = see advancedParams_t
- */
-BMK_benchOutcome_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
-                        void* dstBuffer, size_t dstCapacity,
-                        const size_t* fileSizes, unsigned nbFiles,
-                        int cLevel, const ZSTD_compressionParameters* comprParams,
-                        const void* dictBuffer, size_t dictBufferSize,
-                        int displayLevel, const char* displayName,
-                        const BMK_advancedParams_t* adv);
-
-
-
-#endif   /* BENCH_ZSTD_H_3242387 */
-
-#if defined (__cplusplus)
-}
-#endif

package/src/bsdiff/zstd/programs/datagen.c

@@ -1,186 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-
-
-/*-************************************
-*  Dependencies
-**************************************/
-#include "datagen.h"
-#include "platform.h"  /* SET_BINARY_MODE */
-#include <stdlib.h>    /* malloc, free */
-#include <stdio.h>     /* FILE, fwrite, fprintf */
-#include <string.h>    /* memcpy */
-#include "../lib/common/mem.h"  /* U32 */
-
-
-/*-************************************
-*  Macros
-**************************************/
-#define KB *(1 <<10)
-#define MIN(a,b)  ( (a) < (b) ? (a) : (b) )
-
-#define RDG_DEBUG 0
-#define TRACE(...)   if (RDG_DEBUG) fprintf(stderr, __VA_ARGS__ )
-
-
-/*-************************************
-*  Local constants
-**************************************/
-#define LTLOG 13
-#define LTSIZE (1<<LTLOG)
-#define LTMASK (LTSIZE-1)
-
-
-/*-*******************************************************
-*  Local Functions
-*********************************************************/
-#define RDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
-static U32 RDG_rand(U32* src)
-{
-    static const U32 prime1 = 2654435761U;
-    static const U32 prime2 = 2246822519U;
-    U32 rand32 = *src;
-    rand32 *= prime1;
-    rand32 ^= prime2;
-    rand32  = RDG_rotl32(rand32, 13);
-    *src = rand32;
-    return rand32 >> 5;
-}
-
-typedef U32 fixedPoint_24_8;
-
-static void RDG_fillLiteralDistrib(BYTE* ldt, fixedPoint_24_8 ld)
-{
-    BYTE const firstChar = (ld<=0.0) ?   0 : '(';
-    BYTE const lastChar  = (ld<=0.0) ? 255 : '}';
-    BYTE character = (ld<=0.0) ? 0 : '0';
-    U32 u;
-
-    if (ld<=0) ld = 0;
-    for (u=0; u<LTSIZE; ) {
-        U32 const weight = (((LTSIZE - u) * ld) >> 8) + 1;
-        U32 const end = MIN ( u + weight , LTSIZE);
-        while (u < end) ldt[u++] = character;
-        character++;
-        if (character > lastChar) character = firstChar;
-    }
-}
-
-
-static BYTE RDG_genChar(U32* seed, const BYTE* ldt)
-{
-    U32 const id = RDG_rand(seed) & LTMASK;
-    return ldt[id];  /* memory-sanitizer fails here, stating "uninitialized value" when table initialized with P==0.0. Checked : table is fully initialized */
-}
-
-
-static U32 RDG_rand15Bits (U32* seedPtr)
-{
-    return RDG_rand(seedPtr) & 0x7FFF;
-}
-
-static U32 RDG_randLength(U32* seedPtr)
-{
-    if (RDG_rand(seedPtr) & 7) return (RDG_rand(seedPtr) & 0xF);   /* small length */
-    return (RDG_rand(seedPtr) & 0x1FF) + 0xF;
-}
-
-static void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize,
-                         double matchProba, const BYTE* ldt, U32* seedPtr)
-{
-    BYTE* const buffPtr = (BYTE*)buffer;
-    U32 const matchProba32 = (U32)(32768 * matchProba);
-    size_t pos = prefixSize;
-    U32 prevOffset = 1;
-
-    /* special case : sparse content */
-    while (matchProba >= 1.0) {
-        size_t size0 = RDG_rand(seedPtr) & 3;
-        size0  = (size_t)1 << (16 + size0 * 2);
-        size0 += RDG_rand(seedPtr) & (size0-1);   /* because size0 is power of 2*/
-        if (buffSize < pos + size0) {
-            memset(buffPtr+pos, 0, buffSize-pos);
-            return;
-        }
-        memset(buffPtr+pos, 0, size0);
-        pos += size0;
-        buffPtr[pos-1] = RDG_genChar(seedPtr, ldt);
-        continue;
-    }
-
-    /* init */
-    if (pos==0) buffPtr[0] = RDG_genChar(seedPtr, ldt), pos=1;
-
-    /* Generate compressible data */
-    while (pos < buffSize) {
-        /* Select : Literal (char) or Match (within 32K) */
-        if (RDG_rand15Bits(seedPtr) < matchProba32) {
-            /* Copy (within 32K) */
-            U32 const length = RDG_randLength(seedPtr) + 4;
-            U32 const d = (U32) MIN(pos + length , buffSize);
-            U32 const repeatOffset = (RDG_rand(seedPtr) & 15) == 2;
-            U32 const randOffset = RDG_rand15Bits(seedPtr) + 1;
-            U32 const offset = repeatOffset ? prevOffset : (U32) MIN(randOffset , pos);
-            size_t match = pos - offset;
-            while (pos < d) { buffPtr[pos++] = buffPtr[match++];   /* correctly manages overlaps */ }
-            prevOffset = offset;
-        } else {
-            /* Literal (noise) */
-            U32 const length = RDG_randLength(seedPtr);
-            U32 const d = (U32) MIN(pos + length, buffSize);
-            while (pos < d) { buffPtr[pos++] = RDG_genChar(seedPtr, ldt); }
-    }   }
-}
-
-
-void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed)
-{
-    U32 seed32 = seed;
-    BYTE ldt[LTSIZE];
-    memset(ldt, '0', sizeof(ldt));  /* yes, character '0', this is intentional */
-    if (litProba<=0.0) litProba = matchProba / 4.5;
-    RDG_fillLiteralDistrib(ldt, (fixedPoint_24_8)(litProba * 256 + 0.001));
-    RDG_genBlock(buffer, size, 0, matchProba, ldt, &seed32);
-}
-
-
-void RDG_genStdout(unsigned long long size, double matchProba, double litProba, unsigned seed)
-{
-    U32 seed32 = seed;
-    size_t const stdBlockSize = 128 KB;
-    size_t const stdDictSize = 32 KB;
-    BYTE* const buff = (BYTE*)malloc(stdDictSize + stdBlockSize);
-    U64 total = 0;
-    BYTE ldt[LTSIZE];   /* literals distribution table */
-
-    /* init */
-    if (buff==NULL) { perror("datagen"); exit(1); }
-    if (litProba<=0.0) litProba = matchProba / 4.5;
-    memset(ldt, '0', sizeof(ldt));   /* yes, character '0', this is intentional */
-    RDG_fillLiteralDistrib(ldt, (fixedPoint_24_8)(litProba * 256 + 0.001));
-    SET_BINARY_MODE(stdout);
-
-    /* Generate initial dict */
-    RDG_genBlock(buff, stdDictSize, 0, matchProba, ldt, &seed32);
-
-    /* Generate compressible data */
-    while (total < size) {
-        size_t const genBlockSize = (size_t) (MIN (stdBlockSize, size-total));
-        RDG_genBlock(buff, stdDictSize+stdBlockSize, stdDictSize, matchProba, ldt, &seed32);
-        total += genBlockSize;
-        { size_t const unused = fwrite(buff, 1, genBlockSize, stdout); (void)unused; }
-        /* update dict */
-        memcpy(buff, buff + stdBlockSize, stdDictSize);
-    }
-
-    /* cleanup */
-    free(buff);
-}

package/src/bsdiff/zstd/programs/datagen.h

@@ -1,30 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-
-#ifndef DATAGEN_H
-#define DATAGEN_H
-
-#include <stddef.h>   /* size_t */
-
-void RDG_genStdout(unsigned long long size, double matchProba, double litProba, unsigned seed);
-void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed);
-/*!RDG_genBuffer
-   Generate 'size' bytes of compressible data into 'buffer'.
-   Compressibility can be controlled using 'matchProba', which is floating point value between 0 and 1.
-   'LitProba' is optional, it affect variability of individual bytes. If litProba==0.0, default value will be used.
-   Generated data pattern can be modified using different 'seed'.
-   For a triplet (matchProba, litProba, seed), the function always generate the same content.
-
-   RDG_genStdout
-   Same as RDG_genBuffer, but generates data into stdout
-*/
-
-#endif