zstd-native-ruby 1.0.1 → 1.0.2

data/{vendor → ext/zstd_native}/zstd/contrib/linux-kernel/zstd_compress_module.c

@@ -67,12 +67,6 @@ int zstd_max_clevel(void)
 }
 EXPORT_SYMBOL(zstd_max_clevel);
 
-int zstd_default_clevel(void)
-{
-	return ZSTD_defaultCLevel();
-}
-EXPORT_SYMBOL(zstd_default_clevel);
-
 size_t zstd_compress_bound(size_t src_size)
 {
 	return ZSTD_compressBound(src_size);
@@ -86,13 +80,6 @@ zstd_parameters zstd_get_params(int level,
 }
 EXPORT_SYMBOL(zstd_get_params);
 
-zstd_compression_parameters zstd_get_cparams(int level,
-	unsigned long long estimated_src_size, size_t dict_size)
-{
-	return ZSTD_getCParams(level, estimated_src_size, dict_size);
-}
-EXPORT_SYMBOL(zstd_get_cparams);
-
 size_t zstd_cctx_set_param(zstd_cctx *cctx, ZSTD_cParameter param, int value)
 {
 	return ZSTD_CCtx_setParameter(cctx, param, value);
@@ -159,33 +146,6 @@ zstd_cctx *zstd_init_cctx(void *workspace, size_t workspace_size)
 }
 EXPORT_SYMBOL(zstd_init_cctx);
 
-zstd_cctx *zstd_create_cctx_advanced(zstd_custom_mem custom_mem)
-{
-	return ZSTD_createCCtx_advanced(custom_mem);
-}
-EXPORT_SYMBOL(zstd_create_cctx_advanced);
-
-size_t zstd_free_cctx(zstd_cctx *cctx)
-{
-	return ZSTD_freeCCtx(cctx);
-}
-EXPORT_SYMBOL(zstd_free_cctx);
-
-zstd_cdict *zstd_create_cdict_byreference(const void *dict, size_t dict_size,
-					  zstd_compression_parameters cparams,
-					  zstd_custom_mem custom_mem)
-{
-	return ZSTD_createCDict_advanced(dict, dict_size, ZSTD_dlm_byRef,
-					 ZSTD_dct_auto, cparams, custom_mem);
-}
-EXPORT_SYMBOL(zstd_create_cdict_byreference);
-
-size_t zstd_free_cdict(zstd_cdict *cdict)
-{
-	return ZSTD_freeCDict(cdict);
-}
-EXPORT_SYMBOL(zstd_free_cdict);
-
 size_t zstd_compress_cctx(zstd_cctx *cctx, void *dst, size_t dst_capacity,
 	const void *src, size_t src_size, const zstd_parameters *parameters)
 {
@@ -194,15 +154,6 @@ size_t zstd_compress_cctx(zstd_cctx *cctx, void *dst, size_t dst_capacity,
 }
 EXPORT_SYMBOL(zstd_compress_cctx);
 
-size_t zstd_compress_using_cdict(zstd_cctx *cctx, void *dst,
-	size_t dst_capacity, const void *src, size_t src_size,
-	const ZSTD_CDict *cdict)
-{
-	return ZSTD_compress_usingCDict(cctx, dst, dst_capacity,
-					src, src_size, cdict);
-}
-EXPORT_SYMBOL(zstd_compress_using_cdict);
-
 size_t zstd_cstream_workspace_bound(const zstd_compression_parameters *cparams)
 {
 	return ZSTD_estimateCStreamSize_usingCParams(*cparams);

data/{vendor → ext/zstd_native}/zstd/contrib/linux-kernel/zstd_decompress_module.c

@@ -44,33 +44,6 @@ size_t zstd_dctx_workspace_bound(void)
 }
 EXPORT_SYMBOL(zstd_dctx_workspace_bound);
 
-zstd_dctx *zstd_create_dctx_advanced(zstd_custom_mem custom_mem)
-{
-	return ZSTD_createDCtx_advanced(custom_mem);
-}
-EXPORT_SYMBOL(zstd_create_dctx_advanced);
-
-size_t zstd_free_dctx(zstd_dctx *dctx)
-{
-	return ZSTD_freeDCtx(dctx);
-}
-EXPORT_SYMBOL(zstd_free_dctx);
-
-zstd_ddict *zstd_create_ddict_byreference(const void *dict, size_t dict_size,
-					  zstd_custom_mem custom_mem)
-{
-	return ZSTD_createDDict_advanced(dict, dict_size, ZSTD_dlm_byRef,
-					 ZSTD_dct_auto, custom_mem);
-
-}
-EXPORT_SYMBOL(zstd_create_ddict_byreference);
-
-size_t zstd_free_ddict(zstd_ddict *ddict)
-{
-	return ZSTD_freeDDict(ddict);
-}
-EXPORT_SYMBOL(zstd_free_ddict);
-
 zstd_dctx *zstd_init_dctx(void *workspace, size_t workspace_size)
 {
 	if (workspace == NULL)
@@ -86,15 +59,6 @@ size_t zstd_decompress_dctx(zstd_dctx *dctx, void *dst, size_t dst_capacity,
 }
 EXPORT_SYMBOL(zstd_decompress_dctx);
 
-size_t zstd_decompress_using_ddict(zstd_dctx *dctx,
-	void *dst, size_t dst_capacity, const void* src, size_t src_size,
-	const zstd_ddict* ddict)
-{
-	return ZSTD_decompress_usingDDict(dctx, dst, dst_capacity, src,
-					  src_size, ddict);
-}
-EXPORT_SYMBOL(zstd_decompress_using_ddict);
-
 size_t zstd_dstream_workspace_bound(size_t max_window_size)
 {
 	return ZSTD_estimateDStreamSize(max_window_size);

data/{vendor → ext/zstd_native}/zstd/contrib/premake/zstd.lua

@@ -2,7 +2,6 @@
 -- Basic usage: project_zstd(ZSTD_DIR)
 
 function project_zstd(dir, compression, decompression, deprecated, dictbuilder, legacy)
-	if string.sub(dir, -1, 1) ~= '/' then dir = dir .. '/' end
 	if compression == nil then compression = true end
 	if decompression == nil then decompression = true end
 	if deprecated == nil then deprecated = false end

data/{vendor → ext/zstd_native}/zstd/contrib/pzstd/Logging.h

@@ -10,7 +10,6 @@
 
 #include <cstdio>
 #include <mutex>
-#include <chrono>
 
 namespace pzstd {
 

data/{vendor → ext/zstd_native}/zstd/contrib/seekable_format/examples/Makefile

@@ -11,8 +11,6 @@
 
 ZSTDLIB_PATH = ../../../lib
 ZSTDLIB_NAME = libzstd.a
-# Parallel tools only make sense against multi-threaded libzstd
-ZSTDLIB_TARGET = $(ZSTDLIB_NAME)-mt
 ZSTDLIB = $(ZSTDLIB_PATH)/$(ZSTDLIB_NAME)
 
 CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I../ -I../../../lib -I../../../lib/common
@@ -30,7 +28,7 @@ all: seekable_compression seekable_decompression seekable_decompression_mem \
 	parallel_processing
 
 $(ZSTDLIB):
-	$(MAKE) -C $(ZSTDLIB_PATH) $(ZSTDLIB_TARGET)
+	make -C $(ZSTDLIB_PATH) $(ZSTDLIB_NAME)
 
 seekable_compression : seekable_compression.c $(SEEKABLE_OBJS)
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@

data/{vendor → ext/zstd_native}/zstd/contrib/seekable_format/examples/parallel_compression.c

@@ -23,8 +23,6 @@
 
 #include "xxhash.h"
 
-#define ZSTD_MULTITHREAD 1
-#include "threading.h"
 #include "pool.h"      // use zstd thread pool for demo
 
 #include "../zstd_seekable.h"
@@ -74,166 +72,127 @@ static size_t fclose_orDie(FILE* file)
     exit(6);
 }
 
-struct state {
-    FILE* fout;
-    ZSTD_pthread_mutex_t mutex;
-    size_t nextID;
-    struct job* pending;
-    ZSTD_frameLog* frameLog;
-    const int compressionLevel;
-};
+static void fseek_orDie(FILE* file, long int offset, int origin)
+{
+    if (!fseek(file, offset, origin)) {
+        if (!fflush(file)) return;
+    }
+    /* error */
+    perror("fseek");
+    exit(7);
+}
 
-struct job {
-    size_t id;
-    struct job* next;
-    struct state* state;
+static long int ftell_orDie(FILE* file)
+{
+    long int off = ftell(file);
+    if (off != -1) return off;
+    /* error */
+    perror("ftell");
+    exit(8);
+}
 
-    void* src;
+struct job {
+    const void* src;
     size_t srcSize;
     void* dst;
     size_t dstSize;
 
     unsigned checksum;
-};
-
-static void addPending_inmutex(struct state* state, struct job* job)
-{
-    struct job** p = &state->pending;
-    while (*p && (*p)->id < job->id)
-        p = &(*p)->next;
-    job->next = *p;
-    *p = job;
-}
-
-static void flushFrame(struct state* state, struct job* job)
-{
-    fwrite_orDie(job->dst, job->dstSize, state->fout);
-    free(job->dst);
 
-    size_t ret = ZSTD_seekable_logFrame(state->frameLog, (unsigned)job->dstSize, (unsigned)job->srcSize, job->checksum);
-    if (ZSTD_isError(ret)) {
-        fprintf(stderr, "ZSTD_seekable_logFrame() error : %s \n", ZSTD_getErrorName(ret));
-        exit(12);
-    }
-}
-
-static void flushPending_inmutex(struct state* state)
-{
-    while (state->pending && state->pending->id == state->nextID) {
-        struct job* p = state->pending;
-        state->pending = p->next;
-        flushFrame(state, p);
-        free(p);
-        state->nextID++;
-    }
-}
-
-static void finishFrame(struct job* job)
-{
-    struct state *state = job->state;
-    ZSTD_pthread_mutex_lock(&state->mutex);
-    addPending_inmutex(state, job);
-    flushPending_inmutex(state);
-    ZSTD_pthread_mutex_unlock(&state->mutex);
-}
+    int compressionLevel;
+    int done;
+};
 
 static void compressFrame(void* opaque)
 {
     struct job* job = opaque;
 
-    job->checksum = (unsigned)XXH64(job->src, job->srcSize, 0);
+    job->checksum = XXH64(job->src, job->srcSize, 0);
 
-    size_t ret = ZSTD_compress(job->dst, job->dstSize, job->src, job->srcSize, job->state->compressionLevel);
+    size_t ret = ZSTD_compress(job->dst, job->dstSize, job->src, job->srcSize, job->compressionLevel);
     if (ZSTD_isError(ret)) {
         fprintf(stderr, "ZSTD_compress() error : %s \n", ZSTD_getErrorName(ret));
         exit(20);
     }
-    job->dstSize = ret;
-
-    // No longer need
-    free(job->src);
-    job->src = NULL;
-
-    finishFrame(job);
-}
-
-static const char* createOutFilename_orDie(const char* filename)
-{
-    size_t const inL = strlen(filename);
-    size_t const outL = inL + 5;
-    void* outSpace = malloc_orDie(outL);
-    memset(outSpace, 0, outL);
-    strcat(outSpace, filename);
-    strcat(outSpace, ".zst");
-    return (const char*)outSpace;
-}
 
-static void openInOut_orDie(const char* fname, FILE** fin, FILE** fout) {
-    if (strcmp(fname, "-") == 0) {
-        *fin = stdin;
-        *fout = stdout;
-    } else {
-        *fin = fopen_orDie(fname, "rb");
-        const char* outName = createOutFilename_orDie(fname);
-        *fout = fopen_orDie(outName, "wb");
-    }
+    job->dstSize = ret;
+    job->done = 1;
 }
 
-static void compressFile_orDie(const char* fname, int cLevel, unsigned frameSize, size_t nbThreads)
+static void compressFile_orDie(const char* fname, const char* outName, int cLevel, unsigned frameSize, int nbThreads)
 {
-    struct state state = {
-        .nextID = 0,
-        .pending = NULL,
-        .compressionLevel = cLevel,
-    };
-    ZSTD_pthread_mutex_init(&state.mutex, NULL);
-    state.frameLog = ZSTD_seekable_createFrameLog(1);
-    if (state.frameLog == NULL) { fprintf(stderr, "ZSTD_seekable_createFrameLog() failed \n"); exit(11); }
-
     POOL_ctx* pool = POOL_create(nbThreads, nbThreads);
     if (pool == NULL) { fprintf(stderr, "POOL_create() error \n"); exit(9); }
 
-    FILE* fin;
-    openInOut_orDie(fname, &fin, &state.fout);
+    FILE* const fin  = fopen_orDie(fname, "rb");
+    FILE* const fout = fopen_orDie(outName, "wb");
 
     if (ZSTD_compressBound(frameSize) > 0xFFFFFFFFU) { fprintf(stderr, "Frame size too large \n"); exit(10); }
-    size_t dstSize = ZSTD_compressBound(frameSize);
-
-    for (size_t id = 0; 1; id++) {
-        struct job* job = malloc_orDie(sizeof(struct job));
-        job->id = id;
-        job->next = NULL;
-        job->state = &state;
-        job->src = malloc_orDie(frameSize);
-        job->dst = malloc_orDie(dstSize);
-        job->srcSize = fread_orDie(job->src, frameSize, fin);
-        job->dstSize = dstSize; 
-        POOL_add(pool, compressFrame, job);
-        if (feof(fin))
-            break;
+    unsigned dstSize = ZSTD_compressBound(frameSize);
+
+
+    fseek_orDie(fin, 0, SEEK_END);
+    long int length = ftell_orDie(fin);
+    fseek_orDie(fin, 0, SEEK_SET);
+
+    size_t numFrames = (length + frameSize - 1) / frameSize;
+
+    struct job* jobs = malloc_orDie(sizeof(struct job) * numFrames);
+
+    size_t i;
+    for(i = 0; i < numFrames; i++) {
+        void* in = malloc_orDie(frameSize);
+        void* out = malloc_orDie(dstSize);
+
+        size_t inSize = fread_orDie(in, frameSize, fin);
+
+        jobs[i].src = in;
+        jobs[i].srcSize = inSize;
+        jobs[i].dst = out;
+        jobs[i].dstSize = dstSize;
+        jobs[i].compressionLevel = cLevel;
+        jobs[i].done = 0;
+        POOL_add(pool, compressFrame, &jobs[i]);
     }
 
-    POOL_joinJobs(pool);
-    POOL_free(pool);
-    if (state.pending) {
-        fprintf(stderr, "Unexpected leftover output blocks!\n");
-        exit(13);
+    ZSTD_frameLog* fl = ZSTD_seekable_createFrameLog(1);
+    if (fl == NULL) { fprintf(stderr, "ZSTD_seekable_createFrameLog() failed \n"); exit(11); }
+    for (i = 0; i < numFrames; i++) {
+        while (!jobs[i].done) SLEEP(5); /* wake up every 5 milliseconds to check */
+        fwrite_orDie(jobs[i].dst, jobs[i].dstSize, fout);
+        free((void*)jobs[i].src);
+        free(jobs[i].dst);
+
+        size_t ret = ZSTD_seekable_logFrame(fl, jobs[i].dstSize, jobs[i].srcSize, jobs[i].checksum);
+        if (ZSTD_isError(ret)) { fprintf(stderr, "ZSTD_seekable_logFrame() error : %s \n", ZSTD_getErrorName(ret)); }
     }
 
     {   unsigned char seekTableBuff[1024];
         ZSTD_outBuffer out = {seekTableBuff, 1024, 0};
-        while (ZSTD_seekable_writeSeekTable(state.frameLog, &out) != 0) {
-            fwrite_orDie(seekTableBuff, out.pos, state.fout);
+        while (ZSTD_seekable_writeSeekTable(fl, &out) != 0) {
+            fwrite_orDie(seekTableBuff, out.pos, fout);
             out.pos = 0;
         }
-        fwrite_orDie(seekTableBuff, out.pos, state.fout);
+        fwrite_orDie(seekTableBuff, out.pos, fout);
     }
 
-    ZSTD_seekable_freeFrameLog(state.frameLog);
-    fclose_orDie(state.fout);
+    ZSTD_seekable_freeFrameLog(fl);
+    free(jobs);
+    fclose_orDie(fout);
     fclose_orDie(fin);
 }
 
+static const char* createOutFilename_orDie(const char* filename)
+{
+    size_t const inL = strlen(filename);
+    size_t const outL = inL + 5;
+    void* outSpace = malloc_orDie(outL);
+    memset(outSpace, 0, outL);
+    strcat(outSpace, filename);
+    strcat(outSpace, ".zst");
+    return (const char*)outSpace;
+}
+
 int main(int argc, const char** argv) {
     const char* const exeName = argv[0];
     if (argc!=4) {
@@ -245,9 +204,10 @@ int main(int argc, const char** argv) {
 
     {   const char* const inFileName = argv[1];
         unsigned const frameSize = (unsigned)atoi(argv[2]);
-        size_t const nbThreads = (size_t)atoi(argv[3]);
+        int const nbThreads = atoi(argv[3]);
 
-        compressFile_orDie(inFileName, 5, frameSize, nbThreads);
+        const char* const outFileName = createOutFilename_orDie(inFileName);
+        compressFile_orDie(inFileName, outFileName, 5, frameSize, nbThreads);
     }
 
     return 0;

data/{vendor → ext/zstd_native}/zstd/contrib/seekable_format/examples/parallel_processing.c

@@ -100,11 +100,13 @@ struct sum_job {
     const char* fname;
     unsigned long long sum;
     unsigned frameNb;
+    int done;
 };
 
 static void sumFrame(void* opaque)
 {
     struct sum_job* job = (struct sum_job*)opaque;
+    job->done = 0;
 
     FILE* const fin = fopen_orDie(job->fname, "rb");
 
@@ -126,6 +128,7 @@ static void sumFrame(void* opaque)
         sum += data[i];
     }
     job->sum = sum;
+    job->done = 1;
 
     fclose(fin);
     ZSTD_seekable_free(seekable);
@@ -150,14 +153,14 @@ static void sumFile_orDie(const char* fname, int nbThreads)
 
     unsigned fnb;
     for (fnb = 0; fnb < numFrames; fnb++) {
-        jobs[fnb] = (struct sum_job){ fname, 0, fnb };
+        jobs[fnb] = (struct sum_job){ fname, 0, fnb, 0 };
         POOL_add(pool, sumFrame, &jobs[fnb]);
     }
-    POOL_joinJobs(pool);
 
     unsigned long long total = 0;
 
     for (fnb = 0; fnb < numFrames; fnb++) {
+        while (!jobs[fnb].done) SLEEP(5); /* wake up every 5 milliseconds to check */
         total += jobs[fnb].sum;
     }
 

data/{vendor → ext/zstd_native}/zstd/contrib/seekable_format/tests/Makefile

@@ -24,7 +24,7 @@ SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_decompress.c $(ZSTDLIB)
 .PHONY: default clean test
 default: test
 
-test: seekable_tests parallel_compression_test
+test: seekable_tests
 	./seekable_tests
 
 $(ZSTDLIB):
@@ -32,27 +32,7 @@ $(ZSTDLIB):
 
 seekable_tests : $(SEEKABLE_OBJS)
 
-EXAMPLES_PATH = ../examples
-PARALLEL_COMPRESSION = $(EXAMPLES_PATH)/parallel_compression
-
-DATAGEN_PATH = ../../../tests
-DATAGEN = $(DATAGEN_PATH)/datagen
-
-$(PARALLEL_COMPRESSION):
-	$(MAKE) -C $(EXAMPLES_PATH) parallel_compression
-
-$(DATAGEN):
-	$(MAKE) -C $(DATAGEN_PATH) datagen
-
-data.txt: $(DATAGEN)
-	$(DATAGEN) -g100M > $@
-
-parallel_compression_test: $(PARALLEL_COMPRESSION) data.txt
-	ulimit -Sv 102400; $(PARALLEL_COMPRESSION) data.txt 1048576 2
-
-.PHONY: parallel_compression_test parallel_comp
-
 clean:
 	@$(RM) core *.o tmp* result* *.zst \
-		seekable_tests data.txt
+		seekable_tests
 	@echo Cleaning completed

data/{vendor → ext/zstd_native}/zstd/contrib/seekable_format/tests/seekable_tests.c

@@ -316,10 +316,6 @@ int main(int argc, const char** argv)
         for (pos = 0; pos < inSize; pos += 2) {
             size_t const decStatus = ZSTD_seekable_decompress(stream, outBuffer, 1, pos);
             if (decStatus != 1 || outBuffer[0] != inBuffer[pos]) {
-                free(seekBuffer);
-                free(outBuffer);
-                ZSTD_seekable_freeCStream(zscs);
-                ZSTD_seekable_free(stream);
                 goto _test_error;
             }
         }
@@ -327,10 +323,6 @@ int main(int argc, const char** argv)
             /* We read more than the compressed size, meaning there were some rereads.
                This is unneeded because we only seeked forward. */
             printf("Too much data read: %zu read, with compressed size %zu\n", buffWrapper.totalRead, seekSize);
-            free(seekBuffer);
-            free(outBuffer);
-            ZSTD_seekable_freeCStream(zscs);
-            ZSTD_seekable_free(stream);
             goto _test_error;
         }
 
@@ -350,10 +342,6 @@ int main(int argc, const char** argv)
         for (idx = 0; idx < sizeof(tests) / sizeof(tests[0]); idx++) {
             size_t const decStatus = ZSTD_seekable_decompress(stream, outBuffer, tests[idx].size, tests[idx].offset);
             if (decStatus != tests[idx].size || memcmp(outBuffer, inBuffer + tests[idx].offset, tests[idx].size) != 0) {
-                free(seekBuffer);
-                free(outBuffer);
-                ZSTD_seekable_freeCStream(zscs);
-                ZSTD_seekable_free(stream);
                 goto _test_error;
             }
         }

data/{vendor → ext/zstd_native}/zstd/doc/zstd_compression_format.md

@@ -16,7 +16,7 @@ Distribution of this document is unlimited.
 
 ### Version
 
-0.4.4 (2025-03-22)
+0.4.3 (2024-10-07)
 
 
 Introduction
@@ -390,7 +390,7 @@ __`Block_Content`__ and __`Block_Maximum_Size`__
 The size of `Block_Content` is limited by `Block_Maximum_Size`,
 which is the smallest of:
 -  `Window_Size`
--  128 KiB (131.072 bytes)
+-  128 KB
 
 `Block_Maximum_Size` is constant for a given frame.
 This maximum is applicable to both the decompressed size
@@ -1742,7 +1742,6 @@ or at least provide a meaningful error code explaining for which reason it canno
 
 Version changes
 ---------------
-- 0.4.4 : minor clarification for block size
 - 0.4.3 : clarifications for Huffman prefix code assignment example
 - 0.4.2 : refactor FSE table construction process, inspired by Donald Pian
 - 0.4.1 : clarifications on a few error scenarios, by Eric Lasota