zstd-native-ruby 1.0.0

data/vendor/zstd/examples/multiple_simple_compression.c

@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include <stdio.h>     // printf
+#include <stdlib.h>    // free
+#include <string.h>    // memcpy, strlen
+#include <zstd.h>      // presumes zstd library is installed
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
+
+typedef struct {
+    void* fBuffer;
+    void* cBuffer;
+    size_t fBufferSize;
+    size_t cBufferSize;
+    ZSTD_CCtx* cctx;
+} resources;
+
+/*
+ * allocate memory for buffers big enough to compress all files
+ * as well as memory for output file name (ofn)
+ */
+static resources createResources_orDie(int argc, const char** argv, char **ofn, size_t* ofnBufferLen)
+{
+    size_t maxFilenameLength=0;
+    size_t maxFileSize = 0;
+
+    int argNb;
+    for (argNb = 1; argNb < argc; argNb++) {
+      const char* const filename = argv[argNb];
+      size_t const filenameLength = strlen(filename);
+      size_t const fileSize = fsize_orDie(filename);
+
+      if (filenameLength > maxFilenameLength) maxFilenameLength = filenameLength;
+      if (fileSize > maxFileSize) maxFileSize = fileSize;
+    }
+
+    resources ress;
+    ress.fBufferSize = maxFileSize;
+    ress.cBufferSize = ZSTD_compressBound(maxFileSize);
+
+    *ofnBufferLen = maxFilenameLength + 5;
+    *ofn = (char*)malloc_orDie(*ofnBufferLen);
+    ress.fBuffer = malloc_orDie(ress.fBufferSize);
+    ress.cBuffer = malloc_orDie(ress.cBufferSize);
+    ress.cctx = ZSTD_createCCtx();
+    CHECK(ress.cctx != NULL, "ZSTD_createCCtx() failed!");
+    return ress;
+}
+
+static void freeResources(resources ress, char *outFilename)
+{
+    free(ress.fBuffer);
+    free(ress.cBuffer);
+    ZSTD_freeCCtx(ress.cctx);   /* never fails */
+    free(outFilename);
+}
+
+/* compress with pre-allocated context (ZSTD_CCtx) and input/output buffers*/
+static void compressFile_orDie(resources ress, const char* fname, const char* oname)
+{
+    size_t fSize = loadFile_orDie(fname, ress.fBuffer, ress.fBufferSize);
+
+    /* Compress using the context.
+     * If you need more control over parameters, use the advanced API:
+     * ZSTD_CCtx_setParameter(), and ZSTD_compress2().
+     */
+    size_t const cSize = ZSTD_compressCCtx(ress.cctx, ress.cBuffer, ress.cBufferSize, ress.fBuffer, fSize, 1);
+    CHECK_ZSTD(cSize);
+
+    saveFile_orDie(oname, ress.cBuffer, cSize);
+
+    /* success */
+    printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname);
+}
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+
+    if (argc<2) {
+        printf("wrong arguments\n");
+        printf("usage:\n");
+        printf("%s FILE(s)\n", exeName);
+        return 1;
+    }
+
+    /* memory allocation for outFilename and resources */
+    char* outFilename;
+    size_t outFilenameBufferLen;
+    resources const ress = createResources_orDie(argc, argv, &outFilename, &outFilenameBufferLen);
+
+    /* compress files with shared context, input and output buffers */
+    int argNb;
+    for (argNb = 1; argNb < argc; argNb++) {
+        const char* const inFilename = argv[argNb];
+        size_t const inFilenameLen = strlen(inFilename);
+        CHECK(inFilenameLen + 5 <= outFilenameBufferLen, "File name too long!");
+        memcpy(outFilename, inFilename, inFilenameLen);
+        memcpy(outFilename+inFilenameLen, ".zst", 5);
+        compressFile_orDie(ress, inFilename, outFilename);
+    }
+
+    /* free memory */
+    freeResources(ress,outFilename);
+
+    printf("compressed %i files \n", argc-1);
+
+    return 0;
+}

data/vendor/zstd/examples/multiple_streaming_compression.c

@@ -0,0 +1,133 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+/* The objective of this example is to show of to compress multiple successive files
+*  while preserving memory management.
+*  All structures and buffers will be created only once,
+*  and shared across all compression operations */
+
+#include <stdio.h>     // printf
+#include <stdlib.h>    // free
+#include <string.h>    // memset, strcat
+#include <zstd.h>      // presumes zstd library is installed
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
+
+typedef struct {
+    void* buffIn;
+    void* buffOut;
+    size_t buffInSize;
+    size_t buffOutSize;
+    ZSTD_CCtx* cctx;
+} resources;
+
+static resources createResources_orDie(int cLevel)
+{
+    resources ress;
+    ress.buffInSize = ZSTD_CStreamInSize();   /* can always read one full block */
+    ress.buffOutSize= ZSTD_CStreamOutSize();  /* can always flush a full block */
+    ress.buffIn = malloc_orDie(ress.buffInSize);
+    ress.buffOut= malloc_orDie(ress.buffOutSize);
+    ress.cctx = ZSTD_createCCtx();
+    CHECK(ress.cctx != NULL, "ZSTD_createCCtx() failed!");
+
+    /* Set any compression parameters you want here.
+     * They will persist for every compression operation.
+     * Here we set the compression level, and enable the checksum.
+     */
+    CHECK_ZSTD( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, cLevel) );
+    CHECK_ZSTD( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_checksumFlag, 1) );
+    return ress;
+}
+
+static void freeResources(resources ress)
+{
+    ZSTD_freeCCtx(ress.cctx);
+    free(ress.buffIn);
+    free(ress.buffOut);
+}
+
+static void compressFile_orDie(resources ress, const char* fname, const char* outName)
+{
+    // Open the input and output files.
+    FILE* const fin  = fopen_orDie(fname, "rb");
+    FILE* const fout = fopen_orDie(outName, "wb");
+
+    /* Reset the context to a clean state to start a new compression operation.
+     * The parameters are sticky, so we keep the compression level and extra
+     * parameters that we set in createResources_orDie().
+     */
+    CHECK_ZSTD( ZSTD_CCtx_reset(ress.cctx, ZSTD_reset_session_only) );
+
+    size_t const toRead = ress.buffInSize;
+    size_t read;
+    while ( (read = fread_orDie(ress.buffIn, toRead, fin)) ) {
+        /* This loop is the same as streaming_compression.c.
+         * See that file for detailed comments.
+         */
+        int const lastChunk = (read < toRead);
+        ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue;
+
+        ZSTD_inBuffer input = { ress.buffIn, read, 0 };
+        int finished;
+        do {
+            ZSTD_outBuffer output = { ress.buffOut, ress.buffOutSize, 0 };
+            size_t const remaining = ZSTD_compressStream2(ress.cctx, &output, &input, mode);
+            CHECK_ZSTD(remaining);
+            fwrite_orDie(ress.buffOut, output.pos, fout);
+            finished = lastChunk ? (remaining == 0) : (input.pos == input.size);
+        } while (!finished);
+        CHECK(input.pos == input.size,
+              "Impossible: zstd only returns 0 when the input is completely consumed!");
+    }
+
+    fclose_orDie(fout);
+    fclose_orDie(fin);
+}
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+
+    if (argc<2) {
+        printf("wrong arguments\n");
+        printf("usage:\n");
+        printf("%s FILE(s)\n", exeName);
+        return 1;
+    }
+
+    int const cLevel = 7;
+    resources const ress = createResources_orDie(cLevel);
+    void* ofnBuffer = NULL;
+    size_t ofnbSize = 0;
+
+    int argNb;
+    for (argNb = 1; argNb < argc; argNb++) {
+        const char* const ifn = argv[argNb];
+        size_t const ifnSize = strlen(ifn);
+        size_t const ofnSize = ifnSize + 5;
+        if (ofnbSize <= ofnSize) {
+            ofnbSize = ofnSize + 16;
+            free(ofnBuffer);
+            ofnBuffer = malloc_orDie(ofnbSize);
+        }
+        memset(ofnBuffer, 0, ofnSize);
+        strcat(ofnBuffer, ifn);
+        strcat(ofnBuffer, ".zst");
+        compressFile_orDie(ress, ifn, ofnBuffer);
+    }
+
+    freeResources(ress);
+    free(ofnBuffer);
+
+    printf("compressed %i files \n", argc-1);
+
+    return 0;
+}

data/vendor/zstd/examples/simple_compression.c

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include <stdio.h>     // printf
+#include <stdlib.h>    // free
+#include <string.h>    // strlen, strcat, memset
+#include <zstd.h>      // presumes zstd library is installed
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
+
+static void compress_orDie(const char* fname, const char* oname)
+{
+    size_t fSize;
+    void* const fBuff = mallocAndLoadFile_orDie(fname, &fSize);
+    size_t const cBuffSize = ZSTD_compressBound(fSize);
+    void* const cBuff = malloc_orDie(cBuffSize);
+
+    /* Compress.
+     * If you are doing many compressions, you may want to reuse the context.
+     * See the multiple_simple_compression.c example.
+     */
+    size_t const cSize = ZSTD_compress(cBuff, cBuffSize, fBuff, fSize, 1);
+    CHECK_ZSTD(cSize);
+
+    saveFile_orDie(oname, cBuff, cSize);
+
+    /* success */
+    printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname);
+
+    free(fBuff);
+    free(cBuff);
+}
+
+static char* createOutFilename_orDie(const char* filename)
+{
+    size_t const inL = strlen(filename);
+    size_t const outL = inL + 5;
+    void* const outSpace = malloc_orDie(outL);
+    memset(outSpace, 0, outL);
+    strcat(outSpace, filename);
+    strcat(outSpace, ".zst");
+    return (char*)outSpace;
+}
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+
+    if (argc!=2) {
+        printf("wrong arguments\n");
+        printf("usage:\n");
+        printf("%s FILE\n", exeName);
+        return 1;
+    }
+
+    const char* const inFilename = argv[1];
+
+    char* const outFilename = createOutFilename_orDie(inFilename);
+    compress_orDie(inFilename, outFilename);
+    free(outFilename);
+    return 0;
+}

data/vendor/zstd/examples/simple_decompression.c

@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include <stdio.h>     // printf
+#include <stdlib.h>    // free
+#include <zstd.h>      // presumes zstd library is installed
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
+
+static void decompress(const char* fname)
+{
+    size_t cSize;
+    void* const cBuff = mallocAndLoadFile_orDie(fname, &cSize);
+    /* Read the content size from the frame header. For simplicity we require
+     * that it is always present. By default, zstd will write the content size
+     * in the header when it is known. If you can't guarantee that the frame
+     * content size is always written into the header, either use streaming
+     * decompression, or ZSTD_decompressBound().
+     */
+    unsigned long long const rSize = ZSTD_getFrameContentSize(cBuff, cSize);
+    CHECK(rSize != ZSTD_CONTENTSIZE_ERROR, "%s: not compressed by zstd!", fname);
+    CHECK(rSize != ZSTD_CONTENTSIZE_UNKNOWN, "%s: original size unknown!", fname);
+
+    void* const rBuff = malloc_orDie((size_t)rSize);
+
+    /* Decompress.
+     * If you are doing many decompressions, you may want to reuse the context
+     * and use ZSTD_decompressDCtx(). If you want to set advanced parameters,
+     * use ZSTD_DCtx_setParameter().
+     */
+    size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize);
+    CHECK_ZSTD(dSize);
+    /* When zstd knows the content size, it will error if it doesn't match. */
+    CHECK(dSize == rSize, "Impossible because zstd will check this condition!");
+
+    /* success */
+    printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize);
+
+    free(rBuff);
+    free(cBuff);
+}
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+
+    if (argc!=2) {
+        printf("wrong arguments\n");
+        printf("usage:\n");
+        printf("%s FILE\n", exeName);
+        return 1;
+    }
+
+    decompress(argv[1]);
+
+    printf("%s correctly decoded (in memory). \n", argv[1]);
+
+    return 0;
+}

data/vendor/zstd/examples/streaming_compression.c

@@ -0,0 +1,146 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+#include <stdio.h>     // printf
+#include <stdlib.h>    // free
+#include <string.h>    // memset, strcat, strlen
+#include <zstd.h>      // presumes zstd library is installed
+#include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()
+
+static void compressFile_orDie(const char* fname, const char* outName, int cLevel,
+                               int nbThreads)
+{
+    fprintf (stderr, "Starting compression of %s with level %d, using %d threads\n",
+             fname, cLevel, nbThreads);
+
+    /* Open the input and output files. */
+    FILE* const fin  = fopen_orDie(fname, "rb");
+    FILE* const fout = fopen_orDie(outName, "wb");
+    /* Create the input and output buffers.
+     * They may be any size, but we recommend using these functions to size them.
+     * Performance will only suffer significantly for very tiny buffers.
+     */
+    size_t const buffInSize = ZSTD_CStreamInSize();
+    void*  const buffIn  = malloc_orDie(buffInSize);
+    size_t const buffOutSize = ZSTD_CStreamOutSize();
+    void*  const buffOut = malloc_orDie(buffOutSize);
+
+    /* Create the context. */
+    ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+    CHECK(cctx != NULL, "ZSTD_createCCtx() failed!");
+
+    /* Set any parameters you want.
+     * Here we set the compression level, and enable the checksum.
+     */
+    CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel) );
+    CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) );
+    if (nbThreads > 1) {
+        size_t const r = ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads);
+        if (ZSTD_isError(r)) {
+            fprintf (stderr, "Note: the linked libzstd library doesn't support multithreading. "
+                             "Reverting to single-thread mode. \n");
+        }
+    }
+
+    /* This loop read from the input file, compresses that entire chunk,
+     * and writes all output produced to the output file.
+     */
+    size_t const toRead = buffInSize;
+    for (;;) {
+        size_t read = fread_orDie(buffIn, toRead, fin);
+        /* Select the flush mode.
+         * If the read may not be finished (read == toRead) we use
+         * ZSTD_e_continue. If this is the last chunk, we use ZSTD_e_end.
+         * Zstd optimizes the case where the first flush mode is ZSTD_e_end,
+         * since it knows it is compressing the entire source in one pass.
+         */
+        int const lastChunk = (read < toRead);
+        ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue;
+        /* Set the input buffer to what we just read.
+         * We compress until the input buffer is empty, each time flushing the
+         * output.
+         */
+        ZSTD_inBuffer input = { buffIn, read, 0 };
+        int finished;
+        do {
+            /* Compress into the output buffer and write all of the output to
+             * the file so we can reuse the buffer next iteration.
+             */
+            ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
+            size_t const remaining = ZSTD_compressStream2(cctx, &output , &input, mode);
+            CHECK_ZSTD(remaining);
+            fwrite_orDie(buffOut, output.pos, fout);
+            /* If we're on the last chunk we're finished when zstd returns 0,
+             * which means its consumed all the input AND finished the frame.
+             * Otherwise, we're finished when we've consumed all the input.
+             */
+            finished = lastChunk ? (remaining == 0) : (input.pos == input.size);
+        } while (!finished);
+        CHECK(input.pos == input.size,
+              "Impossible: zstd only returns 0 when the input is completely consumed!");
+
+        if (lastChunk) {
+            break;
+        }
+    }
+
+    ZSTD_freeCCtx(cctx);
+    fclose_orDie(fout);
+    fclose_orDie(fin);
+    free(buffIn);
+    free(buffOut);
+}
+
+
+static char* createOutFilename_orDie(const char* filename)
+{
+    size_t const inL = strlen(filename);
+    size_t const outL = inL + 5;
+    void* const outSpace = malloc_orDie(outL);
+    memset(outSpace, 0, outL);
+    strcat(outSpace, filename);
+    strcat(outSpace, ".zst");
+    return (char*)outSpace;
+}
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+
+    if (argc < 2) {
+        printf("wrong arguments\n");
+        printf("usage:\n");
+        printf("%s FILE [LEVEL] [THREADS]\n", exeName);
+        return 1;
+    }
+
+    int cLevel = 1;
+    int nbThreads = 1;
+
+    if (argc >= 3) {
+      cLevel = atoi (argv[2]);
+      CHECK(cLevel != 0, "can't parse LEVEL!");
+    }
+
+    if (argc >= 4) {
+      nbThreads = atoi (argv[3]);
+      CHECK(nbThreads != 0, "can't parse THREADS!");
+    }
+
+    const char* const inFilename = argv[1];
+
+    char* const outFilename = createOutFilename_orDie(inFilename);
+    compressFile_orDie(inFilename, outFilename, cLevel, nbThreads);
+
+    free(outFilename);   /* not strictly required, since program execution stops there,
+                          * but some static analyzer may complain otherwise */
+    return 0;
+}