zstd-native-ruby 1.0.0

data/vendor/zstd/tests/fuzz/simple_round_trip.c

@@ -0,0 +1,182 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/**
+ * This fuzz target performs a zstd round-trip test (compress & decompress),
+ * compares the result with the original, and calls abort() on corruption.
+ */
+
+#define ZSTD_STATIC_LINKING_ONLY
+
+#include <stddef.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include "fuzz_helpers.h"
+#include "zstd_helpers.h"
+#include "fuzz_data_producer.h"
+#include "fuzz_third_party_seq_prod.h"
+
+static ZSTD_CCtx *cctx = NULL;
+static ZSTD_DCtx *dctx = NULL;
+
+static size_t getDecompressionMargin(void const* compressed, size_t cSize, size_t srcSize, int hasSmallBlocks, int maxBlockSize)
+{
+    size_t margin = ZSTD_decompressionMargin(compressed, cSize);
+    if (!hasSmallBlocks) {
+        /* The macro should be correct in this case, but it may be smaller
+         * because of e.g. block splitting, so take the smaller of the two.
+         */
+        ZSTD_FrameHeader zfh;
+        size_t marginM;
+        FUZZ_ZASSERT(ZSTD_getFrameHeader(&zfh, compressed, cSize));
+        if (maxBlockSize == 0) {
+            maxBlockSize = zfh.blockSizeMax;
+        } else {
+            maxBlockSize = MIN(maxBlockSize, (int)zfh.blockSizeMax);
+        }
+        marginM = ZSTD_DECOMPRESSION_MARGIN(srcSize, maxBlockSize);
+        if (marginM < margin)
+            margin = marginM;
+    }
+    return margin;
+}
+
+static size_t roundTripTest(void *result, size_t resultCapacity,
+                            void *compressed, size_t compressedCapacity,
+                            const void *src, size_t srcSize,
+                            FUZZ_dataProducer_t *producer)
+{
+    size_t cSize;
+    size_t dSize;
+    int targetCBlockSize = 0;
+    int maxBlockSize = 0;
+    if (FUZZ_dataProducer_uint32Range(producer, 0, 1)) {
+        size_t const remainingBytes = FUZZ_dataProducer_remainingBytes(producer);
+        FUZZ_setRandomParameters(cctx, srcSize, producer);
+        cSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize);
+        FUZZ_ZASSERT(cSize);
+        FUZZ_ZASSERT(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetCBlockSize, &targetCBlockSize));
+        FUZZ_ZASSERT(ZSTD_CCtx_getParameter(cctx, ZSTD_c_maxBlockSize, &maxBlockSize));
+        // Compress a second time and check for determinism
+        {
+            size_t const cSize0 = cSize;
+            XXH64_hash_t const hash0 = XXH64(compressed, cSize, 0);
+            FUZZ_dataProducer_rollBack(producer, remainingBytes);
+            FUZZ_setRandomParameters(cctx, srcSize, producer);
+            cSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize);
+            FUZZ_ASSERT(cSize == cSize0);
+            FUZZ_ASSERT(XXH64(compressed, cSize, 0) == hash0);
+        }
+    } else {
+        int const cLevel = FUZZ_dataProducer_int32Range(producer, kMinClevel, kMaxClevel);
+        cSize = ZSTD_compressCCtx(
+            cctx, compressed, compressedCapacity, src, srcSize, cLevel);
+        FUZZ_ZASSERT(cSize);
+        // Compress a second time and check for determinism
+        {
+            size_t const cSize0 = cSize;
+            XXH64_hash_t const hash0 = XXH64(compressed, cSize, 0);
+            cSize = ZSTD_compressCCtx(
+                cctx, compressed, compressedCapacity, src, srcSize, cLevel);
+            FUZZ_ASSERT(cSize == cSize0);
+            FUZZ_ASSERT(XXH64(compressed, cSize, 0) == hash0);
+        }
+    }
+    if (FUZZ_dataProducer_uint32Range(producer, 0, 1)) {
+        FUZZ_ZASSERT(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, maxBlockSize));
+    }
+    dSize = ZSTD_decompressDCtx(dctx, result, resultCapacity, compressed, cSize);
+    FUZZ_ZASSERT(dSize);
+    FUZZ_ASSERT_MSG(dSize == srcSize, "Incorrect regenerated size");
+    FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, result, dSize), "Corruption!");
+
+    {
+        size_t margin = getDecompressionMargin(compressed, cSize, srcSize, targetCBlockSize, maxBlockSize);
+        size_t const outputSize = srcSize + margin;
+        char* const output = (char*)FUZZ_malloc(outputSize);
+        char* const input = output + outputSize - cSize;
+        FUZZ_ASSERT(outputSize >= cSize);
+        memcpy(input, compressed, cSize);
+
+        dSize = ZSTD_decompressDCtx(dctx, output, outputSize, input, cSize);
+        FUZZ_ZASSERT(dSize);
+        FUZZ_ASSERT_MSG(dSize == srcSize, "Incorrect regenerated size");
+        FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, output, srcSize), "Corruption!");
+
+        free(output);
+    }
+
+    /* When superblock is enabled make sure we don't expand the block more than expected.
+     * NOTE: This test is currently disabled because superblock mode can arbitrarily
+     * expand the block in the worst case. Once superblock mode has been improved we can
+     * re-enable this test.
+     */
+    if (0 && targetCBlockSize != 0) {
+        size_t normalCSize;
+        FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 0));
+        normalCSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize);
+        FUZZ_ZASSERT(normalCSize);
+        {
+            size_t const bytesPerBlock = 3 /* block header */
+                + 5 /* Literal header */
+                + 6 /* Huffman jump table */
+                + 3 /* number of sequences */
+                + 1 /* symbol compression modes */;
+            size_t const expectedExpansion = bytesPerBlock * (1 + (normalCSize / MAX(1, targetCBlockSize)));
+            size_t const allowedExpansion = (srcSize >> 3) + 5 * expectedExpansion + 10;
+            FUZZ_ASSERT(cSize <= normalCSize + allowedExpansion);
+        }
+    }
+    return dSize;
+}
+
+int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
+{
+    FUZZ_SEQ_PROD_SETUP();
+
+    size_t const rBufSize = size;
+    void* rBuf = FUZZ_malloc(rBufSize);
+    size_t cBufSize = ZSTD_compressBound(size);
+    void* cBuf;
+
+    /* Give a random portion of src data to the producer, to use for
+    parameter generation. The rest will be used for (de)compression */
+    FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
+    size = FUZZ_dataProducer_reserveDataPrefix(producer);
+
+    /* Half of the time fuzz with a 1 byte smaller output size.
+     * This will still succeed because we don't use a dictionary, so the dictID
+     * field is empty, giving us 4 bytes of overhead.
+     */
+    cBufSize -= FUZZ_dataProducer_uint32Range(producer, 0, 1);
+
+    cBuf = FUZZ_malloc(cBufSize);
+
+    if (!cctx) {
+        cctx = ZSTD_createCCtx();
+        FUZZ_ASSERT(cctx);
+    }
+    if (!dctx) {
+        dctx = ZSTD_createDCtx();
+        FUZZ_ASSERT(dctx);
+    }
+
+    roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size, producer);
+    free(rBuf);
+    free(cBuf);
+    FUZZ_dataProducer_free(producer);
+#ifndef STATEFUL_FUZZING
+    ZSTD_freeCCtx(cctx); cctx = NULL;
+    ZSTD_freeDCtx(dctx); dctx = NULL;
+#endif
+    FUZZ_SEQ_PROD_TEARDOWN();
+    return 0;
+}

data/vendor/zstd/tests/fuzz/stream_decompress.c

@@ -0,0 +1,119 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/**
+ * This fuzz target attempts to decompress the fuzzed data with the simple
+ * decompression function to ensure the decompressor never crashes.
+ */
+
+#define ZSTD_STATIC_LINKING_ONLY
+
+#include <stddef.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include "fuzz_helpers.h"
+#include "zstd.h"
+#include "fuzz_data_producer.h"
+
+static ZSTD_DStream *dstream = NULL;
+uint32_t seed;
+
+static ZSTD_outBuffer makeOutBuffer(FUZZ_dataProducer_t *producer, void* buf, size_t bufSize)
+{
+  ZSTD_outBuffer buffer = { buf, 0, 0 };
+
+  if (FUZZ_dataProducer_empty(producer)) {
+    buffer.size = bufSize;
+  } else {
+    buffer.size = (FUZZ_dataProducer_uint32Range(producer, 0, bufSize));
+  }
+  FUZZ_ASSERT(buffer.size <= bufSize);
+
+  if (buffer.size == 0) {
+    buffer.dst = NULL;
+  }
+
+  return buffer;
+}
+
+static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size,
+                                  FUZZ_dataProducer_t *producer)
+{
+  ZSTD_inBuffer buffer = { *src, 0, 0 };
+
+  FUZZ_ASSERT(*size > 0);
+  if (FUZZ_dataProducer_empty(producer)) {
+    buffer.size = *size;
+  } else {
+    buffer.size = (FUZZ_dataProducer_uint32Range(producer, 0, *size));
+  }
+  FUZZ_ASSERT(buffer.size <= *size);
+  *src += buffer.size;
+  *size -= buffer.size;
+
+  if (buffer.size == 0) {
+    buffer.src = NULL;
+  }
+
+  return buffer;
+}
+
+int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
+{
+    /* Give a random portion of src data to the producer, to use for
+    parameter generation. The rest will be used for (de)compression */
+    FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
+    int stableOutBuffer;
+    ZSTD_outBuffer out;
+    void* buf;
+    size_t bufSize;
+    size = FUZZ_dataProducer_reserveDataPrefix(producer);
+    bufSize = MAX(10 * size, ZSTD_BLOCKSIZE_MAX);
+
+    /* Allocate all buffers and contexts if not already allocated */
+    buf = FUZZ_malloc(bufSize);
+
+    if (!dstream) {
+        dstream = ZSTD_createDStream();
+        FUZZ_ASSERT(dstream);
+    } else {
+        FUZZ_ZASSERT(ZSTD_DCtx_reset(dstream, ZSTD_reset_session_only));
+    }
+
+    stableOutBuffer = FUZZ_dataProducer_uint32Range(producer, 0, 10) == 5;
+    if (stableOutBuffer) {
+      FUZZ_ZASSERT(ZSTD_DCtx_setParameter(dstream, ZSTD_d_stableOutBuffer, 1));
+      out.dst = buf;
+      out.size = bufSize;
+      out.pos = 0;
+    } else {
+      out = makeOutBuffer(producer, buf, bufSize);
+    }
+
+    while (size > 0) {
+        ZSTD_inBuffer in = makeInBuffer(&src, &size, producer);
+        do {
+            size_t const rc = ZSTD_decompressStream(dstream, &out, &in);
+            if (ZSTD_isError(rc)) goto error;
+            if (out.pos == out.size) {
+                if (stableOutBuffer) goto error;
+                out = makeOutBuffer(producer, buf, bufSize);
+            }
+        } while (in.pos != in.size);
+    }
+
+error:
+#ifndef STATEFUL_FUZZING
+    ZSTD_freeDStream(dstream); dstream = NULL;
+#endif
+    FUZZ_dataProducer_free(producer);
+    free(buf);
+    return 0;
+}

data/vendor/zstd/tests/fuzz/stream_round_trip.c

@@ -0,0 +1,218 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/**
+ * This fuzz target performs a zstd round-trip test (compress & decompress),
+ * compares the result with the original, and calls abort() on corruption.
+ */
+
+#define ZSTD_STATIC_LINKING_ONLY
+
+#include <stddef.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include "fuzz_helpers.h"
+#include "zstd_helpers.h"
+#include "fuzz_data_producer.h"
+#include "fuzz_third_party_seq_prod.h"
+
+ZSTD_CCtx *cctx = NULL;
+static ZSTD_DCtx *dctx = NULL;
+static uint8_t* cBuf = NULL;
+static uint8_t* rBuf = NULL;
+static size_t bufSize = 0;
+
+static ZSTD_outBuffer makeOutBuffer(uint8_t *dst, size_t capacity,
+                                    FUZZ_dataProducer_t *producer)
+{
+    ZSTD_outBuffer buffer = { dst, 0, 0 };
+
+    FUZZ_ASSERT(capacity > 0);
+    buffer.size = (FUZZ_dataProducer_uint32Range(producer, 1, capacity));
+    FUZZ_ASSERT(buffer.size <= capacity);
+
+    return buffer;
+}
+
+static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size,
+                                  FUZZ_dataProducer_t *producer)
+{
+    ZSTD_inBuffer buffer = { *src, 0, 0 };
+
+    FUZZ_ASSERT(*size > 0);
+    buffer.size = (FUZZ_dataProducer_uint32Range(producer, 1, *size));
+    FUZZ_ASSERT(buffer.size <= *size);
+    *src += buffer.size;
+    *size -= buffer.size;
+
+    return buffer;
+}
+
+static size_t compress(uint8_t *dst, size_t capacity,
+                       const uint8_t *src, size_t srcSize,
+                     FUZZ_dataProducer_t *producer)
+{
+    size_t dstSize = 0;
+    ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
+    FUZZ_setRandomParameters(cctx, srcSize, producer);
+    int maxBlockSize;
+    FUZZ_ZASSERT(ZSTD_CCtx_getParameter(cctx, ZSTD_c_maxBlockSize, &maxBlockSize));
+
+    while (srcSize > 0) {
+        ZSTD_inBuffer in = makeInBuffer(&src, &srcSize, producer);
+        /* Mode controls the action. If mode == -1 we pick a new mode */
+        int mode = -1;
+        while (in.pos < in.size || mode != -1) {
+            ZSTD_outBuffer out = makeOutBuffer(dst, capacity, producer);
+            /* Previous action finished, pick a new mode. */
+            if (mode == -1) mode = FUZZ_dataProducer_uint32Range(producer, 0, 9);
+            switch (mode) {
+                case 0: /* fall-through */
+                case 1: /* fall-through */
+                case 2: {
+                    size_t const ret =
+                        ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush);
+                    FUZZ_ZASSERT(ret);
+                    if (ret == 0)
+                        mode = -1;
+                    break;
+                }
+                case 3: {
+                    size_t ret =
+                        ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
+                    FUZZ_ZASSERT(ret);
+                    /* Reset the compressor when the frame is finished */
+                    if (ret == 0) {
+                        ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
+                        if (FUZZ_dataProducer_uint32Range(producer, 0, 7) == 0) {
+                            size_t const remaining = in.size - in.pos;
+                            FUZZ_setRandomParameters(cctx, remaining, producer);
+                            /* Always use the same maxBlockSize */
+                            FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, maxBlockSize));
+                        }
+                        mode = -1;
+                    }
+                    break;
+                }
+                case 4: {
+                    ZSTD_inBuffer nullIn = { NULL, 0, 0 };
+                    ZSTD_outBuffer nullOut = { NULL, 0, 0 };
+                    size_t const ret = ZSTD_compressStream2(cctx, &nullOut, &nullIn, ZSTD_e_continue);
+                    FUZZ_ZASSERT(ret);
+                }
+                /* fall-through */
+                default: {
+                    size_t const ret =
+                        ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
+                    FUZZ_ZASSERT(ret);
+                    mode = -1;
+                }
+            }
+            dst += out.pos;
+            dstSize += out.pos;
+            capacity -= out.pos;
+        }
+    }
+    for (;;) {
+        ZSTD_inBuffer in = {NULL, 0, 0};
+        ZSTD_outBuffer out = makeOutBuffer(dst, capacity, producer);
+        size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
+        FUZZ_ZASSERT(ret);
+
+        dst += out.pos;
+        dstSize += out.pos;
+        capacity -= out.pos;
+        if (ret == 0)
+            break;
+    }
+    return dstSize;
+}
+
+static size_t decompress(void* dst, size_t dstCapacity, void const* src, size_t srcSize, FUZZ_dataProducer_t* producer)
+{
+    ZSTD_inBuffer in = {src, srcSize, 0};
+    ZSTD_outBuffer out = {dst, dstCapacity, 0};
+    int maxBlockSize;
+    FUZZ_ZASSERT(ZSTD_CCtx_getParameter(cctx, ZSTD_c_maxBlockSize, &maxBlockSize));
+    if (FUZZ_dataProducer_uint32Range(producer, 0, 1)) {
+        FUZZ_ZASSERT(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, maxBlockSize));
+    }
+    while (in.pos < in.size) {
+        size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
+        FUZZ_ZASSERT(ret);
+        FUZZ_ASSERT(ret == 0);
+    }
+    return out.pos;
+}
+
+int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
+{
+    FUZZ_SEQ_PROD_SETUP();
+    size_t neededBufSize;
+
+    /* Give a random portion of src data to the producer, to use for
+    parameter generation. The rest will be used for (de)compression */
+    FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
+    size = FUZZ_dataProducer_reserveDataPrefix(producer);
+
+    neededBufSize = ZSTD_compressBound(size) * 15;
+
+    /* Allocate all buffers and contexts if not already allocated */
+    if (neededBufSize > bufSize) {
+        free(cBuf);
+        free(rBuf);
+        cBuf = (uint8_t*)FUZZ_malloc(neededBufSize);
+        rBuf = (uint8_t*)FUZZ_malloc(neededBufSize);
+        bufSize = neededBufSize;
+    }
+    if (!cctx) {
+        cctx = ZSTD_createCCtx();
+        FUZZ_ASSERT(cctx);
+    }
+    if (!dctx) {
+        dctx = ZSTD_createDCtx();
+        FUZZ_ASSERT(dctx);
+    }
+
+    {
+        size_t const cSize = compress(cBuf, neededBufSize, src, size, producer);
+        size_t const rSize = decompress(rBuf, neededBufSize, cBuf, cSize, producer);
+        FUZZ_ZASSERT(rSize);
+        FUZZ_ASSERT_MSG(rSize == size, "Incorrect regenerated size");
+        FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, rBuf, size), "Corruption!");
+
+        /* Test in-place decompression (note the macro doesn't work in this case) */
+        {
+            size_t const margin = ZSTD_decompressionMargin(cBuf, cSize);
+            size_t const outputSize = size + margin;
+            char* const output = (char*)FUZZ_malloc(outputSize);
+            char* const input = output + outputSize - cSize;
+            size_t dSize;
+            FUZZ_ASSERT(outputSize >= cSize);
+            memcpy(input, cBuf, cSize);
+
+            dSize = ZSTD_decompressDCtx(dctx, output, outputSize, input, cSize);
+            FUZZ_ZASSERT(dSize);
+            FUZZ_ASSERT_MSG(dSize == size, "Incorrect regenerated size");
+            FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, output, size), "Corruption!");
+
+            free(output);
+        }
+    }
+
+    FUZZ_dataProducer_free(producer);
+#ifndef STATEFUL_FUZZING
+    ZSTD_freeCCtx(cctx); cctx = NULL;
+    ZSTD_freeDCtx(dctx); dctx = NULL;
+#endif
+    FUZZ_SEQ_PROD_TEARDOWN();
+    return 0;
+}

data/vendor/zstd/tests/fuzz/zstd_frame_info.c

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/**
+ * This fuzz target fuzzes all of the helper functions that consume compressed
+ * input.
+ */
+
+#include <stddef.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include "fuzz_helpers.h"
+#include "zstd_helpers.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
+{
+    ZSTD_FrameHeader zfh;
+    if (size == 0) {
+        src = NULL;
+   }
+    /* You can fuzz any helper functions here that are fast, and take zstd
+     * compressed data as input. E.g. don't expect the input to be a dictionary,
+     * so don't fuzz ZSTD_getDictID_fromDict().
+     */
+    ZSTD_getFrameContentSize(src, size);
+    ZSTD_getDecompressedSize(src, size);
+    ZSTD_findFrameCompressedSize(src, size);
+    ZSTD_getDictID_fromFrame(src, size);
+    ZSTD_findDecompressedSize(src, size);
+    ZSTD_decompressBound(src, size);
+    ZSTD_frameHeaderSize(src, size);
+    ZSTD_isFrame(src, size);
+    ZSTD_getFrameHeader(&zfh, src, size);
+    ZSTD_getFrameHeader_advanced(&zfh, src, size, ZSTD_f_zstd1);
+    return 0;
+}