extlz4 0.3 → 0.3.1

data/contrib/lz4/lib/lz4hc.h

@@ -336,6 +336,9 @@ LZ4LIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionL
 #ifndef LZ4_HC_SLO_098092834
 #define LZ4_HC_SLO_098092834
 
+#define LZ4_STATIC_LINKING_ONLY   /* LZ4LIB_STATIC_API */
+#include "lz4.h"
+
 #if defined (__cplusplus)
 extern "C" {
 #endif

data/contrib/lz4/ossfuzz/Makefile

@@ -0,0 +1,74 @@
+# ##########################################################################
+# LZ4 oss fuzzer - Makefile
+#
+# GPL v2 License
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# You can contact the author at :
+#  - LZ4 homepage : http://www.lz4.org
+#  - LZ4 source repository : https://github.com/lz4/lz4
+# ##########################################################################
+# compress_fuzzer : OSS Fuzz test tool
+# decompress_fuzzer : OSS Fuzz test tool
+# ##########################################################################
+
+LZ4DIR  := ../lib
+LIB_FUZZING_ENGINE ?= standaloneengine.o
+
+DEBUGLEVEL?= 1
+DEBUGFLAGS = -g -DLZ4_DEBUG=$(DEBUGLEVEL)
+
+LZ4_CFLAGS  = $(CFLAGS) $(DEBUGFLAGS) $(MOREFLAGS)
+LZ4_CXXFLAGS = $(CXXFLAGS) $(DEBUGFLAGS) $(MOREFLAGS)
+LZ4_CPPFLAGS = $(CPPFLAGS) -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_ \
+               -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+
+FUZZERS := \
+	compress_fuzzer \
+	decompress_fuzzer \
+	round_trip_fuzzer \
+	round_trip_stream_fuzzer \
+	compress_hc_fuzzer \
+	round_trip_hc_fuzzer \
+	compress_frame_fuzzer \
+	round_trip_frame_fuzzer \
+	decompress_frame_fuzzer
+
+all: $(FUZZERS)
+
+# Include a rule to build the static library if calling this target
+# directly.
+$(LZ4DIR)/liblz4.a:
+	$(MAKE) -C $(LZ4DIR) CFLAGS="$(LZ4_CFLAGS)" liblz4.a
+
+%.o: %.c
+	$(CC) -c $(LZ4_CFLAGS) $(LZ4_CPPFLAGS) $< -o $@
+
+# Generic rule for generating fuzzers
+%_fuzzer: %_fuzzer.o lz4_helpers.o $(LZ4DIR)/liblz4.a
+	# Compile the standalone code just in case. The OSS-Fuzz code might
+	# override the LIB_FUZZING_ENGINE value to "-fsanitize=fuzzer"
+	$(CC) -c $(LZ4_CFLAGS) $(LZ4_CPPFLAGS) standaloneengine.c -o standaloneengine.o
+
+	# Now compile the actual fuzzer.
+	$(CXX) $(LZ4_CXXFLAGS) $(LZ4_CPPFLAGS) $(LDFLAGS) $(LIB_FUZZING_ENGINE) $^ -o $@$(EXT)
+
+%_fuzzer_clean:
+	$(RM) $*_fuzzer $*_fuzzer.o standaloneengine.o
+
+.PHONY: clean
+clean: compress_fuzzer_clean decompress_fuzzer_clean
+	$(MAKE) -C $(LZ4DIR) clean

data/contrib/lz4/ossfuzz/compress_frame_fuzzer.c

@@ -0,0 +1,42 @@
+/**
+ * This fuzz target attempts to compress the fuzzed data with the simple
+ * compression function with an output buffer that may be too small to
+ * ensure that the compressor never crashes.
+ */
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "fuzz_helpers.h"
+#include "lz4.h"
+#include "lz4frame.h"
+#include "lz4_helpers.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
+{
+    uint32_t seed = FUZZ_seed(&data, &size);
+    LZ4F_preferences_t const prefs = FUZZ_randomPreferences(&seed);
+    size_t const compressBound = LZ4F_compressFrameBound(size, &prefs);
+    size_t const dstCapacity = FUZZ_rand32(&seed, 0, compressBound);
+    char* const dst = (char*)malloc(dstCapacity);
+    char* const rt = (char*)malloc(size);
+
+    FUZZ_ASSERT(dst);
+    FUZZ_ASSERT(rt);
+
+    /* If compression succeeds it must round trip correctly. */
+    size_t const dstSize =
+            LZ4F_compressFrame(dst, dstCapacity, data, size, &prefs);
+    if (!LZ4F_isError(dstSize)) {
+        size_t const rtSize = FUZZ_decompressFrame(rt, size, dst, dstSize);
+        FUZZ_ASSERT_MSG(rtSize == size, "Incorrect regenerated size");
+        FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!");
+    }
+
+    free(dst);
+    free(rt);
+
+    return 0;
+}

data/contrib/lz4/ossfuzz/compress_fuzzer.c

@@ -0,0 +1,51 @@
+/**
+ * This fuzz target attempts to compress the fuzzed data with the simple
+ * compression function with an output buffer that may be too small to
+ * ensure that the compressor never crashes.
+ */
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "fuzz_helpers.h"
+#include "lz4.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
+{
+    uint32_t seed = FUZZ_seed(&data, &size);
+    size_t const dstCapacity = FUZZ_rand32(&seed, 0, LZ4_compressBound(size));
+    char* const dst = (char*)malloc(dstCapacity);
+    char* const rt = (char*)malloc(size);
+
+    FUZZ_ASSERT(dst);
+    FUZZ_ASSERT(rt);
+
+    /* If compression succeeds it must round trip correctly. */
+    {
+        int const dstSize = LZ4_compress_default((const char*)data, dst,
+                                                 size, dstCapacity);
+        if (dstSize > 0) {
+            int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size);
+            FUZZ_ASSERT_MSG(rtSize == size, "Incorrect regenerated size");
+            FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!");
+        }
+    }
+
+    if (dstCapacity > 0) {
+        /* Compression succeeds and must round trip correctly. */
+        int compressedSize = size;
+        int const dstSize = LZ4_compress_destSize((const char*)data, dst,
+                                                  &compressedSize, dstCapacity);
+        FUZZ_ASSERT(dstSize > 0);
+        int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size);
+        FUZZ_ASSERT_MSG(rtSize == compressedSize, "Incorrect regenerated size");
+        FUZZ_ASSERT_MSG(!memcmp(data, rt, compressedSize), "Corruption!");
+    }
+
+    free(dst);
+    free(rt);
+
+    return 0;
+}

data/contrib/lz4/ossfuzz/compress_hc_fuzzer.c

@@ -0,0 +1,57 @@
+/**
+ * This fuzz target attempts to compress the fuzzed data with the simple
+ * compression function with an output buffer that may be too small to
+ * ensure that the compressor never crashes.
+ */
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "fuzz_helpers.h"
+#include "lz4.h"
+#include "lz4hc.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
+{
+    uint32_t seed = FUZZ_seed(&data, &size);
+    size_t const dstCapacity = FUZZ_rand32(&seed, 0, LZ4_compressBound(size));
+    char* const dst = (char*)malloc(dstCapacity);
+    char* const rt = (char*)malloc(size);
+    int const level = FUZZ_rand32(&seed, LZ4HC_CLEVEL_MIN, LZ4HC_CLEVEL_MAX);
+
+    FUZZ_ASSERT(dst);
+    FUZZ_ASSERT(rt);
+
+    /* If compression succeeds it must round trip correctly. */
+    {
+        int const dstSize = LZ4_compress_HC((const char*)data, dst, size,
+                                            dstCapacity, level);
+        if (dstSize > 0) {
+            int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size);
+            FUZZ_ASSERT_MSG(rtSize == size, "Incorrect regenerated size");
+            FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!");
+        }
+    }
+
+    if (dstCapacity > 0) {
+        /* Compression succeeds and must round trip correctly. */
+        void* state = malloc(LZ4_sizeofStateHC());
+        FUZZ_ASSERT(state);
+        int compressedSize = size;
+        int const dstSize = LZ4_compress_HC_destSize(state, (const char*)data,
+                                                     dst, &compressedSize,
+                                                     dstCapacity, level);
+        FUZZ_ASSERT(dstSize > 0);
+        int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size);
+        FUZZ_ASSERT_MSG(rtSize == compressedSize, "Incorrect regenerated size");
+        FUZZ_ASSERT_MSG(!memcmp(data, rt, compressedSize), "Corruption!");
+        free(state);
+    }
+
+    free(dst);
+    free(rt);
+
+    return 0;
+}

data/contrib/lz4/ossfuzz/decompress_frame_fuzzer.c

@@ -0,0 +1,67 @@
+/**
+ * This fuzz target attempts to decompress the fuzzed data with the simple
+ * decompression function to ensure the decompressor never crashes.
+ */
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "fuzz_helpers.h"
+#include "lz4.h"
+#define LZ4F_STATIC_LINKING_ONLY
+#include "lz4frame.h"
+#include "lz4_helpers.h"
+
+static void decompress(LZ4F_dctx* dctx, void* dst, size_t dstCapacity,
+                       const void* src, size_t srcSize,
+                       const void* dict, size_t dictSize,
+                       const LZ4F_decompressOptions_t* opts)
+{
+    LZ4F_resetDecompressionContext(dctx);
+    if (dictSize == 0)
+        LZ4F_decompress(dctx, dst, &dstCapacity, src, &srcSize, opts);
+    else
+        LZ4F_decompress_usingDict(dctx, dst, &dstCapacity, src, &srcSize,
+                                  dict, dictSize, opts);
+}
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
+{
+
+    uint32_t seed = FUZZ_seed(&data, &size);
+    size_t const dstCapacity = FUZZ_rand32(&seed, 0, 4 * size);
+    size_t const largeDictSize = 64 * 1024;
+    size_t const dictSize = FUZZ_rand32(&seed, 0, largeDictSize);
+    char* const dst = (char*)malloc(dstCapacity);
+    char* const dict = (char*)malloc(dictSize);
+    LZ4F_decompressOptions_t opts;
+    LZ4F_dctx* dctx;
+    LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);
+
+    FUZZ_ASSERT(dctx);
+    FUZZ_ASSERT(dst);
+    FUZZ_ASSERT(dict);
+
+    /* Prepare the dictionary. The data doesn't matter for decompression. */
+    memset(dict, 0, dictSize);
+
+
+    /* Decompress using multiple configurations. */
+    memset(&opts, 0, sizeof(opts));
+    opts.stableDst = 0;
+    decompress(dctx, dst, dstCapacity, data, size, NULL, 0, &opts);
+    opts.stableDst = 1;
+    decompress(dctx, dst, dstCapacity, data, size, NULL, 0, &opts);
+    opts.stableDst = 0;
+    decompress(dctx, dst, dstCapacity, data, size, dict, dictSize, &opts);
+    opts.stableDst = 1;
+    decompress(dctx, dst, dstCapacity, data, size, dict, dictSize, &opts);
+
+    LZ4F_freeDecompressionContext(dctx);
+    free(dst);
+    free(dict);
+
+    return 0;
+}

data/contrib/lz4/ossfuzz/decompress_fuzzer.c

@@ -0,0 +1,58 @@
+/**
+ * This fuzz target attempts to decompress the fuzzed data with the simple
+ * decompression function to ensure the decompressor never crashes.
+ */
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "fuzz_helpers.h"
+#include "lz4.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
+{
+
+    uint32_t seed = FUZZ_seed(&data, &size);
+    size_t const dstCapacity = FUZZ_rand32(&seed, 0, 4 * size);
+    size_t const smallDictSize = size + 1;
+    size_t const largeDictSize = 64 * 1024 - 1;
+    size_t const dictSize = MAX(smallDictSize, largeDictSize);
+    char* const dst = (char*)malloc(dstCapacity);
+    char* const dict = (char*)malloc(dictSize + size);
+    char* const largeDict = dict;
+    char* const dataAfterDict = dict + dictSize;
+    char* const smallDict = dataAfterDict - smallDictSize;
+
+    FUZZ_ASSERT(dst);
+    FUZZ_ASSERT(dict);
+
+    /* Prepare the dictionary. The data doesn't matter for decompression. */
+    memset(dict, 0, dictSize);
+    memcpy(dataAfterDict, data, size);
+
+    /* Decompress using each possible dictionary configuration. */
+    /* No dictionary. */
+    LZ4_decompress_safe_usingDict((char const*)data, dst, size,
+                                  dstCapacity, NULL, 0);
+    /* Small external dictonary. */
+    LZ4_decompress_safe_usingDict((char const*)data, dst, size,
+                                  dstCapacity, smallDict, smallDictSize);
+    /* Large external dictionary. */
+    LZ4_decompress_safe_usingDict((char const*)data, dst, size,
+                                  dstCapacity, largeDict, largeDictSize);
+    /* Small prefix. */
+    LZ4_decompress_safe_usingDict((char const*)dataAfterDict, dst, size,
+                                  dstCapacity, smallDict, smallDictSize);
+    /* Large prefix. */
+    LZ4_decompress_safe_usingDict((char const*)data, dst, size,
+                                  dstCapacity, largeDict, largeDictSize);
+    /* Partial decompression. */
+    LZ4_decompress_safe_partial((char const*)data, dst, size,
+                                dstCapacity, dstCapacity);
+    free(dst);
+    free(dict);
+
+    return 0;
+}

data/contrib/lz4/ossfuzz/fuzz.h

@@ -0,0 +1,48 @@
+/**
+ * Fuzz target interface.
+ * Fuzz targets have some common parameters passed as macros during compilation.
+ * Check the documentation for each individual fuzzer for more parameters.
+ *
+ * @param FUZZ_RNG_SEED_SIZE:
+ *        The number of bytes of the source to look at when constructing a seed
+ *        for the deterministic RNG. These bytes are discarded before passing
+ *        the data to lz4 functions. Every fuzzer initializes the RNG exactly
+ *        once before doing anything else, even if it is unused.
+ *        Default: 4.
+ * @param LZ4_DEBUG:
+ *        This is a parameter for the lz4 library. Defining `LZ4_DEBUG=1`
+ *        enables assert() statements in the lz4 library. Higher levels enable
+ *        logging, so aren't recommended. Defining `LZ4_DEBUG=1` is
+ *        recommended.
+ * @param LZ4_FORCE_MEMORY_ACCESS:
+ *        This flag controls how the zstd library accesses unaligned memory.
+ *        It can be undefined, or 0 through 2. If it is undefined, it selects
+ *        the method to use based on the compiler. If testing with UBSAN set
+ *        MEM_FORCE_MEMORY_ACCESS=0 to use the standard compliant method.
+ * @param FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+ *        This is the canonical flag to enable deterministic builds for fuzzing.
+ *        Changes to zstd for fuzzing are gated behind this define.
+ *        It is recommended to define this when building zstd for fuzzing.
+ */
+
+#ifndef FUZZ_H
+#define FUZZ_H
+
+#ifndef FUZZ_RNG_SEED_SIZE
+#  define FUZZ_RNG_SEED_SIZE 4
+#endif
+
+#include <stddef.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

data/contrib/lz4/ossfuzz/fuzz_helpers.h

@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+
+/**
+ * Helper functions for fuzzing.
+ */
+
+#ifndef FUZZ_HELPERS_H
+#define FUZZ_HELPERS_H
+
+#include "fuzz.h"
+#include "xxhash.h"
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define LZ4_COMMONDEFS_ONLY
+#ifndef LZ4_SRC_INCLUDED
+#include "lz4.c"   /* LZ4_count, constants, mem */
+#endif
+
+#define MIN(a,b)   ( (a) < (b) ? (a) : (b) )
+#define MAX(a,b)   ( (a) > (b) ? (a) : (b) )
+
+#define FUZZ_QUOTE_IMPL(str) #str
+#define FUZZ_QUOTE(str) FUZZ_QUOTE_IMPL(str)
+
+/**
+ * Asserts for fuzzing that are always enabled.
+ */
+#define FUZZ_ASSERT_MSG(cond, msg)                                             \
+  ((cond) ? (void)0                                                            \
+          : (fprintf(stderr, "%s: %u: Assertion: `%s' failed. %s\n", __FILE__, \
+                     __LINE__, FUZZ_QUOTE(cond), (msg)),                       \
+             abort()))
+#define FUZZ_ASSERT(cond) FUZZ_ASSERT_MSG((cond), "");
+
+#if defined(__GNUC__)
+#define FUZZ_STATIC static __inline __attribute__((unused))
+#elif defined(__cplusplus) ||                                                  \
+    (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+#define FUZZ_STATIC static inline
+#elif defined(_MSC_VER)
+#define FUZZ_STATIC static __inline
+#else
+#define FUZZ_STATIC static
+#endif
+
+/**
+ * Deterministically constructs a seed based on the fuzz input.
+ * Consumes up to the first FUZZ_RNG_SEED_SIZE bytes of the input.
+ */
+FUZZ_STATIC uint32_t FUZZ_seed(uint8_t const **src, size_t* size) {
+    uint8_t const *data = *src;
+    size_t const toHash = MIN(FUZZ_RNG_SEED_SIZE, *size);
+    *size -= toHash;
+    *src += toHash;
+    return XXH32(data, toHash, 0);
+}
+
+#define FUZZ_rotl32(x, r) (((x) << (r)) | ((x) >> (32 - (r))))
+
+FUZZ_STATIC uint32_t FUZZ_rand(uint32_t *state) {
+    static const uint32_t prime1 = 2654435761U;
+    static const uint32_t prime2 = 2246822519U;
+    uint32_t rand32 = *state;
+    rand32 *= prime1;
+    rand32 += prime2;
+    rand32 = FUZZ_rotl32(rand32, 13);
+    *state = rand32;
+    return rand32 >> 5;
+}
+
+/* Returns a random numer in the range [min, max]. */
+FUZZ_STATIC uint32_t FUZZ_rand32(uint32_t *state, uint32_t min, uint32_t max) {
+    uint32_t random = FUZZ_rand(state);
+    return min + (random % (max - min + 1));
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif