extlz4 0.3.1 → 0.3.2

data/contrib/lz4/lib/lz4hc.h

@@ -198,57 +198,32 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in
 #define LZ4HC_HASH_MASK (LZ4HC_HASHTABLESIZE - 1)
 
 
-#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
-#include <stdint.h>
-
-typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal;
-struct LZ4HC_CCtx_internal
-{
-    uint32_t   hashTable[LZ4HC_HASHTABLESIZE];
-    uint16_t   chainTable[LZ4HC_MAXD];
-    const uint8_t* end;         /* next block here to continue on current prefix */
-    const uint8_t* base;        /* All index relative to this position */
-    const uint8_t* dictBase;    /* alternate base for extDict */
-    uint32_t   dictLimit;       /* below that point, need extDict */
-    uint32_t   lowLimit;        /* below that point, no more dict */
-    uint32_t   nextToUpdate;    /* index from which to continue dictionary update */
-    short      compressionLevel;
-    int8_t     favorDecSpeed;   /* favor decompression speed if this flag set,
-                                   otherwise, favor compression ratio */
-    int8_t     dirty;           /* stream has to be fully reset if this flag is set */
-    const LZ4HC_CCtx_internal* dictCtx;
-};
-
-#else
-
 typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal;
 struct LZ4HC_CCtx_internal
 {
-    unsigned int   hashTable[LZ4HC_HASHTABLESIZE];
-    unsigned short chainTable[LZ4HC_MAXD];
-    const unsigned char* end;        /* next block here to continue on current prefix */
-    const unsigned char* base;       /* All index relative to this position */
-    const unsigned char* dictBase;   /* alternate base for extDict */
-    unsigned int   dictLimit;        /* below that point, need extDict */
-    unsigned int   lowLimit;         /* below that point, no more dict */
-    unsigned int   nextToUpdate;     /* index from which to continue dictionary update */
-    short          compressionLevel;
-    char           favorDecSpeed;    /* favor decompression speed if this flag set,
-                                        otherwise, favor compression ratio */
-    char           dirty;            /* stream has to be fully reset if this flag is set */
+    LZ4_u32   hashTable[LZ4HC_HASHTABLESIZE];
+    LZ4_u16   chainTable[LZ4HC_MAXD];
+    const LZ4_byte* end;       /* next block here to continue on current prefix */
+    const LZ4_byte* base;      /* All index relative to this position */
+    const LZ4_byte* dictBase;  /* alternate base for extDict */
+    LZ4_u32   dictLimit;       /* below that point, need extDict */
+    LZ4_u32   lowLimit;        /* below that point, no more dict */
+    LZ4_u32   nextToUpdate;    /* index from which to continue dictionary update */
+    short     compressionLevel;
+    LZ4_i8    favorDecSpeed;   /* favor decompression speed if this flag set,
+                                  otherwise, favor compression ratio */
+    LZ4_i8    dirty;           /* stream has to be fully reset if this flag is set */
     const LZ4HC_CCtx_internal* dictCtx;
 };
 
-#endif
-
 
 /* Do not use these definitions directly !
  * Declare or allocate an LZ4_streamHC_t instead.
  */
-#define LZ4_STREAMHCSIZE       (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56 + ((sizeof(void*)==16) ? 56 : 0) /* AS400*/ ) /* 262200 or 262256*/
-#define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t))
+#define LZ4_STREAMHCSIZE       262200  /* static size, for inter-version compatibility */
+#define LZ4_STREAMHCSIZE_VOIDP (LZ4_STREAMHCSIZE / sizeof(void*))
 union LZ4_streamHC_u {
-    size_t table[LZ4_STREAMHCSIZE_SIZET];
+    void* table[LZ4_STREAMHCSIZE_VOIDP];
     LZ4HC_CCtx_internal internal_donotuse;
 }; /* previously typedef'd to LZ4_streamHC_t */
 

data/contrib/lz4/ossfuzz/Makefile

@@ -26,7 +26,7 @@
 # ##########################################################################
 
 LZ4DIR  := ../lib
-LIB_FUZZING_ENGINE ?= standaloneengine.o
+LIB_FUZZING_ENGINE ?=
 
 DEBUGLEVEL?= 1
 DEBUGFLAGS = -g -DLZ4_DEBUG=$(DEBUGLEVEL)
@@ -47,6 +47,7 @@ FUZZERS := \
 	round_trip_frame_fuzzer \
 	decompress_frame_fuzzer
 
+.PHONY: all
 all: $(FUZZERS)
 
 # Include a rule to build the static library if calling this target
@@ -58,17 +59,20 @@ $(LZ4DIR)/liblz4.a:
 	$(CC) -c $(LZ4_CFLAGS) $(LZ4_CPPFLAGS) $< -o $@
 
 # Generic rule for generating fuzzers
-%_fuzzer: %_fuzzer.o lz4_helpers.o $(LZ4DIR)/liblz4.a
-	# Compile the standalone code just in case. The OSS-Fuzz code might
-	# override the LIB_FUZZING_ENGINE value to "-fsanitize=fuzzer"
-	$(CC) -c $(LZ4_CFLAGS) $(LZ4_CPPFLAGS) standaloneengine.c -o standaloneengine.o
-
-	# Now compile the actual fuzzer.
+ifeq ($(LIB_FUZZING_ENGINE),)
+  LIB_FUZZING_DEPS := standaloneengine.o
+else
+  LIB_FUZZING_DEPS :=
+endif
+%_fuzzer: %_fuzzer.o lz4_helpers.o fuzz_data_producer.o $(LZ4DIR)/liblz4.a $(LIB_FUZZING_DEPS)
 	$(CXX) $(LZ4_CXXFLAGS) $(LZ4_CPPFLAGS) $(LDFLAGS) $(LIB_FUZZING_ENGINE) $^ -o $@$(EXT)
 
 %_fuzzer_clean:
 	$(RM) $*_fuzzer $*_fuzzer.o standaloneengine.o
 
 .PHONY: clean
-clean: compress_fuzzer_clean decompress_fuzzer_clean
+clean: compress_fuzzer_clean decompress_fuzzer_clean \
+	compress_frame_fuzzer_clean compress_hc_fuzzer_clean \
+	decompress_frame_fuzzer_clean round_trip_frame_fuzzer_clean \
+	round_trip_fuzzer_clean round_trip_hc_fuzzer_clean round_trip_stream_fuzzer_clean
 	$(MAKE) -C $(LZ4DIR) clean

data/contrib/lz4/ossfuzz/compress_frame_fuzzer.c

@@ -13,18 +13,23 @@
 #include "lz4.h"
 #include "lz4frame.h"
 #include "lz4_helpers.h"
+#include "fuzz_data_producer.h"
 
 int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 {
-    uint32_t seed = FUZZ_seed(&data, &size);
-    LZ4F_preferences_t const prefs = FUZZ_randomPreferences(&seed);
+    FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size);
+    LZ4F_preferences_t const prefs = FUZZ_dataProducer_preferences(producer);
+    size_t const dstCapacitySeed = FUZZ_dataProducer_retrieve32(producer);
+    size = FUZZ_dataProducer_remainingBytes(producer);
+
     size_t const compressBound = LZ4F_compressFrameBound(size, &prefs);
-    size_t const dstCapacity = FUZZ_rand32(&seed, 0, compressBound);
+    size_t const dstCapacity = FUZZ_getRange_from_uint32(dstCapacitySeed, 0, compressBound);
+
     char* const dst = (char*)malloc(dstCapacity);
     char* const rt = (char*)malloc(size);
 
-    FUZZ_ASSERT(dst);
-    FUZZ_ASSERT(rt);
+    FUZZ_ASSERT(dst!=NULL);
+    FUZZ_ASSERT(rt!=NULL);
 
     /* If compression succeeds it must round trip correctly. */
     size_t const dstSize =
@@ -37,6 +42,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 
     free(dst);
     free(rt);
+    FUZZ_dataProducer_free(producer);
 
     return 0;
 }

data/contrib/lz4/ossfuzz/compress_fuzzer.c

@@ -10,12 +10,18 @@
 #include <string.h>
 
 #include "fuzz_helpers.h"
+#include "fuzz_data_producer.h"
 #include "lz4.h"
 
 int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 {
-    uint32_t seed = FUZZ_seed(&data, &size);
-    size_t const dstCapacity = FUZZ_rand32(&seed, 0, LZ4_compressBound(size));
+    FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size);
+    size_t const dstCapacitySeed = FUZZ_dataProducer_retrieve32(producer);
+    size = FUZZ_dataProducer_remainingBytes(producer);
+
+    size_t const compressBound = LZ4_compressBound(size);
+    size_t const dstCapacity = FUZZ_getRange_from_uint32(dstCapacitySeed, 0, compressBound);
+
     char* const dst = (char*)malloc(dstCapacity);
     char* const rt = (char*)malloc(size);
 
@@ -46,6 +52,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 
     free(dst);
     free(rt);
+    FUZZ_dataProducer_free(producer);
 
     return 0;
 }

data/contrib/lz4/ossfuzz/compress_hc_fuzzer.c

@@ -10,16 +10,22 @@
 #include <string.h>
 
 #include "fuzz_helpers.h"
+#include "fuzz_data_producer.h"
 #include "lz4.h"
 #include "lz4hc.h"
 
 int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 {
-    uint32_t seed = FUZZ_seed(&data, &size);
-    size_t const dstCapacity = FUZZ_rand32(&seed, 0, LZ4_compressBound(size));
+    FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size);
+    size_t const dstCapacitySeed = FUZZ_dataProducer_retrieve32(producer);
+    size_t const levelSeed = FUZZ_dataProducer_retrieve32(producer);
+    size = FUZZ_dataProducer_remainingBytes(producer);
+
+    size_t const dstCapacity = FUZZ_getRange_from_uint32(dstCapacitySeed, 0, size);
+    int const level = FUZZ_getRange_from_uint32(levelSeed, LZ4HC_CLEVEL_MIN, LZ4HC_CLEVEL_MAX);
+
     char* const dst = (char*)malloc(dstCapacity);
     char* const rt = (char*)malloc(size);
-    int const level = FUZZ_rand32(&seed, LZ4HC_CLEVEL_MIN, LZ4HC_CLEVEL_MAX);
 
     FUZZ_ASSERT(dst);
     FUZZ_ASSERT(rt);
@@ -52,6 +58,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 
     free(dst);
     free(rt);
+    FUZZ_dataProducer_free(producer);
 
     return 0;
 }

data/contrib/lz4/ossfuzz/decompress_frame_fuzzer.c

@@ -9,6 +9,7 @@
 #include <string.h>
 
 #include "fuzz_helpers.h"
+#include "fuzz_data_producer.h"
 #include "lz4.h"
 #define LZ4F_STATIC_LINKING_ONLY
 #include "lz4frame.h"
@@ -29,11 +30,17 @@ static void decompress(LZ4F_dctx* dctx, void* dst, size_t dstCapacity,
 
 int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 {
+    FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size);
+    size_t const dstCapacitySeed = FUZZ_dataProducer_retrieve32(producer);
+    size_t const dictSizeSeed = FUZZ_dataProducer_retrieve32(producer);
+    size = FUZZ_dataProducer_remainingBytes(producer);
 
-    uint32_t seed = FUZZ_seed(&data, &size);
-    size_t const dstCapacity = FUZZ_rand32(&seed, 0, 4 * size);
+    size_t const dstCapacity = FUZZ_getRange_from_uint32(
+      dstCapacitySeed, 0, 4 * size);
     size_t const largeDictSize = 64 * 1024;
-    size_t const dictSize = FUZZ_rand32(&seed, 0, largeDictSize);
+    size_t const dictSize = FUZZ_getRange_from_uint32(
+      dictSizeSeed, 0, largeDictSize);
+
     char* const dst = (char*)malloc(dstCapacity);
     char* const dict = (char*)malloc(dictSize);
     LZ4F_decompressOptions_t opts;
@@ -62,6 +69,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
     LZ4F_freeDecompressionContext(dctx);
     free(dst);
     free(dict);
+    FUZZ_dataProducer_free(producer);
 
     return 0;
 }

data/contrib/lz4/ossfuzz/decompress_fuzzer.c

@@ -9,13 +9,16 @@
 #include <string.h>
 
 #include "fuzz_helpers.h"
+#include "fuzz_data_producer.h"
 #include "lz4.h"
 
 int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 {
+    FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size);
+    size_t const dstCapacitySeed = FUZZ_dataProducer_retrieve32(producer);
+    size = FUZZ_dataProducer_remainingBytes(producer);
 
-    uint32_t seed = FUZZ_seed(&data, &size);
-    size_t const dstCapacity = FUZZ_rand32(&seed, 0, 4 * size);
+    size_t const dstCapacity = FUZZ_getRange_from_uint32(dstCapacitySeed, 0, 4 * size);
     size_t const smallDictSize = size + 1;
     size_t const largeDictSize = 64 * 1024 - 1;
     size_t const dictSize = MAX(smallDictSize, largeDictSize);
@@ -53,6 +56,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
                                 dstCapacity, dstCapacity);
     free(dst);
     free(dict);
+    FUZZ_dataProducer_free(producer);
 
     return 0;
 }

data/contrib/lz4/ossfuzz/fuzz_data_producer.c

@@ -0,0 +1,77 @@
+#include "fuzz_data_producer.h"
+
+struct FUZZ_dataProducer_s{
+  const uint8_t *data;
+  size_t size;
+};
+
+FUZZ_dataProducer_t* FUZZ_dataProducer_create(const uint8_t* data, size_t size) {
+  FUZZ_dataProducer_t* const producer = malloc(sizeof(FUZZ_dataProducer_t));
+
+  FUZZ_ASSERT(producer != NULL);
+
+  producer->data = data;
+  producer->size = size;
+  return producer;
+}
+
+void FUZZ_dataProducer_free(FUZZ_dataProducer_t *producer) { free(producer); }
+
+uint32_t FUZZ_dataProducer_retrieve32(FUZZ_dataProducer_t *producer) {
+    const uint8_t* data = producer->data;
+    const size_t size = producer->size;
+    if (size == 0) {
+        return 0;
+    } else if (size < 4) {
+        producer->size -= 1;
+        return (uint32_t)data[size - 1];
+    } else {
+        producer->size -= 4;
+        return *(data + size - 4);
+    }
+}
+
+uint32_t FUZZ_getRange_from_uint32(uint32_t seed, uint32_t min, uint32_t max)
+{
+    uint32_t range = max - min;
+    if (range == 0xffffffff) {
+      return seed;
+    }
+    return min + seed % (range + 1);
+}
+
+uint32_t FUZZ_dataProducer_range32(FUZZ_dataProducer_t* producer,
+    uint32_t min, uint32_t max)
+{
+    size_t const seed = FUZZ_dataProducer_retrieve32(producer);
+    return FUZZ_getRange_from_uint32(seed, min, max);
+}
+
+LZ4F_frameInfo_t FUZZ_dataProducer_frameInfo(FUZZ_dataProducer_t* producer)
+{
+    LZ4F_frameInfo_t info = LZ4F_INIT_FRAMEINFO;
+    info.blockSizeID = FUZZ_dataProducer_range32(producer, LZ4F_max64KB - 1, LZ4F_max4MB);
+    if (info.blockSizeID < LZ4F_max64KB) {
+        info.blockSizeID = LZ4F_default;
+    }
+    info.blockMode = FUZZ_dataProducer_range32(producer, LZ4F_blockLinked, LZ4F_blockIndependent);
+    info.contentChecksumFlag = FUZZ_dataProducer_range32(producer, LZ4F_noContentChecksum,
+                                           LZ4F_contentChecksumEnabled);
+    info.blockChecksumFlag = FUZZ_dataProducer_range32(producer, LZ4F_noBlockChecksum,
+                                         LZ4F_blockChecksumEnabled);
+    return info;
+}
+
+LZ4F_preferences_t FUZZ_dataProducer_preferences(FUZZ_dataProducer_t* producer)
+{
+    LZ4F_preferences_t prefs = LZ4F_INIT_PREFERENCES;
+    prefs.frameInfo = FUZZ_dataProducer_frameInfo(producer);
+    prefs.compressionLevel = FUZZ_dataProducer_range32(producer, 0, LZ4HC_CLEVEL_MAX + 3) - 3;
+    prefs.autoFlush = FUZZ_dataProducer_range32(producer, 0, 1);
+    prefs.favorDecSpeed = FUZZ_dataProducer_range32(producer, 0, 1);
+    return prefs;
+}
+
+size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer){
+  return producer->size;
+}

data/contrib/lz4/ossfuzz/fuzz_data_producer.h

@@ -0,0 +1,36 @@
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "fuzz_helpers.h"
+#include "lz4frame.h"
+#include "lz4hc.h"
+
+/* Struct used for maintaining the state of the data */
+typedef struct FUZZ_dataProducer_s FUZZ_dataProducer_t;
+
+/* Returns a data producer state struct. Use for producer initialization. */
+FUZZ_dataProducer_t *FUZZ_dataProducer_create(const uint8_t *data, size_t size);
+
+/* Frees the data producer */
+void FUZZ_dataProducer_free(FUZZ_dataProducer_t *producer);
+
+/* Returns 32 bits from the end of data */
+uint32_t FUZZ_dataProducer_retrieve32(FUZZ_dataProducer_t *producer);
+
+/* Returns value between [min, max] */
+uint32_t FUZZ_getRange_from_uint32(uint32_t seed, uint32_t min, uint32_t max);
+
+/* Combination of above two functions for non adaptive use cases. ie where size is not involved */
+uint32_t FUZZ_dataProducer_range32(FUZZ_dataProducer_t *producer, uint32_t min,
+                                  uint32_t max);
+
+/* Returns lz4 preferences */
+LZ4F_preferences_t FUZZ_dataProducer_preferences(FUZZ_dataProducer_t* producer);
+
+/* Returns lz4 frame info */
+LZ4F_frameInfo_t FUZZ_dataProducer_frameInfo(FUZZ_dataProducer_t* producer);
+
+/* Returns the size of the remaining bytes of data in the producer */
+size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer);

data/contrib/lz4/ossfuzz/round_trip_frame_fuzzer.c

@@ -12,14 +12,17 @@
 #include "lz4.h"
 #include "lz4frame.h"
 #include "lz4_helpers.h"
+#include "fuzz_data_producer.h"
 
 int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 {
-    uint32_t seed = FUZZ_seed(&data, &size);
-    LZ4F_preferences_t const prefs = FUZZ_randomPreferences(&seed);
-    size_t const dstCapacity = LZ4F_compressFrameBound(size, &prefs);
+    FUZZ_dataProducer_t* producer = FUZZ_dataProducer_create(data, size);
+    LZ4F_preferences_t const prefs = FUZZ_dataProducer_preferences(producer);
+    size = FUZZ_dataProducer_remainingBytes(producer);
+
+    size_t const dstCapacity = LZ4F_compressFrameBound(LZ4_compressBound(size), &prefs);
     char* const dst = (char*)malloc(dstCapacity);
-    char* const rt = (char*)malloc(size);
+    char* const rt = (char*)malloc(FUZZ_dataProducer_remainingBytes(producer));
 
     FUZZ_ASSERT(dst);
     FUZZ_ASSERT(rt);
@@ -34,6 +37,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 
     free(dst);
     free(rt);
+    FUZZ_dataProducer_free(producer);
 
     return 0;
 }

data/contrib/lz4/ossfuzz/round_trip_fuzzer.c

@@ -10,11 +10,17 @@
 
 #include "fuzz_helpers.h"
 #include "lz4.h"
+#include "fuzz_data_producer.h"
 
 int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 {
-    uint32_t seed = FUZZ_seed(&data, &size);
+    FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size);
+    size_t const partialCapacitySeed = FUZZ_dataProducer_retrieve32(producer);
+    size = FUZZ_dataProducer_remainingBytes(producer);
+
+    size_t const partialCapacity = FUZZ_getRange_from_uint32(partialCapacitySeed, 0, size);
     size_t const dstCapacity = LZ4_compressBound(size);
+
     char* const dst = (char*)malloc(dstCapacity);
     char* const rt = (char*)malloc(size);
 
@@ -32,7 +38,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 
     /* Partial decompression must succeed. */
     {
-        size_t const partialCapacity = FUZZ_rand32(&seed, 0, size);
         char* const partial = (char*)malloc(partialCapacity);
         FUZZ_ASSERT(partial);
         int const partialSize = LZ4_decompress_safe_partial(
@@ -43,8 +48,10 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
         free(partial);
     }
 
+
     free(dst);
     free(rt);
+    FUZZ_dataProducer_free(producer);
 
     return 0;
 }