extlz4 0.3.3 → 0.3.4

data/contrib/lz4/ossfuzz/decompress_fuzzer.c

@@ -39,7 +39,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
     /* No dictionary. */
     LZ4_decompress_safe_usingDict((char const*)data, dst, size,
                                   dstCapacity, NULL, 0);
-    /* Small external dictonary. */
+    /* Small external dictionary. */
     LZ4_decompress_safe_usingDict((char const*)data, dst, size,
                                   dstCapacity, smallDict, smallDictSize);
     /* Large external dictionary. */
@@ -49,11 +49,27 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
     LZ4_decompress_safe_usingDict((char const*)dataAfterDict, dst, size,
                                   dstCapacity, smallDict, smallDictSize);
     /* Large prefix. */
-    LZ4_decompress_safe_usingDict((char const*)data, dst, size,
+    LZ4_decompress_safe_usingDict((char const*)dataAfterDict, dst, size,
                                   dstCapacity, largeDict, largeDictSize);
     /* Partial decompression. */
     LZ4_decompress_safe_partial((char const*)data, dst, size,
                                 dstCapacity, dstCapacity);
+    /* Partial decompression using each possible dictionary configuration. */
+    /* Partial decompression with no dictionary. */
+    LZ4_decompress_safe_partial_usingDict((char const*)data, dst, size,
+                                  dstCapacity, dstCapacity, NULL, 0);
+    /* Partial decompression with small external dictionary. */
+    LZ4_decompress_safe_partial_usingDict((char const*)data, dst, size,
+                                  dstCapacity, dstCapacity, smallDict, smallDictSize);
+    /* Partial decompression with large external dictionary. */
+    LZ4_decompress_safe_partial_usingDict((char const*)data, dst, size,
+                                  dstCapacity, dstCapacity, largeDict, largeDictSize);
+    /* Partial decompression with small prefix. */
+    LZ4_decompress_safe_partial_usingDict((char const*)dataAfterDict, dst, size,
+                                  dstCapacity, dstCapacity, smallDict, smallDictSize);
+    /* Partial decompression wtih large prefix. */
+    LZ4_decompress_safe_partial_usingDict((char const*)dataAfterDict, dst, size,
+                                  dstCapacity, dstCapacity, largeDict, largeDictSize);
     free(dst);
     free(dict);
     FUZZ_dataProducer_free(producer);

data/contrib/lz4/ossfuzz/fuzz_helpers.h

@@ -4,7 +4,8 @@
  *
  * This source code is licensed under both the BSD-style license (found in the
  * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
+ * in the COPYING file in the root directory of this source tree),
+ * meaning you may select, at your option, one of the above-listed licenses.
  */
 
 /**
@@ -81,7 +82,7 @@ FUZZ_STATIC uint32_t FUZZ_rand(uint32_t *state) {
     return rand32 >> 5;
 }
 
-/* Returns a random numer in the range [min, max]. */
+/* Returns a random number in the range [min, max]. */
 FUZZ_STATIC uint32_t FUZZ_rand32(uint32_t *state, uint32_t min, uint32_t max) {
     uint32_t random = FUZZ_rand(state);
     return min + (random % (max - min + 1));

data/contrib/lz4/ossfuzz/round_trip_frame_uncompressed_fuzzer.c

@@ -0,0 +1,134 @@
+/**
+ * This fuzz target performs a lz4 round-trip test (compress & decompress),
+ * compares the result with the original, and calls abort() on corruption.
+ */
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "fuzz_data_producer.h"
+#include "fuzz_helpers.h"
+#include "lz4.h"
+#include "lz4_helpers.h"
+#include "lz4frame.h"
+#include "lz4frame_static.h"
+
+static void decompress(LZ4F_dctx *dctx, void *src, void *dst,
+                       size_t dstCapacity, size_t readSize) {
+    size_t ret = 1;
+    const void *srcPtr = (const char *) src;
+    void *dstPtr = (char *) dst;
+    const void *const srcEnd = (const char *) srcPtr + readSize;
+
+    while (ret != 0) {
+        while (srcPtr < srcEnd && ret != 0) {
+            /* Any data within dst has been flushed at this stage */
+            size_t dstSize = dstCapacity;
+            size_t srcSize = (const char *) srcEnd - (const char *) srcPtr;
+            ret = LZ4F_decompress(dctx, dstPtr, &dstSize, srcPtr, &srcSize,
+                    /* LZ4F_decompressOptions_t */ NULL);
+            FUZZ_ASSERT(!LZ4F_isError(ret));
+
+            /* Update input */
+            srcPtr = (const char *) srcPtr + srcSize;
+            dstPtr = (char *) dstPtr + dstSize;
+        }
+
+        FUZZ_ASSERT(srcPtr <= srcEnd);
+    }
+}
+
+static void compress_round_trip(const uint8_t *data, size_t size,
+                                FUZZ_dataProducer_t *producer, LZ4F_preferences_t const prefs) {
+
+    // Choose random uncompressed offset start and end by producing seeds from random data, calculate the remaining
+    // data size that will be used for compression later and use the seeds to actually calculate the offsets
+    size_t const uncompressedOffsetSeed = FUZZ_dataProducer_retrieve32(producer);
+    size_t const uncompressedEndOffsetSeed = FUZZ_dataProducer_retrieve32(producer);
+    size = FUZZ_dataProducer_remainingBytes(producer);
+
+    size_t const uncompressedOffset = FUZZ_getRange_from_uint32(uncompressedOffsetSeed, 0, size);
+    size_t const uncompressedEndOffset = FUZZ_getRange_from_uint32(uncompressedEndOffsetSeed, uncompressedOffset, size);
+    size_t const uncompressedSize = uncompressedEndOffset - uncompressedOffset;
+    FUZZ_ASSERT(uncompressedOffset <= uncompressedEndOffset);
+    FUZZ_ASSERT(uncompressedEndOffset <= size);
+
+    const uint8_t *const uncompressedData = data + uncompressedOffset;
+
+    size_t const dstCapacity =
+            LZ4F_compressFrameBound(LZ4_compressBound(size), &prefs) +
+            uncompressedSize;
+    char *const dst = (char *) malloc(dstCapacity);
+    size_t rtCapacity = dstCapacity;
+    char *const rt = (char *) malloc(rtCapacity);
+
+    FUZZ_ASSERT(dst);
+    FUZZ_ASSERT(rt);
+
+    /* Compression must succeed and round trip correctly. */
+    LZ4F_compressionContext_t ctx;
+    size_t const ctxCreation = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
+    FUZZ_ASSERT(!LZ4F_isError(ctxCreation));
+
+    size_t const headerSize = LZ4F_compressBegin(ctx, dst, dstCapacity, &prefs);
+    FUZZ_ASSERT(!LZ4F_isError(headerSize));
+    size_t compressedSize = headerSize;
+
+    /* Compress data before uncompressed offset */
+    size_t lz4Return = LZ4F_compressUpdate(ctx, dst + compressedSize, dstCapacity,
+                                           data, uncompressedOffset, NULL);
+    FUZZ_ASSERT(!LZ4F_isError(lz4Return));
+    compressedSize += lz4Return;
+
+    /* Add uncompressed data */
+    lz4Return = LZ4F_uncompressedUpdate(ctx, dst + compressedSize, dstCapacity,
+                                        uncompressedData, uncompressedSize, NULL);
+    FUZZ_ASSERT(!LZ4F_isError(lz4Return));
+    compressedSize += lz4Return;
+
+    /* Compress data after uncompressed offset */
+    lz4Return = LZ4F_compressUpdate(ctx, dst + compressedSize, dstCapacity,
+                                    data + uncompressedEndOffset,
+                                    size - uncompressedEndOffset, NULL);
+    FUZZ_ASSERT(!LZ4F_isError(lz4Return));
+    compressedSize += lz4Return;
+
+    /* Finish compression */
+    lz4Return = LZ4F_compressEnd(ctx, dst + compressedSize, dstCapacity, NULL);
+    FUZZ_ASSERT(!LZ4F_isError(lz4Return));
+    compressedSize += lz4Return;
+
+    LZ4F_decompressOptions_t opts;
+    memset(&opts, 0, sizeof(opts));
+    opts.stableDst = 1;
+    LZ4F_dctx *dctx;
+    LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);
+    FUZZ_ASSERT(dctx);
+
+    decompress(dctx, dst, rt, rtCapacity, compressedSize);
+
+    LZ4F_freeDecompressionContext(dctx);
+
+    FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!");
+
+    free(dst);
+    free(rt);
+
+    FUZZ_dataProducer_free(producer);
+    LZ4F_freeCompressionContext(ctx);
+}
+
+static void compress_independent_block_mode(const uint8_t *data, size_t size) {
+    FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size);
+    LZ4F_preferences_t prefs = FUZZ_dataProducer_preferences(producer);
+    prefs.frameInfo.blockMode = LZ4F_blockIndependent;
+    compress_round_trip(data, size, producer, prefs);
+}
+
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+    compress_independent_block_mode(data, size);
+    return 0;
+}

data/contrib/lz4/ossfuzz/round_trip_fuzzer.c

@@ -20,11 +20,16 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 
     size_t const partialCapacity = FUZZ_getRange_from_uint32(partialCapacitySeed, 0, size);
     size_t const dstCapacity = LZ4_compressBound(size);
-
-    char* const dst = (char*)malloc(dstCapacity);
+    size_t const largeSize = 64 * 1024 - 1;
+    size_t const smallSize = 1024;
+    char* const dstPlusLargePrefix = (char*)malloc(dstCapacity + largeSize);
+    FUZZ_ASSERT(dstPlusLargePrefix);
+    char* const dstPlusSmallPrefix = dstPlusLargePrefix + largeSize - smallSize;
+    char* const largeDict = (char*)malloc(largeSize);
+    FUZZ_ASSERT(largeDict);
+    char* const smallDict = largeDict + largeSize - smallSize;
+    char* const dst = dstPlusLargePrefix + largeSize;
     char* const rt = (char*)malloc(size);
-
-    FUZZ_ASSERT(dst);
     FUZZ_ASSERT(rt);
 
     /* Compression must succeed and round trip correctly. */
@@ -47,9 +52,64 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
         FUZZ_ASSERT_MSG(!memcmp(data, partial, partialSize), "Corruption!");
         free(partial);
     }
+    /* Partial decompression using dict with no dict. */
+    {
+        char* const partial = (char*)malloc(partialCapacity);
+        FUZZ_ASSERT(partial);
+        int const partialSize = LZ4_decompress_safe_partial_usingDict(
+                dst, partial, dstSize, partialCapacity, partialCapacity, NULL, 0);
+        FUZZ_ASSERT(partialSize >= 0);
+        FUZZ_ASSERT_MSG(partialSize == partialCapacity, "Incorrect size");
+        FUZZ_ASSERT_MSG(!memcmp(data, partial, partialSize), "Corruption!");
+        free(partial);
+    }
+    /* Partial decompression using dict with small prefix as dict */
+    {
+        char* const partial = (char*)malloc(partialCapacity);
+        FUZZ_ASSERT(partial);
+        int const partialSize = LZ4_decompress_safe_partial_usingDict(
+                dst, partial, dstSize, partialCapacity, partialCapacity, dstPlusSmallPrefix, smallSize);
+        FUZZ_ASSERT(partialSize >= 0);
+        FUZZ_ASSERT_MSG(partialSize == partialCapacity, "Incorrect size");
+        FUZZ_ASSERT_MSG(!memcmp(data, partial, partialSize), "Corruption!");
+        free(partial);
+    }
+    /* Partial decompression using dict with large prefix as dict */
+    {
+        char* const partial = (char*)malloc(partialCapacity);
+        FUZZ_ASSERT(partial);
+        int const partialSize = LZ4_decompress_safe_partial_usingDict(
+                dst, partial, dstSize, partialCapacity, partialCapacity, dstPlusLargePrefix, largeSize);
+        FUZZ_ASSERT(partialSize >= 0);
+        FUZZ_ASSERT_MSG(partialSize == partialCapacity, "Incorrect size");
+        FUZZ_ASSERT_MSG(!memcmp(data, partial, partialSize), "Corruption!");
+        free(partial);
+    }
+    /* Partial decompression using dict with small external dict */
+    {
+        char* const partial = (char*)malloc(partialCapacity);
+        FUZZ_ASSERT(partial);
+        int const partialSize = LZ4_decompress_safe_partial_usingDict(
+                dst, partial, dstSize, partialCapacity, partialCapacity, smallDict, smallSize);
+        FUZZ_ASSERT(partialSize >= 0);
+        FUZZ_ASSERT_MSG(partialSize == partialCapacity, "Incorrect size");
+        FUZZ_ASSERT_MSG(!memcmp(data, partial, partialSize), "Corruption!");
+        free(partial);
+    }
+    /* Partial decompression using dict with large external dict */
+    {
+        char* const partial = (char*)malloc(partialCapacity);
+        FUZZ_ASSERT(partial);
+        int const partialSize = LZ4_decompress_safe_partial_usingDict(
+                dst, partial, dstSize, partialCapacity, partialCapacity, largeDict, largeSize);
+        FUZZ_ASSERT(partialSize >= 0);
+        FUZZ_ASSERT_MSG(partialSize == partialCapacity, "Incorrect size");
+        FUZZ_ASSERT_MSG(!memcmp(data, partial, partialSize), "Corruption!");
+        free(partial);
+    }
 
-
-    free(dst);
+    free(dstPlusLargePrefix);
+    free(largeDict);
     free(rt);
     FUZZ_dataProducer_free(producer);
 

data/ext/blockapi.c

@@ -188,7 +188,7 @@ aux_shouldbe_string(VALUE obj)
 static inline size_t
 aux_lz4_compressbound(VALUE src)
 {
-    return LZ4_compressBound(RSTRING_LEN(src));
+    return LZ4_compressBound(rb_long2int(RSTRING_LEN(src)));
 }
 
 enum {
@@ -441,7 +441,7 @@ blkenc_setup(int argc, VALUE argv[], struct blockencoder *p, VALUE predict)
          * NOTE: すぐ下で LZ4_saveDict() を実行するため、
          * NOTE: p->predict のバッファ領域が保持されることはない。
          */
-        p->traits->loaddict(p->context, RSTRING_PTR(p->predict), RSTRING_LEN(p->predict));
+        p->traits->loaddict(p->context, RSTRING_PTR(p->predict), rb_long2int(RSTRING_LEN(p->predict)));
     }
 
     p->prefixsize = p->traits->savedict(p->context, p->prefix, sizeof(p->prefix));
@@ -492,7 +492,7 @@ blkenc_update(int argc, VALUE argv[], VALUE enc)
     char *srcp;
     size_t srcsize;
     RSTRING_GETMEM(src, srcp, srcsize);
-    int s = p->traits->update(p->context, srcp, RSTRING_PTR(dest), srcsize, maxsize, p->level);
+    int s = p->traits->update(p->context, srcp, RSTRING_PTR(dest), aux_size2int(srcsize), aux_size2int(maxsize), p->level);
     if (s <= 0) {
         rb_raise(extlz4_eError,
                 "destsize too small (given destsize is %"PRIuSIZE")",
@@ -673,7 +673,7 @@ blkenc_s_encode(int argc, VALUE argv[], VALUE lz4)
     aux_str_reserve(dest, maxsize);
     rb_str_set_len(dest, 0);
 
-    int size = encoder(RSTRING_PTR(src), RSTRING_PTR(dest), srcsize, maxsize, level);
+    int size = encoder(RSTRING_PTR(src), RSTRING_PTR(dest), aux_size2int(srcsize), aux_size2int(maxsize), level);
     if (size <= 0) {
         rb_raise(extlz4_eError,
                  "failed LZ4 compress - maxsize is too small, or out of memory");
@@ -870,8 +870,8 @@ blkdec_update(int argc, VALUE argv[], VALUE dec)
     const char *srcp;
     size_t srcsize;
     RSTRING_GETMEM(src, srcp, srcsize);
-    LZ4_setStreamDecode(p->context, p->dictbuf, p->dictsize);
-    int s = aux_LZ4_decompress_safe_continue(p->context, srcp, RSTRING_PTR(dest), srcsize, maxsize);
+    LZ4_setStreamDecode(p->context, p->dictbuf, aux_size2int(p->dictsize));
+    int s = aux_LZ4_decompress_safe_continue(p->context, srcp, RSTRING_PTR(dest), aux_size2int(srcsize), aux_size2int(maxsize));
     if (s < 0) {
         rb_raise(extlz4_eError,
                 "`max_dest_size' too small, or corrupt lz4'd data");
@@ -879,7 +879,7 @@ blkdec_update(int argc, VALUE argv[], VALUE dec)
     rb_str_set_len(dest, s);
 
     /* copy prefix */
-    if (s < sizeof(p->dictbuf)) {
+    if ((size_t)s < sizeof(p->dictbuf)) {
         ssize_t discard = (p->dictsize + s) - sizeof(p->dictbuf);
         if (discard > 0) {
             size_t remain = p->dictsize - discard;
@@ -967,7 +967,7 @@ blkdec_s_decode(int argc, VALUE argv[], VALUE lz4)
     aux_str_reserve(dest, maxsize);
     rb_str_set_len(dest, 0);
 
-    int size = LZ4_decompress_safe(RSTRING_PTR(src), RSTRING_PTR(dest), RSTRING_LEN(src), maxsize);
+    int size = LZ4_decompress_safe(RSTRING_PTR(src), RSTRING_PTR(dest), rb_long2int(RSTRING_LEN(src)), aux_size2int(maxsize));
     if (size < 0) {
         rb_raise(extlz4_eError,
                  "failed LZ4_decompress_safe - max_dest_size is too small, or data is corrupted");

data/ext/extlz4.h

@@ -111,4 +111,16 @@ getref(VALUE obj, const rb_data_type_t *type)
     return checkref(obj, getrefp(obj, type));
 }
 
+static inline int
+aux_size2int(size_t n)
+{
+    int m = (int)n;
+
+    if (m < 0 || (size_t)m != n) {
+        rb_raise(rb_eRangeError, "out of range integer conversion (size_t to int)");
+    }
+
+    return m;
+}
+
 #endif /* !EXTLZ4_H */

data/ext/frameapi.c

@@ -423,7 +423,7 @@ aux_read(VALUE obj, size_t size, VALUE buf)
     if (NIL_P(AUX_FUNCALL(obj, id_read, SIZET2NUM(size), buf))) {
         return Qnil;
     } else {
-        if (RSTRING_LEN(buf) > size) {
+        if ((size_t)RSTRING_LEN(buf) > size) {
             rb_raise(rb_eRuntimeError, "most read (%d, but expected to %d)", (int)RSTRING_LEN(buf), (int)size);
         }
         return buf;
@@ -511,7 +511,7 @@ fdec_read_fetch(VALUE dec, struct decoder *p)
         rb_str_set_len(p->inbuf, 0);
     }
 
-    while (RSTRING_LEN(p->inbuf) < p->status) {
+    while ((size_t)RSTRING_LEN(p->inbuf) < p->status) {
         p->readbuf = aux_read(p->inport, p->status - RSTRING_LEN(p->inbuf), p->readbuf);
         if (NIL_P(p->readbuf)) {
             rb_raise(rb_eRuntimeError,
@@ -547,7 +547,7 @@ fdec_read_decode(VALUE dec, struct decoder *p, char *dest, size_t size)
             break;
         }
 
-        if (p->status > 0 && p->outoff >= RSTRING_LEN(p->outbuf)) {
+        if (p->status > 0 && p->outoff >= (size_t)RSTRING_LEN(p->outbuf)) {
             fdec_read_fetch(dec, p);
         }
 

data/ext/hashargs.c

@@ -11,6 +11,12 @@
 
 #include "hashargs.h"
 
+#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
+# define aux_noreturn _Noreturn
+#else
+# define aux_noreturn
+#endif
+
 struct rbx_scanhash_args
 {
     struct rbx_scanhash_arg *args;
@@ -18,7 +24,7 @@ struct rbx_scanhash_args
     VALUE rest;
 };
 
-static void
+aux_noreturn static void
 rbx_scanhash_error(ID given, struct rbx_scanhash_arg *args, const struct rbx_scanhash_arg *end)
 {
     // 引数の数が㌧でもない数の場合、よくないことが起きそう。

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: extlz4
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.4
 platform: ruby
 authors:
 - dearblue
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-10-05 00:00:00.000000000 Z
+date: 2023-06-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -100,13 +100,25 @@ files:
 - contrib/lz4/build/VS2017/lz4.sln
 - contrib/lz4/build/VS2017/lz4/lz4.rc
 - contrib/lz4/build/VS2017/lz4/lz4.vcxproj
+- contrib/lz4/build/VS2022/datagen/datagen.vcxproj
+- contrib/lz4/build/VS2022/frametest/frametest.vcxproj
+- contrib/lz4/build/VS2022/fullbench-dll/fullbench-dll.vcxproj
+- contrib/lz4/build/VS2022/fullbench/fullbench.vcxproj
+- contrib/lz4/build/VS2022/fuzzer/fuzzer.vcxproj
+- contrib/lz4/build/VS2022/liblz4-dll/liblz4-dll.rc
+- contrib/lz4/build/VS2022/liblz4-dll/liblz4-dll.vcxproj
+- contrib/lz4/build/VS2022/liblz4/liblz4.vcxproj
+- contrib/lz4/build/VS2022/lz4.sln
 - contrib/lz4/build/cmake/CMakeLists.txt
+- contrib/lz4/build/cmake/lz4Config.cmake.in
 - contrib/lz4/lib/LICENSE
 - contrib/lz4/lib/README.md
 - contrib/lz4/lib/liblz4-dll.rc.in
 - contrib/lz4/lib/liblz4.pc.in
 - contrib/lz4/lib/lz4.c
 - contrib/lz4/lib/lz4.h
+- contrib/lz4/lib/lz4file.c
+- contrib/lz4/lib/lz4file.h
 - contrib/lz4/lib/lz4frame.c
 - contrib/lz4/lib/lz4frame.h
 - contrib/lz4/lib/lz4frame_static.h
@@ -128,13 +140,12 @@ files:
 - contrib/lz4/ossfuzz/lz4_helpers.h
 - contrib/lz4/ossfuzz/ossfuzz.sh
 - contrib/lz4/ossfuzz/round_trip_frame_fuzzer.c
+- contrib/lz4/ossfuzz/round_trip_frame_uncompressed_fuzzer.c
 - contrib/lz4/ossfuzz/round_trip_fuzzer.c
 - contrib/lz4/ossfuzz/round_trip_hc_fuzzer.c
 - contrib/lz4/ossfuzz/round_trip_stream_fuzzer.c
 - contrib/lz4/ossfuzz/standaloneengine.c
 - contrib/lz4/ossfuzz/travisoss.sh
-- contrib/lz4/tmp
-- contrib/lz4/tmpsparse
 - examples/frameapi.rb
 - ext/blockapi.c
 - ext/depend
@@ -176,7 +187,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.4.13
 signing_key:
 specification_version: 4
 summary: ruby bindings for LZ4