zstd-ruby 1.3.8.0 → 1.4.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +6 -5
  3. data/README.md +1 -1
  4. data/ext/zstdruby/libzstd/Makefile +133 -61
  5. data/ext/zstdruby/libzstd/README.md +51 -18
  6. data/ext/zstdruby/libzstd/common/bitstream.h +38 -39
  7. data/ext/zstdruby/libzstd/common/compiler.h +41 -6
  8. data/ext/zstdruby/libzstd/common/cpu.h +1 -1
  9. data/ext/zstdruby/libzstd/common/debug.c +11 -31
  10. data/ext/zstdruby/libzstd/common/debug.h +11 -31
  11. data/ext/zstdruby/libzstd/common/entropy_common.c +13 -33
  12. data/ext/zstdruby/libzstd/common/error_private.c +2 -1
  13. data/ext/zstdruby/libzstd/common/error_private.h +6 -2
  14. data/ext/zstdruby/libzstd/common/fse.h +13 -33
  15. data/ext/zstdruby/libzstd/common/fse_decompress.c +12 -35
  16. data/ext/zstdruby/libzstd/common/huf.h +15 -33
  17. data/ext/zstdruby/libzstd/common/mem.h +75 -2
  18. data/ext/zstdruby/libzstd/common/pool.c +8 -4
  19. data/ext/zstdruby/libzstd/common/pool.h +2 -2
  20. data/ext/zstdruby/libzstd/common/threading.c +52 -6
  21. data/ext/zstdruby/libzstd/common/threading.h +36 -4
  22. data/ext/zstdruby/libzstd/common/xxhash.c +25 -37
  23. data/ext/zstdruby/libzstd/common/xxhash.h +11 -31
  24. data/ext/zstdruby/libzstd/common/zstd_common.c +1 -1
  25. data/ext/zstdruby/libzstd/common/zstd_errors.h +2 -1
  26. data/ext/zstdruby/libzstd/common/zstd_internal.h +203 -22
  27. data/ext/zstdruby/libzstd/compress/fse_compress.c +19 -42
  28. data/ext/zstdruby/libzstd/compress/hist.c +15 -35
  29. data/ext/zstdruby/libzstd/compress/hist.h +12 -32
  30. data/ext/zstdruby/libzstd/compress/huf_compress.c +92 -92
  31. data/ext/zstdruby/libzstd/compress/zstd_compress.c +1460 -1472
  32. data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +330 -65
  33. data/ext/zstdruby/libzstd/compress/zstd_compress_literals.c +158 -0
  34. data/ext/zstdruby/libzstd/compress/zstd_compress_literals.h +29 -0
  35. data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.c +419 -0
  36. data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.h +54 -0
  37. data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.c +845 -0
  38. data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.h +32 -0
  39. data/ext/zstdruby/libzstd/compress/zstd_cwksp.h +525 -0
  40. data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +65 -43
  41. data/ext/zstdruby/libzstd/compress/zstd_double_fast.h +2 -2
  42. data/ext/zstdruby/libzstd/compress/zstd_fast.c +264 -159
  43. data/ext/zstdruby/libzstd/compress/zstd_fast.h +2 -2
  44. data/ext/zstdruby/libzstd/compress/zstd_lazy.c +74 -42
  45. data/ext/zstdruby/libzstd/compress/zstd_lazy.h +2 -2
  46. data/ext/zstdruby/libzstd/compress/zstd_ldm.c +33 -11
  47. data/ext/zstdruby/libzstd/compress/zstd_ldm.h +7 -2
  48. data/ext/zstdruby/libzstd/compress/zstd_opt.c +108 -125
  49. data/ext/zstdruby/libzstd/compress/zstd_opt.h +1 -1
  50. data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +129 -93
  51. data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +46 -28
  52. data/ext/zstdruby/libzstd/decompress/huf_decompress.c +76 -60
  53. data/ext/zstdruby/libzstd/decompress/zstd_ddict.c +14 -10
  54. data/ext/zstdruby/libzstd/decompress/zstd_ddict.h +2 -2
  55. data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +471 -258
  56. data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.c +471 -346
  57. data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.h +3 -3
  58. data/ext/zstdruby/libzstd/decompress/zstd_decompress_internal.h +25 -4
  59. data/ext/zstdruby/libzstd/deprecated/zbuff.h +9 -8
  60. data/ext/zstdruby/libzstd/deprecated/zbuff_common.c +2 -2
  61. data/ext/zstdruby/libzstd/deprecated/zbuff_compress.c +1 -1
  62. data/ext/zstdruby/libzstd/deprecated/zbuff_decompress.c +1 -1
  63. data/ext/zstdruby/libzstd/dictBuilder/cover.c +220 -65
  64. data/ext/zstdruby/libzstd/dictBuilder/cover.h +81 -7
  65. data/ext/zstdruby/libzstd/dictBuilder/fastcover.c +85 -56
  66. data/ext/zstdruby/libzstd/dictBuilder/zdict.c +43 -19
  67. data/ext/zstdruby/libzstd/dictBuilder/zdict.h +73 -35
  68. data/ext/zstdruby/libzstd/dll/example/Makefile +2 -1
  69. data/ext/zstdruby/libzstd/dll/example/build_package.bat +3 -2
  70. data/ext/zstdruby/libzstd/legacy/zstd_legacy.h +49 -15
  71. data/ext/zstdruby/libzstd/legacy/zstd_v01.c +142 -117
  72. data/ext/zstdruby/libzstd/legacy/zstd_v01.h +13 -8
  73. data/ext/zstdruby/libzstd/legacy/zstd_v02.c +54 -25
  74. data/ext/zstdruby/libzstd/legacy/zstd_v02.h +13 -8
  75. data/ext/zstdruby/libzstd/legacy/zstd_v03.c +55 -25
  76. data/ext/zstdruby/libzstd/legacy/zstd_v03.h +13 -8
  77. data/ext/zstdruby/libzstd/legacy/zstd_v04.c +62 -29
  78. data/ext/zstdruby/libzstd/legacy/zstd_v04.h +13 -8
  79. data/ext/zstdruby/libzstd/legacy/zstd_v05.c +145 -109
  80. data/ext/zstdruby/libzstd/legacy/zstd_v05.h +14 -9
  81. data/ext/zstdruby/libzstd/legacy/zstd_v06.c +56 -26
  82. data/ext/zstdruby/libzstd/legacy/zstd_v06.h +11 -6
  83. data/ext/zstdruby/libzstd/legacy/zstd_v07.c +65 -28
  84. data/ext/zstdruby/libzstd/legacy/zstd_v07.h +11 -6
  85. data/ext/zstdruby/libzstd/libzstd.pc.in +3 -2
  86. data/ext/zstdruby/libzstd/zstd.h +921 -597
  87. data/lib/zstd-ruby/version.rb +1 -1
  88. data/zstd-ruby.gemspec +2 -2
  89. metadata +19 -14
  90. data/ext/zstdruby/libzstd/dll/libzstd.def +0 -87
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -19,7 +19,7 @@ extern "C" {
19
19
  * Dependencies
20
20
  ***************************************/
21
21
  #include <stddef.h> /* size_t */
22
- #include "mem.h" /* U64, U32 */
22
+ #include "../common/mem.h" /* U64, U32 */
23
23
 
24
24
 
25
25
  /* *************************************
@@ -33,13 +33,18 @@ extern "C" {
33
33
  size_t ZSTDv05_decompress( void* dst, size_t dstCapacity,
34
34
  const void* src, size_t compressedSize);
35
35
 
36
- /**
37
- ZSTDv05_getFrameSrcSize() : get the source length of a ZSTD frame
38
- compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
39
- return : the number of bytes that would be read to decompress this frame
40
- or an errorCode if it fails (which can be tested using ZSTDv05_isError())
41
- */
42
- size_t ZSTDv05_findFrameCompressedSize(const void* src, size_t compressedSize);
36
+ /**
37
+ ZSTDv05_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.5.x format
38
+ srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
39
+ cSize (output parameter) : the number of bytes that would be read to decompress this frame
40
+ or an error code if it fails (which can be tested using ZSTDv01_isError())
41
+ dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
42
+ or ZSTD_CONTENTSIZE_ERROR if an error occurs
43
+
44
+ note : assumes `cSize` and `dBound` are _not_ NULL.
45
+ */
46
+ void ZSTDv05_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
47
+ size_t* cSize, unsigned long long* dBound);
43
48
 
44
49
  /* *************************************
45
50
  * Helper functions
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -14,7 +14,7 @@
14
14
  #include <stddef.h> /* size_t, ptrdiff_t */
15
15
  #include <string.h> /* memcpy */
16
16
  #include <stdlib.h> /* malloc, free, qsort */
17
- #include "error_private.h"
17
+ #include "../common/error_private.h"
18
18
 
19
19
 
20
20
 
@@ -506,6 +506,8 @@ typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
506
506
  #define FSEv06_ENCODING_STATIC 2
507
507
  #define FSEv06_ENCODING_DYNAMIC 3
508
508
 
509
+ #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
510
+
509
511
  static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
510
512
  1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12,
511
513
  13,14,15,16 };
@@ -858,7 +860,7 @@ MEM_STATIC unsigned BITv06_highbit32 ( U32 val)
858
860
  _BitScanReverse ( &r, val );
859
861
  return (unsigned) r;
860
862
  # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
861
- return 31 - __builtin_clz (val);
863
+ return __builtin_clz (val) ^ 31;
862
864
  # else /* Software version */
863
865
  static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
864
866
  U32 v = val;
@@ -1860,7 +1862,7 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta
1860
1862
 
1861
1863
  if (!srcSize) return ERROR(srcSize_wrong);
1862
1864
  iSize = ip[0];
1863
- //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
1865
+ /* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */
1864
1866
 
1865
1867
  if (iSize >= 128) { /* special header */
1866
1868
  if (iSize >= (242)) { /* RLE */
@@ -2012,7 +2014,7 @@ size_t HUFv06_readDTableX2 (U16* DTable, const void* src, size_t srcSize)
2012
2014
  HUFv06_DEltX2* const dt = (HUFv06_DEltX2*)dtPtr;
2013
2015
 
2014
2016
  HUFv06_STATIC_ASSERT(sizeof(HUFv06_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */
2015
- //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */
2017
+ /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
2016
2018
 
2017
2019
  iSize = HUFv06_readStats(huffWeight, HUFv06_MAX_SYMBOL_VALUE + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
2018
2020
  if (HUFv06_isError(iSize)) return iSize;
@@ -2338,7 +2340,7 @@ size_t HUFv06_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
2338
2340
 
2339
2341
  HUFv06_STATIC_ASSERT(sizeof(HUFv06_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */
2340
2342
  if (memLog > HUFv06_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
2341
- //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */
2343
+ /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
2342
2344
 
2343
2345
  iSize = HUFv06_readStats(weightList, HUFv06_MAX_SYMBOL_VALUE + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
2344
2346
  if (HUFv06_isError(iSize)) return iSize;
@@ -2662,13 +2664,13 @@ size_t HUFv06_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
2662
2664
 
2663
2665
  { U32 algoNb = 0;
2664
2666
  if (Dtime[1] < Dtime[0]) algoNb = 1;
2665
- // if (Dtime[2] < Dtime[algoNb]) algoNb = 2; /* current speed of HUFv06_decompress4X6 is not good */
2667
+ /* if (Dtime[2] < Dtime[algoNb]) algoNb = 2; */ /* current speed of HUFv06_decompress4X6 is not good */
2666
2668
  return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
2667
2669
  }
2668
2670
 
2669
- //return HUFv06_decompress4X2(dst, dstSize, cSrc, cSrcSize); /* multi-streams single-symbol decoding */
2670
- //return HUFv06_decompress4X4(dst, dstSize, cSrc, cSrcSize); /* multi-streams double-symbols decoding */
2671
- //return HUFv06_decompress4X6(dst, dstSize, cSrc, cSrcSize); /* multi-streams quad-symbols decoding */
2671
+ /* return HUFv06_decompress4X2(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams single-symbol decoding */
2672
+ /* return HUFv06_decompress4X4(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams double-symbols decoding */
2673
+ /* return HUFv06_decompress4X6(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams quad-symbols decoding */
2672
2674
  }
2673
2675
  /*
2674
2676
  Common functions of Zstd compression library
@@ -3240,14 +3242,12 @@ static size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
3240
3242
  }
3241
3243
 
3242
3244
  /* FSE table descriptors */
3245
+ if (ip + 4 > iend) return ERROR(srcSize_wrong); /* min : header byte + all 3 are "raw", hence no header, but at least xxLog bits per type */
3243
3246
  { U32 const LLtype = *ip >> 6;
3244
3247
  U32 const Offtype = (*ip >> 4) & 3;
3245
3248
  U32 const MLtype = (*ip >> 2) & 3;
3246
3249
  ip++;
3247
3250
 
3248
- /* check */
3249
- if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
3250
-
3251
3251
  /* Build DTables */
3252
3252
  { size_t const bhSize = ZSTDv06_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
3253
3253
  if (ZSTDv06_isError(bhSize)) return ERROR(corruption_detected);
@@ -3406,7 +3406,7 @@ static size_t ZSTDv06_execSequence(BYTE* op,
3406
3406
  if (sequence.offset < 8) {
3407
3407
  /* close range match, overlap */
3408
3408
  static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
3409
- static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */
3409
+ static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
3410
3410
  int const sub2 = dec64table[sequence.offset];
3411
3411
  op[0] = match[0];
3412
3412
  op[1] = match[1];
@@ -3501,8 +3501,10 @@ static size_t ZSTDv06_decompressSequences(
3501
3501
  { size_t const lastLLSize = litEnd - litPtr;
3502
3502
  if (litPtr > litEnd) return ERROR(corruption_detected); /* too many literals already used */
3503
3503
  if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
3504
- memcpy(op, litPtr, lastLLSize);
3505
- op += lastLLSize;
3504
+ if (lastLLSize > 0) {
3505
+ memcpy(op, litPtr, lastLLSize);
3506
+ op += lastLLSize;
3507
+ }
3506
3508
  }
3507
3509
 
3508
3510
  return op-ostart;
@@ -3654,36 +3656,62 @@ size_t ZSTDv06_decompress(void* dst, size_t dstCapacity, const void* src, size_t
3654
3656
  #endif
3655
3657
  }
3656
3658
 
3657
- size_t ZSTDv06_findFrameCompressedSize(const void* src, size_t srcSize)
3659
+ /* ZSTD_errorFrameSizeInfoLegacy() :
3660
+ assumes `cSize` and `dBound` are _not_ NULL */
3661
+ static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
3662
+ {
3663
+ *cSize = ret;
3664
+ *dBound = ZSTD_CONTENTSIZE_ERROR;
3665
+ }
3666
+
3667
+ void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
3658
3668
  {
3659
3669
  const BYTE* ip = (const BYTE*)src;
3660
3670
  size_t remainingSize = srcSize;
3671
+ size_t nbBlocks = 0;
3661
3672
  blockProperties_t blockProperties = { bt_compressed, 0 };
3662
3673
 
3663
3674
  /* Frame Header */
3664
- { size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src, ZSTDv06_frameHeaderSize_min);
3665
- if (ZSTDv06_isError(frameHeaderSize)) return frameHeaderSize;
3666
- if (MEM_readLE32(src) != ZSTDv06_MAGICNUMBER) return ERROR(prefix_unknown);
3667
- if (srcSize < frameHeaderSize+ZSTDv06_blockHeaderSize) return ERROR(srcSize_wrong);
3675
+ { size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src, srcSize);
3676
+ if (ZSTDv06_isError(frameHeaderSize)) {
3677
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, frameHeaderSize);
3678
+ return;
3679
+ }
3680
+ if (MEM_readLE32(src) != ZSTDv06_MAGICNUMBER) {
3681
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
3682
+ return;
3683
+ }
3684
+ if (srcSize < frameHeaderSize+ZSTDv06_blockHeaderSize) {
3685
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3686
+ return;
3687
+ }
3668
3688
  ip += frameHeaderSize; remainingSize -= frameHeaderSize;
3669
3689
  }
3670
3690
 
3671
3691
  /* Loop on each block */
3672
3692
  while (1) {
3673
3693
  size_t const cBlockSize = ZSTDv06_getcBlockSize(ip, remainingSize, &blockProperties);
3674
- if (ZSTDv06_isError(cBlockSize)) return cBlockSize;
3694
+ if (ZSTDv06_isError(cBlockSize)) {
3695
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
3696
+ return;
3697
+ }
3675
3698
 
3676
3699
  ip += ZSTDv06_blockHeaderSize;
3677
3700
  remainingSize -= ZSTDv06_blockHeaderSize;
3678
- if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
3701
+ if (cBlockSize > remainingSize) {
3702
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3703
+ return;
3704
+ }
3679
3705
 
3680
3706
  if (cBlockSize == 0) break; /* bt_end */
3681
3707
 
3682
3708
  ip += cBlockSize;
3683
3709
  remainingSize -= cBlockSize;
3710
+ nbBlocks++;
3684
3711
  }
3685
3712
 
3686
- return ip - (const BYTE*)src;
3713
+ *cSize = ip - (const BYTE*)src;
3714
+ *dBound = nbBlocks * ZSTDv06_BLOCKSIZE_MAX;
3687
3715
  }
3688
3716
 
3689
3717
  /*_******************************
@@ -3974,7 +4002,9 @@ size_t ZBUFFv06_decompressInit(ZBUFFv06_DCtx* zbd)
3974
4002
  MEM_STATIC size_t ZBUFFv06_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
3975
4003
  {
3976
4004
  size_t length = MIN(dstCapacity, srcSize);
3977
- memcpy(dst, src, length);
4005
+ if (length > 0) {
4006
+ memcpy(dst, src, length);
4007
+ }
3978
4008
  return length;
3979
4009
  }
3980
4010
 
@@ -4083,7 +4113,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
4083
4113
  if (!decodedSize) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */
4084
4114
  zbd->outEnd = zbd->outStart + decodedSize;
4085
4115
  zbd->stage = ZBUFFds_flush;
4086
- // break; /* ZBUFFds_flush follows */
4116
+ /* break; */ /* ZBUFFds_flush follows */
4087
4117
  }
4088
4118
  }
4089
4119
  /* fall-through */
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -43,12 +43,17 @@ ZSTDLIBv06_API size_t ZSTDv06_decompress( void* dst, size_t dstCapacity,
43
43
  const void* src, size_t compressedSize);
44
44
 
45
45
  /**
46
- ZSTDv06_getFrameSrcSize() : get the source length of a ZSTD frame
47
- compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
48
- return : the number of bytes that would be read to decompress this frame
49
- or an errorCode if it fails (which can be tested using ZSTDv06_isError())
46
+ ZSTDv06_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.6.x format
47
+ srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
48
+ cSize (output parameter) : the number of bytes that would be read to decompress this frame
49
+ or an error code if it fails (which can be tested using ZSTDv01_isError())
50
+ dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
51
+ or ZSTD_CONTENTSIZE_ERROR if an error occurs
52
+
53
+ note : assumes `cSize` and `dBound` are _not_ NULL.
50
54
  */
51
- size_t ZSTDv06_findFrameCompressedSize(const void* src, size_t compressedSize);
55
+ void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
56
+ size_t* cSize, unsigned long long* dBound);
52
57
 
53
58
  /* *************************************
54
59
  * Helper functions
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -17,14 +17,14 @@
17
17
  #ifndef XXH_STATIC_LINKING_ONLY
18
18
  # define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
19
19
  #endif
20
- #include "xxhash.h" /* XXH64_* */
20
+ #include "../common/xxhash.h" /* XXH64_* */
21
21
  #include "zstd_v07.h"
22
22
 
23
23
  #define FSEv07_STATIC_LINKING_ONLY /* FSEv07_MIN_TABLELOG */
24
24
  #define HUFv07_STATIC_LINKING_ONLY /* HUFv07_TABLELOG_ABSOLUTEMAX */
25
25
  #define ZSTDv07_STATIC_LINKING_ONLY
26
26
 
27
- #include "error_private.h"
27
+ #include "../common/error_private.h"
28
28
 
29
29
 
30
30
  #ifdef ZSTDv07_STATIC_LINKING_ONLY
@@ -530,7 +530,7 @@ MEM_STATIC unsigned BITv07_highbit32 (U32 val)
530
530
  _BitScanReverse ( &r, val );
531
531
  return (unsigned) r;
532
532
  # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
533
- return 31 - __builtin_clz (val);
533
+ return __builtin_clz (val) ^ 31;
534
534
  # else /* Software version */
535
535
  static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
536
536
  U32 v = val;
@@ -1314,7 +1314,7 @@ size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
1314
1314
 
1315
1315
  if (!srcSize) return ERROR(srcSize_wrong);
1316
1316
  iSize = ip[0];
1317
- //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
1317
+ /* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */
1318
1318
 
1319
1319
  if (iSize >= 128) { /* special header */
1320
1320
  if (iSize >= (242)) { /* RLE */
@@ -1784,7 +1784,7 @@ size_t HUFv07_readDTableX2 (HUFv07_DTable* DTable, const void* src, size_t srcSi
1784
1784
  HUFv07_DEltX2* const dt = (HUFv07_DEltX2*)dtPtr;
1785
1785
 
1786
1786
  HUFv07_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUFv07_DTable));
1787
- //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */
1787
+ /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
1788
1788
 
1789
1789
  iSize = HUFv07_readStats(huffWeight, HUFv07_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
1790
1790
  if (HUFv07_isError(iSize)) return iSize;
@@ -2148,7 +2148,7 @@ size_t HUFv07_readDTableX4 (HUFv07_DTable* DTable, const void* src, size_t srcSi
2148
2148
 
2149
2149
  HUFv07_STATIC_ASSERT(sizeof(HUFv07_DEltX4) == sizeof(HUFv07_DTable)); /* if compilation fails here, assertion is false */
2150
2150
  if (maxTableLog > HUFv07_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge);
2151
- //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */
2151
+ /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
2152
2152
 
2153
2153
  iSize = HUFv07_readStats(weightList, HUFv07_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
2154
2154
  if (HUFv07_isError(iSize)) return iSize;
@@ -2530,8 +2530,8 @@ size_t HUFv07_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
2530
2530
  return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
2531
2531
  }
2532
2532
 
2533
- //return HUFv07_decompress4X2(dst, dstSize, cSrc, cSrcSize); /* multi-streams single-symbol decoding */
2534
- //return HUFv07_decompress4X4(dst, dstSize, cSrc, cSrcSize); /* multi-streams double-symbols decoding */
2533
+ /* return HUFv07_decompress4X2(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams single-symbol decoding */
2534
+ /* return HUFv07_decompress4X4(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams double-symbols decoding */
2535
2535
  }
2536
2536
 
2537
2537
  size_t HUFv07_decompress4X_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
@@ -2740,6 +2740,8 @@ typedef enum { lbt_huffman, lbt_repeat, lbt_raw, lbt_rle } litBlockType_t;
2740
2740
  #define FSEv07_ENCODING_STATIC 2
2741
2741
  #define FSEv07_ENCODING_DYNAMIC 3
2742
2742
 
2743
+ #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
2744
+
2743
2745
  static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2744
2746
  1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12,
2745
2747
  13,14,15,16 };
@@ -3270,7 +3272,9 @@ static size_t ZSTDv07_getcBlockSize(const void* src, size_t srcSize, blockProper
3270
3272
  static size_t ZSTDv07_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
3271
3273
  {
3272
3274
  if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall);
3273
- memcpy(dst, src, srcSize);
3275
+ if (srcSize > 0) {
3276
+ memcpy(dst, src, srcSize);
3277
+ }
3274
3278
  return srcSize;
3275
3279
  }
3276
3280
 
@@ -3468,14 +3472,12 @@ static size_t ZSTDv07_decodeSeqHeaders(int* nbSeqPtr,
3468
3472
  }
3469
3473
 
3470
3474
  /* FSE table descriptors */
3475
+ if (ip + 4 > iend) return ERROR(srcSize_wrong); /* min : header byte + all 3 are "raw", hence no header, but at least xxLog bits per type */
3471
3476
  { U32 const LLtype = *ip >> 6;
3472
3477
  U32 const OFtype = (*ip >> 4) & 3;
3473
3478
  U32 const MLtype = (*ip >> 2) & 3;
3474
3479
  ip++;
3475
3480
 
3476
- /* check */
3477
- if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
3478
-
3479
3481
  /* Build DTables */
3480
3482
  { size_t const llhSize = ZSTDv07_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
3481
3483
  if (ZSTDv07_isError(llhSize)) return ERROR(corruption_detected);
@@ -3631,7 +3633,7 @@ size_t ZSTDv07_execSequence(BYTE* op,
3631
3633
  if (sequence.offset < 8) {
3632
3634
  /* close range match, overlap */
3633
3635
  static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
3634
- static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */
3636
+ static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
3635
3637
  int const sub2 = dec64table[sequence.offset];
3636
3638
  op[0] = match[0];
3637
3639
  op[1] = match[1];
@@ -3712,10 +3714,12 @@ static size_t ZSTDv07_decompressSequences(
3712
3714
 
3713
3715
  /* last literal segment */
3714
3716
  { size_t const lastLLSize = litEnd - litPtr;
3715
- //if (litPtr > litEnd) return ERROR(corruption_detected); /* too many literals already used */
3717
+ /* if (litPtr > litEnd) return ERROR(corruption_detected); */ /* too many literals already used */
3716
3718
  if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall);
3717
- memcpy(op, litPtr, lastLLSize);
3718
- op += lastLLSize;
3719
+ if (lastLLSize > 0) {
3720
+ memcpy(op, litPtr, lastLLSize);
3721
+ op += lastLLSize;
3722
+ }
3719
3723
  }
3720
3724
 
3721
3725
  return op-ostart;
@@ -3776,7 +3780,9 @@ ZSTDLIBv07_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockS
3776
3780
  static size_t ZSTDv07_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length)
3777
3781
  {
3778
3782
  if (length > dstCapacity) return ERROR(dstSize_tooSmall);
3779
- memset(dst, byte, length);
3783
+ if (length > 0) {
3784
+ memset(dst, byte, length);
3785
+ }
3780
3786
  return length;
3781
3787
  }
3782
3788
 
@@ -3895,19 +3901,40 @@ size_t ZSTDv07_decompress(void* dst, size_t dstCapacity, const void* src, size_t
3895
3901
  #endif
3896
3902
  }
3897
3903
 
3898
- size_t ZSTDv07_findFrameCompressedSize(const void* src, size_t srcSize)
3904
+ /* ZSTD_errorFrameSizeInfoLegacy() :
3905
+ assumes `cSize` and `dBound` are _not_ NULL */
3906
+ static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
3907
+ {
3908
+ *cSize = ret;
3909
+ *dBound = ZSTD_CONTENTSIZE_ERROR;
3910
+ }
3911
+
3912
+ void ZSTDv07_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
3899
3913
  {
3900
3914
  const BYTE* ip = (const BYTE*)src;
3901
3915
  size_t remainingSize = srcSize;
3916
+ size_t nbBlocks = 0;
3902
3917
 
3903
3918
  /* check */
3904
- if (srcSize < ZSTDv07_frameHeaderSize_min+ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong);
3919
+ if (srcSize < ZSTDv07_frameHeaderSize_min+ZSTDv07_blockHeaderSize) {
3920
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3921
+ return;
3922
+ }
3905
3923
 
3906
3924
  /* Frame Header */
3907
- { size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src, ZSTDv07_frameHeaderSize_min);
3908
- if (ZSTDv07_isError(frameHeaderSize)) return frameHeaderSize;
3909
- if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) return ERROR(prefix_unknown);
3910
- if (srcSize < frameHeaderSize+ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong);
3925
+ { size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src, srcSize);
3926
+ if (ZSTDv07_isError(frameHeaderSize)) {
3927
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, frameHeaderSize);
3928
+ return;
3929
+ }
3930
+ if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) {
3931
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
3932
+ return;
3933
+ }
3934
+ if (srcSize < frameHeaderSize+ZSTDv07_blockHeaderSize) {
3935
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3936
+ return;
3937
+ }
3911
3938
  ip += frameHeaderSize; remainingSize -= frameHeaderSize;
3912
3939
  }
3913
3940
 
@@ -3915,20 +3942,28 @@ size_t ZSTDv07_findFrameCompressedSize(const void* src, size_t srcSize)
3915
3942
  while (1) {
3916
3943
  blockProperties_t blockProperties;
3917
3944
  size_t const cBlockSize = ZSTDv07_getcBlockSize(ip, remainingSize, &blockProperties);
3918
- if (ZSTDv07_isError(cBlockSize)) return cBlockSize;
3945
+ if (ZSTDv07_isError(cBlockSize)) {
3946
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
3947
+ return;
3948
+ }
3919
3949
 
3920
3950
  ip += ZSTDv07_blockHeaderSize;
3921
3951
  remainingSize -= ZSTDv07_blockHeaderSize;
3922
3952
 
3923
3953
  if (blockProperties.blockType == bt_end) break;
3924
3954
 
3925
- if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
3955
+ if (cBlockSize > remainingSize) {
3956
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3957
+ return;
3958
+ }
3926
3959
 
3927
3960
  ip += cBlockSize;
3928
3961
  remainingSize -= cBlockSize;
3962
+ nbBlocks++;
3929
3963
  }
3930
3964
 
3931
- return ip - (const BYTE*)src;
3965
+ *cSize = ip - (const BYTE*)src;
3966
+ *dBound = nbBlocks * ZSTDv07_BLOCKSIZE_ABSOLUTEMAX;
3932
3967
  }
3933
3968
 
3934
3969
  /*_******************************
@@ -4349,7 +4384,9 @@ size_t ZBUFFv07_decompressInit(ZBUFFv07_DCtx* zbd)
4349
4384
  MEM_STATIC size_t ZBUFFv07_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
4350
4385
  {
4351
4386
  size_t const length = MIN(dstCapacity, srcSize);
4352
- memcpy(dst, src, length);
4387
+ if (length > 0) {
4388
+ memcpy(dst, src, length);
4389
+ }
4353
4390
  return length;
4354
4391
  }
4355
4392