zstd-ruby 1.3.8.0 → 1.4.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (90) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +6 -5
  3. data/README.md +1 -1
  4. data/ext/zstdruby/libzstd/Makefile +133 -61
  5. data/ext/zstdruby/libzstd/README.md +51 -18
  6. data/ext/zstdruby/libzstd/common/bitstream.h +38 -39
  7. data/ext/zstdruby/libzstd/common/compiler.h +41 -6
  8. data/ext/zstdruby/libzstd/common/cpu.h +1 -1
  9. data/ext/zstdruby/libzstd/common/debug.c +11 -31
  10. data/ext/zstdruby/libzstd/common/debug.h +11 -31
  11. data/ext/zstdruby/libzstd/common/entropy_common.c +13 -33
  12. data/ext/zstdruby/libzstd/common/error_private.c +2 -1
  13. data/ext/zstdruby/libzstd/common/error_private.h +6 -2
  14. data/ext/zstdruby/libzstd/common/fse.h +13 -33
  15. data/ext/zstdruby/libzstd/common/fse_decompress.c +12 -35
  16. data/ext/zstdruby/libzstd/common/huf.h +15 -33
  17. data/ext/zstdruby/libzstd/common/mem.h +75 -2
  18. data/ext/zstdruby/libzstd/common/pool.c +8 -4
  19. data/ext/zstdruby/libzstd/common/pool.h +2 -2
  20. data/ext/zstdruby/libzstd/common/threading.c +52 -6
  21. data/ext/zstdruby/libzstd/common/threading.h +36 -4
  22. data/ext/zstdruby/libzstd/common/xxhash.c +25 -37
  23. data/ext/zstdruby/libzstd/common/xxhash.h +11 -31
  24. data/ext/zstdruby/libzstd/common/zstd_common.c +1 -1
  25. data/ext/zstdruby/libzstd/common/zstd_errors.h +2 -1
  26. data/ext/zstdruby/libzstd/common/zstd_internal.h +203 -22
  27. data/ext/zstdruby/libzstd/compress/fse_compress.c +19 -42
  28. data/ext/zstdruby/libzstd/compress/hist.c +15 -35
  29. data/ext/zstdruby/libzstd/compress/hist.h +12 -32
  30. data/ext/zstdruby/libzstd/compress/huf_compress.c +92 -92
  31. data/ext/zstdruby/libzstd/compress/zstd_compress.c +1460 -1472
  32. data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +330 -65
  33. data/ext/zstdruby/libzstd/compress/zstd_compress_literals.c +158 -0
  34. data/ext/zstdruby/libzstd/compress/zstd_compress_literals.h +29 -0
  35. data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.c +419 -0
  36. data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.h +54 -0
  37. data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.c +845 -0
  38. data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.h +32 -0
  39. data/ext/zstdruby/libzstd/compress/zstd_cwksp.h +525 -0
  40. data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +65 -43
  41. data/ext/zstdruby/libzstd/compress/zstd_double_fast.h +2 -2
  42. data/ext/zstdruby/libzstd/compress/zstd_fast.c +264 -159
  43. data/ext/zstdruby/libzstd/compress/zstd_fast.h +2 -2
  44. data/ext/zstdruby/libzstd/compress/zstd_lazy.c +74 -42
  45. data/ext/zstdruby/libzstd/compress/zstd_lazy.h +2 -2
  46. data/ext/zstdruby/libzstd/compress/zstd_ldm.c +33 -11
  47. data/ext/zstdruby/libzstd/compress/zstd_ldm.h +7 -2
  48. data/ext/zstdruby/libzstd/compress/zstd_opt.c +108 -125
  49. data/ext/zstdruby/libzstd/compress/zstd_opt.h +1 -1
  50. data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +129 -93
  51. data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +46 -28
  52. data/ext/zstdruby/libzstd/decompress/huf_decompress.c +76 -60
  53. data/ext/zstdruby/libzstd/decompress/zstd_ddict.c +14 -10
  54. data/ext/zstdruby/libzstd/decompress/zstd_ddict.h +2 -2
  55. data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +471 -258
  56. data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.c +471 -346
  57. data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.h +3 -3
  58. data/ext/zstdruby/libzstd/decompress/zstd_decompress_internal.h +25 -4
  59. data/ext/zstdruby/libzstd/deprecated/zbuff.h +9 -8
  60. data/ext/zstdruby/libzstd/deprecated/zbuff_common.c +2 -2
  61. data/ext/zstdruby/libzstd/deprecated/zbuff_compress.c +1 -1
  62. data/ext/zstdruby/libzstd/deprecated/zbuff_decompress.c +1 -1
  63. data/ext/zstdruby/libzstd/dictBuilder/cover.c +220 -65
  64. data/ext/zstdruby/libzstd/dictBuilder/cover.h +81 -7
  65. data/ext/zstdruby/libzstd/dictBuilder/fastcover.c +85 -56
  66. data/ext/zstdruby/libzstd/dictBuilder/zdict.c +43 -19
  67. data/ext/zstdruby/libzstd/dictBuilder/zdict.h +73 -35
  68. data/ext/zstdruby/libzstd/dll/example/Makefile +2 -1
  69. data/ext/zstdruby/libzstd/dll/example/build_package.bat +3 -2
  70. data/ext/zstdruby/libzstd/legacy/zstd_legacy.h +49 -15
  71. data/ext/zstdruby/libzstd/legacy/zstd_v01.c +142 -117
  72. data/ext/zstdruby/libzstd/legacy/zstd_v01.h +13 -8
  73. data/ext/zstdruby/libzstd/legacy/zstd_v02.c +54 -25
  74. data/ext/zstdruby/libzstd/legacy/zstd_v02.h +13 -8
  75. data/ext/zstdruby/libzstd/legacy/zstd_v03.c +55 -25
  76. data/ext/zstdruby/libzstd/legacy/zstd_v03.h +13 -8
  77. data/ext/zstdruby/libzstd/legacy/zstd_v04.c +62 -29
  78. data/ext/zstdruby/libzstd/legacy/zstd_v04.h +13 -8
  79. data/ext/zstdruby/libzstd/legacy/zstd_v05.c +145 -109
  80. data/ext/zstdruby/libzstd/legacy/zstd_v05.h +14 -9
  81. data/ext/zstdruby/libzstd/legacy/zstd_v06.c +56 -26
  82. data/ext/zstdruby/libzstd/legacy/zstd_v06.h +11 -6
  83. data/ext/zstdruby/libzstd/legacy/zstd_v07.c +65 -28
  84. data/ext/zstdruby/libzstd/legacy/zstd_v07.h +11 -6
  85. data/ext/zstdruby/libzstd/libzstd.pc.in +3 -2
  86. data/ext/zstdruby/libzstd/zstd.h +921 -597
  87. data/lib/zstd-ruby/version.rb +1 -1
  88. data/zstd-ruby.gemspec +2 -2
  89. metadata +19 -14
  90. data/ext/zstdruby/libzstd/dll/libzstd.def +0 -87
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -19,7 +19,7 @@ extern "C" {
19
19
  * Dependencies
20
20
  ***************************************/
21
21
  #include <stddef.h> /* size_t */
22
- #include "mem.h" /* U64, U32 */
22
+ #include "../common/mem.h" /* U64, U32 */
23
23
 
24
24
 
25
25
  /* *************************************
@@ -33,13 +33,18 @@ extern "C" {
33
33
  size_t ZSTDv05_decompress( void* dst, size_t dstCapacity,
34
34
  const void* src, size_t compressedSize);
35
35
 
36
- /**
37
- ZSTDv05_getFrameSrcSize() : get the source length of a ZSTD frame
38
- compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
39
- return : the number of bytes that would be read to decompress this frame
40
- or an errorCode if it fails (which can be tested using ZSTDv05_isError())
41
- */
42
- size_t ZSTDv05_findFrameCompressedSize(const void* src, size_t compressedSize);
36
+ /**
37
+ ZSTDv05_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.5.x format
38
+ srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
39
+ cSize (output parameter) : the number of bytes that would be read to decompress this frame
40
+ or an error code if it fails (which can be tested using ZSTDv01_isError())
41
+ dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
42
+ or ZSTD_CONTENTSIZE_ERROR if an error occurs
43
+
44
+ note : assumes `cSize` and `dBound` are _not_ NULL.
45
+ */
46
+ void ZSTDv05_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
47
+ size_t* cSize, unsigned long long* dBound);
43
48
 
44
49
  /* *************************************
45
50
  * Helper functions
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -14,7 +14,7 @@
14
14
  #include <stddef.h> /* size_t, ptrdiff_t */
15
15
  #include <string.h> /* memcpy */
16
16
  #include <stdlib.h> /* malloc, free, qsort */
17
- #include "error_private.h"
17
+ #include "../common/error_private.h"
18
18
 
19
19
 
20
20
 
@@ -506,6 +506,8 @@ typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
506
506
  #define FSEv06_ENCODING_STATIC 2
507
507
  #define FSEv06_ENCODING_DYNAMIC 3
508
508
 
509
+ #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
510
+
509
511
  static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
510
512
  1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12,
511
513
  13,14,15,16 };
@@ -858,7 +860,7 @@ MEM_STATIC unsigned BITv06_highbit32 ( U32 val)
858
860
  _BitScanReverse ( &r, val );
859
861
  return (unsigned) r;
860
862
  # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
861
- return 31 - __builtin_clz (val);
863
+ return __builtin_clz (val) ^ 31;
862
864
  # else /* Software version */
863
865
  static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
864
866
  U32 v = val;
@@ -1860,7 +1862,7 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta
1860
1862
 
1861
1863
  if (!srcSize) return ERROR(srcSize_wrong);
1862
1864
  iSize = ip[0];
1863
- //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
1865
+ /* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */
1864
1866
 
1865
1867
  if (iSize >= 128) { /* special header */
1866
1868
  if (iSize >= (242)) { /* RLE */
@@ -2012,7 +2014,7 @@ size_t HUFv06_readDTableX2 (U16* DTable, const void* src, size_t srcSize)
2012
2014
  HUFv06_DEltX2* const dt = (HUFv06_DEltX2*)dtPtr;
2013
2015
 
2014
2016
  HUFv06_STATIC_ASSERT(sizeof(HUFv06_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */
2015
- //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */
2017
+ /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
2016
2018
 
2017
2019
  iSize = HUFv06_readStats(huffWeight, HUFv06_MAX_SYMBOL_VALUE + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
2018
2020
  if (HUFv06_isError(iSize)) return iSize;
@@ -2338,7 +2340,7 @@ size_t HUFv06_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
2338
2340
 
2339
2341
  HUFv06_STATIC_ASSERT(sizeof(HUFv06_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */
2340
2342
  if (memLog > HUFv06_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
2341
- //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */
2343
+ /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
2342
2344
 
2343
2345
  iSize = HUFv06_readStats(weightList, HUFv06_MAX_SYMBOL_VALUE + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
2344
2346
  if (HUFv06_isError(iSize)) return iSize;
@@ -2662,13 +2664,13 @@ size_t HUFv06_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
2662
2664
 
2663
2665
  { U32 algoNb = 0;
2664
2666
  if (Dtime[1] < Dtime[0]) algoNb = 1;
2665
- // if (Dtime[2] < Dtime[algoNb]) algoNb = 2; /* current speed of HUFv06_decompress4X6 is not good */
2667
+ /* if (Dtime[2] < Dtime[algoNb]) algoNb = 2; */ /* current speed of HUFv06_decompress4X6 is not good */
2666
2668
  return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
2667
2669
  }
2668
2670
 
2669
- //return HUFv06_decompress4X2(dst, dstSize, cSrc, cSrcSize); /* multi-streams single-symbol decoding */
2670
- //return HUFv06_decompress4X4(dst, dstSize, cSrc, cSrcSize); /* multi-streams double-symbols decoding */
2671
- //return HUFv06_decompress4X6(dst, dstSize, cSrc, cSrcSize); /* multi-streams quad-symbols decoding */
2671
+ /* return HUFv06_decompress4X2(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams single-symbol decoding */
2672
+ /* return HUFv06_decompress4X4(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams double-symbols decoding */
2673
+ /* return HUFv06_decompress4X6(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams quad-symbols decoding */
2672
2674
  }
2673
2675
  /*
2674
2676
  Common functions of Zstd compression library
@@ -3240,14 +3242,12 @@ static size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
3240
3242
  }
3241
3243
 
3242
3244
  /* FSE table descriptors */
3245
+ if (ip + 4 > iend) return ERROR(srcSize_wrong); /* min : header byte + all 3 are "raw", hence no header, but at least xxLog bits per type */
3243
3246
  { U32 const LLtype = *ip >> 6;
3244
3247
  U32 const Offtype = (*ip >> 4) & 3;
3245
3248
  U32 const MLtype = (*ip >> 2) & 3;
3246
3249
  ip++;
3247
3250
 
3248
- /* check */
3249
- if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
3250
-
3251
3251
  /* Build DTables */
3252
3252
  { size_t const bhSize = ZSTDv06_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
3253
3253
  if (ZSTDv06_isError(bhSize)) return ERROR(corruption_detected);
@@ -3406,7 +3406,7 @@ static size_t ZSTDv06_execSequence(BYTE* op,
3406
3406
  if (sequence.offset < 8) {
3407
3407
  /* close range match, overlap */
3408
3408
  static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
3409
- static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */
3409
+ static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
3410
3410
  int const sub2 = dec64table[sequence.offset];
3411
3411
  op[0] = match[0];
3412
3412
  op[1] = match[1];
@@ -3501,8 +3501,10 @@ static size_t ZSTDv06_decompressSequences(
3501
3501
  { size_t const lastLLSize = litEnd - litPtr;
3502
3502
  if (litPtr > litEnd) return ERROR(corruption_detected); /* too many literals already used */
3503
3503
  if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
3504
- memcpy(op, litPtr, lastLLSize);
3505
- op += lastLLSize;
3504
+ if (lastLLSize > 0) {
3505
+ memcpy(op, litPtr, lastLLSize);
3506
+ op += lastLLSize;
3507
+ }
3506
3508
  }
3507
3509
 
3508
3510
  return op-ostart;
@@ -3654,36 +3656,62 @@ size_t ZSTDv06_decompress(void* dst, size_t dstCapacity, const void* src, size_t
3654
3656
  #endif
3655
3657
  }
3656
3658
 
3657
- size_t ZSTDv06_findFrameCompressedSize(const void* src, size_t srcSize)
3659
+ /* ZSTD_errorFrameSizeInfoLegacy() :
3660
+ assumes `cSize` and `dBound` are _not_ NULL */
3661
+ static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
3662
+ {
3663
+ *cSize = ret;
3664
+ *dBound = ZSTD_CONTENTSIZE_ERROR;
3665
+ }
3666
+
3667
+ void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
3658
3668
  {
3659
3669
  const BYTE* ip = (const BYTE*)src;
3660
3670
  size_t remainingSize = srcSize;
3671
+ size_t nbBlocks = 0;
3661
3672
  blockProperties_t blockProperties = { bt_compressed, 0 };
3662
3673
 
3663
3674
  /* Frame Header */
3664
- { size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src, ZSTDv06_frameHeaderSize_min);
3665
- if (ZSTDv06_isError(frameHeaderSize)) return frameHeaderSize;
3666
- if (MEM_readLE32(src) != ZSTDv06_MAGICNUMBER) return ERROR(prefix_unknown);
3667
- if (srcSize < frameHeaderSize+ZSTDv06_blockHeaderSize) return ERROR(srcSize_wrong);
3675
+ { size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src, srcSize);
3676
+ if (ZSTDv06_isError(frameHeaderSize)) {
3677
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, frameHeaderSize);
3678
+ return;
3679
+ }
3680
+ if (MEM_readLE32(src) != ZSTDv06_MAGICNUMBER) {
3681
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
3682
+ return;
3683
+ }
3684
+ if (srcSize < frameHeaderSize+ZSTDv06_blockHeaderSize) {
3685
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3686
+ return;
3687
+ }
3668
3688
  ip += frameHeaderSize; remainingSize -= frameHeaderSize;
3669
3689
  }
3670
3690
 
3671
3691
  /* Loop on each block */
3672
3692
  while (1) {
3673
3693
  size_t const cBlockSize = ZSTDv06_getcBlockSize(ip, remainingSize, &blockProperties);
3674
- if (ZSTDv06_isError(cBlockSize)) return cBlockSize;
3694
+ if (ZSTDv06_isError(cBlockSize)) {
3695
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
3696
+ return;
3697
+ }
3675
3698
 
3676
3699
  ip += ZSTDv06_blockHeaderSize;
3677
3700
  remainingSize -= ZSTDv06_blockHeaderSize;
3678
- if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
3701
+ if (cBlockSize > remainingSize) {
3702
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3703
+ return;
3704
+ }
3679
3705
 
3680
3706
  if (cBlockSize == 0) break; /* bt_end */
3681
3707
 
3682
3708
  ip += cBlockSize;
3683
3709
  remainingSize -= cBlockSize;
3710
+ nbBlocks++;
3684
3711
  }
3685
3712
 
3686
- return ip - (const BYTE*)src;
3713
+ *cSize = ip - (const BYTE*)src;
3714
+ *dBound = nbBlocks * ZSTDv06_BLOCKSIZE_MAX;
3687
3715
  }
3688
3716
 
3689
3717
  /*_******************************
@@ -3974,7 +4002,9 @@ size_t ZBUFFv06_decompressInit(ZBUFFv06_DCtx* zbd)
3974
4002
  MEM_STATIC size_t ZBUFFv06_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
3975
4003
  {
3976
4004
  size_t length = MIN(dstCapacity, srcSize);
3977
- memcpy(dst, src, length);
4005
+ if (length > 0) {
4006
+ memcpy(dst, src, length);
4007
+ }
3978
4008
  return length;
3979
4009
  }
3980
4010
 
@@ -4083,7 +4113,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
4083
4113
  if (!decodedSize) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */
4084
4114
  zbd->outEnd = zbd->outStart + decodedSize;
4085
4115
  zbd->stage = ZBUFFds_flush;
4086
- // break; /* ZBUFFds_flush follows */
4116
+ /* break; */ /* ZBUFFds_flush follows */
4087
4117
  }
4088
4118
  }
4089
4119
  /* fall-through */
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -43,12 +43,17 @@ ZSTDLIBv06_API size_t ZSTDv06_decompress( void* dst, size_t dstCapacity,
43
43
  const void* src, size_t compressedSize);
44
44
 
45
45
  /**
46
- ZSTDv06_getFrameSrcSize() : get the source length of a ZSTD frame
47
- compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
48
- return : the number of bytes that would be read to decompress this frame
49
- or an errorCode if it fails (which can be tested using ZSTDv06_isError())
46
+ ZSTDv06_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.6.x format
47
+ srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
48
+ cSize (output parameter) : the number of bytes that would be read to decompress this frame
49
+ or an error code if it fails (which can be tested using ZSTDv01_isError())
50
+ dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
51
+ or ZSTD_CONTENTSIZE_ERROR if an error occurs
52
+
53
+ note : assumes `cSize` and `dBound` are _not_ NULL.
50
54
  */
51
- size_t ZSTDv06_findFrameCompressedSize(const void* src, size_t compressedSize);
55
+ void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
56
+ size_t* cSize, unsigned long long* dBound);
52
57
 
53
58
  /* *************************************
54
59
  * Helper functions
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -17,14 +17,14 @@
17
17
  #ifndef XXH_STATIC_LINKING_ONLY
18
18
  # define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
19
19
  #endif
20
- #include "xxhash.h" /* XXH64_* */
20
+ #include "../common/xxhash.h" /* XXH64_* */
21
21
  #include "zstd_v07.h"
22
22
 
23
23
  #define FSEv07_STATIC_LINKING_ONLY /* FSEv07_MIN_TABLELOG */
24
24
  #define HUFv07_STATIC_LINKING_ONLY /* HUFv07_TABLELOG_ABSOLUTEMAX */
25
25
  #define ZSTDv07_STATIC_LINKING_ONLY
26
26
 
27
- #include "error_private.h"
27
+ #include "../common/error_private.h"
28
28
 
29
29
 
30
30
  #ifdef ZSTDv07_STATIC_LINKING_ONLY
@@ -530,7 +530,7 @@ MEM_STATIC unsigned BITv07_highbit32 (U32 val)
530
530
  _BitScanReverse ( &r, val );
531
531
  return (unsigned) r;
532
532
  # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
533
- return 31 - __builtin_clz (val);
533
+ return __builtin_clz (val) ^ 31;
534
534
  # else /* Software version */
535
535
  static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
536
536
  U32 v = val;
@@ -1314,7 +1314,7 @@ size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
1314
1314
 
1315
1315
  if (!srcSize) return ERROR(srcSize_wrong);
1316
1316
  iSize = ip[0];
1317
- //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
1317
+ /* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */
1318
1318
 
1319
1319
  if (iSize >= 128) { /* special header */
1320
1320
  if (iSize >= (242)) { /* RLE */
@@ -1784,7 +1784,7 @@ size_t HUFv07_readDTableX2 (HUFv07_DTable* DTable, const void* src, size_t srcSi
1784
1784
  HUFv07_DEltX2* const dt = (HUFv07_DEltX2*)dtPtr;
1785
1785
 
1786
1786
  HUFv07_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUFv07_DTable));
1787
- //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */
1787
+ /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
1788
1788
 
1789
1789
  iSize = HUFv07_readStats(huffWeight, HUFv07_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
1790
1790
  if (HUFv07_isError(iSize)) return iSize;
@@ -2148,7 +2148,7 @@ size_t HUFv07_readDTableX4 (HUFv07_DTable* DTable, const void* src, size_t srcSi
2148
2148
 
2149
2149
  HUFv07_STATIC_ASSERT(sizeof(HUFv07_DEltX4) == sizeof(HUFv07_DTable)); /* if compilation fails here, assertion is false */
2150
2150
  if (maxTableLog > HUFv07_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge);
2151
- //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */
2151
+ /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
2152
2152
 
2153
2153
  iSize = HUFv07_readStats(weightList, HUFv07_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
2154
2154
  if (HUFv07_isError(iSize)) return iSize;
@@ -2530,8 +2530,8 @@ size_t HUFv07_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
2530
2530
  return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
2531
2531
  }
2532
2532
 
2533
- //return HUFv07_decompress4X2(dst, dstSize, cSrc, cSrcSize); /* multi-streams single-symbol decoding */
2534
- //return HUFv07_decompress4X4(dst, dstSize, cSrc, cSrcSize); /* multi-streams double-symbols decoding */
2533
+ /* return HUFv07_decompress4X2(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams single-symbol decoding */
2534
+ /* return HUFv07_decompress4X4(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams double-symbols decoding */
2535
2535
  }
2536
2536
 
2537
2537
  size_t HUFv07_decompress4X_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
@@ -2740,6 +2740,8 @@ typedef enum { lbt_huffman, lbt_repeat, lbt_raw, lbt_rle } litBlockType_t;
2740
2740
  #define FSEv07_ENCODING_STATIC 2
2741
2741
  #define FSEv07_ENCODING_DYNAMIC 3
2742
2742
 
2743
+ #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
2744
+
2743
2745
  static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2744
2746
  1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12,
2745
2747
  13,14,15,16 };
@@ -3270,7 +3272,9 @@ static size_t ZSTDv07_getcBlockSize(const void* src, size_t srcSize, blockProper
3270
3272
  static size_t ZSTDv07_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
3271
3273
  {
3272
3274
  if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall);
3273
- memcpy(dst, src, srcSize);
3275
+ if (srcSize > 0) {
3276
+ memcpy(dst, src, srcSize);
3277
+ }
3274
3278
  return srcSize;
3275
3279
  }
3276
3280
 
@@ -3468,14 +3472,12 @@ static size_t ZSTDv07_decodeSeqHeaders(int* nbSeqPtr,
3468
3472
  }
3469
3473
 
3470
3474
  /* FSE table descriptors */
3475
+ if (ip + 4 > iend) return ERROR(srcSize_wrong); /* min : header byte + all 3 are "raw", hence no header, but at least xxLog bits per type */
3471
3476
  { U32 const LLtype = *ip >> 6;
3472
3477
  U32 const OFtype = (*ip >> 4) & 3;
3473
3478
  U32 const MLtype = (*ip >> 2) & 3;
3474
3479
  ip++;
3475
3480
 
3476
- /* check */
3477
- if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
3478
-
3479
3481
  /* Build DTables */
3480
3482
  { size_t const llhSize = ZSTDv07_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
3481
3483
  if (ZSTDv07_isError(llhSize)) return ERROR(corruption_detected);
@@ -3631,7 +3633,7 @@ size_t ZSTDv07_execSequence(BYTE* op,
3631
3633
  if (sequence.offset < 8) {
3632
3634
  /* close range match, overlap */
3633
3635
  static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
3634
- static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */
3636
+ static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
3635
3637
  int const sub2 = dec64table[sequence.offset];
3636
3638
  op[0] = match[0];
3637
3639
  op[1] = match[1];
@@ -3712,10 +3714,12 @@ static size_t ZSTDv07_decompressSequences(
3712
3714
 
3713
3715
  /* last literal segment */
3714
3716
  { size_t const lastLLSize = litEnd - litPtr;
3715
- //if (litPtr > litEnd) return ERROR(corruption_detected); /* too many literals already used */
3717
+ /* if (litPtr > litEnd) return ERROR(corruption_detected); */ /* too many literals already used */
3716
3718
  if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall);
3717
- memcpy(op, litPtr, lastLLSize);
3718
- op += lastLLSize;
3719
+ if (lastLLSize > 0) {
3720
+ memcpy(op, litPtr, lastLLSize);
3721
+ op += lastLLSize;
3722
+ }
3719
3723
  }
3720
3724
 
3721
3725
  return op-ostart;
@@ -3776,7 +3780,9 @@ ZSTDLIBv07_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockS
3776
3780
  static size_t ZSTDv07_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length)
3777
3781
  {
3778
3782
  if (length > dstCapacity) return ERROR(dstSize_tooSmall);
3779
- memset(dst, byte, length);
3783
+ if (length > 0) {
3784
+ memset(dst, byte, length);
3785
+ }
3780
3786
  return length;
3781
3787
  }
3782
3788
 
@@ -3895,19 +3901,40 @@ size_t ZSTDv07_decompress(void* dst, size_t dstCapacity, const void* src, size_t
3895
3901
  #endif
3896
3902
  }
3897
3903
 
3898
- size_t ZSTDv07_findFrameCompressedSize(const void* src, size_t srcSize)
3904
+ /* ZSTD_errorFrameSizeInfoLegacy() :
3905
+ assumes `cSize` and `dBound` are _not_ NULL */
3906
+ static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
3907
+ {
3908
+ *cSize = ret;
3909
+ *dBound = ZSTD_CONTENTSIZE_ERROR;
3910
+ }
3911
+
3912
+ void ZSTDv07_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
3899
3913
  {
3900
3914
  const BYTE* ip = (const BYTE*)src;
3901
3915
  size_t remainingSize = srcSize;
3916
+ size_t nbBlocks = 0;
3902
3917
 
3903
3918
  /* check */
3904
- if (srcSize < ZSTDv07_frameHeaderSize_min+ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong);
3919
+ if (srcSize < ZSTDv07_frameHeaderSize_min+ZSTDv07_blockHeaderSize) {
3920
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3921
+ return;
3922
+ }
3905
3923
 
3906
3924
  /* Frame Header */
3907
- { size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src, ZSTDv07_frameHeaderSize_min);
3908
- if (ZSTDv07_isError(frameHeaderSize)) return frameHeaderSize;
3909
- if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) return ERROR(prefix_unknown);
3910
- if (srcSize < frameHeaderSize+ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong);
3925
+ { size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src, srcSize);
3926
+ if (ZSTDv07_isError(frameHeaderSize)) {
3927
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, frameHeaderSize);
3928
+ return;
3929
+ }
3930
+ if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) {
3931
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
3932
+ return;
3933
+ }
3934
+ if (srcSize < frameHeaderSize+ZSTDv07_blockHeaderSize) {
3935
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3936
+ return;
3937
+ }
3911
3938
  ip += frameHeaderSize; remainingSize -= frameHeaderSize;
3912
3939
  }
3913
3940
 
@@ -3915,20 +3942,28 @@ size_t ZSTDv07_findFrameCompressedSize(const void* src, size_t srcSize)
3915
3942
  while (1) {
3916
3943
  blockProperties_t blockProperties;
3917
3944
  size_t const cBlockSize = ZSTDv07_getcBlockSize(ip, remainingSize, &blockProperties);
3918
- if (ZSTDv07_isError(cBlockSize)) return cBlockSize;
3945
+ if (ZSTDv07_isError(cBlockSize)) {
3946
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
3947
+ return;
3948
+ }
3919
3949
 
3920
3950
  ip += ZSTDv07_blockHeaderSize;
3921
3951
  remainingSize -= ZSTDv07_blockHeaderSize;
3922
3952
 
3923
3953
  if (blockProperties.blockType == bt_end) break;
3924
3954
 
3925
- if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
3955
+ if (cBlockSize > remainingSize) {
3956
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3957
+ return;
3958
+ }
3926
3959
 
3927
3960
  ip += cBlockSize;
3928
3961
  remainingSize -= cBlockSize;
3962
+ nbBlocks++;
3929
3963
  }
3930
3964
 
3931
- return ip - (const BYTE*)src;
3965
+ *cSize = ip - (const BYTE*)src;
3966
+ *dBound = nbBlocks * ZSTDv07_BLOCKSIZE_ABSOLUTEMAX;
3932
3967
  }
3933
3968
 
3934
3969
  /*_******************************
@@ -4349,7 +4384,9 @@ size_t ZBUFFv07_decompressInit(ZBUFFv07_DCtx* zbd)
4349
4384
  MEM_STATIC size_t ZBUFFv07_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
4350
4385
  {
4351
4386
  size_t const length = MIN(dstCapacity, srcSize);
4352
- memcpy(dst, src, length);
4387
+ if (length > 0) {
4388
+ memcpy(dst, src, length);
4389
+ }
4353
4390
  return length;
4354
4391
  }
4355
4392