extzstd 0.1 → 0.3.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (134) hide show
  1. checksums.yaml +5 -5
  2. data/HISTORY.ja.md +39 -0
  3. data/README.md +38 -56
  4. data/contrib/zstd/CHANGELOG +613 -0
  5. data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
  6. data/contrib/zstd/CONTRIBUTING.md +406 -0
  7. data/contrib/zstd/COPYING +339 -0
  8. data/contrib/zstd/Makefile +420 -0
  9. data/contrib/zstd/README.md +179 -41
  10. data/contrib/zstd/TESTING.md +44 -0
  11. data/contrib/zstd/appveyor.yml +292 -0
  12. data/contrib/zstd/lib/BUCK +234 -0
  13. data/contrib/zstd/lib/Makefile +451 -0
  14. data/contrib/zstd/lib/README.md +207 -0
  15. data/contrib/zstd/{common → lib/common}/bitstream.h +187 -138
  16. data/contrib/zstd/lib/common/compiler.h +288 -0
  17. data/contrib/zstd/lib/common/cpu.h +213 -0
  18. data/contrib/zstd/lib/common/debug.c +24 -0
  19. data/contrib/zstd/lib/common/debug.h +107 -0
  20. data/contrib/zstd/lib/common/entropy_common.c +362 -0
  21. data/contrib/zstd/{common → lib/common}/error_private.c +25 -12
  22. data/contrib/zstd/{common → lib/common}/error_private.h +14 -10
  23. data/contrib/zstd/{common → lib/common}/fse.h +173 -92
  24. data/contrib/zstd/{common → lib/common}/fse_decompress.c +149 -85
  25. data/contrib/zstd/lib/common/huf.h +361 -0
  26. data/contrib/zstd/{common → lib/common}/mem.h +115 -59
  27. data/contrib/zstd/lib/common/pool.c +350 -0
  28. data/contrib/zstd/lib/common/pool.h +84 -0
  29. data/contrib/zstd/lib/common/threading.c +122 -0
  30. data/contrib/zstd/lib/common/threading.h +155 -0
  31. data/contrib/zstd/{common → lib/common}/xxhash.c +55 -96
  32. data/contrib/zstd/{common → lib/common}/xxhash.h +23 -47
  33. data/contrib/zstd/lib/common/zstd_common.c +83 -0
  34. data/contrib/zstd/lib/common/zstd_deps.h +111 -0
  35. data/contrib/zstd/lib/common/zstd_errors.h +95 -0
  36. data/contrib/zstd/lib/common/zstd_internal.h +478 -0
  37. data/contrib/zstd/{compress → lib/compress}/fse_compress.c +214 -319
  38. data/contrib/zstd/lib/compress/hist.c +181 -0
  39. data/contrib/zstd/lib/compress/hist.h +75 -0
  40. data/contrib/zstd/lib/compress/huf_compress.c +913 -0
  41. data/contrib/zstd/lib/compress/zstd_compress.c +5208 -0
  42. data/contrib/zstd/lib/compress/zstd_compress_internal.h +1203 -0
  43. data/contrib/zstd/lib/compress/zstd_compress_literals.c +158 -0
  44. data/contrib/zstd/lib/compress/zstd_compress_literals.h +29 -0
  45. data/contrib/zstd/lib/compress/zstd_compress_sequences.c +433 -0
  46. data/contrib/zstd/lib/compress/zstd_compress_sequences.h +54 -0
  47. data/contrib/zstd/lib/compress/zstd_compress_superblock.c +849 -0
  48. data/contrib/zstd/lib/compress/zstd_compress_superblock.h +32 -0
  49. data/contrib/zstd/lib/compress/zstd_cwksp.h +561 -0
  50. data/contrib/zstd/lib/compress/zstd_double_fast.c +521 -0
  51. data/contrib/zstd/lib/compress/zstd_double_fast.h +38 -0
  52. data/contrib/zstd/lib/compress/zstd_fast.c +496 -0
  53. data/contrib/zstd/lib/compress/zstd_fast.h +37 -0
  54. data/contrib/zstd/lib/compress/zstd_lazy.c +1412 -0
  55. data/contrib/zstd/lib/compress/zstd_lazy.h +87 -0
  56. data/contrib/zstd/lib/compress/zstd_ldm.c +660 -0
  57. data/contrib/zstd/lib/compress/zstd_ldm.h +116 -0
  58. data/contrib/zstd/lib/compress/zstd_opt.c +1345 -0
  59. data/contrib/zstd/lib/compress/zstd_opt.h +56 -0
  60. data/contrib/zstd/lib/compress/zstdmt_compress.c +1811 -0
  61. data/contrib/zstd/lib/compress/zstdmt_compress.h +110 -0
  62. data/contrib/zstd/lib/decompress/huf_decompress.c +1350 -0
  63. data/contrib/zstd/lib/decompress/zstd_ddict.c +244 -0
  64. data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
  65. data/contrib/zstd/lib/decompress/zstd_decompress.c +1930 -0
  66. data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1540 -0
  67. data/contrib/zstd/lib/decompress/zstd_decompress_block.h +62 -0
  68. data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +190 -0
  69. data/contrib/zstd/{common → lib/deprecated}/zbuff.h +68 -45
  70. data/contrib/zstd/lib/deprecated/zbuff_common.c +26 -0
  71. data/contrib/zstd/lib/deprecated/zbuff_compress.c +147 -0
  72. data/contrib/zstd/lib/deprecated/zbuff_decompress.c +75 -0
  73. data/contrib/zstd/lib/dictBuilder/cover.c +1245 -0
  74. data/contrib/zstd/lib/dictBuilder/cover.h +157 -0
  75. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.c +3 -3
  76. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.h +0 -0
  77. data/contrib/zstd/lib/dictBuilder/fastcover.c +758 -0
  78. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/zdict.c +318 -194
  79. data/contrib/zstd/lib/dictBuilder/zdict.h +305 -0
  80. data/contrib/zstd/{legacy → lib/legacy}/zstd_legacy.h +171 -15
  81. data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.c +191 -124
  82. data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.h +19 -5
  83. data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.c +125 -125
  84. data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.h +19 -5
  85. data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.c +125 -124
  86. data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.h +20 -6
  87. data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.c +151 -299
  88. data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.h +19 -5
  89. data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.c +237 -243
  90. data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.h +19 -6
  91. data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.c +130 -143
  92. data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.h +18 -5
  93. data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.c +158 -157
  94. data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.h +19 -5
  95. data/contrib/zstd/lib/libzstd.pc.in +15 -0
  96. data/contrib/zstd/lib/zstd.h +2391 -0
  97. data/ext/depend +2 -0
  98. data/ext/extconf.rb +15 -6
  99. data/ext/extzstd.c +76 -145
  100. data/ext/extzstd.h +80 -31
  101. data/ext/extzstd_stream.c +417 -142
  102. data/ext/libzstd_conf.h +8 -0
  103. data/ext/zstd_common.c +10 -7
  104. data/ext/zstd_compress.c +14 -5
  105. data/ext/zstd_decompress.c +5 -4
  106. data/ext/zstd_dictbuilder.c +9 -4
  107. data/ext/zstd_dictbuilder_fastcover.c +3 -0
  108. data/ext/zstd_legacy_v01.c +3 -1
  109. data/ext/zstd_legacy_v02.c +3 -1
  110. data/ext/zstd_legacy_v03.c +3 -1
  111. data/ext/zstd_legacy_v04.c +3 -1
  112. data/ext/zstd_legacy_v05.c +3 -1
  113. data/ext/zstd_legacy_v06.c +3 -1
  114. data/ext/zstd_legacy_v07.c +3 -1
  115. data/gemstub.rb +10 -24
  116. data/lib/extzstd.rb +64 -179
  117. data/lib/extzstd/version.rb +6 -1
  118. data/test/test_basic.rb +9 -6
  119. metadata +113 -57
  120. data/HISTORY.ja +0 -5
  121. data/contrib/zstd/common/entropy_common.c +0 -225
  122. data/contrib/zstd/common/huf.h +0 -228
  123. data/contrib/zstd/common/zstd_common.c +0 -83
  124. data/contrib/zstd/common/zstd_errors.h +0 -60
  125. data/contrib/zstd/common/zstd_internal.h +0 -267
  126. data/contrib/zstd/compress/huf_compress.c +0 -533
  127. data/contrib/zstd/compress/zbuff_compress.c +0 -319
  128. data/contrib/zstd/compress/zstd_compress.c +0 -3264
  129. data/contrib/zstd/compress/zstd_opt.h +0 -900
  130. data/contrib/zstd/decompress/huf_decompress.c +0 -883
  131. data/contrib/zstd/decompress/zbuff_decompress.c +0 -252
  132. data/contrib/zstd/decompress/zstd_decompress.c +0 -1842
  133. data/contrib/zstd/dictBuilder/zdict.h +0 -111
  134. data/contrib/zstd/zstd.h +0 -640
@@ -1,10 +1,11 @@
1
- /**
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
1
+ /*
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
- * This source code is licensed under the BSD-style license found in the
6
- * LICENSE file in the root directory of this source tree. An additional grant
7
- * of patent rights can be found in the PATENTS file in the same directory.
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
8
9
  */
9
10
 
10
11
  #ifndef ZSTD_V04_H_91868324769238
@@ -34,6 +35,19 @@ ZSTDv04_decompress() : decompress ZSTD frames compliant with v0.4.x format
34
35
  size_t ZSTDv04_decompress( void* dst, size_t maxOriginalSize,
35
36
  const void* src, size_t compressedSize);
36
37
 
38
+ /**
39
+ ZSTDv04_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.4.x format
40
+ srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
41
+ cSize (output parameter) : the number of bytes that would be read to decompress this frame
42
+ or an error code if it fails (which can be tested using ZSTDv01_isError())
43
+ dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
44
+ or ZSTD_CONTENTSIZE_ERROR if an error occurs
45
+
46
+ note : assumes `cSize` and `dBound` are _not_ NULL.
47
+ */
48
+ void ZSTDv04_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
49
+ size_t* cSize, unsigned long long* dBound);
50
+
37
51
  /**
38
52
  ZSTDv04_isError() : tells if the result of ZSTDv04_decompress() is an error
39
53
  */
@@ -1,16 +1,17 @@
1
- /**
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
1
+ /*
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
- * This source code is licensed under the BSD-style license found in the
6
- * LICENSE file in the root directory of this source tree. An additional grant
7
- * of patent rights can be found in the PATENTS file in the same directory.
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
8
9
  */
9
10
 
10
11
 
11
12
  /*- Dependencies -*/
12
13
  #include "zstd_v05.h"
13
- #include "error_private.h"
14
+ #include "../common/error_private.h"
14
15
 
15
16
 
16
17
  /* ******************************************************************
@@ -79,7 +80,11 @@ extern "C" {
79
80
  * Basic Types
80
81
  *****************************************************************/
81
82
  #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
82
- # include <stdint.h>
83
+ # if defined(_AIX)
84
+ # include <inttypes.h>
85
+ # else
86
+ # include <stdint.h> /* intptr_t */
87
+ # endif
83
88
  typedef uint8_t BYTE;
84
89
  typedef uint16_t U16;
85
90
  typedef int16_t S16;
@@ -326,13 +331,6 @@ size_t ZSTDv05_decompress_usingPreparedDCtx(
326
331
  * Streaming functions (direct mode)
327
332
  ****************************************/
328
333
  size_t ZSTDv05_decompressBegin(ZSTDv05_DCtx* dctx);
329
- size_t ZSTDv05_decompressBegin_usingDict(ZSTDv05_DCtx* dctx, const void* dict, size_t dictSize);
330
- void ZSTDv05_copyDCtx(ZSTDv05_DCtx* dctx, const ZSTDv05_DCtx* preparedDCtx);
331
-
332
- size_t ZSTDv05_getFrameParams(ZSTDv05_parameters* params, const void* src, size_t srcSize);
333
-
334
- size_t ZSTDv05_nextSrcSizeToDecompress(ZSTDv05_DCtx* dctx);
335
- size_t ZSTDv05_decompressContinue(ZSTDv05_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
336
334
 
337
335
  /*
338
336
  Streaming decompression, direct mode (bufferless)
@@ -497,6 +495,8 @@ static const size_t ZSTDv05_frameHeaderSize_min = 5;
497
495
 
498
496
  #define WILDCOPY_OVERLENGTH 8
499
497
 
498
+ #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
499
+
500
500
  typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
501
501
 
502
502
 
@@ -509,7 +509,7 @@ static void ZSTDv05_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
509
509
 
510
510
  /*! ZSTDv05_wildcopy() :
511
511
  * custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
512
- MEM_STATIC void ZSTDv05_wildcopy(void* dst, const void* src, size_t length)
512
+ MEM_STATIC void ZSTDv05_wildcopy(void* dst, const void* src, ptrdiff_t length)
513
513
  {
514
514
  const BYTE* ip = (const BYTE*)src;
515
515
  BYTE* op = (BYTE*)dst;
@@ -742,18 +742,6 @@ MEM_STATIC BITv05_DStream_status BITv05_reloadDStream(BITv05_DStream_t* bitD);
742
742
  MEM_STATIC unsigned BITv05_endOfDStream(const BITv05_DStream_t* bitD);
743
743
 
744
744
 
745
- /*!
746
- * Start by invoking BITv05_initDStream().
747
- * A chunk of the bitStream is then stored into a local register.
748
- * Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
749
- * You can then retrieve bitFields stored into the local register, **in reverse order**.
750
- * Local register is explicitly reloaded from memory by the BITv05_reloadDStream() method.
751
- * A reload guarantee a minimum of ((8*sizeof(size_t))-7) bits when its result is BITv05_DStream_unfinished.
752
- * Otherwise, it can be less than that, so proceed accordingly.
753
- * Checking if DStream has reached its end can be performed with BITv05_endOfDStream()
754
- */
755
-
756
-
757
745
  /*-****************************************
758
746
  * unsafe API
759
747
  ******************************************/
@@ -765,14 +753,14 @@ MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, unsigned nbBits);
765
753
  /*-**************************************************************
766
754
  * Helper functions
767
755
  ****************************************************************/
768
- MEM_STATIC unsigned BITv05_highbit32 (register U32 val)
756
+ MEM_STATIC unsigned BITv05_highbit32 (U32 val)
769
757
  {
770
758
  # if defined(_MSC_VER) /* Visual */
771
759
  unsigned long r=0;
772
760
  _BitScanReverse ( &r, val );
773
761
  return (unsigned) r;
774
762
  # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
775
- return 31 - __builtin_clz (val);
763
+ return __builtin_clz (val) ^ 31;
776
764
  # else /* Software version */
777
765
  static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
778
766
  U32 v = val;
@@ -818,13 +806,13 @@ MEM_STATIC size_t BITv05_initDStream(BITv05_DStream_t* bitD, const void* srcBuff
818
806
  bitD->bitContainer = *(const BYTE*)(bitD->start);
819
807
  switch(srcSize)
820
808
  {
821
- case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
822
- case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
823
- case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
824
- case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
825
- case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
826
- case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
827
- default:;
809
+ case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);/* fall-through */
810
+ case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);/* fall-through */
811
+ case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);/* fall-through */
812
+ case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24; /* fall-through */
813
+ case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16; /* fall-through */
814
+ case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8; /* fall-through */
815
+ default: break;
828
816
  }
829
817
  contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
830
818
  if (contain32 == 0) return ERROR(GENERIC); /* endMark not present */
@@ -835,13 +823,6 @@ MEM_STATIC size_t BITv05_initDStream(BITv05_DStream_t* bitD, const void* srcBuff
835
823
  return srcSize;
836
824
  }
837
825
 
838
- /*!BITv05_lookBits
839
- * Provides next n bits from local register
840
- * local register is not modified (bits are still present for next read/look)
841
- * On 32-bits, maxNbBits==25
842
- * On 64-bits, maxNbBits==57
843
- * @return : value extracted
844
- */
845
826
  MEM_STATIC size_t BITv05_lookBits(BITv05_DStream_t* bitD, U32 nbBits)
846
827
  {
847
828
  const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
@@ -861,12 +842,7 @@ MEM_STATIC void BITv05_skipBits(BITv05_DStream_t* bitD, U32 nbBits)
861
842
  bitD->bitsConsumed += nbBits;
862
843
  }
863
844
 
864
- /*!BITv05_readBits
865
- * Read next n bits from local register.
866
- * pay attention to not read more than nbBits contained into local register.
867
- * @return : extracted value.
868
- */
869
- MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, U32 nbBits)
845
+ MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, unsigned nbBits)
870
846
  {
871
847
  size_t value = BITv05_lookBits(bitD, nbBits);
872
848
  BITv05_skipBits(bitD, nbBits);
@@ -875,7 +851,7 @@ MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, U32 nbBits)
875
851
 
876
852
  /*!BITv05_readBitsFast :
877
853
  * unsafe version; only works only if nbBits >= 1 */
878
- MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, U32 nbBits)
854
+ MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, unsigned nbBits)
879
855
  {
880
856
  size_t value = BITv05_lookBitsFast(bitD, nbBits);
881
857
  BITv05_skipBits(bitD, nbBits);
@@ -884,8 +860,8 @@ MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, U32 nbBits)
884
860
 
885
861
  MEM_STATIC BITv05_DStream_status BITv05_reloadDStream(BITv05_DStream_t* bitD)
886
862
  {
887
- if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
888
- return BITv05_DStream_overflow;
863
+ if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
864
+ return BITv05_DStream_overflow;
889
865
 
890
866
  if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) {
891
867
  bitD->ptr -= bitD->bitsConsumed >> 3;
@@ -1001,54 +977,6 @@ static unsigned char FSEv05_decodeSymbol(FSEv05_DState_t* DStatePtr, BITv05_DStr
1001
977
 
1002
978
  static unsigned FSEv05_endOfDState(const FSEv05_DState_t* DStatePtr);
1003
979
 
1004
- /*!
1005
- Let's now decompose FSEv05_decompress_usingDTable() into its unitary components.
1006
- You will decode FSEv05-encoded symbols from the bitStream,
1007
- and also any other bitFields you put in, **in reverse order**.
1008
-
1009
- You will need a few variables to track your bitStream. They are :
1010
-
1011
- BITv05_DStream_t DStream; // Stream context
1012
- FSEv05_DState_t DState; // State context. Multiple ones are possible
1013
- FSEv05_DTable* DTablePtr; // Decoding table, provided by FSEv05_buildDTable()
1014
-
1015
- The first thing to do is to init the bitStream.
1016
- errorCode = BITv05_initDStream(&DStream, srcBuffer, srcSize);
1017
-
1018
- You should then retrieve your initial state(s)
1019
- (in reverse flushing order if you have several ones) :
1020
- errorCode = FSEv05_initDState(&DState, &DStream, DTablePtr);
1021
-
1022
- You can then decode your data, symbol after symbol.
1023
- For information the maximum number of bits read by FSEv05_decodeSymbol() is 'tableLog'.
1024
- Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
1025
- unsigned char symbol = FSEv05_decodeSymbol(&DState, &DStream);
1026
-
1027
- You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
1028
- Note : maximum allowed nbBits is 25, for 32-bits compatibility
1029
- size_t bitField = BITv05_readBits(&DStream, nbBits);
1030
-
1031
- All above operations only read from local register (which size depends on size_t).
1032
- Refueling the register from memory is manually performed by the reload method.
1033
- endSignal = FSEv05_reloadDStream(&DStream);
1034
-
1035
- BITv05_reloadDStream() result tells if there is still some more data to read from DStream.
1036
- BITv05_DStream_unfinished : there is still some data left into the DStream.
1037
- BITv05_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
1038
- BITv05_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
1039
- BITv05_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
1040
-
1041
- When reaching end of buffer (BITv05_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
1042
- to properly detect the exact end of stream.
1043
- After each decoded symbol, check if DStream is fully consumed using this simple test :
1044
- BITv05_reloadDStream(&DStream) >= BITv05_DStream_completed
1045
-
1046
- When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
1047
- Checking if DStream has reached its end is performed by :
1048
- BITv05_endOfDStream(&DStream);
1049
- Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
1050
- FSEv05_endOfDState(&DState);
1051
- */
1052
980
 
1053
981
 
1054
982
  /* *****************************************
@@ -1240,7 +1168,7 @@ MEM_STATIC unsigned FSEv05_endOfDState(const FSEv05_DState_t* DStatePtr)
1240
1168
  /* **************************************************************
1241
1169
  * Complex types
1242
1170
  ****************************************************************/
1243
- typedef U32 DTable_max_t[FSEv05_DTABLE_SIZE_U32(FSEv05_MAX_TABLELOG)];
1171
+ typedef unsigned DTable_max_t[FSEv05_DTABLE_SIZE_U32(FSEv05_MAX_TABLELOG)];
1244
1172
 
1245
1173
 
1246
1174
  /* **************************************************************
@@ -1302,6 +1230,7 @@ size_t FSEv05_buildDTable(FSEv05_DTable* dt, const short* normalizedCounter, uns
1302
1230
  if (tableLog > FSEv05_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
1303
1231
 
1304
1232
  /* Init, lay down lowprob symbols */
1233
+ memset(tableDecode, 0, sizeof(FSEv05_FUNCTION_TYPE) * (maxSymbolValue+1) ); /* useless init, but keep static analyzer happy, and we don't need to performance optimize legacy decoders */
1305
1234
  DTableH.tableLog = (U16)tableLog;
1306
1235
  for (s=0; s<=maxSymbolValue; s++) {
1307
1236
  if (normalizedCounter[s]==-1) {
@@ -1879,7 +1808,7 @@ static size_t HUFv05_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
1879
1808
 
1880
1809
  if (!srcSize) return ERROR(srcSize_wrong);
1881
1810
  iSize = ip[0];
1882
- //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
1811
+ /* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */
1883
1812
 
1884
1813
  if (iSize >= 128) { /* special header */
1885
1814
  if (iSize >= (242)) { /* RLE */
@@ -1954,7 +1883,7 @@ size_t HUFv05_readDTableX2 (U16* DTable, const void* src, size_t srcSize)
1954
1883
  HUFv05_DEltX2* const dt = (HUFv05_DEltX2*)dtPtr;
1955
1884
 
1956
1885
  HUFv05_STATIC_ASSERT(sizeof(HUFv05_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */
1957
- //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */
1886
+ /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
1958
1887
 
1959
1888
  iSize = HUFv05_readStats(huffWeight, HUFv05_MAX_SYMBOL_VALUE + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
1960
1889
  if (HUFv05_isError(iSize)) return iSize;
@@ -2073,91 +2002,92 @@ size_t HUFv05_decompress4X2_usingDTable(
2073
2002
  const void* cSrc, size_t cSrcSize,
2074
2003
  const U16* DTable)
2075
2004
  {
2076
- const BYTE* const istart = (const BYTE*) cSrc;
2077
- BYTE* const ostart = (BYTE*) dst;
2078
- BYTE* const oend = ostart + dstSize;
2079
- const void* const dtPtr = DTable;
2080
- const HUFv05_DEltX2* const dt = ((const HUFv05_DEltX2*)dtPtr) +1;
2081
- const U32 dtLog = DTable[0];
2082
- size_t errorCode;
2083
-
2084
- /* Init */
2085
- BITv05_DStream_t bitD1;
2086
- BITv05_DStream_t bitD2;
2087
- BITv05_DStream_t bitD3;
2088
- BITv05_DStream_t bitD4;
2089
- const size_t length1 = MEM_readLE16(istart);
2090
- const size_t length2 = MEM_readLE16(istart+2);
2091
- const size_t length3 = MEM_readLE16(istart+4);
2092
- size_t length4;
2093
- const BYTE* const istart1 = istart + 6; /* jumpTable */
2094
- const BYTE* const istart2 = istart1 + length1;
2095
- const BYTE* const istart3 = istart2 + length2;
2096
- const BYTE* const istart4 = istart3 + length3;
2097
- const size_t segmentSize = (dstSize+3) / 4;
2098
- BYTE* const opStart2 = ostart + segmentSize;
2099
- BYTE* const opStart3 = opStart2 + segmentSize;
2100
- BYTE* const opStart4 = opStart3 + segmentSize;
2101
- BYTE* op1 = ostart;
2102
- BYTE* op2 = opStart2;
2103
- BYTE* op3 = opStart3;
2104
- BYTE* op4 = opStart4;
2105
- U32 endSignal;
2106
-
2107
2005
  /* Check */
2108
2006
  if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
2007
+ {
2008
+ const BYTE* const istart = (const BYTE*) cSrc;
2009
+ BYTE* const ostart = (BYTE*) dst;
2010
+ BYTE* const oend = ostart + dstSize;
2011
+ const void* const dtPtr = DTable;
2012
+ const HUFv05_DEltX2* const dt = ((const HUFv05_DEltX2*)dtPtr) +1;
2013
+ const U32 dtLog = DTable[0];
2014
+ size_t errorCode;
2109
2015
 
2110
- length4 = cSrcSize - (length1 + length2 + length3 + 6);
2111
- if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
2112
- errorCode = BITv05_initDStream(&bitD1, istart1, length1);
2113
- if (HUFv05_isError(errorCode)) return errorCode;
2114
- errorCode = BITv05_initDStream(&bitD2, istart2, length2);
2115
- if (HUFv05_isError(errorCode)) return errorCode;
2116
- errorCode = BITv05_initDStream(&bitD3, istart3, length3);
2117
- if (HUFv05_isError(errorCode)) return errorCode;
2118
- errorCode = BITv05_initDStream(&bitD4, istart4, length4);
2119
- if (HUFv05_isError(errorCode)) return errorCode;
2016
+ /* Init */
2017
+ BITv05_DStream_t bitD1;
2018
+ BITv05_DStream_t bitD2;
2019
+ BITv05_DStream_t bitD3;
2020
+ BITv05_DStream_t bitD4;
2021
+ const size_t length1 = MEM_readLE16(istart);
2022
+ const size_t length2 = MEM_readLE16(istart+2);
2023
+ const size_t length3 = MEM_readLE16(istart+4);
2024
+ size_t length4;
2025
+ const BYTE* const istart1 = istart + 6; /* jumpTable */
2026
+ const BYTE* const istart2 = istart1 + length1;
2027
+ const BYTE* const istart3 = istart2 + length2;
2028
+ const BYTE* const istart4 = istart3 + length3;
2029
+ const size_t segmentSize = (dstSize+3) / 4;
2030
+ BYTE* const opStart2 = ostart + segmentSize;
2031
+ BYTE* const opStart3 = opStart2 + segmentSize;
2032
+ BYTE* const opStart4 = opStart3 + segmentSize;
2033
+ BYTE* op1 = ostart;
2034
+ BYTE* op2 = opStart2;
2035
+ BYTE* op3 = opStart3;
2036
+ BYTE* op4 = opStart4;
2037
+ U32 endSignal;
2120
2038
 
2121
- /* 16-32 symbols per loop (4-8 symbols per stream) */
2122
- endSignal = BITv05_reloadDStream(&bitD1) | BITv05_reloadDStream(&bitD2) | BITv05_reloadDStream(&bitD3) | BITv05_reloadDStream(&bitD4);
2123
- for ( ; (endSignal==BITv05_DStream_unfinished) && (op4<(oend-7)) ; ) {
2124
- HUFv05_DECODE_SYMBOLX2_2(op1, &bitD1);
2125
- HUFv05_DECODE_SYMBOLX2_2(op2, &bitD2);
2126
- HUFv05_DECODE_SYMBOLX2_2(op3, &bitD3);
2127
- HUFv05_DECODE_SYMBOLX2_2(op4, &bitD4);
2128
- HUFv05_DECODE_SYMBOLX2_1(op1, &bitD1);
2129
- HUFv05_DECODE_SYMBOLX2_1(op2, &bitD2);
2130
- HUFv05_DECODE_SYMBOLX2_1(op3, &bitD3);
2131
- HUFv05_DECODE_SYMBOLX2_1(op4, &bitD4);
2132
- HUFv05_DECODE_SYMBOLX2_2(op1, &bitD1);
2133
- HUFv05_DECODE_SYMBOLX2_2(op2, &bitD2);
2134
- HUFv05_DECODE_SYMBOLX2_2(op3, &bitD3);
2135
- HUFv05_DECODE_SYMBOLX2_2(op4, &bitD4);
2136
- HUFv05_DECODE_SYMBOLX2_0(op1, &bitD1);
2137
- HUFv05_DECODE_SYMBOLX2_0(op2, &bitD2);
2138
- HUFv05_DECODE_SYMBOLX2_0(op3, &bitD3);
2139
- HUFv05_DECODE_SYMBOLX2_0(op4, &bitD4);
2039
+ length4 = cSrcSize - (length1 + length2 + length3 + 6);
2040
+ if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
2041
+ errorCode = BITv05_initDStream(&bitD1, istart1, length1);
2042
+ if (HUFv05_isError(errorCode)) return errorCode;
2043
+ errorCode = BITv05_initDStream(&bitD2, istart2, length2);
2044
+ if (HUFv05_isError(errorCode)) return errorCode;
2045
+ errorCode = BITv05_initDStream(&bitD3, istart3, length3);
2046
+ if (HUFv05_isError(errorCode)) return errorCode;
2047
+ errorCode = BITv05_initDStream(&bitD4, istart4, length4);
2048
+ if (HUFv05_isError(errorCode)) return errorCode;
2049
+
2050
+ /* 16-32 symbols per loop (4-8 symbols per stream) */
2140
2051
  endSignal = BITv05_reloadDStream(&bitD1) | BITv05_reloadDStream(&bitD2) | BITv05_reloadDStream(&bitD3) | BITv05_reloadDStream(&bitD4);
2141
- }
2052
+ for ( ; (endSignal==BITv05_DStream_unfinished) && (op4<(oend-7)) ; ) {
2053
+ HUFv05_DECODE_SYMBOLX2_2(op1, &bitD1);
2054
+ HUFv05_DECODE_SYMBOLX2_2(op2, &bitD2);
2055
+ HUFv05_DECODE_SYMBOLX2_2(op3, &bitD3);
2056
+ HUFv05_DECODE_SYMBOLX2_2(op4, &bitD4);
2057
+ HUFv05_DECODE_SYMBOLX2_1(op1, &bitD1);
2058
+ HUFv05_DECODE_SYMBOLX2_1(op2, &bitD2);
2059
+ HUFv05_DECODE_SYMBOLX2_1(op3, &bitD3);
2060
+ HUFv05_DECODE_SYMBOLX2_1(op4, &bitD4);
2061
+ HUFv05_DECODE_SYMBOLX2_2(op1, &bitD1);
2062
+ HUFv05_DECODE_SYMBOLX2_2(op2, &bitD2);
2063
+ HUFv05_DECODE_SYMBOLX2_2(op3, &bitD3);
2064
+ HUFv05_DECODE_SYMBOLX2_2(op4, &bitD4);
2065
+ HUFv05_DECODE_SYMBOLX2_0(op1, &bitD1);
2066
+ HUFv05_DECODE_SYMBOLX2_0(op2, &bitD2);
2067
+ HUFv05_DECODE_SYMBOLX2_0(op3, &bitD3);
2068
+ HUFv05_DECODE_SYMBOLX2_0(op4, &bitD4);
2069
+ endSignal = BITv05_reloadDStream(&bitD1) | BITv05_reloadDStream(&bitD2) | BITv05_reloadDStream(&bitD3) | BITv05_reloadDStream(&bitD4);
2070
+ }
2142
2071
 
2143
- /* check corruption */
2144
- if (op1 > opStart2) return ERROR(corruption_detected);
2145
- if (op2 > opStart3) return ERROR(corruption_detected);
2146
- if (op3 > opStart4) return ERROR(corruption_detected);
2147
- /* note : op4 supposed already verified within main loop */
2072
+ /* check corruption */
2073
+ if (op1 > opStart2) return ERROR(corruption_detected);
2074
+ if (op2 > opStart3) return ERROR(corruption_detected);
2075
+ if (op3 > opStart4) return ERROR(corruption_detected);
2076
+ /* note : op4 supposed already verified within main loop */
2148
2077
 
2149
- /* finish bitStreams one by one */
2150
- HUFv05_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog);
2151
- HUFv05_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog);
2152
- HUFv05_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog);
2153
- HUFv05_decodeStreamX2(op4, &bitD4, oend, dt, dtLog);
2078
+ /* finish bitStreams one by one */
2079
+ HUFv05_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog);
2080
+ HUFv05_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog);
2081
+ HUFv05_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog);
2082
+ HUFv05_decodeStreamX2(op4, &bitD4, oend, dt, dtLog);
2154
2083
 
2155
- /* check */
2156
- endSignal = BITv05_endOfDStream(&bitD1) & BITv05_endOfDStream(&bitD2) & BITv05_endOfDStream(&bitD3) & BITv05_endOfDStream(&bitD4);
2157
- if (!endSignal) return ERROR(corruption_detected);
2084
+ /* check */
2085
+ endSignal = BITv05_endOfDStream(&bitD1) & BITv05_endOfDStream(&bitD2) & BITv05_endOfDStream(&bitD3) & BITv05_endOfDStream(&bitD4);
2086
+ if (!endSignal) return ERROR(corruption_detected);
2158
2087
 
2159
- /* decoded size */
2160
- return dstSize;
2088
+ /* decoded size */
2089
+ return dstSize;
2090
+ }
2161
2091
  }
2162
2092
 
2163
2093
 
@@ -2268,7 +2198,7 @@ static void HUFv05_fillDTableX4(HUFv05_DEltX4* DTable, const U32 targetLog,
2268
2198
  }
2269
2199
  }
2270
2200
 
2271
- size_t HUFv05_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
2201
+ size_t HUFv05_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize)
2272
2202
  {
2273
2203
  BYTE weightList[HUFv05_MAX_SYMBOL_VALUE + 1];
2274
2204
  sortedSymbol_t sortedSymbol[HUFv05_MAX_SYMBOL_VALUE + 1];
@@ -2282,9 +2212,9 @@ size_t HUFv05_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
2282
2212
  void* dtPtr = DTable;
2283
2213
  HUFv05_DEltX4* const dt = ((HUFv05_DEltX4*)dtPtr) + 1;
2284
2214
 
2285
- HUFv05_STATIC_ASSERT(sizeof(HUFv05_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */
2215
+ HUFv05_STATIC_ASSERT(sizeof(HUFv05_DEltX4) == sizeof(unsigned)); /* if compilation fails here, assertion is false */
2286
2216
  if (memLog > HUFv05_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
2287
- //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */
2217
+ /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
2288
2218
 
2289
2219
  iSize = HUFv05_readStats(weightList, HUFv05_MAX_SYMBOL_VALUE + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
2290
2220
  if (HUFv05_isError(iSize)) return iSize;
@@ -2409,7 +2339,7 @@ static inline size_t HUFv05_decodeStreamX4(BYTE* p, BITv05_DStream_t* bitDPtr, B
2409
2339
  size_t HUFv05_decompress1X4_usingDTable(
2410
2340
  void* dst, size_t dstSize,
2411
2341
  const void* cSrc, size_t cSrcSize,
2412
- const U32* DTable)
2342
+ const unsigned* DTable)
2413
2343
  {
2414
2344
  const BYTE* const istart = (const BYTE*) cSrc;
2415
2345
  BYTE* const ostart = (BYTE*) dst;
@@ -2452,7 +2382,7 @@ size_t HUFv05_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t
2452
2382
  size_t HUFv05_decompress4X4_usingDTable(
2453
2383
  void* dst, size_t dstSize,
2454
2384
  const void* cSrc, size_t cSrcSize,
2455
- const U32* DTable)
2385
+ const unsigned* DTable)
2456
2386
  {
2457
2387
  if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
2458
2388
 
@@ -2613,9 +2543,9 @@ size_t HUFv05_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
2613
2543
 
2614
2544
  return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
2615
2545
 
2616
- //return HUFv05_decompress4X2(dst, dstSize, cSrc, cSrcSize); /* multi-streams single-symbol decoding */
2617
- //return HUFv05_decompress4X4(dst, dstSize, cSrc, cSrcSize); /* multi-streams double-symbols decoding */
2618
- //return HUFv05_decompress4X6(dst, dstSize, cSrc, cSrcSize); /* multi-streams quad-symbols decoding */
2546
+ /* return HUFv05_decompress4X2(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams single-symbol decoding */
2547
+ /* return HUFv05_decompress4X4(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams double-symbols decoding */
2548
+ /* return HUFv05_decompress4X6(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams quad-symbols decoding */
2619
2549
  }
2620
2550
  /*
2621
2551
  zstd - standard compression library
@@ -2731,12 +2661,12 @@ struct ZSTDv05_DCtx_s
2731
2661
  ZSTDv05_dStage stage;
2732
2662
  U32 flagStaticTables;
2733
2663
  const BYTE* litPtr;
2734
- size_t litBufSize;
2735
2664
  size_t litSize;
2736
2665
  BYTE litBuffer[BLOCKSIZE + WILDCOPY_OVERLENGTH];
2737
2666
  BYTE headerBuffer[ZSTDv05_frameHeaderSize_max];
2738
2667
  }; /* typedef'd to ZSTDv05_DCtx within "zstd_static.h" */
2739
2668
 
2669
+ size_t ZSTDv05_sizeofDCtx (void); /* Hidden declaration */
2740
2670
  size_t ZSTDv05_sizeofDCtx (void) { return sizeof(ZSTDv05_DCtx); }
2741
2671
 
2742
2672
  size_t ZSTDv05_decompressBegin(ZSTDv05_DCtx* dctx)
@@ -2896,12 +2826,12 @@ static size_t ZSTDv05_decodeFrameHeader_Part2(ZSTDv05_DCtx* zc, const void* src,
2896
2826
  if (srcSize != zc->headerSize)
2897
2827
  return ERROR(srcSize_wrong);
2898
2828
  result = ZSTDv05_getFrameParams(&(zc->params), src, srcSize);
2899
- if ((MEM_32bits()) && (zc->params.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits);
2829
+ if ((MEM_32bits()) && (zc->params.windowLog > 25)) return ERROR(frameParameter_unsupported);
2900
2830
  return result;
2901
2831
  }
2902
2832
 
2903
2833
 
2904
- size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
2834
+ static size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
2905
2835
  {
2906
2836
  const BYTE* const in = (const BYTE* const)src;
2907
2837
  BYTE headerFlags;
@@ -2924,6 +2854,7 @@ size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t*
2924
2854
 
2925
2855
  static size_t ZSTDv05_copyRawBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
2926
2856
  {
2857
+ if (dst==NULL) return ERROR(dstSize_tooSmall);
2927
2858
  if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
2928
2859
  memcpy(dst, src, srcSize);
2929
2860
  return srcSize;
@@ -2932,8 +2863,8 @@ static size_t ZSTDv05_copyRawBlock(void* dst, size_t maxDstSize, const void* src
2932
2863
 
2933
2864
  /*! ZSTDv05_decodeLiteralsBlock() :
2934
2865
  @return : nb of bytes read from src (< srcSize ) */
2935
- size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
2936
- const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
2866
+ static size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
2867
+ const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
2937
2868
  {
2938
2869
  const BYTE* const istart = (const BYTE*) src;
2939
2870
 
@@ -2978,8 +2909,8 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
2978
2909
  return ERROR(corruption_detected);
2979
2910
 
2980
2911
  dctx->litPtr = dctx->litBuffer;
2981
- dctx->litBufSize = BLOCKSIZE+8;
2982
2912
  dctx->litSize = litSize;
2913
+ memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
2983
2914
  return litCSize + lhSize;
2984
2915
  }
2985
2916
  case IS_PCH:
@@ -2996,14 +2927,14 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
2996
2927
  lhSize=3;
2997
2928
  litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2);
2998
2929
  litCSize = ((istart[1] & 3) << 8) + istart[2];
2999
- if (litCSize + litSize > srcSize) return ERROR(corruption_detected);
2930
+ if (litCSize + lhSize > srcSize) return ERROR(corruption_detected);
3000
2931
 
3001
2932
  errorCode = HUFv05_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTableX4);
3002
2933
  if (HUFv05_isError(errorCode)) return ERROR(corruption_detected);
3003
2934
 
3004
2935
  dctx->litPtr = dctx->litBuffer;
3005
- dctx->litBufSize = BLOCKSIZE+WILDCOPY_OVERLENGTH;
3006
2936
  dctx->litSize = litSize;
2937
+ memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
3007
2938
  return litCSize + lhSize;
3008
2939
  }
3009
2940
  case IS_RAW:
@@ -3028,13 +2959,12 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
3028
2959
  if (litSize+lhSize > srcSize) return ERROR(corruption_detected);
3029
2960
  memcpy(dctx->litBuffer, istart+lhSize, litSize);
3030
2961
  dctx->litPtr = dctx->litBuffer;
3031
- dctx->litBufSize = BLOCKSIZE+8;
3032
2962
  dctx->litSize = litSize;
2963
+ memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
3033
2964
  return lhSize+litSize;
3034
2965
  }
3035
2966
  /* direct reference into compressed stream */
3036
2967
  dctx->litPtr = istart+lhSize;
3037
- dctx->litBufSize = srcSize-lhSize;
3038
2968
  dctx->litSize = litSize;
3039
2969
  return lhSize+litSize;
3040
2970
  }
@@ -3057,9 +2987,8 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
3057
2987
  break;
3058
2988
  }
3059
2989
  if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
3060
- memset(dctx->litBuffer, istart[lhSize], litSize);
2990
+ memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH);
3061
2991
  dctx->litPtr = dctx->litBuffer;
3062
- dctx->litBufSize = BLOCKSIZE+WILDCOPY_OVERLENGTH;
3063
2992
  dctx->litSize = litSize;
3064
2993
  return lhSize+1;
3065
2994
  }
@@ -3069,7 +2998,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
3069
2998
  }
3070
2999
 
3071
3000
 
3072
- size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
3001
+ static size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
3073
3002
  FSEv05_DTable* DTableLL, FSEv05_DTable* DTableML, FSEv05_DTable* DTableOffb,
3074
3003
  const void* src, size_t srcSize, U32 flagStaticTable)
3075
3004
  {
@@ -3077,7 +3006,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
3077
3006
  const BYTE* ip = istart;
3078
3007
  const BYTE* const iend = istart + srcSize;
3079
3008
  U32 LLtype, Offtype, MLtype;
3080
- U32 LLlog, Offlog, MLlog;
3009
+ unsigned LLlog, Offlog, MLlog;
3081
3010
  size_t dumpsLength;
3082
3011
 
3083
3012
  /* check */
@@ -3135,7 +3064,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
3135
3064
  break;
3136
3065
  case FSEv05_ENCODING_DYNAMIC :
3137
3066
  default : /* impossible */
3138
- { U32 max = MaxLL;
3067
+ { unsigned max = MaxLL;
3139
3068
  headerSize = FSEv05_readNCount(norm, &max, &LLlog, ip, iend-ip);
3140
3069
  if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
3141
3070
  if (LLlog > LLFSEv05Log) return ERROR(corruption_detected);
@@ -3159,7 +3088,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
3159
3088
  break;
3160
3089
  case FSEv05_ENCODING_DYNAMIC :
3161
3090
  default : /* impossible */
3162
- { U32 max = MaxOff;
3091
+ { unsigned max = MaxOff;
3163
3092
  headerSize = FSEv05_readNCount(norm, &max, &Offlog, ip, iend-ip);
3164
3093
  if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
3165
3094
  if (Offlog > OffFSEv05Log) return ERROR(corruption_detected);
@@ -3183,7 +3112,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
3183
3112
  break;
3184
3113
  case FSEv05_ENCODING_DYNAMIC :
3185
3114
  default : /* impossible */
3186
- { U32 max = MaxML;
3115
+ { unsigned max = MaxML;
3187
3116
  headerSize = FSEv05_readNCount(norm, &max, &MLlog, ip, iend-ip);
3188
3117
  if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
3189
3118
  if (MLlog > MLFSEv05Log) return ERROR(corruption_detected);
@@ -3226,14 +3155,18 @@ static void ZSTDv05_decodeSequence(seq_t* seq, seqState_t* seqState)
3226
3155
  litLength = FSEv05_peakSymbol(&(seqState->stateLL));
3227
3156
  prevOffset = litLength ? seq->offset : seqState->prevOffset;
3228
3157
  if (litLength == MaxLL) {
3229
- U32 add = *dumps++;
3158
+ const U32 add = *dumps++;
3230
3159
  if (add < 255) litLength += add;
3231
- else {
3232
- litLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no risk : dumps is always followed by seq tables > 1 byte */
3233
- if (litLength&1) litLength>>=1, dumps += 3;
3234
- else litLength = (U16)(litLength)>>1, dumps += 2;
3160
+ else if (dumps + 2 <= de) {
3161
+ litLength = MEM_readLE16(dumps);
3162
+ dumps += 2;
3163
+ if ((litLength & 1) && dumps < de) {
3164
+ litLength += *dumps << 16;
3165
+ dumps += 1;
3166
+ }
3167
+ litLength>>=1;
3235
3168
  }
3236
- if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
3169
+ if (dumps >= de) { dumps = de-1; } /* late correction, to avoid read overflow (data is now corrupted anyway) */
3237
3170
  }
3238
3171
 
3239
3172
  /* Offset */
@@ -3259,14 +3192,18 @@ static void ZSTDv05_decodeSequence(seq_t* seq, seqState_t* seqState)
3259
3192
  /* MatchLength */
3260
3193
  matchLength = FSEv05_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
3261
3194
  if (matchLength == MaxML) {
3262
- U32 add = *dumps++;
3195
+ const U32 add = dumps<de ? *dumps++ : 0;
3263
3196
  if (add < 255) matchLength += add;
3264
- else {
3265
- matchLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
3266
- if (matchLength&1) matchLength>>=1, dumps += 3;
3267
- else matchLength = (U16)(matchLength)>>1, dumps += 2;
3197
+ else if (dumps + 2 <= de) {
3198
+ matchLength = MEM_readLE16(dumps);
3199
+ dumps += 2;
3200
+ if ((matchLength & 1) && dumps < de) {
3201
+ matchLength += *dumps << 16;
3202
+ dumps += 1;
3203
+ }
3204
+ matchLength >>= 1;
3268
3205
  }
3269
- if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
3206
+ if (dumps >= de) { dumps = de-1; } /* late correction, to avoid read overflow (data is now corrupted anyway) */
3270
3207
  }
3271
3208
  matchLength += MINMATCH;
3272
3209
 
@@ -3289,11 +3226,11 @@ static void ZSTDv05_decodeSequence(seq_t* seq, seqState_t* seqState)
3289
3226
 
3290
3227
  static size_t ZSTDv05_execSequence(BYTE* op,
3291
3228
  BYTE* const oend, seq_t sequence,
3292
- const BYTE** litPtr, const BYTE* const litLimit_8,
3229
+ const BYTE** litPtr, const BYTE* const litLimit,
3293
3230
  const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
3294
3231
  {
3295
3232
  static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
3296
- static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */
3233
+ static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
3297
3234
  BYTE* const oLitEnd = op + sequence.litLength;
3298
3235
  const size_t sequenceLength = sequence.litLength + sequence.matchLength;
3299
3236
  BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
@@ -3304,7 +3241,7 @@ static size_t ZSTDv05_execSequence(BYTE* op,
3304
3241
  /* check */
3305
3242
  if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
3306
3243
  if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
3307
- if (litEnd > litLimit_8) return ERROR(corruption_detected); /* risk read beyond lit buffer */
3244
+ if (litEnd > litLimit) return ERROR(corruption_detected); /* risk read beyond lit buffer */
3308
3245
 
3309
3246
  /* copy Literals */
3310
3247
  ZSTDv05_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@@ -3328,7 +3265,7 @@ static size_t ZSTDv05_execSequence(BYTE* op,
3328
3265
  op = oLitEnd + length1;
3329
3266
  sequence.matchLength -= length1;
3330
3267
  match = base;
3331
- if (op > oend_8) {
3268
+ if (op > oend_8 || sequence.matchLength < MINMATCH) {
3332
3269
  while (op < oMatchEnd) *op++ = *match++;
3333
3270
  return sequenceLength;
3334
3271
  }
@@ -3351,7 +3288,7 @@ static size_t ZSTDv05_execSequence(BYTE* op,
3351
3288
  }
3352
3289
  op += 8; match += 8;
3353
3290
 
3354
- if (oMatchEnd > oend-12) {
3291
+ if (oMatchEnd > oend-(16-MINMATCH)) {
3355
3292
  if (op < oend_8) {
3356
3293
  ZSTDv05_wildcopy(op, match, oend_8 - op);
3357
3294
  match += oend_8 - op;
@@ -3360,7 +3297,7 @@ static size_t ZSTDv05_execSequence(BYTE* op,
3360
3297
  while (op < oMatchEnd)
3361
3298
  *op++ = *match++;
3362
3299
  } else {
3363
- ZSTDv05_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
3300
+ ZSTDv05_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
3364
3301
  }
3365
3302
  return sequenceLength;
3366
3303
  }
@@ -3376,15 +3313,14 @@ static size_t ZSTDv05_decompressSequences(
3376
3313
  BYTE* const ostart = (BYTE* const)dst;
3377
3314
  BYTE* op = ostart;
3378
3315
  BYTE* const oend = ostart + maxDstSize;
3379
- size_t errorCode, dumpsLength;
3316
+ size_t errorCode, dumpsLength=0;
3380
3317
  const BYTE* litPtr = dctx->litPtr;
3381
- const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8;
3382
3318
  const BYTE* const litEnd = litPtr + dctx->litSize;
3383
- int nbSeq;
3384
- const BYTE* dumps;
3385
- U32* DTableLL = dctx->LLTable;
3386
- U32* DTableML = dctx->MLTable;
3387
- U32* DTableOffb = dctx->OffTable;
3319
+ int nbSeq=0;
3320
+ const BYTE* dumps = NULL;
3321
+ unsigned* DTableLL = dctx->LLTable;
3322
+ unsigned* DTableML = dctx->MLTable;
3323
+ unsigned* DTableOffb = dctx->OffTable;
3388
3324
  const BYTE* const base = (const BYTE*) (dctx->base);
3389
3325
  const BYTE* const vBase = (const BYTE*) (dctx->vBase);
3390
3326
  const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
@@ -3416,7 +3352,7 @@ static size_t ZSTDv05_decompressSequences(
3416
3352
  size_t oneSeqSize;
3417
3353
  nbSeq--;
3418
3354
  ZSTDv05_decodeSequence(&sequence, &seqState);
3419
- oneSeqSize = ZSTDv05_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd);
3355
+ oneSeqSize = ZSTDv05_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
3420
3356
  if (ZSTDv05_isError(oneSeqSize)) return oneSeqSize;
3421
3357
  op += oneSeqSize;
3422
3358
  }
@@ -3430,8 +3366,10 @@ static size_t ZSTDv05_decompressSequences(
3430
3366
  size_t lastLLSize = litEnd - litPtr;
3431
3367
  if (litPtr > litEnd) return ERROR(corruption_detected); /* too many literals already used */
3432
3368
  if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
3433
- memcpy(op, litPtr, lastLLSize);
3434
- op += lastLLSize;
3369
+ if (lastLLSize > 0) {
3370
+ memcpy(op, litPtr, lastLLSize);
3371
+ op += lastLLSize;
3372
+ }
3435
3373
  }
3436
3374
 
3437
3375
  return op-ostart;
@@ -3490,10 +3428,10 @@ static size_t ZSTDv05_decompress_continueDCtx(ZSTDv05_DCtx* dctx,
3490
3428
  BYTE* const oend = ostart + maxDstSize;
3491
3429
  size_t remainingSize = srcSize;
3492
3430
  blockProperties_t blockProperties;
3431
+ memset(&blockProperties, 0, sizeof(blockProperties));
3493
3432
 
3494
3433
  /* Frame Header */
3495
- {
3496
- size_t frameHeaderSize;
3434
+ { size_t frameHeaderSize;
3497
3435
  if (srcSize < ZSTDv05_frameHeaderSize_min+ZSTDv05_blockHeaderSize) return ERROR(srcSize_wrong);
3498
3436
  frameHeaderSize = ZSTDv05_decodeFrameHeader_Part1(dctx, src, ZSTDv05_frameHeaderSize_min);
3499
3437
  if (ZSTDv05_isError(frameHeaderSize)) return frameHeaderSize;
@@ -3585,6 +3523,58 @@ size_t ZSTDv05_decompress(void* dst, size_t maxDstSize, const void* src, size_t
3585
3523
  #endif
3586
3524
  }
3587
3525
 
3526
+ /* ZSTD_errorFrameSizeInfoLegacy() :
3527
+ assumes `cSize` and `dBound` are _not_ NULL */
3528
+ static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
3529
+ {
3530
+ *cSize = ret;
3531
+ *dBound = ZSTD_CONTENTSIZE_ERROR;
3532
+ }
3533
+
3534
+ void ZSTDv05_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
3535
+ {
3536
+ const BYTE* ip = (const BYTE*)src;
3537
+ size_t remainingSize = srcSize;
3538
+ size_t nbBlocks = 0;
3539
+ blockProperties_t blockProperties;
3540
+
3541
+ /* Frame Header */
3542
+ if (srcSize < ZSTDv05_frameHeaderSize_min) {
3543
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3544
+ return;
3545
+ }
3546
+ if (MEM_readLE32(src) != ZSTDv05_MAGICNUMBER) {
3547
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
3548
+ return;
3549
+ }
3550
+ ip += ZSTDv05_frameHeaderSize_min; remainingSize -= ZSTDv05_frameHeaderSize_min;
3551
+
3552
+ /* Loop on each block */
3553
+ while (1)
3554
+ {
3555
+ size_t cBlockSize = ZSTDv05_getcBlockSize(ip, remainingSize, &blockProperties);
3556
+ if (ZSTDv05_isError(cBlockSize)) {
3557
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
3558
+ return;
3559
+ }
3560
+
3561
+ ip += ZSTDv05_blockHeaderSize;
3562
+ remainingSize -= ZSTDv05_blockHeaderSize;
3563
+ if (cBlockSize > remainingSize) {
3564
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3565
+ return;
3566
+ }
3567
+
3568
+ if (cBlockSize == 0) break; /* bt_end */
3569
+
3570
+ ip += cBlockSize;
3571
+ remainingSize -= cBlockSize;
3572
+ nbBlocks++;
3573
+ }
3574
+
3575
+ *cSize = ip - (const BYTE*)src;
3576
+ *dBound = nbBlocks * BLOCKSIZE;
3577
+ }
3588
3578
 
3589
3579
  /* ******************************
3590
3580
  * Streaming Decompression API
@@ -3681,7 +3671,7 @@ static size_t ZSTDv05_loadEntropy(ZSTDv05_DCtx* dctx, const void* dict, size_t d
3681
3671
  {
3682
3672
  size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, errorCode, litlengthHeaderSize;
3683
3673
  short offcodeNCount[MaxOff+1];
3684
- U32 offcodeMaxValue=MaxOff, offcodeLog;
3674
+ unsigned offcodeMaxValue=MaxOff, offcodeLog;
3685
3675
  short matchlengthNCount[MaxML+1];
3686
3676
  unsigned matchlengthMaxValue = MaxML, matchlengthLog;
3687
3677
  short litlengthNCount[MaxLL+1];
@@ -3807,7 +3797,9 @@ static size_t ZBUFFv05_blockHeaderSize = 3;
3807
3797
  static size_t ZBUFFv05_limitCopy(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
3808
3798
  {
3809
3799
  size_t length = MIN(maxDstSize, srcSize);
3810
- memcpy(dst, src, length);
3800
+ if (length > 0) {
3801
+ memcpy(dst, src, length);
3802
+ }
3811
3803
  return length;
3812
3804
  }
3813
3805
 
@@ -3928,7 +3920,7 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst
3928
3920
  zbc->stage = ZBUFFv05ds_decodeHeader;
3929
3921
  break;
3930
3922
  }
3931
-
3923
+ /* fall-through */
3932
3924
  case ZBUFFv05ds_loadHeader:
3933
3925
  /* complete header from src */
3934
3926
  {
@@ -3944,9 +3936,9 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst
3944
3936
  *maxDstSizePtr = 0;
3945
3937
  return headerSize - zbc->hPos;
3946
3938
  }
3947
- // zbc->stage = ZBUFFv05ds_decodeHeader; break; /* useless : stage follows */
3939
+ /* zbc->stage = ZBUFFv05ds_decodeHeader; break; */ /* useless : stage follows */
3948
3940
  }
3949
-
3941
+ /* fall-through */
3950
3942
  case ZBUFFv05ds_decodeHeader:
3951
3943
  /* apply header to create / resize buffers */
3952
3944
  {
@@ -3973,7 +3965,7 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst
3973
3965
  break;
3974
3966
  }
3975
3967
  zbc->stage = ZBUFFv05ds_read;
3976
-
3968
+ /* fall-through */
3977
3969
  case ZBUFFv05ds_read:
3978
3970
  {
3979
3971
  size_t neededInSize = ZSTDv05_nextSrcSizeToDecompress(zbc->zc);
@@ -3997,7 +3989,7 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst
3997
3989
  if (ip==iend) { notDone = 0; break; } /* no more input */
3998
3990
  zbc->stage = ZBUFFv05ds_load;
3999
3991
  }
4000
-
3992
+ /* fall-through */
4001
3993
  case ZBUFFv05ds_load:
4002
3994
  {
4003
3995
  size_t neededInSize = ZSTDv05_nextSrcSizeToDecompress(zbc->zc);
@@ -4017,8 +4009,10 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst
4017
4009
  if (!decodedSize) { zbc->stage = ZBUFFv05ds_read; break; } /* this was just a header */
4018
4010
  zbc->outEnd = zbc->outStart + decodedSize;
4019
4011
  zbc->stage = ZBUFFv05ds_flush;
4020
- // break; /* ZBUFFv05ds_flush follows */
4021
- } }
4012
+ /* break; */ /* ZBUFFv05ds_flush follows */
4013
+ }
4014
+ }
4015
+ /* fall-through */
4022
4016
  case ZBUFFv05ds_flush:
4023
4017
  {
4024
4018
  size_t toFlushSize = zbc->outEnd - zbc->outStart;