extzstd 0.1 → 0.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. checksums.yaml +5 -5
  2. data/HISTORY.ja.md +39 -0
  3. data/README.md +38 -56
  4. data/contrib/zstd/CHANGELOG +613 -0
  5. data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
  6. data/contrib/zstd/CONTRIBUTING.md +406 -0
  7. data/contrib/zstd/COPYING +339 -0
  8. data/contrib/zstd/Makefile +420 -0
  9. data/contrib/zstd/README.md +179 -41
  10. data/contrib/zstd/TESTING.md +44 -0
  11. data/contrib/zstd/appveyor.yml +292 -0
  12. data/contrib/zstd/lib/BUCK +234 -0
  13. data/contrib/zstd/lib/Makefile +451 -0
  14. data/contrib/zstd/lib/README.md +207 -0
  15. data/contrib/zstd/{common → lib/common}/bitstream.h +187 -138
  16. data/contrib/zstd/lib/common/compiler.h +288 -0
  17. data/contrib/zstd/lib/common/cpu.h +213 -0
  18. data/contrib/zstd/lib/common/debug.c +24 -0
  19. data/contrib/zstd/lib/common/debug.h +107 -0
  20. data/contrib/zstd/lib/common/entropy_common.c +362 -0
  21. data/contrib/zstd/{common → lib/common}/error_private.c +25 -12
  22. data/contrib/zstd/{common → lib/common}/error_private.h +14 -10
  23. data/contrib/zstd/{common → lib/common}/fse.h +173 -92
  24. data/contrib/zstd/{common → lib/common}/fse_decompress.c +149 -85
  25. data/contrib/zstd/lib/common/huf.h +361 -0
  26. data/contrib/zstd/{common → lib/common}/mem.h +115 -59
  27. data/contrib/zstd/lib/common/pool.c +350 -0
  28. data/contrib/zstd/lib/common/pool.h +84 -0
  29. data/contrib/zstd/lib/common/threading.c +122 -0
  30. data/contrib/zstd/lib/common/threading.h +155 -0
  31. data/contrib/zstd/{common → lib/common}/xxhash.c +55 -96
  32. data/contrib/zstd/{common → lib/common}/xxhash.h +23 -47
  33. data/contrib/zstd/lib/common/zstd_common.c +83 -0
  34. data/contrib/zstd/lib/common/zstd_deps.h +111 -0
  35. data/contrib/zstd/lib/common/zstd_errors.h +95 -0
  36. data/contrib/zstd/lib/common/zstd_internal.h +478 -0
  37. data/contrib/zstd/{compress → lib/compress}/fse_compress.c +214 -319
  38. data/contrib/zstd/lib/compress/hist.c +181 -0
  39. data/contrib/zstd/lib/compress/hist.h +75 -0
  40. data/contrib/zstd/lib/compress/huf_compress.c +913 -0
  41. data/contrib/zstd/lib/compress/zstd_compress.c +5208 -0
  42. data/contrib/zstd/lib/compress/zstd_compress_internal.h +1203 -0
  43. data/contrib/zstd/lib/compress/zstd_compress_literals.c +158 -0
  44. data/contrib/zstd/lib/compress/zstd_compress_literals.h +29 -0
  45. data/contrib/zstd/lib/compress/zstd_compress_sequences.c +433 -0
  46. data/contrib/zstd/lib/compress/zstd_compress_sequences.h +54 -0
  47. data/contrib/zstd/lib/compress/zstd_compress_superblock.c +849 -0
  48. data/contrib/zstd/lib/compress/zstd_compress_superblock.h +32 -0
  49. data/contrib/zstd/lib/compress/zstd_cwksp.h +561 -0
  50. data/contrib/zstd/lib/compress/zstd_double_fast.c +521 -0
  51. data/contrib/zstd/lib/compress/zstd_double_fast.h +38 -0
  52. data/contrib/zstd/lib/compress/zstd_fast.c +496 -0
  53. data/contrib/zstd/lib/compress/zstd_fast.h +37 -0
  54. data/contrib/zstd/lib/compress/zstd_lazy.c +1412 -0
  55. data/contrib/zstd/lib/compress/zstd_lazy.h +87 -0
  56. data/contrib/zstd/lib/compress/zstd_ldm.c +660 -0
  57. data/contrib/zstd/lib/compress/zstd_ldm.h +116 -0
  58. data/contrib/zstd/lib/compress/zstd_opt.c +1345 -0
  59. data/contrib/zstd/lib/compress/zstd_opt.h +56 -0
  60. data/contrib/zstd/lib/compress/zstdmt_compress.c +1811 -0
  61. data/contrib/zstd/lib/compress/zstdmt_compress.h +110 -0
  62. data/contrib/zstd/lib/decompress/huf_decompress.c +1350 -0
  63. data/contrib/zstd/lib/decompress/zstd_ddict.c +244 -0
  64. data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
  65. data/contrib/zstd/lib/decompress/zstd_decompress.c +1930 -0
  66. data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1540 -0
  67. data/contrib/zstd/lib/decompress/zstd_decompress_block.h +62 -0
  68. data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +190 -0
  69. data/contrib/zstd/{common → lib/deprecated}/zbuff.h +68 -45
  70. data/contrib/zstd/lib/deprecated/zbuff_common.c +26 -0
  71. data/contrib/zstd/lib/deprecated/zbuff_compress.c +147 -0
  72. data/contrib/zstd/lib/deprecated/zbuff_decompress.c +75 -0
  73. data/contrib/zstd/lib/dictBuilder/cover.c +1245 -0
  74. data/contrib/zstd/lib/dictBuilder/cover.h +157 -0
  75. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.c +3 -3
  76. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.h +0 -0
  77. data/contrib/zstd/lib/dictBuilder/fastcover.c +758 -0
  78. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/zdict.c +318 -194
  79. data/contrib/zstd/lib/dictBuilder/zdict.h +305 -0
  80. data/contrib/zstd/{legacy → lib/legacy}/zstd_legacy.h +171 -15
  81. data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.c +191 -124
  82. data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.h +19 -5
  83. data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.c +125 -125
  84. data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.h +19 -5
  85. data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.c +125 -124
  86. data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.h +20 -6
  87. data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.c +151 -299
  88. data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.h +19 -5
  89. data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.c +237 -243
  90. data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.h +19 -6
  91. data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.c +130 -143
  92. data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.h +18 -5
  93. data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.c +158 -157
  94. data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.h +19 -5
  95. data/contrib/zstd/lib/libzstd.pc.in +15 -0
  96. data/contrib/zstd/lib/zstd.h +2391 -0
  97. data/ext/depend +2 -0
  98. data/ext/extconf.rb +15 -6
  99. data/ext/extzstd.c +76 -145
  100. data/ext/extzstd.h +80 -31
  101. data/ext/extzstd_stream.c +417 -142
  102. data/ext/libzstd_conf.h +8 -0
  103. data/ext/zstd_common.c +10 -7
  104. data/ext/zstd_compress.c +14 -5
  105. data/ext/zstd_decompress.c +5 -4
  106. data/ext/zstd_dictbuilder.c +9 -4
  107. data/ext/zstd_dictbuilder_fastcover.c +3 -0
  108. data/ext/zstd_legacy_v01.c +3 -1
  109. data/ext/zstd_legacy_v02.c +3 -1
  110. data/ext/zstd_legacy_v03.c +3 -1
  111. data/ext/zstd_legacy_v04.c +3 -1
  112. data/ext/zstd_legacy_v05.c +3 -1
  113. data/ext/zstd_legacy_v06.c +3 -1
  114. data/ext/zstd_legacy_v07.c +3 -1
  115. data/gemstub.rb +10 -24
  116. data/lib/extzstd.rb +64 -179
  117. data/lib/extzstd/version.rb +6 -1
  118. data/test/test_basic.rb +9 -6
  119. metadata +113 -57
  120. data/HISTORY.ja +0 -5
  121. data/contrib/zstd/common/entropy_common.c +0 -225
  122. data/contrib/zstd/common/huf.h +0 -228
  123. data/contrib/zstd/common/zstd_common.c +0 -83
  124. data/contrib/zstd/common/zstd_errors.h +0 -60
  125. data/contrib/zstd/common/zstd_internal.h +0 -267
  126. data/contrib/zstd/compress/huf_compress.c +0 -533
  127. data/contrib/zstd/compress/zbuff_compress.c +0 -319
  128. data/contrib/zstd/compress/zstd_compress.c +0 -3264
  129. data/contrib/zstd/compress/zstd_opt.h +0 -900
  130. data/contrib/zstd/decompress/huf_decompress.c +0 -883
  131. data/contrib/zstd/decompress/zbuff_decompress.c +0 -252
  132. data/contrib/zstd/decompress/zstd_decompress.c +0 -1842
  133. data/contrib/zstd/dictBuilder/zdict.h +0 -111
  134. data/contrib/zstd/zstd.h +0 -640
@@ -1,10 +1,11 @@
1
- /**
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
1
+ /*
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
- * This source code is licensed under the BSD-style license found in the
6
- * LICENSE file in the root directory of this source tree. An additional grant
7
- * of patent rights can be found in the PATENTS file in the same directory.
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
8
9
  */
9
10
 
10
11
  #ifndef ZSTD_V04_H_91868324769238
@@ -34,6 +35,19 @@ ZSTDv04_decompress() : decompress ZSTD frames compliant with v0.4.x format
34
35
  size_t ZSTDv04_decompress( void* dst, size_t maxOriginalSize,
35
36
  const void* src, size_t compressedSize);
36
37
 
38
+ /**
39
+ ZSTDv04_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.4.x format
40
+ srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
41
+ cSize (output parameter) : the number of bytes that would be read to decompress this frame
42
+ or an error code if it fails (which can be tested using ZSTDv01_isError())
43
+ dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
44
+ or ZSTD_CONTENTSIZE_ERROR if an error occurs
45
+
46
+ note : assumes `cSize` and `dBound` are _not_ NULL.
47
+ */
48
+ void ZSTDv04_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
49
+ size_t* cSize, unsigned long long* dBound);
50
+
37
51
  /**
38
52
  ZSTDv04_isError() : tells if the result of ZSTDv04_decompress() is an error
39
53
  */
@@ -1,16 +1,17 @@
1
- /**
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
1
+ /*
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
- * This source code is licensed under the BSD-style license found in the
6
- * LICENSE file in the root directory of this source tree. An additional grant
7
- * of patent rights can be found in the PATENTS file in the same directory.
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
8
9
  */
9
10
 
10
11
 
11
12
  /*- Dependencies -*/
12
13
  #include "zstd_v05.h"
13
- #include "error_private.h"
14
+ #include "../common/error_private.h"
14
15
 
15
16
 
16
17
  /* ******************************************************************
@@ -79,7 +80,11 @@ extern "C" {
79
80
  * Basic Types
80
81
  *****************************************************************/
81
82
  #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
82
- # include <stdint.h>
83
+ # if defined(_AIX)
84
+ # include <inttypes.h>
85
+ # else
86
+ # include <stdint.h> /* intptr_t */
87
+ # endif
83
88
  typedef uint8_t BYTE;
84
89
  typedef uint16_t U16;
85
90
  typedef int16_t S16;
@@ -326,13 +331,6 @@ size_t ZSTDv05_decompress_usingPreparedDCtx(
326
331
  * Streaming functions (direct mode)
327
332
  ****************************************/
328
333
  size_t ZSTDv05_decompressBegin(ZSTDv05_DCtx* dctx);
329
- size_t ZSTDv05_decompressBegin_usingDict(ZSTDv05_DCtx* dctx, const void* dict, size_t dictSize);
330
- void ZSTDv05_copyDCtx(ZSTDv05_DCtx* dctx, const ZSTDv05_DCtx* preparedDCtx);
331
-
332
- size_t ZSTDv05_getFrameParams(ZSTDv05_parameters* params, const void* src, size_t srcSize);
333
-
334
- size_t ZSTDv05_nextSrcSizeToDecompress(ZSTDv05_DCtx* dctx);
335
- size_t ZSTDv05_decompressContinue(ZSTDv05_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
336
334
 
337
335
  /*
338
336
  Streaming decompression, direct mode (bufferless)
@@ -497,6 +495,8 @@ static const size_t ZSTDv05_frameHeaderSize_min = 5;
497
495
 
498
496
  #define WILDCOPY_OVERLENGTH 8
499
497
 
498
+ #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
499
+
500
500
  typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
501
501
 
502
502
 
@@ -509,7 +509,7 @@ static void ZSTDv05_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
509
509
 
510
510
  /*! ZSTDv05_wildcopy() :
511
511
  * custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
512
- MEM_STATIC void ZSTDv05_wildcopy(void* dst, const void* src, size_t length)
512
+ MEM_STATIC void ZSTDv05_wildcopy(void* dst, const void* src, ptrdiff_t length)
513
513
  {
514
514
  const BYTE* ip = (const BYTE*)src;
515
515
  BYTE* op = (BYTE*)dst;
@@ -742,18 +742,6 @@ MEM_STATIC BITv05_DStream_status BITv05_reloadDStream(BITv05_DStream_t* bitD);
742
742
  MEM_STATIC unsigned BITv05_endOfDStream(const BITv05_DStream_t* bitD);
743
743
 
744
744
 
745
- /*!
746
- * Start by invoking BITv05_initDStream().
747
- * A chunk of the bitStream is then stored into a local register.
748
- * Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
749
- * You can then retrieve bitFields stored into the local register, **in reverse order**.
750
- * Local register is explicitly reloaded from memory by the BITv05_reloadDStream() method.
751
- * A reload guarantee a minimum of ((8*sizeof(size_t))-7) bits when its result is BITv05_DStream_unfinished.
752
- * Otherwise, it can be less than that, so proceed accordingly.
753
- * Checking if DStream has reached its end can be performed with BITv05_endOfDStream()
754
- */
755
-
756
-
757
745
  /*-****************************************
758
746
  * unsafe API
759
747
  ******************************************/
@@ -765,14 +753,14 @@ MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, unsigned nbBits);
765
753
  /*-**************************************************************
766
754
  * Helper functions
767
755
  ****************************************************************/
768
- MEM_STATIC unsigned BITv05_highbit32 (register U32 val)
756
+ MEM_STATIC unsigned BITv05_highbit32 (U32 val)
769
757
  {
770
758
  # if defined(_MSC_VER) /* Visual */
771
759
  unsigned long r=0;
772
760
  _BitScanReverse ( &r, val );
773
761
  return (unsigned) r;
774
762
  # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
775
- return 31 - __builtin_clz (val);
763
+ return __builtin_clz (val) ^ 31;
776
764
  # else /* Software version */
777
765
  static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
778
766
  U32 v = val;
@@ -818,13 +806,13 @@ MEM_STATIC size_t BITv05_initDStream(BITv05_DStream_t* bitD, const void* srcBuff
818
806
  bitD->bitContainer = *(const BYTE*)(bitD->start);
819
807
  switch(srcSize)
820
808
  {
821
- case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
822
- case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
823
- case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
824
- case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
825
- case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
826
- case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
827
- default:;
809
+ case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);/* fall-through */
810
+ case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);/* fall-through */
811
+ case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);/* fall-through */
812
+ case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24; /* fall-through */
813
+ case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16; /* fall-through */
814
+ case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8; /* fall-through */
815
+ default: break;
828
816
  }
829
817
  contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
830
818
  if (contain32 == 0) return ERROR(GENERIC); /* endMark not present */
@@ -835,13 +823,6 @@ MEM_STATIC size_t BITv05_initDStream(BITv05_DStream_t* bitD, const void* srcBuff
835
823
  return srcSize;
836
824
  }
837
825
 
838
- /*!BITv05_lookBits
839
- * Provides next n bits from local register
840
- * local register is not modified (bits are still present for next read/look)
841
- * On 32-bits, maxNbBits==25
842
- * On 64-bits, maxNbBits==57
843
- * @return : value extracted
844
- */
845
826
  MEM_STATIC size_t BITv05_lookBits(BITv05_DStream_t* bitD, U32 nbBits)
846
827
  {
847
828
  const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
@@ -861,12 +842,7 @@ MEM_STATIC void BITv05_skipBits(BITv05_DStream_t* bitD, U32 nbBits)
861
842
  bitD->bitsConsumed += nbBits;
862
843
  }
863
844
 
864
- /*!BITv05_readBits
865
- * Read next n bits from local register.
866
- * pay attention to not read more than nbBits contained into local register.
867
- * @return : extracted value.
868
- */
869
- MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, U32 nbBits)
845
+ MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, unsigned nbBits)
870
846
  {
871
847
  size_t value = BITv05_lookBits(bitD, nbBits);
872
848
  BITv05_skipBits(bitD, nbBits);
@@ -875,7 +851,7 @@ MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, U32 nbBits)
875
851
 
876
852
  /*!BITv05_readBitsFast :
877
853
  * unsafe version; only works only if nbBits >= 1 */
878
- MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, U32 nbBits)
854
+ MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, unsigned nbBits)
879
855
  {
880
856
  size_t value = BITv05_lookBitsFast(bitD, nbBits);
881
857
  BITv05_skipBits(bitD, nbBits);
@@ -884,8 +860,8 @@ MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, U32 nbBits)
884
860
 
885
861
  MEM_STATIC BITv05_DStream_status BITv05_reloadDStream(BITv05_DStream_t* bitD)
886
862
  {
887
- if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
888
- return BITv05_DStream_overflow;
863
+ if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
864
+ return BITv05_DStream_overflow;
889
865
 
890
866
  if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) {
891
867
  bitD->ptr -= bitD->bitsConsumed >> 3;
@@ -1001,54 +977,6 @@ static unsigned char FSEv05_decodeSymbol(FSEv05_DState_t* DStatePtr, BITv05_DStr
1001
977
 
1002
978
  static unsigned FSEv05_endOfDState(const FSEv05_DState_t* DStatePtr);
1003
979
 
1004
- /*!
1005
- Let's now decompose FSEv05_decompress_usingDTable() into its unitary components.
1006
- You will decode FSEv05-encoded symbols from the bitStream,
1007
- and also any other bitFields you put in, **in reverse order**.
1008
-
1009
- You will need a few variables to track your bitStream. They are :
1010
-
1011
- BITv05_DStream_t DStream; // Stream context
1012
- FSEv05_DState_t DState; // State context. Multiple ones are possible
1013
- FSEv05_DTable* DTablePtr; // Decoding table, provided by FSEv05_buildDTable()
1014
-
1015
- The first thing to do is to init the bitStream.
1016
- errorCode = BITv05_initDStream(&DStream, srcBuffer, srcSize);
1017
-
1018
- You should then retrieve your initial state(s)
1019
- (in reverse flushing order if you have several ones) :
1020
- errorCode = FSEv05_initDState(&DState, &DStream, DTablePtr);
1021
-
1022
- You can then decode your data, symbol after symbol.
1023
- For information the maximum number of bits read by FSEv05_decodeSymbol() is 'tableLog'.
1024
- Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
1025
- unsigned char symbol = FSEv05_decodeSymbol(&DState, &DStream);
1026
-
1027
- You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
1028
- Note : maximum allowed nbBits is 25, for 32-bits compatibility
1029
- size_t bitField = BITv05_readBits(&DStream, nbBits);
1030
-
1031
- All above operations only read from local register (which size depends on size_t).
1032
- Refueling the register from memory is manually performed by the reload method.
1033
- endSignal = FSEv05_reloadDStream(&DStream);
1034
-
1035
- BITv05_reloadDStream() result tells if there is still some more data to read from DStream.
1036
- BITv05_DStream_unfinished : there is still some data left into the DStream.
1037
- BITv05_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
1038
- BITv05_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
1039
- BITv05_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
1040
-
1041
- When reaching end of buffer (BITv05_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
1042
- to properly detect the exact end of stream.
1043
- After each decoded symbol, check if DStream is fully consumed using this simple test :
1044
- BITv05_reloadDStream(&DStream) >= BITv05_DStream_completed
1045
-
1046
- When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
1047
- Checking if DStream has reached its end is performed by :
1048
- BITv05_endOfDStream(&DStream);
1049
- Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
1050
- FSEv05_endOfDState(&DState);
1051
- */
1052
980
 
1053
981
 
1054
982
  /* *****************************************
@@ -1240,7 +1168,7 @@ MEM_STATIC unsigned FSEv05_endOfDState(const FSEv05_DState_t* DStatePtr)
1240
1168
  /* **************************************************************
1241
1169
  * Complex types
1242
1170
  ****************************************************************/
1243
- typedef U32 DTable_max_t[FSEv05_DTABLE_SIZE_U32(FSEv05_MAX_TABLELOG)];
1171
+ typedef unsigned DTable_max_t[FSEv05_DTABLE_SIZE_U32(FSEv05_MAX_TABLELOG)];
1244
1172
 
1245
1173
 
1246
1174
  /* **************************************************************
@@ -1302,6 +1230,7 @@ size_t FSEv05_buildDTable(FSEv05_DTable* dt, const short* normalizedCounter, uns
1302
1230
  if (tableLog > FSEv05_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
1303
1231
 
1304
1232
  /* Init, lay down lowprob symbols */
1233
+ memset(tableDecode, 0, sizeof(FSEv05_FUNCTION_TYPE) * (maxSymbolValue+1) ); /* useless init, but keep static analyzer happy, and we don't need to performance optimize legacy decoders */
1305
1234
  DTableH.tableLog = (U16)tableLog;
1306
1235
  for (s=0; s<=maxSymbolValue; s++) {
1307
1236
  if (normalizedCounter[s]==-1) {
@@ -1879,7 +1808,7 @@ static size_t HUFv05_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
1879
1808
 
1880
1809
  if (!srcSize) return ERROR(srcSize_wrong);
1881
1810
  iSize = ip[0];
1882
- //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
1811
+ /* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */
1883
1812
 
1884
1813
  if (iSize >= 128) { /* special header */
1885
1814
  if (iSize >= (242)) { /* RLE */
@@ -1954,7 +1883,7 @@ size_t HUFv05_readDTableX2 (U16* DTable, const void* src, size_t srcSize)
1954
1883
  HUFv05_DEltX2* const dt = (HUFv05_DEltX2*)dtPtr;
1955
1884
 
1956
1885
  HUFv05_STATIC_ASSERT(sizeof(HUFv05_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */
1957
- //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */
1886
+ /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
1958
1887
 
1959
1888
  iSize = HUFv05_readStats(huffWeight, HUFv05_MAX_SYMBOL_VALUE + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
1960
1889
  if (HUFv05_isError(iSize)) return iSize;
@@ -2073,91 +2002,92 @@ size_t HUFv05_decompress4X2_usingDTable(
2073
2002
  const void* cSrc, size_t cSrcSize,
2074
2003
  const U16* DTable)
2075
2004
  {
2076
- const BYTE* const istart = (const BYTE*) cSrc;
2077
- BYTE* const ostart = (BYTE*) dst;
2078
- BYTE* const oend = ostart + dstSize;
2079
- const void* const dtPtr = DTable;
2080
- const HUFv05_DEltX2* const dt = ((const HUFv05_DEltX2*)dtPtr) +1;
2081
- const U32 dtLog = DTable[0];
2082
- size_t errorCode;
2083
-
2084
- /* Init */
2085
- BITv05_DStream_t bitD1;
2086
- BITv05_DStream_t bitD2;
2087
- BITv05_DStream_t bitD3;
2088
- BITv05_DStream_t bitD4;
2089
- const size_t length1 = MEM_readLE16(istart);
2090
- const size_t length2 = MEM_readLE16(istart+2);
2091
- const size_t length3 = MEM_readLE16(istart+4);
2092
- size_t length4;
2093
- const BYTE* const istart1 = istart + 6; /* jumpTable */
2094
- const BYTE* const istart2 = istart1 + length1;
2095
- const BYTE* const istart3 = istart2 + length2;
2096
- const BYTE* const istart4 = istart3 + length3;
2097
- const size_t segmentSize = (dstSize+3) / 4;
2098
- BYTE* const opStart2 = ostart + segmentSize;
2099
- BYTE* const opStart3 = opStart2 + segmentSize;
2100
- BYTE* const opStart4 = opStart3 + segmentSize;
2101
- BYTE* op1 = ostart;
2102
- BYTE* op2 = opStart2;
2103
- BYTE* op3 = opStart3;
2104
- BYTE* op4 = opStart4;
2105
- U32 endSignal;
2106
-
2107
2005
  /* Check */
2108
2006
  if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
2007
+ {
2008
+ const BYTE* const istart = (const BYTE*) cSrc;
2009
+ BYTE* const ostart = (BYTE*) dst;
2010
+ BYTE* const oend = ostart + dstSize;
2011
+ const void* const dtPtr = DTable;
2012
+ const HUFv05_DEltX2* const dt = ((const HUFv05_DEltX2*)dtPtr) +1;
2013
+ const U32 dtLog = DTable[0];
2014
+ size_t errorCode;
2109
2015
 
2110
- length4 = cSrcSize - (length1 + length2 + length3 + 6);
2111
- if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
2112
- errorCode = BITv05_initDStream(&bitD1, istart1, length1);
2113
- if (HUFv05_isError(errorCode)) return errorCode;
2114
- errorCode = BITv05_initDStream(&bitD2, istart2, length2);
2115
- if (HUFv05_isError(errorCode)) return errorCode;
2116
- errorCode = BITv05_initDStream(&bitD3, istart3, length3);
2117
- if (HUFv05_isError(errorCode)) return errorCode;
2118
- errorCode = BITv05_initDStream(&bitD4, istart4, length4);
2119
- if (HUFv05_isError(errorCode)) return errorCode;
2016
+ /* Init */
2017
+ BITv05_DStream_t bitD1;
2018
+ BITv05_DStream_t bitD2;
2019
+ BITv05_DStream_t bitD3;
2020
+ BITv05_DStream_t bitD4;
2021
+ const size_t length1 = MEM_readLE16(istart);
2022
+ const size_t length2 = MEM_readLE16(istart+2);
2023
+ const size_t length3 = MEM_readLE16(istart+4);
2024
+ size_t length4;
2025
+ const BYTE* const istart1 = istart + 6; /* jumpTable */
2026
+ const BYTE* const istart2 = istart1 + length1;
2027
+ const BYTE* const istart3 = istart2 + length2;
2028
+ const BYTE* const istart4 = istart3 + length3;
2029
+ const size_t segmentSize = (dstSize+3) / 4;
2030
+ BYTE* const opStart2 = ostart + segmentSize;
2031
+ BYTE* const opStart3 = opStart2 + segmentSize;
2032
+ BYTE* const opStart4 = opStart3 + segmentSize;
2033
+ BYTE* op1 = ostart;
2034
+ BYTE* op2 = opStart2;
2035
+ BYTE* op3 = opStart3;
2036
+ BYTE* op4 = opStart4;
2037
+ U32 endSignal;
2120
2038
 
2121
- /* 16-32 symbols per loop (4-8 symbols per stream) */
2122
- endSignal = BITv05_reloadDStream(&bitD1) | BITv05_reloadDStream(&bitD2) | BITv05_reloadDStream(&bitD3) | BITv05_reloadDStream(&bitD4);
2123
- for ( ; (endSignal==BITv05_DStream_unfinished) && (op4<(oend-7)) ; ) {
2124
- HUFv05_DECODE_SYMBOLX2_2(op1, &bitD1);
2125
- HUFv05_DECODE_SYMBOLX2_2(op2, &bitD2);
2126
- HUFv05_DECODE_SYMBOLX2_2(op3, &bitD3);
2127
- HUFv05_DECODE_SYMBOLX2_2(op4, &bitD4);
2128
- HUFv05_DECODE_SYMBOLX2_1(op1, &bitD1);
2129
- HUFv05_DECODE_SYMBOLX2_1(op2, &bitD2);
2130
- HUFv05_DECODE_SYMBOLX2_1(op3, &bitD3);
2131
- HUFv05_DECODE_SYMBOLX2_1(op4, &bitD4);
2132
- HUFv05_DECODE_SYMBOLX2_2(op1, &bitD1);
2133
- HUFv05_DECODE_SYMBOLX2_2(op2, &bitD2);
2134
- HUFv05_DECODE_SYMBOLX2_2(op3, &bitD3);
2135
- HUFv05_DECODE_SYMBOLX2_2(op4, &bitD4);
2136
- HUFv05_DECODE_SYMBOLX2_0(op1, &bitD1);
2137
- HUFv05_DECODE_SYMBOLX2_0(op2, &bitD2);
2138
- HUFv05_DECODE_SYMBOLX2_0(op3, &bitD3);
2139
- HUFv05_DECODE_SYMBOLX2_0(op4, &bitD4);
2039
+ length4 = cSrcSize - (length1 + length2 + length3 + 6);
2040
+ if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
2041
+ errorCode = BITv05_initDStream(&bitD1, istart1, length1);
2042
+ if (HUFv05_isError(errorCode)) return errorCode;
2043
+ errorCode = BITv05_initDStream(&bitD2, istart2, length2);
2044
+ if (HUFv05_isError(errorCode)) return errorCode;
2045
+ errorCode = BITv05_initDStream(&bitD3, istart3, length3);
2046
+ if (HUFv05_isError(errorCode)) return errorCode;
2047
+ errorCode = BITv05_initDStream(&bitD4, istart4, length4);
2048
+ if (HUFv05_isError(errorCode)) return errorCode;
2049
+
2050
+ /* 16-32 symbols per loop (4-8 symbols per stream) */
2140
2051
  endSignal = BITv05_reloadDStream(&bitD1) | BITv05_reloadDStream(&bitD2) | BITv05_reloadDStream(&bitD3) | BITv05_reloadDStream(&bitD4);
2141
- }
2052
+ for ( ; (endSignal==BITv05_DStream_unfinished) && (op4<(oend-7)) ; ) {
2053
+ HUFv05_DECODE_SYMBOLX2_2(op1, &bitD1);
2054
+ HUFv05_DECODE_SYMBOLX2_2(op2, &bitD2);
2055
+ HUFv05_DECODE_SYMBOLX2_2(op3, &bitD3);
2056
+ HUFv05_DECODE_SYMBOLX2_2(op4, &bitD4);
2057
+ HUFv05_DECODE_SYMBOLX2_1(op1, &bitD1);
2058
+ HUFv05_DECODE_SYMBOLX2_1(op2, &bitD2);
2059
+ HUFv05_DECODE_SYMBOLX2_1(op3, &bitD3);
2060
+ HUFv05_DECODE_SYMBOLX2_1(op4, &bitD4);
2061
+ HUFv05_DECODE_SYMBOLX2_2(op1, &bitD1);
2062
+ HUFv05_DECODE_SYMBOLX2_2(op2, &bitD2);
2063
+ HUFv05_DECODE_SYMBOLX2_2(op3, &bitD3);
2064
+ HUFv05_DECODE_SYMBOLX2_2(op4, &bitD4);
2065
+ HUFv05_DECODE_SYMBOLX2_0(op1, &bitD1);
2066
+ HUFv05_DECODE_SYMBOLX2_0(op2, &bitD2);
2067
+ HUFv05_DECODE_SYMBOLX2_0(op3, &bitD3);
2068
+ HUFv05_DECODE_SYMBOLX2_0(op4, &bitD4);
2069
+ endSignal = BITv05_reloadDStream(&bitD1) | BITv05_reloadDStream(&bitD2) | BITv05_reloadDStream(&bitD3) | BITv05_reloadDStream(&bitD4);
2070
+ }
2142
2071
 
2143
- /* check corruption */
2144
- if (op1 > opStart2) return ERROR(corruption_detected);
2145
- if (op2 > opStart3) return ERROR(corruption_detected);
2146
- if (op3 > opStart4) return ERROR(corruption_detected);
2147
- /* note : op4 supposed already verified within main loop */
2072
+ /* check corruption */
2073
+ if (op1 > opStart2) return ERROR(corruption_detected);
2074
+ if (op2 > opStart3) return ERROR(corruption_detected);
2075
+ if (op3 > opStart4) return ERROR(corruption_detected);
2076
+ /* note : op4 supposed already verified within main loop */
2148
2077
 
2149
- /* finish bitStreams one by one */
2150
- HUFv05_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog);
2151
- HUFv05_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog);
2152
- HUFv05_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog);
2153
- HUFv05_decodeStreamX2(op4, &bitD4, oend, dt, dtLog);
2078
+ /* finish bitStreams one by one */
2079
+ HUFv05_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog);
2080
+ HUFv05_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog);
2081
+ HUFv05_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog);
2082
+ HUFv05_decodeStreamX2(op4, &bitD4, oend, dt, dtLog);
2154
2083
 
2155
- /* check */
2156
- endSignal = BITv05_endOfDStream(&bitD1) & BITv05_endOfDStream(&bitD2) & BITv05_endOfDStream(&bitD3) & BITv05_endOfDStream(&bitD4);
2157
- if (!endSignal) return ERROR(corruption_detected);
2084
+ /* check */
2085
+ endSignal = BITv05_endOfDStream(&bitD1) & BITv05_endOfDStream(&bitD2) & BITv05_endOfDStream(&bitD3) & BITv05_endOfDStream(&bitD4);
2086
+ if (!endSignal) return ERROR(corruption_detected);
2158
2087
 
2159
- /* decoded size */
2160
- return dstSize;
2088
+ /* decoded size */
2089
+ return dstSize;
2090
+ }
2161
2091
  }
2162
2092
 
2163
2093
 
@@ -2268,7 +2198,7 @@ static void HUFv05_fillDTableX4(HUFv05_DEltX4* DTable, const U32 targetLog,
2268
2198
  }
2269
2199
  }
2270
2200
 
2271
- size_t HUFv05_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
2201
+ size_t HUFv05_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize)
2272
2202
  {
2273
2203
  BYTE weightList[HUFv05_MAX_SYMBOL_VALUE + 1];
2274
2204
  sortedSymbol_t sortedSymbol[HUFv05_MAX_SYMBOL_VALUE + 1];
@@ -2282,9 +2212,9 @@ size_t HUFv05_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
2282
2212
  void* dtPtr = DTable;
2283
2213
  HUFv05_DEltX4* const dt = ((HUFv05_DEltX4*)dtPtr) + 1;
2284
2214
 
2285
- HUFv05_STATIC_ASSERT(sizeof(HUFv05_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */
2215
+ HUFv05_STATIC_ASSERT(sizeof(HUFv05_DEltX4) == sizeof(unsigned)); /* if compilation fails here, assertion is false */
2286
2216
  if (memLog > HUFv05_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
2287
- //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */
2217
+ /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
2288
2218
 
2289
2219
  iSize = HUFv05_readStats(weightList, HUFv05_MAX_SYMBOL_VALUE + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
2290
2220
  if (HUFv05_isError(iSize)) return iSize;
@@ -2409,7 +2339,7 @@ static inline size_t HUFv05_decodeStreamX4(BYTE* p, BITv05_DStream_t* bitDPtr, B
2409
2339
  size_t HUFv05_decompress1X4_usingDTable(
2410
2340
  void* dst, size_t dstSize,
2411
2341
  const void* cSrc, size_t cSrcSize,
2412
- const U32* DTable)
2342
+ const unsigned* DTable)
2413
2343
  {
2414
2344
  const BYTE* const istart = (const BYTE*) cSrc;
2415
2345
  BYTE* const ostart = (BYTE*) dst;
@@ -2452,7 +2382,7 @@ size_t HUFv05_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t
2452
2382
  size_t HUFv05_decompress4X4_usingDTable(
2453
2383
  void* dst, size_t dstSize,
2454
2384
  const void* cSrc, size_t cSrcSize,
2455
- const U32* DTable)
2385
+ const unsigned* DTable)
2456
2386
  {
2457
2387
  if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
2458
2388
 
@@ -2613,9 +2543,9 @@ size_t HUFv05_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
2613
2543
 
2614
2544
  return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
2615
2545
 
2616
- //return HUFv05_decompress4X2(dst, dstSize, cSrc, cSrcSize); /* multi-streams single-symbol decoding */
2617
- //return HUFv05_decompress4X4(dst, dstSize, cSrc, cSrcSize); /* multi-streams double-symbols decoding */
2618
- //return HUFv05_decompress4X6(dst, dstSize, cSrc, cSrcSize); /* multi-streams quad-symbols decoding */
2546
+ /* return HUFv05_decompress4X2(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams single-symbol decoding */
2547
+ /* return HUFv05_decompress4X4(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams double-symbols decoding */
2548
+ /* return HUFv05_decompress4X6(dst, dstSize, cSrc, cSrcSize); */ /* multi-streams quad-symbols decoding */
2619
2549
  }
2620
2550
  /*
2621
2551
  zstd - standard compression library
@@ -2731,12 +2661,12 @@ struct ZSTDv05_DCtx_s
2731
2661
  ZSTDv05_dStage stage;
2732
2662
  U32 flagStaticTables;
2733
2663
  const BYTE* litPtr;
2734
- size_t litBufSize;
2735
2664
  size_t litSize;
2736
2665
  BYTE litBuffer[BLOCKSIZE + WILDCOPY_OVERLENGTH];
2737
2666
  BYTE headerBuffer[ZSTDv05_frameHeaderSize_max];
2738
2667
  }; /* typedef'd to ZSTDv05_DCtx within "zstd_static.h" */
2739
2668
 
2669
+ size_t ZSTDv05_sizeofDCtx (void); /* Hidden declaration */
2740
2670
  size_t ZSTDv05_sizeofDCtx (void) { return sizeof(ZSTDv05_DCtx); }
2741
2671
 
2742
2672
  size_t ZSTDv05_decompressBegin(ZSTDv05_DCtx* dctx)
@@ -2896,12 +2826,12 @@ static size_t ZSTDv05_decodeFrameHeader_Part2(ZSTDv05_DCtx* zc, const void* src,
2896
2826
  if (srcSize != zc->headerSize)
2897
2827
  return ERROR(srcSize_wrong);
2898
2828
  result = ZSTDv05_getFrameParams(&(zc->params), src, srcSize);
2899
- if ((MEM_32bits()) && (zc->params.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits);
2829
+ if ((MEM_32bits()) && (zc->params.windowLog > 25)) return ERROR(frameParameter_unsupported);
2900
2830
  return result;
2901
2831
  }
2902
2832
 
2903
2833
 
2904
- size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
2834
+ static size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
2905
2835
  {
2906
2836
  const BYTE* const in = (const BYTE* const)src;
2907
2837
  BYTE headerFlags;
@@ -2924,6 +2854,7 @@ size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t*
2924
2854
 
2925
2855
  static size_t ZSTDv05_copyRawBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
2926
2856
  {
2857
+ if (dst==NULL) return ERROR(dstSize_tooSmall);
2927
2858
  if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
2928
2859
  memcpy(dst, src, srcSize);
2929
2860
  return srcSize;
@@ -2932,8 +2863,8 @@ static size_t ZSTDv05_copyRawBlock(void* dst, size_t maxDstSize, const void* src
2932
2863
 
2933
2864
  /*! ZSTDv05_decodeLiteralsBlock() :
2934
2865
  @return : nb of bytes read from src (< srcSize ) */
2935
- size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
2936
- const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
2866
+ static size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
2867
+ const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
2937
2868
  {
2938
2869
  const BYTE* const istart = (const BYTE*) src;
2939
2870
 
@@ -2978,8 +2909,8 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
2978
2909
  return ERROR(corruption_detected);
2979
2910
 
2980
2911
  dctx->litPtr = dctx->litBuffer;
2981
- dctx->litBufSize = BLOCKSIZE+8;
2982
2912
  dctx->litSize = litSize;
2913
+ memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
2983
2914
  return litCSize + lhSize;
2984
2915
  }
2985
2916
  case IS_PCH:
@@ -2996,14 +2927,14 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
2996
2927
  lhSize=3;
2997
2928
  litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2);
2998
2929
  litCSize = ((istart[1] & 3) << 8) + istart[2];
2999
- if (litCSize + litSize > srcSize) return ERROR(corruption_detected);
2930
+ if (litCSize + lhSize > srcSize) return ERROR(corruption_detected);
3000
2931
 
3001
2932
  errorCode = HUFv05_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTableX4);
3002
2933
  if (HUFv05_isError(errorCode)) return ERROR(corruption_detected);
3003
2934
 
3004
2935
  dctx->litPtr = dctx->litBuffer;
3005
- dctx->litBufSize = BLOCKSIZE+WILDCOPY_OVERLENGTH;
3006
2936
  dctx->litSize = litSize;
2937
+ memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
3007
2938
  return litCSize + lhSize;
3008
2939
  }
3009
2940
  case IS_RAW:
@@ -3028,13 +2959,12 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
3028
2959
  if (litSize+lhSize > srcSize) return ERROR(corruption_detected);
3029
2960
  memcpy(dctx->litBuffer, istart+lhSize, litSize);
3030
2961
  dctx->litPtr = dctx->litBuffer;
3031
- dctx->litBufSize = BLOCKSIZE+8;
3032
2962
  dctx->litSize = litSize;
2963
+ memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
3033
2964
  return lhSize+litSize;
3034
2965
  }
3035
2966
  /* direct reference into compressed stream */
3036
2967
  dctx->litPtr = istart+lhSize;
3037
- dctx->litBufSize = srcSize-lhSize;
3038
2968
  dctx->litSize = litSize;
3039
2969
  return lhSize+litSize;
3040
2970
  }
@@ -3057,9 +2987,8 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
3057
2987
  break;
3058
2988
  }
3059
2989
  if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
3060
- memset(dctx->litBuffer, istart[lhSize], litSize);
2990
+ memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH);
3061
2991
  dctx->litPtr = dctx->litBuffer;
3062
- dctx->litBufSize = BLOCKSIZE+WILDCOPY_OVERLENGTH;
3063
2992
  dctx->litSize = litSize;
3064
2993
  return lhSize+1;
3065
2994
  }
@@ -3069,7 +2998,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
3069
2998
  }
3070
2999
 
3071
3000
 
3072
- size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
3001
+ static size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
3073
3002
  FSEv05_DTable* DTableLL, FSEv05_DTable* DTableML, FSEv05_DTable* DTableOffb,
3074
3003
  const void* src, size_t srcSize, U32 flagStaticTable)
3075
3004
  {
@@ -3077,7 +3006,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
3077
3006
  const BYTE* ip = istart;
3078
3007
  const BYTE* const iend = istart + srcSize;
3079
3008
  U32 LLtype, Offtype, MLtype;
3080
- U32 LLlog, Offlog, MLlog;
3009
+ unsigned LLlog, Offlog, MLlog;
3081
3010
  size_t dumpsLength;
3082
3011
 
3083
3012
  /* check */
@@ -3135,7 +3064,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
3135
3064
  break;
3136
3065
  case FSEv05_ENCODING_DYNAMIC :
3137
3066
  default : /* impossible */
3138
- { U32 max = MaxLL;
3067
+ { unsigned max = MaxLL;
3139
3068
  headerSize = FSEv05_readNCount(norm, &max, &LLlog, ip, iend-ip);
3140
3069
  if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
3141
3070
  if (LLlog > LLFSEv05Log) return ERROR(corruption_detected);
@@ -3159,7 +3088,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
3159
3088
  break;
3160
3089
  case FSEv05_ENCODING_DYNAMIC :
3161
3090
  default : /* impossible */
3162
- { U32 max = MaxOff;
3091
+ { unsigned max = MaxOff;
3163
3092
  headerSize = FSEv05_readNCount(norm, &max, &Offlog, ip, iend-ip);
3164
3093
  if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
3165
3094
  if (Offlog > OffFSEv05Log) return ERROR(corruption_detected);
@@ -3183,7 +3112,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
3183
3112
  break;
3184
3113
  case FSEv05_ENCODING_DYNAMIC :
3185
3114
  default : /* impossible */
3186
- { U32 max = MaxML;
3115
+ { unsigned max = MaxML;
3187
3116
  headerSize = FSEv05_readNCount(norm, &max, &MLlog, ip, iend-ip);
3188
3117
  if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
3189
3118
  if (MLlog > MLFSEv05Log) return ERROR(corruption_detected);
@@ -3226,14 +3155,18 @@ static void ZSTDv05_decodeSequence(seq_t* seq, seqState_t* seqState)
3226
3155
  litLength = FSEv05_peakSymbol(&(seqState->stateLL));
3227
3156
  prevOffset = litLength ? seq->offset : seqState->prevOffset;
3228
3157
  if (litLength == MaxLL) {
3229
- U32 add = *dumps++;
3158
+ const U32 add = *dumps++;
3230
3159
  if (add < 255) litLength += add;
3231
- else {
3232
- litLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no risk : dumps is always followed by seq tables > 1 byte */
3233
- if (litLength&1) litLength>>=1, dumps += 3;
3234
- else litLength = (U16)(litLength)>>1, dumps += 2;
3160
+ else if (dumps + 2 <= de) {
3161
+ litLength = MEM_readLE16(dumps);
3162
+ dumps += 2;
3163
+ if ((litLength & 1) && dumps < de) {
3164
+ litLength += *dumps << 16;
3165
+ dumps += 1;
3166
+ }
3167
+ litLength>>=1;
3235
3168
  }
3236
- if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
3169
+ if (dumps >= de) { dumps = de-1; } /* late correction, to avoid read overflow (data is now corrupted anyway) */
3237
3170
  }
3238
3171
 
3239
3172
  /* Offset */
@@ -3259,14 +3192,18 @@ static void ZSTDv05_decodeSequence(seq_t* seq, seqState_t* seqState)
3259
3192
  /* MatchLength */
3260
3193
  matchLength = FSEv05_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
3261
3194
  if (matchLength == MaxML) {
3262
- U32 add = *dumps++;
3195
+ const U32 add = dumps<de ? *dumps++ : 0;
3263
3196
  if (add < 255) matchLength += add;
3264
- else {
3265
- matchLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
3266
- if (matchLength&1) matchLength>>=1, dumps += 3;
3267
- else matchLength = (U16)(matchLength)>>1, dumps += 2;
3197
+ else if (dumps + 2 <= de) {
3198
+ matchLength = MEM_readLE16(dumps);
3199
+ dumps += 2;
3200
+ if ((matchLength & 1) && dumps < de) {
3201
+ matchLength += *dumps << 16;
3202
+ dumps += 1;
3203
+ }
3204
+ matchLength >>= 1;
3268
3205
  }
3269
- if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
3206
+ if (dumps >= de) { dumps = de-1; } /* late correction, to avoid read overflow (data is now corrupted anyway) */
3270
3207
  }
3271
3208
  matchLength += MINMATCH;
3272
3209
 
@@ -3289,11 +3226,11 @@ static void ZSTDv05_decodeSequence(seq_t* seq, seqState_t* seqState)
3289
3226
 
3290
3227
  static size_t ZSTDv05_execSequence(BYTE* op,
3291
3228
  BYTE* const oend, seq_t sequence,
3292
- const BYTE** litPtr, const BYTE* const litLimit_8,
3229
+ const BYTE** litPtr, const BYTE* const litLimit,
3293
3230
  const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
3294
3231
  {
3295
3232
  static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
3296
- static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */
3233
+ static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
3297
3234
  BYTE* const oLitEnd = op + sequence.litLength;
3298
3235
  const size_t sequenceLength = sequence.litLength + sequence.matchLength;
3299
3236
  BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
@@ -3304,7 +3241,7 @@ static size_t ZSTDv05_execSequence(BYTE* op,
3304
3241
  /* check */
3305
3242
  if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
3306
3243
  if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
3307
- if (litEnd > litLimit_8) return ERROR(corruption_detected); /* risk read beyond lit buffer */
3244
+ if (litEnd > litLimit) return ERROR(corruption_detected); /* risk read beyond lit buffer */
3308
3245
 
3309
3246
  /* copy Literals */
3310
3247
  ZSTDv05_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@@ -3328,7 +3265,7 @@ static size_t ZSTDv05_execSequence(BYTE* op,
3328
3265
  op = oLitEnd + length1;
3329
3266
  sequence.matchLength -= length1;
3330
3267
  match = base;
3331
- if (op > oend_8) {
3268
+ if (op > oend_8 || sequence.matchLength < MINMATCH) {
3332
3269
  while (op < oMatchEnd) *op++ = *match++;
3333
3270
  return sequenceLength;
3334
3271
  }
@@ -3351,7 +3288,7 @@ static size_t ZSTDv05_execSequence(BYTE* op,
3351
3288
  }
3352
3289
  op += 8; match += 8;
3353
3290
 
3354
- if (oMatchEnd > oend-12) {
3291
+ if (oMatchEnd > oend-(16-MINMATCH)) {
3355
3292
  if (op < oend_8) {
3356
3293
  ZSTDv05_wildcopy(op, match, oend_8 - op);
3357
3294
  match += oend_8 - op;
@@ -3360,7 +3297,7 @@ static size_t ZSTDv05_execSequence(BYTE* op,
3360
3297
  while (op < oMatchEnd)
3361
3298
  *op++ = *match++;
3362
3299
  } else {
3363
- ZSTDv05_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
3300
+ ZSTDv05_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
3364
3301
  }
3365
3302
  return sequenceLength;
3366
3303
  }
@@ -3376,15 +3313,14 @@ static size_t ZSTDv05_decompressSequences(
3376
3313
  BYTE* const ostart = (BYTE* const)dst;
3377
3314
  BYTE* op = ostart;
3378
3315
  BYTE* const oend = ostart + maxDstSize;
3379
- size_t errorCode, dumpsLength;
3316
+ size_t errorCode, dumpsLength=0;
3380
3317
  const BYTE* litPtr = dctx->litPtr;
3381
- const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8;
3382
3318
  const BYTE* const litEnd = litPtr + dctx->litSize;
3383
- int nbSeq;
3384
- const BYTE* dumps;
3385
- U32* DTableLL = dctx->LLTable;
3386
- U32* DTableML = dctx->MLTable;
3387
- U32* DTableOffb = dctx->OffTable;
3319
+ int nbSeq=0;
3320
+ const BYTE* dumps = NULL;
3321
+ unsigned* DTableLL = dctx->LLTable;
3322
+ unsigned* DTableML = dctx->MLTable;
3323
+ unsigned* DTableOffb = dctx->OffTable;
3388
3324
  const BYTE* const base = (const BYTE*) (dctx->base);
3389
3325
  const BYTE* const vBase = (const BYTE*) (dctx->vBase);
3390
3326
  const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
@@ -3416,7 +3352,7 @@ static size_t ZSTDv05_decompressSequences(
3416
3352
  size_t oneSeqSize;
3417
3353
  nbSeq--;
3418
3354
  ZSTDv05_decodeSequence(&sequence, &seqState);
3419
- oneSeqSize = ZSTDv05_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd);
3355
+ oneSeqSize = ZSTDv05_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
3420
3356
  if (ZSTDv05_isError(oneSeqSize)) return oneSeqSize;
3421
3357
  op += oneSeqSize;
3422
3358
  }
@@ -3430,8 +3366,10 @@ static size_t ZSTDv05_decompressSequences(
3430
3366
  size_t lastLLSize = litEnd - litPtr;
3431
3367
  if (litPtr > litEnd) return ERROR(corruption_detected); /* too many literals already used */
3432
3368
  if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
3433
- memcpy(op, litPtr, lastLLSize);
3434
- op += lastLLSize;
3369
+ if (lastLLSize > 0) {
3370
+ memcpy(op, litPtr, lastLLSize);
3371
+ op += lastLLSize;
3372
+ }
3435
3373
  }
3436
3374
 
3437
3375
  return op-ostart;
@@ -3490,10 +3428,10 @@ static size_t ZSTDv05_decompress_continueDCtx(ZSTDv05_DCtx* dctx,
3490
3428
  BYTE* const oend = ostart + maxDstSize;
3491
3429
  size_t remainingSize = srcSize;
3492
3430
  blockProperties_t blockProperties;
3431
+ memset(&blockProperties, 0, sizeof(blockProperties));
3493
3432
 
3494
3433
  /* Frame Header */
3495
- {
3496
- size_t frameHeaderSize;
3434
+ { size_t frameHeaderSize;
3497
3435
  if (srcSize < ZSTDv05_frameHeaderSize_min+ZSTDv05_blockHeaderSize) return ERROR(srcSize_wrong);
3498
3436
  frameHeaderSize = ZSTDv05_decodeFrameHeader_Part1(dctx, src, ZSTDv05_frameHeaderSize_min);
3499
3437
  if (ZSTDv05_isError(frameHeaderSize)) return frameHeaderSize;
@@ -3585,6 +3523,58 @@ size_t ZSTDv05_decompress(void* dst, size_t maxDstSize, const void* src, size_t
3585
3523
  #endif
3586
3524
  }
3587
3525
 
3526
+ /* ZSTD_errorFrameSizeInfoLegacy() :
3527
+ assumes `cSize` and `dBound` are _not_ NULL */
3528
+ static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
3529
+ {
3530
+ *cSize = ret;
3531
+ *dBound = ZSTD_CONTENTSIZE_ERROR;
3532
+ }
3533
+
3534
+ void ZSTDv05_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
3535
+ {
3536
+ const BYTE* ip = (const BYTE*)src;
3537
+ size_t remainingSize = srcSize;
3538
+ size_t nbBlocks = 0;
3539
+ blockProperties_t blockProperties;
3540
+
3541
+ /* Frame Header */
3542
+ if (srcSize < ZSTDv05_frameHeaderSize_min) {
3543
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3544
+ return;
3545
+ }
3546
+ if (MEM_readLE32(src) != ZSTDv05_MAGICNUMBER) {
3547
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
3548
+ return;
3549
+ }
3550
+ ip += ZSTDv05_frameHeaderSize_min; remainingSize -= ZSTDv05_frameHeaderSize_min;
3551
+
3552
+ /* Loop on each block */
3553
+ while (1)
3554
+ {
3555
+ size_t cBlockSize = ZSTDv05_getcBlockSize(ip, remainingSize, &blockProperties);
3556
+ if (ZSTDv05_isError(cBlockSize)) {
3557
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
3558
+ return;
3559
+ }
3560
+
3561
+ ip += ZSTDv05_blockHeaderSize;
3562
+ remainingSize -= ZSTDv05_blockHeaderSize;
3563
+ if (cBlockSize > remainingSize) {
3564
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3565
+ return;
3566
+ }
3567
+
3568
+ if (cBlockSize == 0) break; /* bt_end */
3569
+
3570
+ ip += cBlockSize;
3571
+ remainingSize -= cBlockSize;
3572
+ nbBlocks++;
3573
+ }
3574
+
3575
+ *cSize = ip - (const BYTE*)src;
3576
+ *dBound = nbBlocks * BLOCKSIZE;
3577
+ }
3588
3578
 
3589
3579
  /* ******************************
3590
3580
  * Streaming Decompression API
@@ -3681,7 +3671,7 @@ static size_t ZSTDv05_loadEntropy(ZSTDv05_DCtx* dctx, const void* dict, size_t d
3681
3671
  {
3682
3672
  size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, errorCode, litlengthHeaderSize;
3683
3673
  short offcodeNCount[MaxOff+1];
3684
- U32 offcodeMaxValue=MaxOff, offcodeLog;
3674
+ unsigned offcodeMaxValue=MaxOff, offcodeLog;
3685
3675
  short matchlengthNCount[MaxML+1];
3686
3676
  unsigned matchlengthMaxValue = MaxML, matchlengthLog;
3687
3677
  short litlengthNCount[MaxLL+1];
@@ -3807,7 +3797,9 @@ static size_t ZBUFFv05_blockHeaderSize = 3;
3807
3797
  static size_t ZBUFFv05_limitCopy(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
3808
3798
  {
3809
3799
  size_t length = MIN(maxDstSize, srcSize);
3810
- memcpy(dst, src, length);
3800
+ if (length > 0) {
3801
+ memcpy(dst, src, length);
3802
+ }
3811
3803
  return length;
3812
3804
  }
3813
3805
 
@@ -3928,7 +3920,7 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst
3928
3920
  zbc->stage = ZBUFFv05ds_decodeHeader;
3929
3921
  break;
3930
3922
  }
3931
-
3923
+ /* fall-through */
3932
3924
  case ZBUFFv05ds_loadHeader:
3933
3925
  /* complete header from src */
3934
3926
  {
@@ -3944,9 +3936,9 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst
3944
3936
  *maxDstSizePtr = 0;
3945
3937
  return headerSize - zbc->hPos;
3946
3938
  }
3947
- // zbc->stage = ZBUFFv05ds_decodeHeader; break; /* useless : stage follows */
3939
+ /* zbc->stage = ZBUFFv05ds_decodeHeader; break; */ /* useless : stage follows */
3948
3940
  }
3949
-
3941
+ /* fall-through */
3950
3942
  case ZBUFFv05ds_decodeHeader:
3951
3943
  /* apply header to create / resize buffers */
3952
3944
  {
@@ -3973,7 +3965,7 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst
3973
3965
  break;
3974
3966
  }
3975
3967
  zbc->stage = ZBUFFv05ds_read;
3976
-
3968
+ /* fall-through */
3977
3969
  case ZBUFFv05ds_read:
3978
3970
  {
3979
3971
  size_t neededInSize = ZSTDv05_nextSrcSizeToDecompress(zbc->zc);
@@ -3997,7 +3989,7 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst
3997
3989
  if (ip==iend) { notDone = 0; break; } /* no more input */
3998
3990
  zbc->stage = ZBUFFv05ds_load;
3999
3991
  }
4000
-
3992
+ /* fall-through */
4001
3993
  case ZBUFFv05ds_load:
4002
3994
  {
4003
3995
  size_t neededInSize = ZSTDv05_nextSrcSizeToDecompress(zbc->zc);
@@ -4017,8 +4009,10 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst
4017
4009
  if (!decodedSize) { zbc->stage = ZBUFFv05ds_read; break; } /* this was just a header */
4018
4010
  zbc->outEnd = zbc->outStart + decodedSize;
4019
4011
  zbc->stage = ZBUFFv05ds_flush;
4020
- // break; /* ZBUFFv05ds_flush follows */
4021
- } }
4012
+ /* break; */ /* ZBUFFv05ds_flush follows */
4013
+ }
4014
+ }
4015
+ /* fall-through */
4022
4016
  case ZBUFFv05ds_flush:
4023
4017
  {
4024
4018
  size_t toFlushSize = zbc->outEnd - zbc->outStart;