extzstd 0.1 → 0.3.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (134) hide show
  1. checksums.yaml +5 -5
  2. data/HISTORY.ja.md +39 -0
  3. data/README.md +38 -56
  4. data/contrib/zstd/CHANGELOG +613 -0
  5. data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
  6. data/contrib/zstd/CONTRIBUTING.md +406 -0
  7. data/contrib/zstd/COPYING +339 -0
  8. data/contrib/zstd/Makefile +420 -0
  9. data/contrib/zstd/README.md +179 -41
  10. data/contrib/zstd/TESTING.md +44 -0
  11. data/contrib/zstd/appveyor.yml +292 -0
  12. data/contrib/zstd/lib/BUCK +234 -0
  13. data/contrib/zstd/lib/Makefile +451 -0
  14. data/contrib/zstd/lib/README.md +207 -0
  15. data/contrib/zstd/{common → lib/common}/bitstream.h +187 -138
  16. data/contrib/zstd/lib/common/compiler.h +288 -0
  17. data/contrib/zstd/lib/common/cpu.h +213 -0
  18. data/contrib/zstd/lib/common/debug.c +24 -0
  19. data/contrib/zstd/lib/common/debug.h +107 -0
  20. data/contrib/zstd/lib/common/entropy_common.c +362 -0
  21. data/contrib/zstd/{common → lib/common}/error_private.c +25 -12
  22. data/contrib/zstd/{common → lib/common}/error_private.h +14 -10
  23. data/contrib/zstd/{common → lib/common}/fse.h +173 -92
  24. data/contrib/zstd/{common → lib/common}/fse_decompress.c +149 -85
  25. data/contrib/zstd/lib/common/huf.h +361 -0
  26. data/contrib/zstd/{common → lib/common}/mem.h +115 -59
  27. data/contrib/zstd/lib/common/pool.c +350 -0
  28. data/contrib/zstd/lib/common/pool.h +84 -0
  29. data/contrib/zstd/lib/common/threading.c +122 -0
  30. data/contrib/zstd/lib/common/threading.h +155 -0
  31. data/contrib/zstd/{common → lib/common}/xxhash.c +55 -96
  32. data/contrib/zstd/{common → lib/common}/xxhash.h +23 -47
  33. data/contrib/zstd/lib/common/zstd_common.c +83 -0
  34. data/contrib/zstd/lib/common/zstd_deps.h +111 -0
  35. data/contrib/zstd/lib/common/zstd_errors.h +95 -0
  36. data/contrib/zstd/lib/common/zstd_internal.h +478 -0
  37. data/contrib/zstd/{compress → lib/compress}/fse_compress.c +214 -319
  38. data/contrib/zstd/lib/compress/hist.c +181 -0
  39. data/contrib/zstd/lib/compress/hist.h +75 -0
  40. data/contrib/zstd/lib/compress/huf_compress.c +913 -0
  41. data/contrib/zstd/lib/compress/zstd_compress.c +5208 -0
  42. data/contrib/zstd/lib/compress/zstd_compress_internal.h +1203 -0
  43. data/contrib/zstd/lib/compress/zstd_compress_literals.c +158 -0
  44. data/contrib/zstd/lib/compress/zstd_compress_literals.h +29 -0
  45. data/contrib/zstd/lib/compress/zstd_compress_sequences.c +433 -0
  46. data/contrib/zstd/lib/compress/zstd_compress_sequences.h +54 -0
  47. data/contrib/zstd/lib/compress/zstd_compress_superblock.c +849 -0
  48. data/contrib/zstd/lib/compress/zstd_compress_superblock.h +32 -0
  49. data/contrib/zstd/lib/compress/zstd_cwksp.h +561 -0
  50. data/contrib/zstd/lib/compress/zstd_double_fast.c +521 -0
  51. data/contrib/zstd/lib/compress/zstd_double_fast.h +38 -0
  52. data/contrib/zstd/lib/compress/zstd_fast.c +496 -0
  53. data/contrib/zstd/lib/compress/zstd_fast.h +37 -0
  54. data/contrib/zstd/lib/compress/zstd_lazy.c +1412 -0
  55. data/contrib/zstd/lib/compress/zstd_lazy.h +87 -0
  56. data/contrib/zstd/lib/compress/zstd_ldm.c +660 -0
  57. data/contrib/zstd/lib/compress/zstd_ldm.h +116 -0
  58. data/contrib/zstd/lib/compress/zstd_opt.c +1345 -0
  59. data/contrib/zstd/lib/compress/zstd_opt.h +56 -0
  60. data/contrib/zstd/lib/compress/zstdmt_compress.c +1811 -0
  61. data/contrib/zstd/lib/compress/zstdmt_compress.h +110 -0
  62. data/contrib/zstd/lib/decompress/huf_decompress.c +1350 -0
  63. data/contrib/zstd/lib/decompress/zstd_ddict.c +244 -0
  64. data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
  65. data/contrib/zstd/lib/decompress/zstd_decompress.c +1930 -0
  66. data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1540 -0
  67. data/contrib/zstd/lib/decompress/zstd_decompress_block.h +62 -0
  68. data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +190 -0
  69. data/contrib/zstd/{common → lib/deprecated}/zbuff.h +68 -45
  70. data/contrib/zstd/lib/deprecated/zbuff_common.c +26 -0
  71. data/contrib/zstd/lib/deprecated/zbuff_compress.c +147 -0
  72. data/contrib/zstd/lib/deprecated/zbuff_decompress.c +75 -0
  73. data/contrib/zstd/lib/dictBuilder/cover.c +1245 -0
  74. data/contrib/zstd/lib/dictBuilder/cover.h +157 -0
  75. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.c +3 -3
  76. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.h +0 -0
  77. data/contrib/zstd/lib/dictBuilder/fastcover.c +758 -0
  78. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/zdict.c +318 -194
  79. data/contrib/zstd/lib/dictBuilder/zdict.h +305 -0
  80. data/contrib/zstd/{legacy → lib/legacy}/zstd_legacy.h +171 -15
  81. data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.c +191 -124
  82. data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.h +19 -5
  83. data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.c +125 -125
  84. data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.h +19 -5
  85. data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.c +125 -124
  86. data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.h +20 -6
  87. data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.c +151 -299
  88. data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.h +19 -5
  89. data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.c +237 -243
  90. data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.h +19 -6
  91. data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.c +130 -143
  92. data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.h +18 -5
  93. data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.c +158 -157
  94. data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.h +19 -5
  95. data/contrib/zstd/lib/libzstd.pc.in +15 -0
  96. data/contrib/zstd/lib/zstd.h +2391 -0
  97. data/ext/depend +2 -0
  98. data/ext/extconf.rb +15 -6
  99. data/ext/extzstd.c +76 -145
  100. data/ext/extzstd.h +80 -31
  101. data/ext/extzstd_stream.c +417 -142
  102. data/ext/libzstd_conf.h +8 -0
  103. data/ext/zstd_common.c +10 -7
  104. data/ext/zstd_compress.c +14 -5
  105. data/ext/zstd_decompress.c +5 -4
  106. data/ext/zstd_dictbuilder.c +9 -4
  107. data/ext/zstd_dictbuilder_fastcover.c +3 -0
  108. data/ext/zstd_legacy_v01.c +3 -1
  109. data/ext/zstd_legacy_v02.c +3 -1
  110. data/ext/zstd_legacy_v03.c +3 -1
  111. data/ext/zstd_legacy_v04.c +3 -1
  112. data/ext/zstd_legacy_v05.c +3 -1
  113. data/ext/zstd_legacy_v06.c +3 -1
  114. data/ext/zstd_legacy_v07.c +3 -1
  115. data/gemstub.rb +10 -24
  116. data/lib/extzstd.rb +64 -179
  117. data/lib/extzstd/version.rb +6 -1
  118. data/test/test_basic.rb +9 -6
  119. metadata +113 -57
  120. data/HISTORY.ja +0 -5
  121. data/contrib/zstd/common/entropy_common.c +0 -225
  122. data/contrib/zstd/common/huf.h +0 -228
  123. data/contrib/zstd/common/zstd_common.c +0 -83
  124. data/contrib/zstd/common/zstd_errors.h +0 -60
  125. data/contrib/zstd/common/zstd_internal.h +0 -267
  126. data/contrib/zstd/compress/huf_compress.c +0 -533
  127. data/contrib/zstd/compress/zbuff_compress.c +0 -319
  128. data/contrib/zstd/compress/zstd_compress.c +0 -3264
  129. data/contrib/zstd/compress/zstd_opt.h +0 -900
  130. data/contrib/zstd/decompress/huf_decompress.c +0 -883
  131. data/contrib/zstd/decompress/zbuff_decompress.c +0 -252
  132. data/contrib/zstd/decompress/zstd_decompress.c +0 -1842
  133. data/contrib/zstd/dictBuilder/zdict.h +0 -111
  134. data/contrib/zstd/zstd.h +0 -640
@@ -1,10 +1,11 @@
1
- /**
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
1
+ /*
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
- * This source code is licensed under the BSD-style license found in the
6
- * LICENSE file in the root directory of this source tree. An additional grant
7
- * of patent rights can be found in the PATENTS file in the same directory.
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
8
9
  */
9
10
 
10
11
  #ifndef ZSTD_V01_H_28739879432
@@ -34,6 +35,19 @@ ZSTDv01_decompress() : decompress ZSTD frames compliant with v0.1.x format
34
35
  size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize,
35
36
  const void* src, size_t compressedSize);
36
37
 
38
+ /**
39
+ ZSTDv01_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.1.x format
40
+ srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
41
+ cSize (output parameter) : the number of bytes that would be read to decompress this frame
42
+ or an error code if it fails (which can be tested using ZSTDv01_isError())
43
+ dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
44
+ or ZSTD_CONTENTSIZE_ERROR if an error occurs
45
+
46
+ note : assumes `cSize` and `dBound` are _not_ NULL.
47
+ */
48
+ void ZSTDv01_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
49
+ size_t* cSize, unsigned long long* dBound);
50
+
37
51
  /**
38
52
  ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error
39
53
  */
@@ -1,16 +1,17 @@
1
- /**
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
1
+ /*
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
- * This source code is licensed under the BSD-style license found in the
6
- * LICENSE file in the root directory of this source tree. An additional grant
7
- * of patent rights can be found in the PATENTS file in the same directory.
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
8
9
  */
9
10
 
10
11
 
11
12
  #include <stddef.h> /* size_t, ptrdiff_t */
12
13
  #include "zstd_v02.h"
13
- #include "error_private.h"
14
+ #include "../common/error_private.h"
14
15
 
15
16
 
16
17
  /******************************************
@@ -88,7 +89,11 @@ extern "C" {
88
89
  * Basic Types
89
90
  *****************************************************************/
90
91
  #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
91
- # include <stdint.h>
92
+ # if defined(_AIX)
93
+ # include <inttypes.h>
94
+ # else
95
+ # include <stdint.h> /* intptr_t */
96
+ # endif
92
97
  typedef uint8_t BYTE;
93
98
  typedef uint16_t U16;
94
99
  typedef int16_t S16;
@@ -188,7 +193,7 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
188
193
  memcpy(memPtr, &value, sizeof(value));
189
194
  }
190
195
 
191
- #endif // MEM_FORCE_MEMORY_ACCESS
196
+ #endif /* MEM_FORCE_MEMORY_ACCESS */
192
197
 
193
198
 
194
199
  MEM_STATIC U16 MEM_readLE16(const void* memPtr)
@@ -216,6 +221,11 @@ MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
216
221
  }
217
222
  }
218
223
 
224
+ MEM_STATIC U32 MEM_readLE24(const void* memPtr)
225
+ {
226
+ return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
227
+ }
228
+
219
229
  MEM_STATIC U32 MEM_readLE32(const void* memPtr)
220
230
  {
221
231
  if (MEM_isLittleEndian())
@@ -329,18 +339,6 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
329
339
  MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
330
340
 
331
341
 
332
- /*
333
- * Start by invoking BIT_initDStream().
334
- * A chunk of the bitStream is then stored into a local register.
335
- * Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
336
- * You can then retrieve bitFields stored into the local register, **in reverse order**.
337
- * Local register is manually filled from memory by the BIT_reloadDStream() method.
338
- * A reload guarantee a minimum of ((8*sizeof(size_t))-7) bits when its result is BIT_DStream_unfinished.
339
- * Otherwise, it can be less than that, so proceed accordingly.
340
- * Checking if DStream has reached its end can be performed with BIT_endOfDStream()
341
- */
342
-
343
-
344
342
  /******************************************
345
343
  * unsafe API
346
344
  ******************************************/
@@ -352,14 +350,14 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
352
350
  /****************************************************************
353
351
  * Helper functions
354
352
  ****************************************************************/
355
- MEM_STATIC unsigned BIT_highbit32 (register U32 val)
353
+ MEM_STATIC unsigned BIT_highbit32 (U32 val)
356
354
  {
357
355
  # if defined(_MSC_VER) /* Visual */
358
356
  unsigned long r=0;
359
357
  _BitScanReverse ( &r, val );
360
358
  return (unsigned) r;
361
359
  # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
362
- return 31 - __builtin_clz (val);
360
+ return __builtin_clz (val) ^ 31;
363
361
  # else /* Software version */
364
362
  static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
365
363
  U32 v = val;
@@ -410,11 +408,17 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
410
408
  switch(srcSize)
411
409
  {
412
410
  case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
411
+ /* fallthrough */
413
412
  case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
413
+ /* fallthrough */
414
414
  case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
415
+ /* fallthrough */
415
416
  case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
417
+ /* fallthrough */
416
418
  case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
419
+ /* fallthrough */
417
420
  case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
421
+ /* fallthrough */
418
422
  default:;
419
423
  }
420
424
  contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
@@ -426,13 +430,6 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
426
430
  return srcSize;
427
431
  }
428
432
 
429
- /*!BIT_lookBits
430
- * Provides next n bits from local register
431
- * local register is not modified (bits are still present for next read/look)
432
- * On 32-bits, maxNbBits==25
433
- * On 64-bits, maxNbBits==57
434
- * @return : value extracted
435
- */
436
433
  MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits)
437
434
  {
438
435
  const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
@@ -452,11 +449,6 @@ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
452
449
  bitD->bitsConsumed += nbBits;
453
450
  }
454
451
 
455
- /*!BIT_readBits
456
- * Read next n bits from local register.
457
- * pay attention to not read more than nbBits contained into local register.
458
- * @return : extracted value.
459
- */
460
452
  MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
461
453
  {
462
454
  size_t value = BIT_lookBits(bitD, nbBits);
@@ -475,8 +467,8 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
475
467
 
476
468
  MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
477
469
  {
478
- if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
479
- return BIT_DStream_overflow;
470
+ if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
471
+ return BIT_DStream_overflow;
480
472
 
481
473
  if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer))
482
474
  {
@@ -694,55 +686,6 @@ static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bi
694
686
 
695
687
  static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
696
688
 
697
- /*
698
- Let's now decompose FSE_decompress_usingDTable() into its unitary components.
699
- You will decode FSE-encoded symbols from the bitStream,
700
- and also any other bitFields you put in, **in reverse order**.
701
-
702
- You will need a few variables to track your bitStream. They are :
703
-
704
- BIT_DStream_t DStream; // Stream context
705
- FSE_DState_t DState; // State context. Multiple ones are possible
706
- FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
707
-
708
- The first thing to do is to init the bitStream.
709
- errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
710
-
711
- You should then retrieve your initial state(s)
712
- (in reverse flushing order if you have several ones) :
713
- errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
714
-
715
- You can then decode your data, symbol after symbol.
716
- For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
717
- Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
718
- unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
719
-
720
- You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
721
- Note : maximum allowed nbBits is 25, for 32-bits compatibility
722
- size_t bitField = BIT_readBits(&DStream, nbBits);
723
-
724
- All above operations only read from local register (which size depends on size_t).
725
- Refueling the register from memory is manually performed by the reload method.
726
- endSignal = FSE_reloadDStream(&DStream);
727
-
728
- BIT_reloadDStream() result tells if there is still some more data to read from DStream.
729
- BIT_DStream_unfinished : there is still some data left into the DStream.
730
- BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
731
- BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
732
- BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
733
-
734
- When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
735
- to properly detect the exact end of stream.
736
- After each decoded symbol, check if DStream is fully consumed using this simple test :
737
- BIT_reloadDStream(&DStream) >= BIT_DStream_completed
738
-
739
- When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
740
- Checking if DStream has reached its end is performed by :
741
- BIT_endOfDStream(&DStream);
742
- Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
743
- FSE_endOfDState(&DState);
744
- */
745
-
746
689
 
747
690
  /******************************************
748
691
  * FSE unsafe API
@@ -1334,8 +1277,8 @@ static size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsi
1334
1277
  else
1335
1278
  {
1336
1279
  bitCount -= (int)(8 * (iend - 4 - ip));
1337
- ip = iend - 4;
1338
- }
1280
+ ip = iend - 4;
1281
+ }
1339
1282
  bitStream = MEM_readLE32(ip) >> (bitCount & 31);
1340
1283
  }
1341
1284
  }
@@ -2040,7 +1983,7 @@ static size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
2040
1983
  rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
2041
1984
  }
2042
1985
 
2043
- /* Build rankVal */
1986
+ /* Build rankVal */
2044
1987
  {
2045
1988
  const U32 minBits = tableLog+1 - maxW;
2046
1989
  U32 nextRankVal = 0;
@@ -2374,7 +2317,7 @@ static size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
2374
2317
  rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
2375
2318
  }
2376
2319
 
2377
- /* Build rankVal */
2320
+ /* Build rankVal */
2378
2321
  {
2379
2322
  const U32 minBits = tableLog+1 - maxW;
2380
2323
  U32 nextRankVal = 0;
@@ -2794,6 +2737,8 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
2794
2737
  #define LITERAL_NOENTROPY 63
2795
2738
  #define COMMAND_NOENTROPY 7 /* to remove */
2796
2739
 
2740
+ #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
2741
+
2797
2742
  static const size_t ZSTD_blockHeaderSize = 3;
2798
2743
  static const size_t ZSTD_frameHeaderSize = 4;
2799
2744
 
@@ -2808,7 +2753,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
2808
2753
  #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
2809
2754
 
2810
2755
  /*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
2811
- static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
2756
+ static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
2812
2757
  {
2813
2758
  const BYTE* ip = (const BYTE*)src;
2814
2759
  BYTE* op = (BYTE*)dst;
@@ -2868,7 +2813,6 @@ struct ZSTD_DCtx_s
2868
2813
  blockType_t bType;
2869
2814
  U32 phase;
2870
2815
  const BYTE* litPtr;
2871
- size_t litBufSize;
2872
2816
  size_t litSize;
2873
2817
  BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
2874
2818
  }; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
@@ -2896,7 +2840,9 @@ static size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockPropertie
2896
2840
  static size_t ZSTD_copyUncompressedBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
2897
2841
  {
2898
2842
  if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
2899
- memcpy(dst, src, srcSize);
2843
+ if (srcSize > 0) {
2844
+ memcpy(dst, src, srcSize);
2845
+ }
2900
2846
  return srcSize;
2901
2847
  }
2902
2848
 
@@ -2940,8 +2886,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
2940
2886
  size_t litSize = BLOCKSIZE;
2941
2887
  const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
2942
2888
  dctx->litPtr = dctx->litBuffer;
2943
- dctx->litBufSize = BLOCKSIZE;
2944
2889
  dctx->litSize = litSize;
2890
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
2945
2891
  return readSize; /* works if it's an error too */
2946
2892
  }
2947
2893
  case IS_RAW:
@@ -2949,16 +2895,16 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
2949
2895
  const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
2950
2896
  if (litSize > srcSize-11) /* risk of reading too far with wildcopy */
2951
2897
  {
2952
- if (litSize > srcSize-3) return ERROR(corruption_detected);
2953
- memcpy(dctx->litBuffer, istart, litSize);
2954
- dctx->litPtr = dctx->litBuffer;
2955
- dctx->litBufSize = BLOCKSIZE;
2956
- dctx->litSize = litSize;
2957
- return litSize+3;
2958
- }
2959
- /* direct reference into compressed stream */
2898
+ if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
2899
+ if (litSize > srcSize-3) return ERROR(corruption_detected);
2900
+ memcpy(dctx->litBuffer, istart, litSize);
2901
+ dctx->litPtr = dctx->litBuffer;
2902
+ dctx->litSize = litSize;
2903
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
2904
+ return litSize+3;
2905
+ }
2906
+ /* direct reference into compressed stream */
2960
2907
  dctx->litPtr = istart+3;
2961
- dctx->litBufSize = srcSize-3;
2962
2908
  dctx->litSize = litSize;
2963
2909
  return litSize+3;
2964
2910
  }
@@ -2966,9 +2912,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
2966
2912
  {
2967
2913
  const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
2968
2914
  if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
2969
- memset(dctx->litBuffer, istart[3], litSize);
2915
+ memset(dctx->litBuffer, istart[3], litSize + 8);
2970
2916
  dctx->litPtr = dctx->litBuffer;
2971
- dctx->litBufSize = BLOCKSIZE;
2972
2917
  dctx->litSize = litSize;
2973
2918
  return 4;
2974
2919
  }
@@ -3110,11 +3055,11 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
3110
3055
  seqState->prevOffset = seq->offset;
3111
3056
  if (litLength == MaxLL)
3112
3057
  {
3113
- U32 add = *dumps++;
3058
+ const U32 add = dumps<de ? *dumps++ : 0;
3114
3059
  if (add < 255) litLength += add;
3115
- else
3060
+ else if (dumps + 3 <= de)
3116
3061
  {
3117
- litLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
3062
+ litLength = MEM_readLE24(dumps);
3118
3063
  dumps += 3;
3119
3064
  }
3120
3065
  if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
@@ -3140,11 +3085,11 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
3140
3085
  matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
3141
3086
  if (matchLength == MaxML)
3142
3087
  {
3143
- U32 add = *dumps++;
3088
+ const U32 add = dumps<de ? *dumps++ : 0;
3144
3089
  if (add < 255) matchLength += add;
3145
- else
3090
+ else if (dumps + 3 <= de)
3146
3091
  {
3147
- matchLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
3092
+ matchLength = MEM_readLE24(dumps);
3148
3093
  dumps += 3;
3149
3094
  }
3150
3095
  if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
@@ -3165,7 +3110,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3165
3110
  BYTE* const base, BYTE* const oend)
3166
3111
  {
3167
3112
  static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
3168
- static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */
3113
+ static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* subtracted */
3169
3114
  const BYTE* const ostart = op;
3170
3115
  BYTE* const oLitEnd = op + sequence.litLength;
3171
3116
  BYTE* const oMatchEnd = op + sequence.litLength + sequence.matchLength; /* risk : address space overflow (32-bits) */
@@ -3175,7 +3120,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3175
3120
  /* checks */
3176
3121
  if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
3177
3122
  if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
3178
- if (litEnd > litLimit-8) return ERROR(corruption_detected); /* overRead beyond lit buffer */
3123
+ if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
3179
3124
 
3180
3125
  /* copy Literals */
3181
3126
  ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@@ -3209,7 +3154,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3209
3154
  }
3210
3155
  op += 8; match += 8;
3211
3156
 
3212
- if (oMatchEnd > oend-12)
3157
+ if (oMatchEnd > oend-(16-MINMATCH))
3213
3158
  {
3214
3159
  if (op < oend_8)
3215
3160
  {
@@ -3221,7 +3166,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3221
3166
  }
3222
3167
  else
3223
3168
  {
3224
- ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
3169
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
3225
3170
  }
3226
3171
  }
3227
3172
 
@@ -3241,7 +3186,6 @@ static size_t ZSTD_decompressSequences(
3241
3186
  BYTE* const oend = ostart + maxDstSize;
3242
3187
  size_t errorCode, dumpsLength;
3243
3188
  const BYTE* litPtr = dctx->litPtr;
3244
- const BYTE* const litMax = litPtr + dctx->litBufSize;
3245
3189
  const BYTE* const litEnd = litPtr + dctx->litSize;
3246
3190
  int nbSeq;
3247
3191
  const BYTE* dumps;
@@ -3277,7 +3221,7 @@ static size_t ZSTD_decompressSequences(
3277
3221
  size_t oneSeqSize;
3278
3222
  nbSeq--;
3279
3223
  ZSTD_decodeSequence(&sequence, &seqState);
3280
- oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litMax, base, oend);
3224
+ oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
3281
3225
  if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
3282
3226
  op += oneSeqSize;
3283
3227
  }
@@ -3291,8 +3235,10 @@ static size_t ZSTD_decompressSequences(
3291
3235
  size_t lastLLSize = litEnd - litPtr;
3292
3236
  if (litPtr > litEnd) return ERROR(corruption_detected);
3293
3237
  if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
3294
- if (op != litPtr) memmove(op, litPtr, lastLLSize);
3295
- op += lastLLSize;
3238
+ if (lastLLSize > 0) {
3239
+ if (op != litPtr) memmove(op, litPtr, lastLLSize);
3240
+ op += lastLLSize;
3241
+ }
3296
3242
  }
3297
3243
  }
3298
3244
 
@@ -3382,6 +3328,60 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
3382
3328
  return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
3383
3329
  }
3384
3330
 
3331
+ /* ZSTD_errorFrameSizeInfoLegacy() :
3332
+ assumes `cSize` and `dBound` are _not_ NULL */
3333
+ static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
3334
+ {
3335
+ *cSize = ret;
3336
+ *dBound = ZSTD_CONTENTSIZE_ERROR;
3337
+ }
3338
+
3339
+ void ZSTDv02_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
3340
+ {
3341
+ const BYTE* ip = (const BYTE*)src;
3342
+ size_t remainingSize = srcSize;
3343
+ size_t nbBlocks = 0;
3344
+ U32 magicNumber;
3345
+ blockProperties_t blockProperties;
3346
+
3347
+ /* Frame Header */
3348
+ if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) {
3349
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3350
+ return;
3351
+ }
3352
+ magicNumber = MEM_readLE32(src);
3353
+ if (magicNumber != ZSTD_magicNumber) {
3354
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
3355
+ return;
3356
+ }
3357
+ ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
3358
+
3359
+ /* Loop on each block */
3360
+ while (1)
3361
+ {
3362
+ size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
3363
+ if (ZSTD_isError(cBlockSize)) {
3364
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
3365
+ return;
3366
+ }
3367
+
3368
+ ip += ZSTD_blockHeaderSize;
3369
+ remainingSize -= ZSTD_blockHeaderSize;
3370
+ if (cBlockSize > remainingSize) {
3371
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3372
+ return;
3373
+ }
3374
+
3375
+ if (cBlockSize == 0) break; /* bt_end */
3376
+
3377
+ ip += cBlockSize;
3378
+ remainingSize -= cBlockSize;
3379
+ nbBlocks++;
3380
+ }
3381
+
3382
+ *cSize = ip - (const BYTE*)src;
3383
+ *dBound = nbBlocks * BLOCKSIZE;
3384
+ }
3385
3385
 
3386
3386
  /*******************************
3387
3387
  * Streaming Decompression API
@@ -3487,36 +3487,36 @@ static size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSi
3487
3487
 
3488
3488
  unsigned ZSTDv02_isError(size_t code)
3489
3489
  {
3490
- return ZSTD_isError(code);
3490
+ return ZSTD_isError(code);
3491
3491
  }
3492
3492
 
3493
3493
  size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
3494
3494
  const void* src, size_t compressedSize)
3495
3495
  {
3496
- return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
3496
+ return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
3497
3497
  }
3498
3498
 
3499
3499
  ZSTDv02_Dctx* ZSTDv02_createDCtx(void)
3500
3500
  {
3501
- return (ZSTDv02_Dctx*)ZSTD_createDCtx();
3501
+ return (ZSTDv02_Dctx*)ZSTD_createDCtx();
3502
3502
  }
3503
3503
 
3504
3504
  size_t ZSTDv02_freeDCtx(ZSTDv02_Dctx* dctx)
3505
3505
  {
3506
- return ZSTD_freeDCtx((ZSTD_DCtx*)dctx);
3506
+ return ZSTD_freeDCtx((ZSTD_DCtx*)dctx);
3507
3507
  }
3508
3508
 
3509
3509
  size_t ZSTDv02_resetDCtx(ZSTDv02_Dctx* dctx)
3510
3510
  {
3511
- return ZSTD_resetDCtx((ZSTD_DCtx*)dctx);
3511
+ return ZSTD_resetDCtx((ZSTD_DCtx*)dctx);
3512
3512
  }
3513
3513
 
3514
3514
  size_t ZSTDv02_nextSrcSizeToDecompress(ZSTDv02_Dctx* dctx)
3515
3515
  {
3516
- return ZSTD_nextSrcSizeToDecompress((ZSTD_DCtx*)dctx);
3516
+ return ZSTD_nextSrcSizeToDecompress((ZSTD_DCtx*)dctx);
3517
3517
  }
3518
3518
 
3519
3519
  size_t ZSTDv02_decompressContinue(ZSTDv02_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
3520
3520
  {
3521
- return ZSTD_decompressContinue((ZSTD_DCtx*)dctx, dst, maxDstSize, src, srcSize);
3521
+ return ZSTD_decompressContinue((ZSTD_DCtx*)dctx, dst, maxDstSize, src, srcSize);
3522
3522
  }