extzstd 0.1 → 0.3.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (134) hide show
  1. checksums.yaml +5 -5
  2. data/HISTORY.ja.md +39 -0
  3. data/README.md +38 -56
  4. data/contrib/zstd/CHANGELOG +613 -0
  5. data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
  6. data/contrib/zstd/CONTRIBUTING.md +406 -0
  7. data/contrib/zstd/COPYING +339 -0
  8. data/contrib/zstd/Makefile +420 -0
  9. data/contrib/zstd/README.md +179 -41
  10. data/contrib/zstd/TESTING.md +44 -0
  11. data/contrib/zstd/appveyor.yml +292 -0
  12. data/contrib/zstd/lib/BUCK +234 -0
  13. data/contrib/zstd/lib/Makefile +451 -0
  14. data/contrib/zstd/lib/README.md +207 -0
  15. data/contrib/zstd/{common → lib/common}/bitstream.h +187 -138
  16. data/contrib/zstd/lib/common/compiler.h +288 -0
  17. data/contrib/zstd/lib/common/cpu.h +213 -0
  18. data/contrib/zstd/lib/common/debug.c +24 -0
  19. data/contrib/zstd/lib/common/debug.h +107 -0
  20. data/contrib/zstd/lib/common/entropy_common.c +362 -0
  21. data/contrib/zstd/{common → lib/common}/error_private.c +25 -12
  22. data/contrib/zstd/{common → lib/common}/error_private.h +14 -10
  23. data/contrib/zstd/{common → lib/common}/fse.h +173 -92
  24. data/contrib/zstd/{common → lib/common}/fse_decompress.c +149 -85
  25. data/contrib/zstd/lib/common/huf.h +361 -0
  26. data/contrib/zstd/{common → lib/common}/mem.h +115 -59
  27. data/contrib/zstd/lib/common/pool.c +350 -0
  28. data/contrib/zstd/lib/common/pool.h +84 -0
  29. data/contrib/zstd/lib/common/threading.c +122 -0
  30. data/contrib/zstd/lib/common/threading.h +155 -0
  31. data/contrib/zstd/{common → lib/common}/xxhash.c +55 -96
  32. data/contrib/zstd/{common → lib/common}/xxhash.h +23 -47
  33. data/contrib/zstd/lib/common/zstd_common.c +83 -0
  34. data/contrib/zstd/lib/common/zstd_deps.h +111 -0
  35. data/contrib/zstd/lib/common/zstd_errors.h +95 -0
  36. data/contrib/zstd/lib/common/zstd_internal.h +478 -0
  37. data/contrib/zstd/{compress → lib/compress}/fse_compress.c +214 -319
  38. data/contrib/zstd/lib/compress/hist.c +181 -0
  39. data/contrib/zstd/lib/compress/hist.h +75 -0
  40. data/contrib/zstd/lib/compress/huf_compress.c +913 -0
  41. data/contrib/zstd/lib/compress/zstd_compress.c +5208 -0
  42. data/contrib/zstd/lib/compress/zstd_compress_internal.h +1203 -0
  43. data/contrib/zstd/lib/compress/zstd_compress_literals.c +158 -0
  44. data/contrib/zstd/lib/compress/zstd_compress_literals.h +29 -0
  45. data/contrib/zstd/lib/compress/zstd_compress_sequences.c +433 -0
  46. data/contrib/zstd/lib/compress/zstd_compress_sequences.h +54 -0
  47. data/contrib/zstd/lib/compress/zstd_compress_superblock.c +849 -0
  48. data/contrib/zstd/lib/compress/zstd_compress_superblock.h +32 -0
  49. data/contrib/zstd/lib/compress/zstd_cwksp.h +561 -0
  50. data/contrib/zstd/lib/compress/zstd_double_fast.c +521 -0
  51. data/contrib/zstd/lib/compress/zstd_double_fast.h +38 -0
  52. data/contrib/zstd/lib/compress/zstd_fast.c +496 -0
  53. data/contrib/zstd/lib/compress/zstd_fast.h +37 -0
  54. data/contrib/zstd/lib/compress/zstd_lazy.c +1412 -0
  55. data/contrib/zstd/lib/compress/zstd_lazy.h +87 -0
  56. data/contrib/zstd/lib/compress/zstd_ldm.c +660 -0
  57. data/contrib/zstd/lib/compress/zstd_ldm.h +116 -0
  58. data/contrib/zstd/lib/compress/zstd_opt.c +1345 -0
  59. data/contrib/zstd/lib/compress/zstd_opt.h +56 -0
  60. data/contrib/zstd/lib/compress/zstdmt_compress.c +1811 -0
  61. data/contrib/zstd/lib/compress/zstdmt_compress.h +110 -0
  62. data/contrib/zstd/lib/decompress/huf_decompress.c +1350 -0
  63. data/contrib/zstd/lib/decompress/zstd_ddict.c +244 -0
  64. data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
  65. data/contrib/zstd/lib/decompress/zstd_decompress.c +1930 -0
  66. data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1540 -0
  67. data/contrib/zstd/lib/decompress/zstd_decompress_block.h +62 -0
  68. data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +190 -0
  69. data/contrib/zstd/{common → lib/deprecated}/zbuff.h +68 -45
  70. data/contrib/zstd/lib/deprecated/zbuff_common.c +26 -0
  71. data/contrib/zstd/lib/deprecated/zbuff_compress.c +147 -0
  72. data/contrib/zstd/lib/deprecated/zbuff_decompress.c +75 -0
  73. data/contrib/zstd/lib/dictBuilder/cover.c +1245 -0
  74. data/contrib/zstd/lib/dictBuilder/cover.h +157 -0
  75. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.c +3 -3
  76. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.h +0 -0
  77. data/contrib/zstd/lib/dictBuilder/fastcover.c +758 -0
  78. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/zdict.c +318 -194
  79. data/contrib/zstd/lib/dictBuilder/zdict.h +305 -0
  80. data/contrib/zstd/{legacy → lib/legacy}/zstd_legacy.h +171 -15
  81. data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.c +191 -124
  82. data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.h +19 -5
  83. data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.c +125 -125
  84. data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.h +19 -5
  85. data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.c +125 -124
  86. data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.h +20 -6
  87. data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.c +151 -299
  88. data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.h +19 -5
  89. data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.c +237 -243
  90. data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.h +19 -6
  91. data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.c +130 -143
  92. data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.h +18 -5
  93. data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.c +158 -157
  94. data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.h +19 -5
  95. data/contrib/zstd/lib/libzstd.pc.in +15 -0
  96. data/contrib/zstd/lib/zstd.h +2391 -0
  97. data/ext/depend +2 -0
  98. data/ext/extconf.rb +15 -6
  99. data/ext/extzstd.c +76 -145
  100. data/ext/extzstd.h +80 -31
  101. data/ext/extzstd_stream.c +417 -142
  102. data/ext/libzstd_conf.h +8 -0
  103. data/ext/zstd_common.c +10 -7
  104. data/ext/zstd_compress.c +14 -5
  105. data/ext/zstd_decompress.c +5 -4
  106. data/ext/zstd_dictbuilder.c +9 -4
  107. data/ext/zstd_dictbuilder_fastcover.c +3 -0
  108. data/ext/zstd_legacy_v01.c +3 -1
  109. data/ext/zstd_legacy_v02.c +3 -1
  110. data/ext/zstd_legacy_v03.c +3 -1
  111. data/ext/zstd_legacy_v04.c +3 -1
  112. data/ext/zstd_legacy_v05.c +3 -1
  113. data/ext/zstd_legacy_v06.c +3 -1
  114. data/ext/zstd_legacy_v07.c +3 -1
  115. data/gemstub.rb +10 -24
  116. data/lib/extzstd.rb +64 -179
  117. data/lib/extzstd/version.rb +6 -1
  118. data/test/test_basic.rb +9 -6
  119. metadata +113 -57
  120. data/HISTORY.ja +0 -5
  121. data/contrib/zstd/common/entropy_common.c +0 -225
  122. data/contrib/zstd/common/huf.h +0 -228
  123. data/contrib/zstd/common/zstd_common.c +0 -83
  124. data/contrib/zstd/common/zstd_errors.h +0 -60
  125. data/contrib/zstd/common/zstd_internal.h +0 -267
  126. data/contrib/zstd/compress/huf_compress.c +0 -533
  127. data/contrib/zstd/compress/zbuff_compress.c +0 -319
  128. data/contrib/zstd/compress/zstd_compress.c +0 -3264
  129. data/contrib/zstd/compress/zstd_opt.h +0 -900
  130. data/contrib/zstd/decompress/huf_decompress.c +0 -883
  131. data/contrib/zstd/decompress/zbuff_decompress.c +0 -252
  132. data/contrib/zstd/decompress/zstd_decompress.c +0 -1842
  133. data/contrib/zstd/dictBuilder/zdict.h +0 -111
  134. data/contrib/zstd/zstd.h +0 -640
@@ -1,10 +1,11 @@
1
- /**
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
1
+ /*
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
- * This source code is licensed under the BSD-style license found in the
6
- * LICENSE file in the root directory of this source tree. An additional grant
7
- * of patent rights can be found in the PATENTS file in the same directory.
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
8
9
  */
9
10
 
10
11
  #ifndef ZSTD_V02_H_4174539423
@@ -34,6 +35,19 @@ ZSTDv02_decompress() : decompress ZSTD frames compliant with v0.2.x format
34
35
  size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
35
36
  const void* src, size_t compressedSize);
36
37
 
38
+ /**
39
+ ZSTDv02_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.2.x format
40
+ srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
41
+ cSize (output parameter) : the number of bytes that would be read to decompress this frame
42
+ or an error code if it fails (which can be tested using ZSTDv01_isError())
43
+ dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
44
+ or ZSTD_CONTENTSIZE_ERROR if an error occurs
45
+
46
+ note : assumes `cSize` and `dBound` are _not_ NULL.
47
+ */
48
+ void ZSTDv02_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
49
+ size_t* cSize, unsigned long long* dBound);
50
+
37
51
  /**
38
52
  ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error
39
53
  */
@@ -1,16 +1,17 @@
1
- /**
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
1
+ /*
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
- * This source code is licensed under the BSD-style license found in the
6
- * LICENSE file in the root directory of this source tree. An additional grant
7
- * of patent rights can be found in the PATENTS file in the same directory.
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
8
9
  */
9
10
 
10
11
 
11
12
  #include <stddef.h> /* size_t, ptrdiff_t */
12
13
  #include "zstd_v03.h"
13
- #include "error_private.h"
14
+ #include "../common/error_private.h"
14
15
 
15
16
 
16
17
  /******************************************
@@ -89,7 +90,11 @@ extern "C" {
89
90
  * Basic Types
90
91
  *****************************************************************/
91
92
  #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
92
- # include <stdint.h>
93
+ # if defined(_AIX)
94
+ # include <inttypes.h>
95
+ # else
96
+ # include <stdint.h> /* intptr_t */
97
+ # endif
93
98
  typedef uint8_t BYTE;
94
99
  typedef uint16_t U16;
95
100
  typedef int16_t S16;
@@ -190,7 +195,7 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
190
195
  }
191
196
 
192
197
 
193
- #endif // MEM_FORCE_MEMORY_ACCESS
198
+ #endif /* MEM_FORCE_MEMORY_ACCESS */
194
199
 
195
200
 
196
201
  MEM_STATIC U16 MEM_readLE16(const void* memPtr)
@@ -218,6 +223,11 @@ MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
218
223
  }
219
224
  }
220
225
 
226
+ MEM_STATIC U32 MEM_readLE24(const void* memPtr)
227
+ {
228
+ return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
229
+ }
230
+
221
231
  MEM_STATIC U32 MEM_readLE32(const void* memPtr)
222
232
  {
223
233
  if (MEM_isLittleEndian())
@@ -331,17 +341,6 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
331
341
  MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
332
342
 
333
343
 
334
- /*
335
- * Start by invoking BIT_initDStream().
336
- * A chunk of the bitStream is then stored into a local register.
337
- * Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
338
- * You can then retrieve bitFields stored into the local register, **in reverse order**.
339
- * Local register is manually filled from memory by the BIT_reloadDStream() method.
340
- * A reload guarantee a minimum of ((8*sizeof(size_t))-7) bits when its result is BIT_DStream_unfinished.
341
- * Otherwise, it can be less than that, so proceed accordingly.
342
- * Checking if DStream has reached its end can be performed with BIT_endOfDStream()
343
- */
344
-
345
344
 
346
345
  /******************************************
347
346
  * unsafe API
@@ -354,14 +353,14 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
354
353
  /****************************************************************
355
354
  * Helper functions
356
355
  ****************************************************************/
357
- MEM_STATIC unsigned BIT_highbit32 (register U32 val)
356
+ MEM_STATIC unsigned BIT_highbit32 (U32 val)
358
357
  {
359
358
  # if defined(_MSC_VER) /* Visual */
360
359
  unsigned long r=0;
361
360
  _BitScanReverse ( &r, val );
362
361
  return (unsigned) r;
363
362
  # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
364
- return 31 - __builtin_clz (val);
363
+ return __builtin_clz (val) ^ 31;
365
364
  # else /* Software version */
366
365
  static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
367
366
  U32 v = val;
@@ -412,11 +411,17 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
412
411
  switch(srcSize)
413
412
  {
414
413
  case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
414
+ /* fallthrough */
415
415
  case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
416
+ /* fallthrough */
416
417
  case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
418
+ /* fallthrough */
417
419
  case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
420
+ /* fallthrough */
418
421
  case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
422
+ /* fallthrough */
419
423
  case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
424
+ /* fallthrough */
420
425
  default:;
421
426
  }
422
427
  contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
@@ -427,14 +432,6 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
427
432
 
428
433
  return srcSize;
429
434
  }
430
-
431
- /*!BIT_lookBits
432
- * Provides next n bits from local register
433
- * local register is not modified (bits are still present for next read/look)
434
- * On 32-bits, maxNbBits==25
435
- * On 64-bits, maxNbBits==57
436
- * @return : value extracted
437
- */
438
435
  MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits)
439
436
  {
440
437
  const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
@@ -454,11 +451,6 @@ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
454
451
  bitD->bitsConsumed += nbBits;
455
452
  }
456
453
 
457
- /*!BIT_readBits
458
- * Read next n bits from local register.
459
- * pay attention to not read more than nbBits contained into local register.
460
- * @return : extracted value.
461
- */
462
454
  MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
463
455
  {
464
456
  size_t value = BIT_lookBits(bitD, nbBits);
@@ -477,8 +469,8 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
477
469
 
478
470
  MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
479
471
  {
480
- if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
481
- return BIT_DStream_overflow;
472
+ if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
473
+ return BIT_DStream_overflow;
482
474
 
483
475
  if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer))
484
476
  {
@@ -696,55 +688,6 @@ static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bi
696
688
 
697
689
  static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
698
690
 
699
- /*
700
- Let's now decompose FSE_decompress_usingDTable() into its unitary components.
701
- You will decode FSE-encoded symbols from the bitStream,
702
- and also any other bitFields you put in, **in reverse order**.
703
-
704
- You will need a few variables to track your bitStream. They are :
705
-
706
- BIT_DStream_t DStream; // Stream context
707
- FSE_DState_t DState; // State context. Multiple ones are possible
708
- FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
709
-
710
- The first thing to do is to init the bitStream.
711
- errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
712
-
713
- You should then retrieve your initial state(s)
714
- (in reverse flushing order if you have several ones) :
715
- errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
716
-
717
- You can then decode your data, symbol after symbol.
718
- For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
719
- Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
720
- unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
721
-
722
- You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
723
- Note : maximum allowed nbBits is 25, for 32-bits compatibility
724
- size_t bitField = BIT_readBits(&DStream, nbBits);
725
-
726
- All above operations only read from local register (which size depends on size_t).
727
- Refueling the register from memory is manually performed by the reload method.
728
- endSignal = FSE_reloadDStream(&DStream);
729
-
730
- BIT_reloadDStream() result tells if there is still some more data to read from DStream.
731
- BIT_DStream_unfinished : there is still some data left into the DStream.
732
- BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
733
- BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
734
- BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
735
-
736
- When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
737
- to properly detect the exact end of stream.
738
- After each decoded symbol, check if DStream is fully consumed using this simple test :
739
- BIT_reloadDStream(&DStream) >= BIT_DStream_completed
740
-
741
- When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
742
- Checking if DStream has reached its end is performed by :
743
- BIT_endOfDStream(&DStream);
744
- Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
745
- FSE_endOfDState(&DState);
746
- */
747
-
748
691
 
749
692
  /******************************************
750
693
  * FSE unsafe API
@@ -1335,8 +1278,8 @@ static size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsi
1335
1278
  else
1336
1279
  {
1337
1280
  bitCount -= (int)(8 * (iend - 4 - ip));
1338
- ip = iend - 4;
1339
- }
1281
+ ip = iend - 4;
1282
+ }
1340
1283
  bitStream = MEM_readLE32(ip) >> (bitCount & 31);
1341
1284
  }
1342
1285
  }
@@ -2037,7 +1980,7 @@ static size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
2037
1980
  rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
2038
1981
  }
2039
1982
 
2040
- /* Build rankVal */
1983
+ /* Build rankVal */
2041
1984
  {
2042
1985
  const U32 minBits = tableLog+1 - maxW;
2043
1986
  U32 nextRankVal = 0;
@@ -2435,6 +2378,8 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
2435
2378
  #define LITERAL_NOENTROPY 63
2436
2379
  #define COMMAND_NOENTROPY 7 /* to remove */
2437
2380
 
2381
+ #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
2382
+
2438
2383
  static const size_t ZSTD_blockHeaderSize = 3;
2439
2384
  static const size_t ZSTD_frameHeaderSize = 4;
2440
2385
 
@@ -2449,7 +2394,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
2449
2394
  #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
2450
2395
 
2451
2396
  /*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
2452
- static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
2397
+ static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
2453
2398
  {
2454
2399
  const BYTE* ip = (const BYTE*)src;
2455
2400
  BYTE* op = (BYTE*)dst;
@@ -2509,7 +2454,6 @@ struct ZSTD_DCtx_s
2509
2454
  blockType_t bType;
2510
2455
  U32 phase;
2511
2456
  const BYTE* litPtr;
2512
- size_t litBufSize;
2513
2457
  size_t litSize;
2514
2458
  BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
2515
2459
  }; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
@@ -2537,7 +2481,9 @@ static size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockPropertie
2537
2481
  static size_t ZSTD_copyUncompressedBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
2538
2482
  {
2539
2483
  if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
2540
- memcpy(dst, src, srcSize);
2484
+ if (srcSize > 0) {
2485
+ memcpy(dst, src, srcSize);
2486
+ }
2541
2487
  return srcSize;
2542
2488
  }
2543
2489
 
@@ -2581,8 +2527,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
2581
2527
  size_t litSize = BLOCKSIZE;
2582
2528
  const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
2583
2529
  dctx->litPtr = dctx->litBuffer;
2584
- dctx->litBufSize = BLOCKSIZE;
2585
2530
  dctx->litSize = litSize;
2531
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
2586
2532
  return readSize; /* works if it's an error too */
2587
2533
  }
2588
2534
  case IS_RAW:
@@ -2590,16 +2536,16 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
2590
2536
  const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
2591
2537
  if (litSize > srcSize-11) /* risk of reading too far with wildcopy */
2592
2538
  {
2593
- if (litSize > srcSize-3) return ERROR(corruption_detected);
2594
- memcpy(dctx->litBuffer, istart, litSize);
2595
- dctx->litPtr = dctx->litBuffer;
2596
- dctx->litBufSize = BLOCKSIZE;
2597
- dctx->litSize = litSize;
2598
- return litSize+3;
2599
- }
2600
- /* direct reference into compressed stream */
2539
+ if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
2540
+ if (litSize > srcSize-3) return ERROR(corruption_detected);
2541
+ memcpy(dctx->litBuffer, istart, litSize);
2542
+ dctx->litPtr = dctx->litBuffer;
2543
+ dctx->litSize = litSize;
2544
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
2545
+ return litSize+3;
2546
+ }
2547
+ /* direct reference into compressed stream */
2601
2548
  dctx->litPtr = istart+3;
2602
- dctx->litBufSize = srcSize-3;
2603
2549
  dctx->litSize = litSize;
2604
2550
  return litSize+3;
2605
2551
  }
@@ -2607,9 +2553,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
2607
2553
  {
2608
2554
  const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
2609
2555
  if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
2610
- memset(dctx->litBuffer, istart[3], litSize);
2556
+ memset(dctx->litBuffer, istart[3], litSize + 8);
2611
2557
  dctx->litPtr = dctx->litBuffer;
2612
- dctx->litBufSize = BLOCKSIZE;
2613
2558
  dctx->litSize = litSize;
2614
2559
  return 4;
2615
2560
  }
@@ -2751,11 +2696,11 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
2751
2696
  seqState->prevOffset = seq->offset;
2752
2697
  if (litLength == MaxLL)
2753
2698
  {
2754
- U32 add = *dumps++;
2699
+ const U32 add = dumps<de ? *dumps++ : 0;
2755
2700
  if (add < 255) litLength += add;
2756
- else
2701
+ else if (dumps + 3 <= de)
2757
2702
  {
2758
- litLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
2703
+ litLength = MEM_readLE24(dumps);
2759
2704
  dumps += 3;
2760
2705
  }
2761
2706
  if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
@@ -2781,11 +2726,11 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
2781
2726
  matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
2782
2727
  if (matchLength == MaxML)
2783
2728
  {
2784
- U32 add = *dumps++;
2729
+ const U32 add = dumps<de ? *dumps++ : 0;
2785
2730
  if (add < 255) matchLength += add;
2786
- else
2731
+ else if (dumps + 3 <= de)
2787
2732
  {
2788
- matchLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
2733
+ matchLength = MEM_readLE24(dumps);
2789
2734
  dumps += 3;
2790
2735
  }
2791
2736
  if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
@@ -2806,7 +2751,7 @@ static size_t ZSTD_execSequence(BYTE* op,
2806
2751
  BYTE* const base, BYTE* const oend)
2807
2752
  {
2808
2753
  static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
2809
- static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */
2754
+ static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* subtracted */
2810
2755
  const BYTE* const ostart = op;
2811
2756
  BYTE* const oLitEnd = op + sequence.litLength;
2812
2757
  BYTE* const oMatchEnd = op + sequence.litLength + sequence.matchLength; /* risk : address space overflow (32-bits) */
@@ -2816,7 +2761,7 @@ static size_t ZSTD_execSequence(BYTE* op,
2816
2761
  /* checks */
2817
2762
  if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
2818
2763
  if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
2819
- if (litEnd > litLimit-8) return ERROR(corruption_detected); /* overRead beyond lit buffer */
2764
+ if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
2820
2765
 
2821
2766
  /* copy Literals */
2822
2767
  ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@@ -2850,7 +2795,7 @@ static size_t ZSTD_execSequence(BYTE* op,
2850
2795
  }
2851
2796
  op += 8; match += 8;
2852
2797
 
2853
- if (oMatchEnd > oend-12)
2798
+ if (oMatchEnd > oend-(16-MINMATCH))
2854
2799
  {
2855
2800
  if (op < oend_8)
2856
2801
  {
@@ -2862,7 +2807,7 @@ static size_t ZSTD_execSequence(BYTE* op,
2862
2807
  }
2863
2808
  else
2864
2809
  {
2865
- ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
2810
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
2866
2811
  }
2867
2812
  }
2868
2813
 
@@ -2882,7 +2827,6 @@ static size_t ZSTD_decompressSequences(
2882
2827
  BYTE* const oend = ostart + maxDstSize;
2883
2828
  size_t errorCode, dumpsLength;
2884
2829
  const BYTE* litPtr = dctx->litPtr;
2885
- const BYTE* const litMax = litPtr + dctx->litBufSize;
2886
2830
  const BYTE* const litEnd = litPtr + dctx->litSize;
2887
2831
  int nbSeq;
2888
2832
  const BYTE* dumps;
@@ -2918,7 +2862,7 @@ static size_t ZSTD_decompressSequences(
2918
2862
  size_t oneSeqSize;
2919
2863
  nbSeq--;
2920
2864
  ZSTD_decodeSequence(&sequence, &seqState);
2921
- oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litMax, base, oend);
2865
+ oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
2922
2866
  if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
2923
2867
  op += oneSeqSize;
2924
2868
  }
@@ -2932,8 +2876,10 @@ static size_t ZSTD_decompressSequences(
2932
2876
  size_t lastLLSize = litEnd - litPtr;
2933
2877
  if (litPtr > litEnd) return ERROR(corruption_detected);
2934
2878
  if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
2935
- if (op != litPtr) memmove(op, litPtr, lastLLSize);
2936
- op += lastLLSize;
2879
+ if (lastLLSize > 0) {
2880
+ if (op != litPtr) memmove(op, litPtr, lastLLSize);
2881
+ op += lastLLSize;
2882
+ }
2937
2883
  }
2938
2884
  }
2939
2885
 
@@ -3023,6 +2969,61 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
3023
2969
  return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
3024
2970
  }
3025
2971
 
2972
+ /* ZSTD_errorFrameSizeInfoLegacy() :
2973
+ assumes `cSize` and `dBound` are _not_ NULL */
2974
+ MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
2975
+ {
2976
+ *cSize = ret;
2977
+ *dBound = ZSTD_CONTENTSIZE_ERROR;
2978
+ }
2979
+
2980
+ void ZSTDv03_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
2981
+ {
2982
+ const BYTE* ip = (const BYTE*)src;
2983
+ size_t remainingSize = srcSize;
2984
+ size_t nbBlocks = 0;
2985
+ U32 magicNumber;
2986
+ blockProperties_t blockProperties;
2987
+
2988
+ /* Frame Header */
2989
+ if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) {
2990
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
2991
+ return;
2992
+ }
2993
+ magicNumber = MEM_readLE32(src);
2994
+ if (magicNumber != ZSTD_magicNumber) {
2995
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
2996
+ return;
2997
+ }
2998
+ ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
2999
+
3000
+ /* Loop on each block */
3001
+ while (1)
3002
+ {
3003
+ size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
3004
+ if (ZSTD_isError(cBlockSize)) {
3005
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
3006
+ return;
3007
+ }
3008
+
3009
+ ip += ZSTD_blockHeaderSize;
3010
+ remainingSize -= ZSTD_blockHeaderSize;
3011
+ if (cBlockSize > remainingSize) {
3012
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3013
+ return;
3014
+ }
3015
+
3016
+ if (cBlockSize == 0) break; /* bt_end */
3017
+
3018
+ ip += cBlockSize;
3019
+ remainingSize -= cBlockSize;
3020
+ nbBlocks++;
3021
+ }
3022
+
3023
+ *cSize = ip - (const BYTE*)src;
3024
+ *dBound = nbBlocks * BLOCKSIZE;
3025
+ }
3026
+
3026
3027
 
3027
3028
  /*******************************
3028
3029
  * Streaming Decompression API
@@ -3128,36 +3129,36 @@ static size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSi
3128
3129
 
3129
3130
  unsigned ZSTDv03_isError(size_t code)
3130
3131
  {
3131
- return ZSTD_isError(code);
3132
+ return ZSTD_isError(code);
3132
3133
  }
3133
3134
 
3134
3135
  size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
3135
3136
  const void* src, size_t compressedSize)
3136
3137
  {
3137
- return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
3138
+ return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
3138
3139
  }
3139
3140
 
3140
3141
  ZSTDv03_Dctx* ZSTDv03_createDCtx(void)
3141
3142
  {
3142
- return (ZSTDv03_Dctx*)ZSTD_createDCtx();
3143
+ return (ZSTDv03_Dctx*)ZSTD_createDCtx();
3143
3144
  }
3144
3145
 
3145
3146
  size_t ZSTDv03_freeDCtx(ZSTDv03_Dctx* dctx)
3146
3147
  {
3147
- return ZSTD_freeDCtx((ZSTD_DCtx*)dctx);
3148
+ return ZSTD_freeDCtx((ZSTD_DCtx*)dctx);
3148
3149
  }
3149
3150
 
3150
3151
  size_t ZSTDv03_resetDCtx(ZSTDv03_Dctx* dctx)
3151
3152
  {
3152
- return ZSTD_resetDCtx((ZSTD_DCtx*)dctx);
3153
+ return ZSTD_resetDCtx((ZSTD_DCtx*)dctx);
3153
3154
  }
3154
3155
 
3155
3156
  size_t ZSTDv03_nextSrcSizeToDecompress(ZSTDv03_Dctx* dctx)
3156
3157
  {
3157
- return ZSTD_nextSrcSizeToDecompress((ZSTD_DCtx*)dctx);
3158
+ return ZSTD_nextSrcSizeToDecompress((ZSTD_DCtx*)dctx);
3158
3159
  }
3159
3160
 
3160
3161
  size_t ZSTDv03_decompressContinue(ZSTDv03_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
3161
3162
  {
3162
- return ZSTD_decompressContinue((ZSTD_DCtx*)dctx, dst, maxDstSize, src, srcSize);
3163
+ return ZSTD_decompressContinue((ZSTD_DCtx*)dctx, dst, maxDstSize, src, srcSize);
3163
3164
  }