extzstd 0.1 → 0.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. checksums.yaml +5 -5
  2. data/HISTORY.ja.md +39 -0
  3. data/README.md +38 -56
  4. data/contrib/zstd/CHANGELOG +613 -0
  5. data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
  6. data/contrib/zstd/CONTRIBUTING.md +406 -0
  7. data/contrib/zstd/COPYING +339 -0
  8. data/contrib/zstd/Makefile +420 -0
  9. data/contrib/zstd/README.md +179 -41
  10. data/contrib/zstd/TESTING.md +44 -0
  11. data/contrib/zstd/appveyor.yml +292 -0
  12. data/contrib/zstd/lib/BUCK +234 -0
  13. data/contrib/zstd/lib/Makefile +451 -0
  14. data/contrib/zstd/lib/README.md +207 -0
  15. data/contrib/zstd/{common → lib/common}/bitstream.h +187 -138
  16. data/contrib/zstd/lib/common/compiler.h +288 -0
  17. data/contrib/zstd/lib/common/cpu.h +213 -0
  18. data/contrib/zstd/lib/common/debug.c +24 -0
  19. data/contrib/zstd/lib/common/debug.h +107 -0
  20. data/contrib/zstd/lib/common/entropy_common.c +362 -0
  21. data/contrib/zstd/{common → lib/common}/error_private.c +25 -12
  22. data/contrib/zstd/{common → lib/common}/error_private.h +14 -10
  23. data/contrib/zstd/{common → lib/common}/fse.h +173 -92
  24. data/contrib/zstd/{common → lib/common}/fse_decompress.c +149 -85
  25. data/contrib/zstd/lib/common/huf.h +361 -0
  26. data/contrib/zstd/{common → lib/common}/mem.h +115 -59
  27. data/contrib/zstd/lib/common/pool.c +350 -0
  28. data/contrib/zstd/lib/common/pool.h +84 -0
  29. data/contrib/zstd/lib/common/threading.c +122 -0
  30. data/contrib/zstd/lib/common/threading.h +155 -0
  31. data/contrib/zstd/{common → lib/common}/xxhash.c +55 -96
  32. data/contrib/zstd/{common → lib/common}/xxhash.h +23 -47
  33. data/contrib/zstd/lib/common/zstd_common.c +83 -0
  34. data/contrib/zstd/lib/common/zstd_deps.h +111 -0
  35. data/contrib/zstd/lib/common/zstd_errors.h +95 -0
  36. data/contrib/zstd/lib/common/zstd_internal.h +478 -0
  37. data/contrib/zstd/{compress → lib/compress}/fse_compress.c +214 -319
  38. data/contrib/zstd/lib/compress/hist.c +181 -0
  39. data/contrib/zstd/lib/compress/hist.h +75 -0
  40. data/contrib/zstd/lib/compress/huf_compress.c +913 -0
  41. data/contrib/zstd/lib/compress/zstd_compress.c +5208 -0
  42. data/contrib/zstd/lib/compress/zstd_compress_internal.h +1203 -0
  43. data/contrib/zstd/lib/compress/zstd_compress_literals.c +158 -0
  44. data/contrib/zstd/lib/compress/zstd_compress_literals.h +29 -0
  45. data/contrib/zstd/lib/compress/zstd_compress_sequences.c +433 -0
  46. data/contrib/zstd/lib/compress/zstd_compress_sequences.h +54 -0
  47. data/contrib/zstd/lib/compress/zstd_compress_superblock.c +849 -0
  48. data/contrib/zstd/lib/compress/zstd_compress_superblock.h +32 -0
  49. data/contrib/zstd/lib/compress/zstd_cwksp.h +561 -0
  50. data/contrib/zstd/lib/compress/zstd_double_fast.c +521 -0
  51. data/contrib/zstd/lib/compress/zstd_double_fast.h +38 -0
  52. data/contrib/zstd/lib/compress/zstd_fast.c +496 -0
  53. data/contrib/zstd/lib/compress/zstd_fast.h +37 -0
  54. data/contrib/zstd/lib/compress/zstd_lazy.c +1412 -0
  55. data/contrib/zstd/lib/compress/zstd_lazy.h +87 -0
  56. data/contrib/zstd/lib/compress/zstd_ldm.c +660 -0
  57. data/contrib/zstd/lib/compress/zstd_ldm.h +116 -0
  58. data/contrib/zstd/lib/compress/zstd_opt.c +1345 -0
  59. data/contrib/zstd/lib/compress/zstd_opt.h +56 -0
  60. data/contrib/zstd/lib/compress/zstdmt_compress.c +1811 -0
  61. data/contrib/zstd/lib/compress/zstdmt_compress.h +110 -0
  62. data/contrib/zstd/lib/decompress/huf_decompress.c +1350 -0
  63. data/contrib/zstd/lib/decompress/zstd_ddict.c +244 -0
  64. data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
  65. data/contrib/zstd/lib/decompress/zstd_decompress.c +1930 -0
  66. data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1540 -0
  67. data/contrib/zstd/lib/decompress/zstd_decompress_block.h +62 -0
  68. data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +190 -0
  69. data/contrib/zstd/{common → lib/deprecated}/zbuff.h +68 -45
  70. data/contrib/zstd/lib/deprecated/zbuff_common.c +26 -0
  71. data/contrib/zstd/lib/deprecated/zbuff_compress.c +147 -0
  72. data/contrib/zstd/lib/deprecated/zbuff_decompress.c +75 -0
  73. data/contrib/zstd/lib/dictBuilder/cover.c +1245 -0
  74. data/contrib/zstd/lib/dictBuilder/cover.h +157 -0
  75. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.c +3 -3
  76. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.h +0 -0
  77. data/contrib/zstd/lib/dictBuilder/fastcover.c +758 -0
  78. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/zdict.c +318 -194
  79. data/contrib/zstd/lib/dictBuilder/zdict.h +305 -0
  80. data/contrib/zstd/{legacy → lib/legacy}/zstd_legacy.h +171 -15
  81. data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.c +191 -124
  82. data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.h +19 -5
  83. data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.c +125 -125
  84. data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.h +19 -5
  85. data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.c +125 -124
  86. data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.h +20 -6
  87. data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.c +151 -299
  88. data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.h +19 -5
  89. data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.c +237 -243
  90. data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.h +19 -6
  91. data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.c +130 -143
  92. data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.h +18 -5
  93. data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.c +158 -157
  94. data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.h +19 -5
  95. data/contrib/zstd/lib/libzstd.pc.in +15 -0
  96. data/contrib/zstd/lib/zstd.h +2391 -0
  97. data/ext/depend +2 -0
  98. data/ext/extconf.rb +15 -6
  99. data/ext/extzstd.c +76 -145
  100. data/ext/extzstd.h +80 -31
  101. data/ext/extzstd_stream.c +417 -142
  102. data/ext/libzstd_conf.h +8 -0
  103. data/ext/zstd_common.c +10 -7
  104. data/ext/zstd_compress.c +14 -5
  105. data/ext/zstd_decompress.c +5 -4
  106. data/ext/zstd_dictbuilder.c +9 -4
  107. data/ext/zstd_dictbuilder_fastcover.c +3 -0
  108. data/ext/zstd_legacy_v01.c +3 -1
  109. data/ext/zstd_legacy_v02.c +3 -1
  110. data/ext/zstd_legacy_v03.c +3 -1
  111. data/ext/zstd_legacy_v04.c +3 -1
  112. data/ext/zstd_legacy_v05.c +3 -1
  113. data/ext/zstd_legacy_v06.c +3 -1
  114. data/ext/zstd_legacy_v07.c +3 -1
  115. data/gemstub.rb +10 -24
  116. data/lib/extzstd.rb +64 -179
  117. data/lib/extzstd/version.rb +6 -1
  118. data/test/test_basic.rb +9 -6
  119. metadata +113 -57
  120. data/HISTORY.ja +0 -5
  121. data/contrib/zstd/common/entropy_common.c +0 -225
  122. data/contrib/zstd/common/huf.h +0 -228
  123. data/contrib/zstd/common/zstd_common.c +0 -83
  124. data/contrib/zstd/common/zstd_errors.h +0 -60
  125. data/contrib/zstd/common/zstd_internal.h +0 -267
  126. data/contrib/zstd/compress/huf_compress.c +0 -533
  127. data/contrib/zstd/compress/zbuff_compress.c +0 -319
  128. data/contrib/zstd/compress/zstd_compress.c +0 -3264
  129. data/contrib/zstd/compress/zstd_opt.h +0 -900
  130. data/contrib/zstd/decompress/huf_decompress.c +0 -883
  131. data/contrib/zstd/decompress/zbuff_decompress.c +0 -252
  132. data/contrib/zstd/decompress/zstd_decompress.c +0 -1842
  133. data/contrib/zstd/dictBuilder/zdict.h +0 -111
  134. data/contrib/zstd/zstd.h +0 -640
@@ -1,10 +1,11 @@
1
- /**
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
1
+ /*
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
- * This source code is licensed under the BSD-style license found in the
6
- * LICENSE file in the root directory of this source tree. An additional grant
7
- * of patent rights can be found in the PATENTS file in the same directory.
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
8
9
  */
9
10
 
10
11
  #ifndef ZSTD_V02_H_4174539423
@@ -34,6 +35,19 @@ ZSTDv02_decompress() : decompress ZSTD frames compliant with v0.2.x format
34
35
  size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
35
36
  const void* src, size_t compressedSize);
36
37
 
38
+ /**
39
+ ZSTDv02_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.2.x format
40
+ srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
41
+ cSize (output parameter) : the number of bytes that would be read to decompress this frame
42
+ or an error code if it fails (which can be tested using ZSTDv01_isError())
43
+ dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
44
+ or ZSTD_CONTENTSIZE_ERROR if an error occurs
45
+
46
+ note : assumes `cSize` and `dBound` are _not_ NULL.
47
+ */
48
+ void ZSTDv02_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
49
+ size_t* cSize, unsigned long long* dBound);
50
+
37
51
  /**
38
52
  ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error
39
53
  */
@@ -1,16 +1,17 @@
1
- /**
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
1
+ /*
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
- * This source code is licensed under the BSD-style license found in the
6
- * LICENSE file in the root directory of this source tree. An additional grant
7
- * of patent rights can be found in the PATENTS file in the same directory.
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
8
9
  */
9
10
 
10
11
 
11
12
  #include <stddef.h> /* size_t, ptrdiff_t */
12
13
  #include "zstd_v03.h"
13
- #include "error_private.h"
14
+ #include "../common/error_private.h"
14
15
 
15
16
 
16
17
  /******************************************
@@ -89,7 +90,11 @@ extern "C" {
89
90
  * Basic Types
90
91
  *****************************************************************/
91
92
  #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
92
- # include <stdint.h>
93
+ # if defined(_AIX)
94
+ # include <inttypes.h>
95
+ # else
96
+ # include <stdint.h> /* intptr_t */
97
+ # endif
93
98
  typedef uint8_t BYTE;
94
99
  typedef uint16_t U16;
95
100
  typedef int16_t S16;
@@ -190,7 +195,7 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
190
195
  }
191
196
 
192
197
 
193
- #endif // MEM_FORCE_MEMORY_ACCESS
198
+ #endif /* MEM_FORCE_MEMORY_ACCESS */
194
199
 
195
200
 
196
201
  MEM_STATIC U16 MEM_readLE16(const void* memPtr)
@@ -218,6 +223,11 @@ MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
218
223
  }
219
224
  }
220
225
 
226
+ MEM_STATIC U32 MEM_readLE24(const void* memPtr)
227
+ {
228
+ return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
229
+ }
230
+
221
231
  MEM_STATIC U32 MEM_readLE32(const void* memPtr)
222
232
  {
223
233
  if (MEM_isLittleEndian())
@@ -331,17 +341,6 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
331
341
  MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
332
342
 
333
343
 
334
- /*
335
- * Start by invoking BIT_initDStream().
336
- * A chunk of the bitStream is then stored into a local register.
337
- * Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
338
- * You can then retrieve bitFields stored into the local register, **in reverse order**.
339
- * Local register is manually filled from memory by the BIT_reloadDStream() method.
340
- * A reload guarantee a minimum of ((8*sizeof(size_t))-7) bits when its result is BIT_DStream_unfinished.
341
- * Otherwise, it can be less than that, so proceed accordingly.
342
- * Checking if DStream has reached its end can be performed with BIT_endOfDStream()
343
- */
344
-
345
344
 
346
345
  /******************************************
347
346
  * unsafe API
@@ -354,14 +353,14 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
354
353
  /****************************************************************
355
354
  * Helper functions
356
355
  ****************************************************************/
357
- MEM_STATIC unsigned BIT_highbit32 (register U32 val)
356
+ MEM_STATIC unsigned BIT_highbit32 (U32 val)
358
357
  {
359
358
  # if defined(_MSC_VER) /* Visual */
360
359
  unsigned long r=0;
361
360
  _BitScanReverse ( &r, val );
362
361
  return (unsigned) r;
363
362
  # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
364
- return 31 - __builtin_clz (val);
363
+ return __builtin_clz (val) ^ 31;
365
364
  # else /* Software version */
366
365
  static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
367
366
  U32 v = val;
@@ -412,11 +411,17 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
412
411
  switch(srcSize)
413
412
  {
414
413
  case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
414
+ /* fallthrough */
415
415
  case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
416
+ /* fallthrough */
416
417
  case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
418
+ /* fallthrough */
417
419
  case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
420
+ /* fallthrough */
418
421
  case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
422
+ /* fallthrough */
419
423
  case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
424
+ /* fallthrough */
420
425
  default:;
421
426
  }
422
427
  contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
@@ -427,14 +432,6 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
427
432
 
428
433
  return srcSize;
429
434
  }
430
-
431
- /*!BIT_lookBits
432
- * Provides next n bits from local register
433
- * local register is not modified (bits are still present for next read/look)
434
- * On 32-bits, maxNbBits==25
435
- * On 64-bits, maxNbBits==57
436
- * @return : value extracted
437
- */
438
435
  MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits)
439
436
  {
440
437
  const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
@@ -454,11 +451,6 @@ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
454
451
  bitD->bitsConsumed += nbBits;
455
452
  }
456
453
 
457
- /*!BIT_readBits
458
- * Read next n bits from local register.
459
- * pay attention to not read more than nbBits contained into local register.
460
- * @return : extracted value.
461
- */
462
454
  MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
463
455
  {
464
456
  size_t value = BIT_lookBits(bitD, nbBits);
@@ -477,8 +469,8 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
477
469
 
478
470
  MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
479
471
  {
480
- if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
481
- return BIT_DStream_overflow;
472
+ if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
473
+ return BIT_DStream_overflow;
482
474
 
483
475
  if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer))
484
476
  {
@@ -696,55 +688,6 @@ static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bi
696
688
 
697
689
  static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
698
690
 
699
- /*
700
- Let's now decompose FSE_decompress_usingDTable() into its unitary components.
701
- You will decode FSE-encoded symbols from the bitStream,
702
- and also any other bitFields you put in, **in reverse order**.
703
-
704
- You will need a few variables to track your bitStream. They are :
705
-
706
- BIT_DStream_t DStream; // Stream context
707
- FSE_DState_t DState; // State context. Multiple ones are possible
708
- FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
709
-
710
- The first thing to do is to init the bitStream.
711
- errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
712
-
713
- You should then retrieve your initial state(s)
714
- (in reverse flushing order if you have several ones) :
715
- errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
716
-
717
- You can then decode your data, symbol after symbol.
718
- For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
719
- Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
720
- unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
721
-
722
- You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
723
- Note : maximum allowed nbBits is 25, for 32-bits compatibility
724
- size_t bitField = BIT_readBits(&DStream, nbBits);
725
-
726
- All above operations only read from local register (which size depends on size_t).
727
- Refueling the register from memory is manually performed by the reload method.
728
- endSignal = FSE_reloadDStream(&DStream);
729
-
730
- BIT_reloadDStream() result tells if there is still some more data to read from DStream.
731
- BIT_DStream_unfinished : there is still some data left into the DStream.
732
- BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
733
- BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
734
- BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
735
-
736
- When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
737
- to properly detect the exact end of stream.
738
- After each decoded symbol, check if DStream is fully consumed using this simple test :
739
- BIT_reloadDStream(&DStream) >= BIT_DStream_completed
740
-
741
- When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
742
- Checking if DStream has reached its end is performed by :
743
- BIT_endOfDStream(&DStream);
744
- Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
745
- FSE_endOfDState(&DState);
746
- */
747
-
748
691
 
749
692
  /******************************************
750
693
  * FSE unsafe API
@@ -1335,8 +1278,8 @@ static size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsi
1335
1278
  else
1336
1279
  {
1337
1280
  bitCount -= (int)(8 * (iend - 4 - ip));
1338
- ip = iend - 4;
1339
- }
1281
+ ip = iend - 4;
1282
+ }
1340
1283
  bitStream = MEM_readLE32(ip) >> (bitCount & 31);
1341
1284
  }
1342
1285
  }
@@ -2037,7 +1980,7 @@ static size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
2037
1980
  rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
2038
1981
  }
2039
1982
 
2040
- /* Build rankVal */
1983
+ /* Build rankVal */
2041
1984
  {
2042
1985
  const U32 minBits = tableLog+1 - maxW;
2043
1986
  U32 nextRankVal = 0;
@@ -2435,6 +2378,8 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
2435
2378
  #define LITERAL_NOENTROPY 63
2436
2379
  #define COMMAND_NOENTROPY 7 /* to remove */
2437
2380
 
2381
+ #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
2382
+
2438
2383
  static const size_t ZSTD_blockHeaderSize = 3;
2439
2384
  static const size_t ZSTD_frameHeaderSize = 4;
2440
2385
 
@@ -2449,7 +2394,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
2449
2394
  #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
2450
2395
 
2451
2396
  /*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
2452
- static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
2397
+ static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
2453
2398
  {
2454
2399
  const BYTE* ip = (const BYTE*)src;
2455
2400
  BYTE* op = (BYTE*)dst;
@@ -2509,7 +2454,6 @@ struct ZSTD_DCtx_s
2509
2454
  blockType_t bType;
2510
2455
  U32 phase;
2511
2456
  const BYTE* litPtr;
2512
- size_t litBufSize;
2513
2457
  size_t litSize;
2514
2458
  BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
2515
2459
  }; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
@@ -2537,7 +2481,9 @@ static size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockPropertie
2537
2481
  static size_t ZSTD_copyUncompressedBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
2538
2482
  {
2539
2483
  if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
2540
- memcpy(dst, src, srcSize);
2484
+ if (srcSize > 0) {
2485
+ memcpy(dst, src, srcSize);
2486
+ }
2541
2487
  return srcSize;
2542
2488
  }
2543
2489
 
@@ -2581,8 +2527,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
2581
2527
  size_t litSize = BLOCKSIZE;
2582
2528
  const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
2583
2529
  dctx->litPtr = dctx->litBuffer;
2584
- dctx->litBufSize = BLOCKSIZE;
2585
2530
  dctx->litSize = litSize;
2531
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
2586
2532
  return readSize; /* works if it's an error too */
2587
2533
  }
2588
2534
  case IS_RAW:
@@ -2590,16 +2536,16 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
2590
2536
  const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
2591
2537
  if (litSize > srcSize-11) /* risk of reading too far with wildcopy */
2592
2538
  {
2593
- if (litSize > srcSize-3) return ERROR(corruption_detected);
2594
- memcpy(dctx->litBuffer, istart, litSize);
2595
- dctx->litPtr = dctx->litBuffer;
2596
- dctx->litBufSize = BLOCKSIZE;
2597
- dctx->litSize = litSize;
2598
- return litSize+3;
2599
- }
2600
- /* direct reference into compressed stream */
2539
+ if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
2540
+ if (litSize > srcSize-3) return ERROR(corruption_detected);
2541
+ memcpy(dctx->litBuffer, istart, litSize);
2542
+ dctx->litPtr = dctx->litBuffer;
2543
+ dctx->litSize = litSize;
2544
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
2545
+ return litSize+3;
2546
+ }
2547
+ /* direct reference into compressed stream */
2601
2548
  dctx->litPtr = istart+3;
2602
- dctx->litBufSize = srcSize-3;
2603
2549
  dctx->litSize = litSize;
2604
2550
  return litSize+3;
2605
2551
  }
@@ -2607,9 +2553,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
2607
2553
  {
2608
2554
  const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
2609
2555
  if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
2610
- memset(dctx->litBuffer, istart[3], litSize);
2556
+ memset(dctx->litBuffer, istart[3], litSize + 8);
2611
2557
  dctx->litPtr = dctx->litBuffer;
2612
- dctx->litBufSize = BLOCKSIZE;
2613
2558
  dctx->litSize = litSize;
2614
2559
  return 4;
2615
2560
  }
@@ -2751,11 +2696,11 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
2751
2696
  seqState->prevOffset = seq->offset;
2752
2697
  if (litLength == MaxLL)
2753
2698
  {
2754
- U32 add = *dumps++;
2699
+ const U32 add = dumps<de ? *dumps++ : 0;
2755
2700
  if (add < 255) litLength += add;
2756
- else
2701
+ else if (dumps + 3 <= de)
2757
2702
  {
2758
- litLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
2703
+ litLength = MEM_readLE24(dumps);
2759
2704
  dumps += 3;
2760
2705
  }
2761
2706
  if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
@@ -2781,11 +2726,11 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
2781
2726
  matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
2782
2727
  if (matchLength == MaxML)
2783
2728
  {
2784
- U32 add = *dumps++;
2729
+ const U32 add = dumps<de ? *dumps++ : 0;
2785
2730
  if (add < 255) matchLength += add;
2786
- else
2731
+ else if (dumps + 3 <= de)
2787
2732
  {
2788
- matchLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
2733
+ matchLength = MEM_readLE24(dumps);
2789
2734
  dumps += 3;
2790
2735
  }
2791
2736
  if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
@@ -2806,7 +2751,7 @@ static size_t ZSTD_execSequence(BYTE* op,
2806
2751
  BYTE* const base, BYTE* const oend)
2807
2752
  {
2808
2753
  static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
2809
- static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */
2754
+ static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* subtracted */
2810
2755
  const BYTE* const ostart = op;
2811
2756
  BYTE* const oLitEnd = op + sequence.litLength;
2812
2757
  BYTE* const oMatchEnd = op + sequence.litLength + sequence.matchLength; /* risk : address space overflow (32-bits) */
@@ -2816,7 +2761,7 @@ static size_t ZSTD_execSequence(BYTE* op,
2816
2761
  /* checks */
2817
2762
  if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
2818
2763
  if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
2819
- if (litEnd > litLimit-8) return ERROR(corruption_detected); /* overRead beyond lit buffer */
2764
+ if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
2820
2765
 
2821
2766
  /* copy Literals */
2822
2767
  ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@@ -2850,7 +2795,7 @@ static size_t ZSTD_execSequence(BYTE* op,
2850
2795
  }
2851
2796
  op += 8; match += 8;
2852
2797
 
2853
- if (oMatchEnd > oend-12)
2798
+ if (oMatchEnd > oend-(16-MINMATCH))
2854
2799
  {
2855
2800
  if (op < oend_8)
2856
2801
  {
@@ -2862,7 +2807,7 @@ static size_t ZSTD_execSequence(BYTE* op,
2862
2807
  }
2863
2808
  else
2864
2809
  {
2865
- ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
2810
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
2866
2811
  }
2867
2812
  }
2868
2813
 
@@ -2882,7 +2827,6 @@ static size_t ZSTD_decompressSequences(
2882
2827
  BYTE* const oend = ostart + maxDstSize;
2883
2828
  size_t errorCode, dumpsLength;
2884
2829
  const BYTE* litPtr = dctx->litPtr;
2885
- const BYTE* const litMax = litPtr + dctx->litBufSize;
2886
2830
  const BYTE* const litEnd = litPtr + dctx->litSize;
2887
2831
  int nbSeq;
2888
2832
  const BYTE* dumps;
@@ -2918,7 +2862,7 @@ static size_t ZSTD_decompressSequences(
2918
2862
  size_t oneSeqSize;
2919
2863
  nbSeq--;
2920
2864
  ZSTD_decodeSequence(&sequence, &seqState);
2921
- oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litMax, base, oend);
2865
+ oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
2922
2866
  if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
2923
2867
  op += oneSeqSize;
2924
2868
  }
@@ -2932,8 +2876,10 @@ static size_t ZSTD_decompressSequences(
2932
2876
  size_t lastLLSize = litEnd - litPtr;
2933
2877
  if (litPtr > litEnd) return ERROR(corruption_detected);
2934
2878
  if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
2935
- if (op != litPtr) memmove(op, litPtr, lastLLSize);
2936
- op += lastLLSize;
2879
+ if (lastLLSize > 0) {
2880
+ if (op != litPtr) memmove(op, litPtr, lastLLSize);
2881
+ op += lastLLSize;
2882
+ }
2937
2883
  }
2938
2884
  }
2939
2885
 
@@ -3023,6 +2969,61 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
3023
2969
  return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
3024
2970
  }
3025
2971
 
2972
+ /* ZSTD_errorFrameSizeInfoLegacy() :
2973
+ assumes `cSize` and `dBound` are _not_ NULL */
2974
+ MEM_STATIC void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
2975
+ {
2976
+ *cSize = ret;
2977
+ *dBound = ZSTD_CONTENTSIZE_ERROR;
2978
+ }
2979
+
2980
+ void ZSTDv03_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
2981
+ {
2982
+ const BYTE* ip = (const BYTE*)src;
2983
+ size_t remainingSize = srcSize;
2984
+ size_t nbBlocks = 0;
2985
+ U32 magicNumber;
2986
+ blockProperties_t blockProperties;
2987
+
2988
+ /* Frame Header */
2989
+ if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) {
2990
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
2991
+ return;
2992
+ }
2993
+ magicNumber = MEM_readLE32(src);
2994
+ if (magicNumber != ZSTD_magicNumber) {
2995
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
2996
+ return;
2997
+ }
2998
+ ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
2999
+
3000
+ /* Loop on each block */
3001
+ while (1)
3002
+ {
3003
+ size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
3004
+ if (ZSTD_isError(cBlockSize)) {
3005
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
3006
+ return;
3007
+ }
3008
+
3009
+ ip += ZSTD_blockHeaderSize;
3010
+ remainingSize -= ZSTD_blockHeaderSize;
3011
+ if (cBlockSize > remainingSize) {
3012
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3013
+ return;
3014
+ }
3015
+
3016
+ if (cBlockSize == 0) break; /* bt_end */
3017
+
3018
+ ip += cBlockSize;
3019
+ remainingSize -= cBlockSize;
3020
+ nbBlocks++;
3021
+ }
3022
+
3023
+ *cSize = ip - (const BYTE*)src;
3024
+ *dBound = nbBlocks * BLOCKSIZE;
3025
+ }
3026
+
3026
3027
 
3027
3028
  /*******************************
3028
3029
  * Streaming Decompression API
@@ -3128,36 +3129,36 @@ static size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSi
3128
3129
 
3129
3130
  unsigned ZSTDv03_isError(size_t code)
3130
3131
  {
3131
- return ZSTD_isError(code);
3132
+ return ZSTD_isError(code);
3132
3133
  }
3133
3134
 
3134
3135
  size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
3135
3136
  const void* src, size_t compressedSize)
3136
3137
  {
3137
- return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
3138
+ return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
3138
3139
  }
3139
3140
 
3140
3141
  ZSTDv03_Dctx* ZSTDv03_createDCtx(void)
3141
3142
  {
3142
- return (ZSTDv03_Dctx*)ZSTD_createDCtx();
3143
+ return (ZSTDv03_Dctx*)ZSTD_createDCtx();
3143
3144
  }
3144
3145
 
3145
3146
  size_t ZSTDv03_freeDCtx(ZSTDv03_Dctx* dctx)
3146
3147
  {
3147
- return ZSTD_freeDCtx((ZSTD_DCtx*)dctx);
3148
+ return ZSTD_freeDCtx((ZSTD_DCtx*)dctx);
3148
3149
  }
3149
3150
 
3150
3151
  size_t ZSTDv03_resetDCtx(ZSTDv03_Dctx* dctx)
3151
3152
  {
3152
- return ZSTD_resetDCtx((ZSTD_DCtx*)dctx);
3153
+ return ZSTD_resetDCtx((ZSTD_DCtx*)dctx);
3153
3154
  }
3154
3155
 
3155
3156
  size_t ZSTDv03_nextSrcSizeToDecompress(ZSTDv03_Dctx* dctx)
3156
3157
  {
3157
- return ZSTD_nextSrcSizeToDecompress((ZSTD_DCtx*)dctx);
3158
+ return ZSTD_nextSrcSizeToDecompress((ZSTD_DCtx*)dctx);
3158
3159
  }
3159
3160
 
3160
3161
  size_t ZSTDv03_decompressContinue(ZSTDv03_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
3161
3162
  {
3162
- return ZSTD_decompressContinue((ZSTD_DCtx*)dctx, dst, maxDstSize, src, srcSize);
3163
+ return ZSTD_decompressContinue((ZSTD_DCtx*)dctx, dst, maxDstSize, src, srcSize);
3163
3164
  }