extzstd 0.1 → 0.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. checksums.yaml +5 -5
  2. data/HISTORY.ja.md +39 -0
  3. data/README.md +38 -56
  4. data/contrib/zstd/CHANGELOG +613 -0
  5. data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
  6. data/contrib/zstd/CONTRIBUTING.md +406 -0
  7. data/contrib/zstd/COPYING +339 -0
  8. data/contrib/zstd/Makefile +420 -0
  9. data/contrib/zstd/README.md +179 -41
  10. data/contrib/zstd/TESTING.md +44 -0
  11. data/contrib/zstd/appveyor.yml +292 -0
  12. data/contrib/zstd/lib/BUCK +234 -0
  13. data/contrib/zstd/lib/Makefile +451 -0
  14. data/contrib/zstd/lib/README.md +207 -0
  15. data/contrib/zstd/{common → lib/common}/bitstream.h +187 -138
  16. data/contrib/zstd/lib/common/compiler.h +288 -0
  17. data/contrib/zstd/lib/common/cpu.h +213 -0
  18. data/contrib/zstd/lib/common/debug.c +24 -0
  19. data/contrib/zstd/lib/common/debug.h +107 -0
  20. data/contrib/zstd/lib/common/entropy_common.c +362 -0
  21. data/contrib/zstd/{common → lib/common}/error_private.c +25 -12
  22. data/contrib/zstd/{common → lib/common}/error_private.h +14 -10
  23. data/contrib/zstd/{common → lib/common}/fse.h +173 -92
  24. data/contrib/zstd/{common → lib/common}/fse_decompress.c +149 -85
  25. data/contrib/zstd/lib/common/huf.h +361 -0
  26. data/contrib/zstd/{common → lib/common}/mem.h +115 -59
  27. data/contrib/zstd/lib/common/pool.c +350 -0
  28. data/contrib/zstd/lib/common/pool.h +84 -0
  29. data/contrib/zstd/lib/common/threading.c +122 -0
  30. data/contrib/zstd/lib/common/threading.h +155 -0
  31. data/contrib/zstd/{common → lib/common}/xxhash.c +55 -96
  32. data/contrib/zstd/{common → lib/common}/xxhash.h +23 -47
  33. data/contrib/zstd/lib/common/zstd_common.c +83 -0
  34. data/contrib/zstd/lib/common/zstd_deps.h +111 -0
  35. data/contrib/zstd/lib/common/zstd_errors.h +95 -0
  36. data/contrib/zstd/lib/common/zstd_internal.h +478 -0
  37. data/contrib/zstd/{compress → lib/compress}/fse_compress.c +214 -319
  38. data/contrib/zstd/lib/compress/hist.c +181 -0
  39. data/contrib/zstd/lib/compress/hist.h +75 -0
  40. data/contrib/zstd/lib/compress/huf_compress.c +913 -0
  41. data/contrib/zstd/lib/compress/zstd_compress.c +5208 -0
  42. data/contrib/zstd/lib/compress/zstd_compress_internal.h +1203 -0
  43. data/contrib/zstd/lib/compress/zstd_compress_literals.c +158 -0
  44. data/contrib/zstd/lib/compress/zstd_compress_literals.h +29 -0
  45. data/contrib/zstd/lib/compress/zstd_compress_sequences.c +433 -0
  46. data/contrib/zstd/lib/compress/zstd_compress_sequences.h +54 -0
  47. data/contrib/zstd/lib/compress/zstd_compress_superblock.c +849 -0
  48. data/contrib/zstd/lib/compress/zstd_compress_superblock.h +32 -0
  49. data/contrib/zstd/lib/compress/zstd_cwksp.h +561 -0
  50. data/contrib/zstd/lib/compress/zstd_double_fast.c +521 -0
  51. data/contrib/zstd/lib/compress/zstd_double_fast.h +38 -0
  52. data/contrib/zstd/lib/compress/zstd_fast.c +496 -0
  53. data/contrib/zstd/lib/compress/zstd_fast.h +37 -0
  54. data/contrib/zstd/lib/compress/zstd_lazy.c +1412 -0
  55. data/contrib/zstd/lib/compress/zstd_lazy.h +87 -0
  56. data/contrib/zstd/lib/compress/zstd_ldm.c +660 -0
  57. data/contrib/zstd/lib/compress/zstd_ldm.h +116 -0
  58. data/contrib/zstd/lib/compress/zstd_opt.c +1345 -0
  59. data/contrib/zstd/lib/compress/zstd_opt.h +56 -0
  60. data/contrib/zstd/lib/compress/zstdmt_compress.c +1811 -0
  61. data/contrib/zstd/lib/compress/zstdmt_compress.h +110 -0
  62. data/contrib/zstd/lib/decompress/huf_decompress.c +1350 -0
  63. data/contrib/zstd/lib/decompress/zstd_ddict.c +244 -0
  64. data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
  65. data/contrib/zstd/lib/decompress/zstd_decompress.c +1930 -0
  66. data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1540 -0
  67. data/contrib/zstd/lib/decompress/zstd_decompress_block.h +62 -0
  68. data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +190 -0
  69. data/contrib/zstd/{common → lib/deprecated}/zbuff.h +68 -45
  70. data/contrib/zstd/lib/deprecated/zbuff_common.c +26 -0
  71. data/contrib/zstd/lib/deprecated/zbuff_compress.c +147 -0
  72. data/contrib/zstd/lib/deprecated/zbuff_decompress.c +75 -0
  73. data/contrib/zstd/lib/dictBuilder/cover.c +1245 -0
  74. data/contrib/zstd/lib/dictBuilder/cover.h +157 -0
  75. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.c +3 -3
  76. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.h +0 -0
  77. data/contrib/zstd/lib/dictBuilder/fastcover.c +758 -0
  78. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/zdict.c +318 -194
  79. data/contrib/zstd/lib/dictBuilder/zdict.h +305 -0
  80. data/contrib/zstd/{legacy → lib/legacy}/zstd_legacy.h +171 -15
  81. data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.c +191 -124
  82. data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.h +19 -5
  83. data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.c +125 -125
  84. data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.h +19 -5
  85. data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.c +125 -124
  86. data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.h +20 -6
  87. data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.c +151 -299
  88. data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.h +19 -5
  89. data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.c +237 -243
  90. data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.h +19 -6
  91. data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.c +130 -143
  92. data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.h +18 -5
  93. data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.c +158 -157
  94. data/contrib/zstd/{legacy → lib/legacy}/zstd_v07.h +19 -5
  95. data/contrib/zstd/lib/libzstd.pc.in +15 -0
  96. data/contrib/zstd/lib/zstd.h +2391 -0
  97. data/ext/depend +2 -0
  98. data/ext/extconf.rb +15 -6
  99. data/ext/extzstd.c +76 -145
  100. data/ext/extzstd.h +80 -31
  101. data/ext/extzstd_stream.c +417 -142
  102. data/ext/libzstd_conf.h +8 -0
  103. data/ext/zstd_common.c +10 -7
  104. data/ext/zstd_compress.c +14 -5
  105. data/ext/zstd_decompress.c +5 -4
  106. data/ext/zstd_dictbuilder.c +9 -4
  107. data/ext/zstd_dictbuilder_fastcover.c +3 -0
  108. data/ext/zstd_legacy_v01.c +3 -1
  109. data/ext/zstd_legacy_v02.c +3 -1
  110. data/ext/zstd_legacy_v03.c +3 -1
  111. data/ext/zstd_legacy_v04.c +3 -1
  112. data/ext/zstd_legacy_v05.c +3 -1
  113. data/ext/zstd_legacy_v06.c +3 -1
  114. data/ext/zstd_legacy_v07.c +3 -1
  115. data/gemstub.rb +10 -24
  116. data/lib/extzstd.rb +64 -179
  117. data/lib/extzstd/version.rb +6 -1
  118. data/test/test_basic.rb +9 -6
  119. metadata +113 -57
  120. data/HISTORY.ja +0 -5
  121. data/contrib/zstd/common/entropy_common.c +0 -225
  122. data/contrib/zstd/common/huf.h +0 -228
  123. data/contrib/zstd/common/zstd_common.c +0 -83
  124. data/contrib/zstd/common/zstd_errors.h +0 -60
  125. data/contrib/zstd/common/zstd_internal.h +0 -267
  126. data/contrib/zstd/compress/huf_compress.c +0 -533
  127. data/contrib/zstd/compress/zbuff_compress.c +0 -319
  128. data/contrib/zstd/compress/zstd_compress.c +0 -3264
  129. data/contrib/zstd/compress/zstd_opt.h +0 -900
  130. data/contrib/zstd/decompress/huf_decompress.c +0 -883
  131. data/contrib/zstd/decompress/zbuff_decompress.c +0 -252
  132. data/contrib/zstd/decompress/zstd_decompress.c +0 -1842
  133. data/contrib/zstd/dictBuilder/zdict.h +0 -111
  134. data/contrib/zstd/zstd.h +0 -640
@@ -1,10 +1,11 @@
1
- /**
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
1
+ /*
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
- * This source code is licensed under the BSD-style license found in the
6
- * LICENSE file in the root directory of this source tree. An additional grant
7
- * of patent rights can be found in the PATENTS file in the same directory.
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
8
9
  */
9
10
 
10
11
  #ifndef ZSTD_V01_H_28739879432
@@ -34,6 +35,19 @@ ZSTDv01_decompress() : decompress ZSTD frames compliant with v0.1.x format
34
35
  size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize,
35
36
  const void* src, size_t compressedSize);
36
37
 
38
+ /**
39
+ ZSTDv01_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.1.x format
40
+ srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
41
+ cSize (output parameter) : the number of bytes that would be read to decompress this frame
42
+ or an error code if it fails (which can be tested using ZSTDv01_isError())
43
+ dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
44
+ or ZSTD_CONTENTSIZE_ERROR if an error occurs
45
+
46
+ note : assumes `cSize` and `dBound` are _not_ NULL.
47
+ */
48
+ void ZSTDv01_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
49
+ size_t* cSize, unsigned long long* dBound);
50
+
37
51
  /**
38
52
  ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error
39
53
  */
@@ -1,16 +1,17 @@
1
- /**
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
1
+ /*
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
- * This source code is licensed under the BSD-style license found in the
6
- * LICENSE file in the root directory of this source tree. An additional grant
7
- * of patent rights can be found in the PATENTS file in the same directory.
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
8
9
  */
9
10
 
10
11
 
11
12
  #include <stddef.h> /* size_t, ptrdiff_t */
12
13
  #include "zstd_v02.h"
13
- #include "error_private.h"
14
+ #include "../common/error_private.h"
14
15
 
15
16
 
16
17
  /******************************************
@@ -88,7 +89,11 @@ extern "C" {
88
89
  * Basic Types
89
90
  *****************************************************************/
90
91
  #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
91
- # include <stdint.h>
92
+ # if defined(_AIX)
93
+ # include <inttypes.h>
94
+ # else
95
+ # include <stdint.h> /* intptr_t */
96
+ # endif
92
97
  typedef uint8_t BYTE;
93
98
  typedef uint16_t U16;
94
99
  typedef int16_t S16;
@@ -188,7 +193,7 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
188
193
  memcpy(memPtr, &value, sizeof(value));
189
194
  }
190
195
 
191
- #endif // MEM_FORCE_MEMORY_ACCESS
196
+ #endif /* MEM_FORCE_MEMORY_ACCESS */
192
197
 
193
198
 
194
199
  MEM_STATIC U16 MEM_readLE16(const void* memPtr)
@@ -216,6 +221,11 @@ MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
216
221
  }
217
222
  }
218
223
 
224
+ MEM_STATIC U32 MEM_readLE24(const void* memPtr)
225
+ {
226
+ return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
227
+ }
228
+
219
229
  MEM_STATIC U32 MEM_readLE32(const void* memPtr)
220
230
  {
221
231
  if (MEM_isLittleEndian())
@@ -329,18 +339,6 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
329
339
  MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
330
340
 
331
341
 
332
- /*
333
- * Start by invoking BIT_initDStream().
334
- * A chunk of the bitStream is then stored into a local register.
335
- * Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
336
- * You can then retrieve bitFields stored into the local register, **in reverse order**.
337
- * Local register is manually filled from memory by the BIT_reloadDStream() method.
338
- * A reload guarantee a minimum of ((8*sizeof(size_t))-7) bits when its result is BIT_DStream_unfinished.
339
- * Otherwise, it can be less than that, so proceed accordingly.
340
- * Checking if DStream has reached its end can be performed with BIT_endOfDStream()
341
- */
342
-
343
-
344
342
  /******************************************
345
343
  * unsafe API
346
344
  ******************************************/
@@ -352,14 +350,14 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
352
350
  /****************************************************************
353
351
  * Helper functions
354
352
  ****************************************************************/
355
- MEM_STATIC unsigned BIT_highbit32 (register U32 val)
353
+ MEM_STATIC unsigned BIT_highbit32 (U32 val)
356
354
  {
357
355
  # if defined(_MSC_VER) /* Visual */
358
356
  unsigned long r=0;
359
357
  _BitScanReverse ( &r, val );
360
358
  return (unsigned) r;
361
359
  # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
362
- return 31 - __builtin_clz (val);
360
+ return __builtin_clz (val) ^ 31;
363
361
  # else /* Software version */
364
362
  static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
365
363
  U32 v = val;
@@ -410,11 +408,17 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
410
408
  switch(srcSize)
411
409
  {
412
410
  case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
411
+ /* fallthrough */
413
412
  case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
413
+ /* fallthrough */
414
414
  case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
415
+ /* fallthrough */
415
416
  case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
417
+ /* fallthrough */
416
418
  case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
419
+ /* fallthrough */
417
420
  case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
421
+ /* fallthrough */
418
422
  default:;
419
423
  }
420
424
  contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
@@ -426,13 +430,6 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
426
430
  return srcSize;
427
431
  }
428
432
 
429
- /*!BIT_lookBits
430
- * Provides next n bits from local register
431
- * local register is not modified (bits are still present for next read/look)
432
- * On 32-bits, maxNbBits==25
433
- * On 64-bits, maxNbBits==57
434
- * @return : value extracted
435
- */
436
433
  MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits)
437
434
  {
438
435
  const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
@@ -452,11 +449,6 @@ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
452
449
  bitD->bitsConsumed += nbBits;
453
450
  }
454
451
 
455
- /*!BIT_readBits
456
- * Read next n bits from local register.
457
- * pay attention to not read more than nbBits contained into local register.
458
- * @return : extracted value.
459
- */
460
452
  MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
461
453
  {
462
454
  size_t value = BIT_lookBits(bitD, nbBits);
@@ -475,8 +467,8 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
475
467
 
476
468
  MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
477
469
  {
478
- if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
479
- return BIT_DStream_overflow;
470
+ if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
471
+ return BIT_DStream_overflow;
480
472
 
481
473
  if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer))
482
474
  {
@@ -694,55 +686,6 @@ static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bi
694
686
 
695
687
  static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
696
688
 
697
- /*
698
- Let's now decompose FSE_decompress_usingDTable() into its unitary components.
699
- You will decode FSE-encoded symbols from the bitStream,
700
- and also any other bitFields you put in, **in reverse order**.
701
-
702
- You will need a few variables to track your bitStream. They are :
703
-
704
- BIT_DStream_t DStream; // Stream context
705
- FSE_DState_t DState; // State context. Multiple ones are possible
706
- FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
707
-
708
- The first thing to do is to init the bitStream.
709
- errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
710
-
711
- You should then retrieve your initial state(s)
712
- (in reverse flushing order if you have several ones) :
713
- errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
714
-
715
- You can then decode your data, symbol after symbol.
716
- For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
717
- Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
718
- unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
719
-
720
- You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
721
- Note : maximum allowed nbBits is 25, for 32-bits compatibility
722
- size_t bitField = BIT_readBits(&DStream, nbBits);
723
-
724
- All above operations only read from local register (which size depends on size_t).
725
- Refueling the register from memory is manually performed by the reload method.
726
- endSignal = FSE_reloadDStream(&DStream);
727
-
728
- BIT_reloadDStream() result tells if there is still some more data to read from DStream.
729
- BIT_DStream_unfinished : there is still some data left into the DStream.
730
- BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
731
- BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
732
- BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
733
-
734
- When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
735
- to properly detect the exact end of stream.
736
- After each decoded symbol, check if DStream is fully consumed using this simple test :
737
- BIT_reloadDStream(&DStream) >= BIT_DStream_completed
738
-
739
- When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
740
- Checking if DStream has reached its end is performed by :
741
- BIT_endOfDStream(&DStream);
742
- Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
743
- FSE_endOfDState(&DState);
744
- */
745
-
746
689
 
747
690
  /******************************************
748
691
  * FSE unsafe API
@@ -1334,8 +1277,8 @@ static size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsi
1334
1277
  else
1335
1278
  {
1336
1279
  bitCount -= (int)(8 * (iend - 4 - ip));
1337
- ip = iend - 4;
1338
- }
1280
+ ip = iend - 4;
1281
+ }
1339
1282
  bitStream = MEM_readLE32(ip) >> (bitCount & 31);
1340
1283
  }
1341
1284
  }
@@ -2040,7 +1983,7 @@ static size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
2040
1983
  rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
2041
1984
  }
2042
1985
 
2043
- /* Build rankVal */
1986
+ /* Build rankVal */
2044
1987
  {
2045
1988
  const U32 minBits = tableLog+1 - maxW;
2046
1989
  U32 nextRankVal = 0;
@@ -2374,7 +2317,7 @@ static size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
2374
2317
  rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
2375
2318
  }
2376
2319
 
2377
- /* Build rankVal */
2320
+ /* Build rankVal */
2378
2321
  {
2379
2322
  const U32 minBits = tableLog+1 - maxW;
2380
2323
  U32 nextRankVal = 0;
@@ -2794,6 +2737,8 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
2794
2737
  #define LITERAL_NOENTROPY 63
2795
2738
  #define COMMAND_NOENTROPY 7 /* to remove */
2796
2739
 
2740
+ #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
2741
+
2797
2742
  static const size_t ZSTD_blockHeaderSize = 3;
2798
2743
  static const size_t ZSTD_frameHeaderSize = 4;
2799
2744
 
@@ -2808,7 +2753,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
2808
2753
  #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
2809
2754
 
2810
2755
  /*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
2811
- static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
2756
+ static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
2812
2757
  {
2813
2758
  const BYTE* ip = (const BYTE*)src;
2814
2759
  BYTE* op = (BYTE*)dst;
@@ -2868,7 +2813,6 @@ struct ZSTD_DCtx_s
2868
2813
  blockType_t bType;
2869
2814
  U32 phase;
2870
2815
  const BYTE* litPtr;
2871
- size_t litBufSize;
2872
2816
  size_t litSize;
2873
2817
  BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
2874
2818
  }; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
@@ -2896,7 +2840,9 @@ static size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockPropertie
2896
2840
  static size_t ZSTD_copyUncompressedBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
2897
2841
  {
2898
2842
  if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
2899
- memcpy(dst, src, srcSize);
2843
+ if (srcSize > 0) {
2844
+ memcpy(dst, src, srcSize);
2845
+ }
2900
2846
  return srcSize;
2901
2847
  }
2902
2848
 
@@ -2940,8 +2886,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
2940
2886
  size_t litSize = BLOCKSIZE;
2941
2887
  const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
2942
2888
  dctx->litPtr = dctx->litBuffer;
2943
- dctx->litBufSize = BLOCKSIZE;
2944
2889
  dctx->litSize = litSize;
2890
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
2945
2891
  return readSize; /* works if it's an error too */
2946
2892
  }
2947
2893
  case IS_RAW:
@@ -2949,16 +2895,16 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
2949
2895
  const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
2950
2896
  if (litSize > srcSize-11) /* risk of reading too far with wildcopy */
2951
2897
  {
2952
- if (litSize > srcSize-3) return ERROR(corruption_detected);
2953
- memcpy(dctx->litBuffer, istart, litSize);
2954
- dctx->litPtr = dctx->litBuffer;
2955
- dctx->litBufSize = BLOCKSIZE;
2956
- dctx->litSize = litSize;
2957
- return litSize+3;
2958
- }
2959
- /* direct reference into compressed stream */
2898
+ if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
2899
+ if (litSize > srcSize-3) return ERROR(corruption_detected);
2900
+ memcpy(dctx->litBuffer, istart, litSize);
2901
+ dctx->litPtr = dctx->litBuffer;
2902
+ dctx->litSize = litSize;
2903
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
2904
+ return litSize+3;
2905
+ }
2906
+ /* direct reference into compressed stream */
2960
2907
  dctx->litPtr = istart+3;
2961
- dctx->litBufSize = srcSize-3;
2962
2908
  dctx->litSize = litSize;
2963
2909
  return litSize+3;
2964
2910
  }
@@ -2966,9 +2912,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
2966
2912
  {
2967
2913
  const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
2968
2914
  if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
2969
- memset(dctx->litBuffer, istart[3], litSize);
2915
+ memset(dctx->litBuffer, istart[3], litSize + 8);
2970
2916
  dctx->litPtr = dctx->litBuffer;
2971
- dctx->litBufSize = BLOCKSIZE;
2972
2917
  dctx->litSize = litSize;
2973
2918
  return 4;
2974
2919
  }
@@ -3110,11 +3055,11 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
3110
3055
  seqState->prevOffset = seq->offset;
3111
3056
  if (litLength == MaxLL)
3112
3057
  {
3113
- U32 add = *dumps++;
3058
+ const U32 add = dumps<de ? *dumps++ : 0;
3114
3059
  if (add < 255) litLength += add;
3115
- else
3060
+ else if (dumps + 3 <= de)
3116
3061
  {
3117
- litLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
3062
+ litLength = MEM_readLE24(dumps);
3118
3063
  dumps += 3;
3119
3064
  }
3120
3065
  if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
@@ -3140,11 +3085,11 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
3140
3085
  matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
3141
3086
  if (matchLength == MaxML)
3142
3087
  {
3143
- U32 add = *dumps++;
3088
+ const U32 add = dumps<de ? *dumps++ : 0;
3144
3089
  if (add < 255) matchLength += add;
3145
- else
3090
+ else if (dumps + 3 <= de)
3146
3091
  {
3147
- matchLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
3092
+ matchLength = MEM_readLE24(dumps);
3148
3093
  dumps += 3;
3149
3094
  }
3150
3095
  if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
@@ -3165,7 +3110,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3165
3110
  BYTE* const base, BYTE* const oend)
3166
3111
  {
3167
3112
  static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
3168
- static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */
3113
+ static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* subtracted */
3169
3114
  const BYTE* const ostart = op;
3170
3115
  BYTE* const oLitEnd = op + sequence.litLength;
3171
3116
  BYTE* const oMatchEnd = op + sequence.litLength + sequence.matchLength; /* risk : address space overflow (32-bits) */
@@ -3175,7 +3120,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3175
3120
  /* checks */
3176
3121
  if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
3177
3122
  if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
3178
- if (litEnd > litLimit-8) return ERROR(corruption_detected); /* overRead beyond lit buffer */
3123
+ if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
3179
3124
 
3180
3125
  /* copy Literals */
3181
3126
  ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@@ -3209,7 +3154,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3209
3154
  }
3210
3155
  op += 8; match += 8;
3211
3156
 
3212
- if (oMatchEnd > oend-12)
3157
+ if (oMatchEnd > oend-(16-MINMATCH))
3213
3158
  {
3214
3159
  if (op < oend_8)
3215
3160
  {
@@ -3221,7 +3166,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3221
3166
  }
3222
3167
  else
3223
3168
  {
3224
- ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
3169
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
3225
3170
  }
3226
3171
  }
3227
3172
 
@@ -3241,7 +3186,6 @@ static size_t ZSTD_decompressSequences(
3241
3186
  BYTE* const oend = ostart + maxDstSize;
3242
3187
  size_t errorCode, dumpsLength;
3243
3188
  const BYTE* litPtr = dctx->litPtr;
3244
- const BYTE* const litMax = litPtr + dctx->litBufSize;
3245
3189
  const BYTE* const litEnd = litPtr + dctx->litSize;
3246
3190
  int nbSeq;
3247
3191
  const BYTE* dumps;
@@ -3277,7 +3221,7 @@ static size_t ZSTD_decompressSequences(
3277
3221
  size_t oneSeqSize;
3278
3222
  nbSeq--;
3279
3223
  ZSTD_decodeSequence(&sequence, &seqState);
3280
- oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litMax, base, oend);
3224
+ oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
3281
3225
  if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
3282
3226
  op += oneSeqSize;
3283
3227
  }
@@ -3291,8 +3235,10 @@ static size_t ZSTD_decompressSequences(
3291
3235
  size_t lastLLSize = litEnd - litPtr;
3292
3236
  if (litPtr > litEnd) return ERROR(corruption_detected);
3293
3237
  if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
3294
- if (op != litPtr) memmove(op, litPtr, lastLLSize);
3295
- op += lastLLSize;
3238
+ if (lastLLSize > 0) {
3239
+ if (op != litPtr) memmove(op, litPtr, lastLLSize);
3240
+ op += lastLLSize;
3241
+ }
3296
3242
  }
3297
3243
  }
3298
3244
 
@@ -3382,6 +3328,60 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
3382
3328
  return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
3383
3329
  }
3384
3330
 
3331
+ /* ZSTD_errorFrameSizeInfoLegacy() :
3332
+ assumes `cSize` and `dBound` are _not_ NULL */
3333
+ static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
3334
+ {
3335
+ *cSize = ret;
3336
+ *dBound = ZSTD_CONTENTSIZE_ERROR;
3337
+ }
3338
+
3339
+ void ZSTDv02_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
3340
+ {
3341
+ const BYTE* ip = (const BYTE*)src;
3342
+ size_t remainingSize = srcSize;
3343
+ size_t nbBlocks = 0;
3344
+ U32 magicNumber;
3345
+ blockProperties_t blockProperties;
3346
+
3347
+ /* Frame Header */
3348
+ if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) {
3349
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3350
+ return;
3351
+ }
3352
+ magicNumber = MEM_readLE32(src);
3353
+ if (magicNumber != ZSTD_magicNumber) {
3354
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
3355
+ return;
3356
+ }
3357
+ ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
3358
+
3359
+ /* Loop on each block */
3360
+ while (1)
3361
+ {
3362
+ size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
3363
+ if (ZSTD_isError(cBlockSize)) {
3364
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
3365
+ return;
3366
+ }
3367
+
3368
+ ip += ZSTD_blockHeaderSize;
3369
+ remainingSize -= ZSTD_blockHeaderSize;
3370
+ if (cBlockSize > remainingSize) {
3371
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3372
+ return;
3373
+ }
3374
+
3375
+ if (cBlockSize == 0) break; /* bt_end */
3376
+
3377
+ ip += cBlockSize;
3378
+ remainingSize -= cBlockSize;
3379
+ nbBlocks++;
3380
+ }
3381
+
3382
+ *cSize = ip - (const BYTE*)src;
3383
+ *dBound = nbBlocks * BLOCKSIZE;
3384
+ }
3385
3385
 
3386
3386
  /*******************************
3387
3387
  * Streaming Decompression API
@@ -3487,36 +3487,36 @@ static size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSi
3487
3487
 
3488
3488
  unsigned ZSTDv02_isError(size_t code)
3489
3489
  {
3490
- return ZSTD_isError(code);
3490
+ return ZSTD_isError(code);
3491
3491
  }
3492
3492
 
3493
3493
  size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
3494
3494
  const void* src, size_t compressedSize)
3495
3495
  {
3496
- return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
3496
+ return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
3497
3497
  }
3498
3498
 
3499
3499
  ZSTDv02_Dctx* ZSTDv02_createDCtx(void)
3500
3500
  {
3501
- return (ZSTDv02_Dctx*)ZSTD_createDCtx();
3501
+ return (ZSTDv02_Dctx*)ZSTD_createDCtx();
3502
3502
  }
3503
3503
 
3504
3504
  size_t ZSTDv02_freeDCtx(ZSTDv02_Dctx* dctx)
3505
3505
  {
3506
- return ZSTD_freeDCtx((ZSTD_DCtx*)dctx);
3506
+ return ZSTD_freeDCtx((ZSTD_DCtx*)dctx);
3507
3507
  }
3508
3508
 
3509
3509
  size_t ZSTDv02_resetDCtx(ZSTDv02_Dctx* dctx)
3510
3510
  {
3511
- return ZSTD_resetDCtx((ZSTD_DCtx*)dctx);
3511
+ return ZSTD_resetDCtx((ZSTD_DCtx*)dctx);
3512
3512
  }
3513
3513
 
3514
3514
  size_t ZSTDv02_nextSrcSizeToDecompress(ZSTDv02_Dctx* dctx)
3515
3515
  {
3516
- return ZSTD_nextSrcSizeToDecompress((ZSTD_DCtx*)dctx);
3516
+ return ZSTD_nextSrcSizeToDecompress((ZSTD_DCtx*)dctx);
3517
3517
  }
3518
3518
 
3519
3519
  size_t ZSTDv02_decompressContinue(ZSTDv02_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
3520
3520
  {
3521
- return ZSTD_decompressContinue((ZSTD_DCtx*)dctx, dst, maxDstSize, src, srcSize);
3521
+ return ZSTD_decompressContinue((ZSTD_DCtx*)dctx, dst, maxDstSize, src, srcSize);
3522
3522
  }