extzstd 0.0.3.CONCEPT → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. checksums.yaml +5 -5
  2. data/HISTORY.ja.md +39 -0
  3. data/LICENSE +6 -6
  4. data/README.md +26 -45
  5. data/contrib/zstd/CHANGELOG +555 -0
  6. data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
  7. data/contrib/zstd/CONTRIBUTING.md +392 -0
  8. data/contrib/zstd/COPYING +339 -0
  9. data/contrib/zstd/LICENSE +13 -9
  10. data/contrib/zstd/Makefile +414 -0
  11. data/contrib/zstd/README.md +170 -45
  12. data/contrib/zstd/TESTING.md +44 -0
  13. data/contrib/zstd/appveyor.yml +289 -0
  14. data/contrib/zstd/lib/BUCK +234 -0
  15. data/contrib/zstd/lib/Makefile +354 -0
  16. data/contrib/zstd/lib/README.md +179 -0
  17. data/contrib/zstd/{common → lib/common}/bitstream.h +170 -130
  18. data/contrib/zstd/lib/common/compiler.h +175 -0
  19. data/contrib/zstd/lib/common/cpu.h +215 -0
  20. data/contrib/zstd/lib/common/debug.c +24 -0
  21. data/contrib/zstd/lib/common/debug.h +114 -0
  22. data/contrib/zstd/{common → lib/common}/entropy_common.c +79 -94
  23. data/contrib/zstd/lib/common/error_private.c +55 -0
  24. data/contrib/zstd/lib/common/error_private.h +80 -0
  25. data/contrib/zstd/{common → lib/common}/fse.h +153 -93
  26. data/contrib/zstd/{common → lib/common}/fse_decompress.c +37 -82
  27. data/contrib/zstd/lib/common/huf.h +340 -0
  28. data/contrib/zstd/{common → lib/common}/mem.h +154 -78
  29. data/contrib/zstd/lib/common/pool.c +344 -0
  30. data/contrib/zstd/lib/common/pool.h +84 -0
  31. data/contrib/zstd/lib/common/threading.c +121 -0
  32. data/contrib/zstd/lib/common/threading.h +155 -0
  33. data/contrib/zstd/{common → lib/common}/xxhash.c +85 -75
  34. data/contrib/zstd/{common → lib/common}/xxhash.h +85 -73
  35. data/contrib/zstd/lib/common/zstd_common.c +83 -0
  36. data/contrib/zstd/lib/common/zstd_errors.h +94 -0
  37. data/contrib/zstd/lib/common/zstd_internal.h +447 -0
  38. data/contrib/zstd/{compress → lib/compress}/fse_compress.c +194 -303
  39. data/contrib/zstd/lib/compress/hist.c +183 -0
  40. data/contrib/zstd/lib/compress/hist.h +75 -0
  41. data/contrib/zstd/lib/compress/huf_compress.c +798 -0
  42. data/contrib/zstd/lib/compress/zstd_compress.c +4278 -0
  43. data/contrib/zstd/lib/compress/zstd_compress_internal.h +1125 -0
  44. data/contrib/zstd/lib/compress/zstd_compress_literals.c +158 -0
  45. data/contrib/zstd/lib/compress/zstd_compress_literals.h +29 -0
  46. data/contrib/zstd/lib/compress/zstd_compress_sequences.c +419 -0
  47. data/contrib/zstd/lib/compress/zstd_compress_sequences.h +54 -0
  48. data/contrib/zstd/lib/compress/zstd_compress_superblock.c +845 -0
  49. data/contrib/zstd/lib/compress/zstd_compress_superblock.h +32 -0
  50. data/contrib/zstd/lib/compress/zstd_cwksp.h +525 -0
  51. data/contrib/zstd/lib/compress/zstd_double_fast.c +521 -0
  52. data/contrib/zstd/lib/compress/zstd_double_fast.h +38 -0
  53. data/contrib/zstd/lib/compress/zstd_fast.c +496 -0
  54. data/contrib/zstd/lib/compress/zstd_fast.h +37 -0
  55. data/contrib/zstd/lib/compress/zstd_lazy.c +1138 -0
  56. data/contrib/zstd/lib/compress/zstd_lazy.h +67 -0
  57. data/contrib/zstd/lib/compress/zstd_ldm.c +619 -0
  58. data/contrib/zstd/lib/compress/zstd_ldm.h +110 -0
  59. data/contrib/zstd/lib/compress/zstd_opt.c +1200 -0
  60. data/contrib/zstd/lib/compress/zstd_opt.h +56 -0
  61. data/contrib/zstd/lib/compress/zstdmt_compress.c +2143 -0
  62. data/contrib/zstd/lib/compress/zstdmt_compress.h +192 -0
  63. data/contrib/zstd/lib/decompress/huf_decompress.c +1248 -0
  64. data/contrib/zstd/lib/decompress/zstd_ddict.c +244 -0
  65. data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
  66. data/contrib/zstd/lib/decompress/zstd_decompress.c +1885 -0
  67. data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1432 -0
  68. data/contrib/zstd/lib/decompress/zstd_decompress_block.h +59 -0
  69. data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +189 -0
  70. data/contrib/zstd/{common → lib/deprecated}/zbuff.h +86 -69
  71. data/contrib/zstd/lib/deprecated/zbuff_common.c +26 -0
  72. data/contrib/zstd/lib/deprecated/zbuff_compress.c +147 -0
  73. data/contrib/zstd/lib/deprecated/zbuff_decompress.c +75 -0
  74. data/contrib/zstd/lib/dictBuilder/cover.c +1236 -0
  75. data/contrib/zstd/lib/dictBuilder/cover.h +157 -0
  76. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.c +3 -3
  77. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/divsufsort.h +5 -5
  78. data/contrib/zstd/lib/dictBuilder/fastcover.c +757 -0
  79. data/contrib/zstd/{dictBuilder → lib/dictBuilder}/zdict.c +437 -347
  80. data/contrib/zstd/lib/dictBuilder/zdict.h +305 -0
  81. data/contrib/zstd/lib/legacy/zstd_legacy.h +415 -0
  82. data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.c +272 -292
  83. data/contrib/zstd/{legacy → lib/legacy}/zstd_v01.h +26 -32
  84. data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.c +162 -392
  85. data/contrib/zstd/{legacy → lib/legacy}/zstd_v02.h +26 -32
  86. data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.c +162 -391
  87. data/contrib/zstd/{legacy → lib/legacy}/zstd_v03.h +27 -33
  88. data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.c +195 -604
  89. data/contrib/zstd/{legacy → lib/legacy}/zstd_v04.h +26 -32
  90. data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.c +300 -575
  91. data/contrib/zstd/{legacy → lib/legacy}/zstd_v05.h +22 -31
  92. data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.c +165 -592
  93. data/contrib/zstd/{legacy → lib/legacy}/zstd_v06.h +54 -67
  94. data/contrib/zstd/lib/legacy/zstd_v07.c +4541 -0
  95. data/contrib/zstd/lib/legacy/zstd_v07.h +187 -0
  96. data/contrib/zstd/lib/libzstd.pc.in +15 -0
  97. data/contrib/zstd/lib/zstd.h +2090 -0
  98. data/ext/depend +2 -0
  99. data/ext/extconf.rb +18 -5
  100. data/ext/extzstd.c +296 -214
  101. data/ext/extzstd.h +81 -36
  102. data/ext/extzstd_nogvls.h +0 -117
  103. data/ext/extzstd_stream.c +622 -0
  104. data/ext/libzstd_conf.h +8 -0
  105. data/ext/zstd_common.c +11 -0
  106. data/ext/zstd_compress.c +15 -0
  107. data/ext/zstd_decompress.c +6 -0
  108. data/ext/zstd_dictbuilder.c +10 -0
  109. data/ext/zstd_dictbuilder_fastcover.c +3 -0
  110. data/ext/zstd_legacy_v01.c +3 -1
  111. data/ext/zstd_legacy_v02.c +3 -1
  112. data/ext/zstd_legacy_v03.c +3 -1
  113. data/ext/zstd_legacy_v04.c +3 -1
  114. data/ext/zstd_legacy_v05.c +3 -1
  115. data/ext/zstd_legacy_v06.c +3 -1
  116. data/ext/zstd_legacy_v07.c +3 -0
  117. data/gemstub.rb +27 -21
  118. data/lib/extzstd.rb +82 -161
  119. data/lib/extzstd/version.rb +1 -1
  120. data/test/test_basic.rb +19 -6
  121. metadata +127 -59
  122. data/contrib/zstd/common/error_private.h +0 -125
  123. data/contrib/zstd/common/error_public.h +0 -77
  124. data/contrib/zstd/common/huf.h +0 -228
  125. data/contrib/zstd/common/zstd.h +0 -475
  126. data/contrib/zstd/common/zstd_common.c +0 -91
  127. data/contrib/zstd/common/zstd_internal.h +0 -238
  128. data/contrib/zstd/compress/huf_compress.c +0 -577
  129. data/contrib/zstd/compress/zbuff_compress.c +0 -327
  130. data/contrib/zstd/compress/zstd_compress.c +0 -3074
  131. data/contrib/zstd/compress/zstd_opt.h +0 -1046
  132. data/contrib/zstd/decompress/huf_decompress.c +0 -894
  133. data/contrib/zstd/decompress/zbuff_decompress.c +0 -294
  134. data/contrib/zstd/decompress/zstd_decompress.c +0 -1362
  135. data/contrib/zstd/dictBuilder/zdict.h +0 -113
  136. data/contrib/zstd/legacy/zstd_legacy.h +0 -140
  137. data/ext/extzstd_buffered.c +0 -265
  138. data/ext/zstd_amalgam.c +0 -18
@@ -1,36 +1,15 @@
1
1
  /*
2
- zstd - standard compression library
3
- Header File
4
- Copyright (C) 2014-2015, Yann Collet.
5
-
6
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
7
-
8
- Redistribution and use in source and binary forms, with or without
9
- modification, are permitted provided that the following conditions are
10
- met:
11
- * Redistributions of source code must retain the above copyright
12
- notice, this list of conditions and the following disclaimer.
13
- * Redistributions in binary form must reproduce the above
14
- copyright notice, this list of conditions and the following disclaimer
15
- in the documentation and/or other materials provided with the
16
- distribution.
17
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
-
29
- You can contact the author at :
30
- - zstd source repository : https://github.com/Cyan4973/zstd
31
- - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
32
- */
33
- #pragma once
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
+ * All rights reserved.
4
+ *
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
9
+ */
10
+
11
+ #ifndef ZSTD_V01_H_28739879432
12
+ #define ZSTD_V01_H_28739879432
34
13
 
35
14
  #if defined (__cplusplus)
36
15
  extern "C" {
@@ -56,6 +35,19 @@ ZSTDv01_decompress() : decompress ZSTD frames compliant with v0.1.x format
56
35
  size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize,
57
36
  const void* src, size_t compressedSize);
58
37
 
38
+ /**
39
+ ZSTDv01_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.1.x format
40
+ srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
41
+ cSize (output parameter) : the number of bytes that would be read to decompress this frame
42
+ or an error code if it fails (which can be tested using ZSTDv01_isError())
43
+ dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame
44
+ or ZSTD_CONTENTSIZE_ERROR if an error occurs
45
+
46
+ note : assumes `cSize` and `dBound` are _not_ NULL.
47
+ */
48
+ void ZSTDv01_findFrameSizeInfoLegacy(const void *src, size_t srcSize,
49
+ size_t* cSize, unsigned long long* dBound);
50
+
59
51
  /**
60
52
  ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error
61
53
  */
@@ -98,3 +90,5 @@ size_t ZSTDv01_decompressContinue(ZSTDv01_Dctx* dctx, void* dst, size_t maxDstSi
98
90
  #if defined (__cplusplus)
99
91
  }
100
92
  #endif
93
+
94
+ #endif /* ZSTD_V01_H_28739879432 */
@@ -1,90 +1,27 @@
1
- /* ******************************************************************
2
- Error codes and messages
3
- Copyright (C) 2013-2015, Yann Collet
4
-
5
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
6
-
7
- Redistribution and use in source and binary forms, with or without
8
- modification, are permitted provided that the following conditions are
9
- met:
10
-
11
- * Redistributions of source code must retain the above copyright
12
- notice, this list of conditions and the following disclaimer.
13
- * Redistributions in binary form must reproduce the above
14
- copyright notice, this list of conditions and the following disclaimer
15
- in the documentation and/or other materials provided with the
16
- distribution.
17
-
18
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
1
+ /*
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
+ * All rights reserved.
4
+ *
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
9
+ */
29
10
 
30
- You can contact the author at :
31
- - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
32
- - Public forum : https://groups.google.com/forum/#!forum/lz4c
33
- ****************************************************************** */
34
- #ifndef ERROR_H_MODULE
35
- #define ERROR_H_MODULE
36
-
37
- #if defined (__cplusplus)
38
- extern "C" {
39
- #endif
40
11
 
41
12
  #include <stddef.h> /* size_t, ptrdiff_t */
42
13
  #include "zstd_v02.h"
43
-
44
- /******************************************
45
- * Compiler-specific
46
- ******************************************/
47
- #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
48
- # define ERR_STATIC static inline
49
- #elif defined(_MSC_VER)
50
- # define ERR_STATIC static __inline
51
- #elif defined(__GNUC__)
52
- # define ERR_STATIC static __attribute__((unused))
53
- #else
54
- # define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
55
- #endif
14
+ #include "../common/error_private.h"
56
15
 
57
16
 
58
17
  /******************************************
59
- * Error Management
18
+ * Compiler-specific
60
19
  ******************************************/
61
- #define PREFIX(name) ZSTD_error_##name
62
-
63
- #define ERROR(name) (size_t)-PREFIX(name)
64
-
65
- #define ERROR_LIST(ITEM) \
66
- ITEM(PREFIX(No_Error)) ITEM(PREFIX(GENERIC)) \
67
- ITEM(PREFIX(memory_allocation)) \
68
- ITEM(PREFIX(dstSize_tooSmall)) ITEM(PREFIX(srcSize_wrong)) \
69
- ITEM(PREFIX(prefix_unknown)) ITEM(PREFIX(corruption_detected)) \
70
- ITEM(PREFIX(tableLog_tooLarge)) ITEM(PREFIX(maxSymbolValue_tooLarge)) ITEM(PREFIX(maxSymbolValue_tooSmall)) \
71
- ITEM(PREFIX(maxCode))
72
-
73
- #define ERROR_GENERATE_ENUM(ENUM) ENUM,
74
- typedef enum { ERROR_LIST(ERROR_GENERATE_ENUM) } ERR_codes; /* enum is exposed, to detect & handle specific errors; compare function result to -enum value */
75
-
76
- #define ERROR_CONVERTTOSTRING(STRING) #STRING,
77
- #define ERROR_GENERATE_STRING(EXPR) ERROR_CONVERTTOSTRING(EXPR)
78
-
79
- ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
80
-
81
-
82
- #if defined (__cplusplus)
83
- }
20
+ #if defined(_MSC_VER) /* Visual Studio */
21
+ # include <stdlib.h> /* _byteswap_ulong */
22
+ # include <intrin.h> /* _byteswap_* */
84
23
  #endif
85
24
 
86
- #endif /* ERROR_H_MODULE */
87
-
88
25
 
89
26
  /* ******************************************************************
90
27
  mem.h
@@ -190,7 +127,7 @@ extern "C" {
190
127
  #ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
191
128
  # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
192
129
  # define MEM_FORCE_MEMORY_ACCESS 2
193
- # elif defined(__INTEL_COMPILER) || \
130
+ # elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
194
131
  (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
195
132
  # define MEM_FORCE_MEMORY_ACCESS 1
196
133
  # endif
@@ -214,8 +151,6 @@ MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
214
151
  MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
215
152
 
216
153
  MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
217
- MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
218
- MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
219
154
 
220
155
  #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
221
156
 
@@ -228,8 +163,6 @@ MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32;
228
163
  MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
229
164
 
230
165
  MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
231
- MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
232
- MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
233
166
 
234
167
  #else
235
168
 
@@ -256,17 +189,7 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
256
189
  memcpy(memPtr, &value, sizeof(value));
257
190
  }
258
191
 
259
- MEM_STATIC void MEM_write32(void* memPtr, U32 value)
260
- {
261
- memcpy(memPtr, &value, sizeof(value));
262
- }
263
-
264
- MEM_STATIC void MEM_write64(void* memPtr, U64 value)
265
- {
266
- memcpy(memPtr, &value, sizeof(value));
267
- }
268
-
269
- #endif // MEM_FORCE_MEMORY_ACCESS
192
+ #endif /* MEM_FORCE_MEMORY_ACCESS */
270
193
 
271
194
 
272
195
  MEM_STATIC U16 MEM_readLE16(const void* memPtr)
@@ -294,6 +217,11 @@ MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
294
217
  }
295
218
  }
296
219
 
220
+ MEM_STATIC U32 MEM_readLE24(const void* memPtr)
221
+ {
222
+ return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
223
+ }
224
+
297
225
  MEM_STATIC U32 MEM_readLE32(const void* memPtr)
298
226
  {
299
227
  if (MEM_isLittleEndian())
@@ -305,21 +233,6 @@ MEM_STATIC U32 MEM_readLE32(const void* memPtr)
305
233
  }
306
234
  }
307
235
 
308
- MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
309
- {
310
- if (MEM_isLittleEndian())
311
- {
312
- MEM_write32(memPtr, val32);
313
- }
314
- else
315
- {
316
- BYTE* p = (BYTE*)memPtr;
317
- p[0] = (BYTE)val32;
318
- p[1] = (BYTE)(val32>>8);
319
- p[2] = (BYTE)(val32>>16);
320
- p[3] = (BYTE)(val32>>24);
321
- }
322
- }
323
236
 
324
237
  MEM_STATIC U64 MEM_readLE64(const void* memPtr)
325
238
  {
@@ -333,25 +246,6 @@ MEM_STATIC U64 MEM_readLE64(const void* memPtr)
333
246
  }
334
247
  }
335
248
 
336
- MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
337
- {
338
- if (MEM_isLittleEndian())
339
- {
340
- MEM_write64(memPtr, val64);
341
- }
342
- else
343
- {
344
- BYTE* p = (BYTE*)memPtr;
345
- p[0] = (BYTE)val64;
346
- p[1] = (BYTE)(val64>>8);
347
- p[2] = (BYTE)(val64>>16);
348
- p[3] = (BYTE)(val64>>24);
349
- p[4] = (BYTE)(val64>>32);
350
- p[5] = (BYTE)(val64>>40);
351
- p[6] = (BYTE)(val64>>48);
352
- p[7] = (BYTE)(val64>>56);
353
- }
354
- }
355
249
 
356
250
  MEM_STATIC size_t MEM_readLEST(const void* memPtr)
357
251
  {
@@ -361,14 +255,6 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
361
255
  return (size_t)MEM_readLE64(memPtr);
362
256
  }
363
257
 
364
- MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
365
- {
366
- if (MEM_32bits())
367
- MEM_writeLE32(memPtr, (U32)val);
368
- else
369
- MEM_writeLE64(memPtr, (U64)val);
370
- }
371
-
372
258
  #if defined (__cplusplus)
373
259
  }
374
260
  #endif
@@ -449,18 +335,6 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
449
335
  MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
450
336
 
451
337
 
452
- /*
453
- * Start by invoking BIT_initDStream().
454
- * A chunk of the bitStream is then stored into a local register.
455
- * Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
456
- * You can then retrieve bitFields stored into the local register, **in reverse order**.
457
- * Local register is manually filled from memory by the BIT_reloadDStream() method.
458
- * A reload guarantee a minimum of ((8*sizeof(size_t))-7) bits when its result is BIT_DStream_unfinished.
459
- * Otherwise, it can be less than that, so proceed accordingly.
460
- * Checking if DStream has reached its end can be performed with BIT_endOfDStream()
461
- */
462
-
463
-
464
338
  /******************************************
465
339
  * unsafe API
466
340
  ******************************************/
@@ -472,14 +346,14 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
472
346
  /****************************************************************
473
347
  * Helper functions
474
348
  ****************************************************************/
475
- MEM_STATIC unsigned BIT_highbit32 (register U32 val)
349
+ MEM_STATIC unsigned BIT_highbit32 (U32 val)
476
350
  {
477
351
  # if defined(_MSC_VER) /* Visual */
478
352
  unsigned long r=0;
479
353
  _BitScanReverse ( &r, val );
480
354
  return (unsigned) r;
481
355
  # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
482
- return 31 - __builtin_clz (val);
356
+ return __builtin_clz (val) ^ 31;
483
357
  # else /* Software version */
484
358
  static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
485
359
  U32 v = val;
@@ -530,11 +404,17 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
530
404
  switch(srcSize)
531
405
  {
532
406
  case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
407
+ /* fallthrough */
533
408
  case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
409
+ /* fallthrough */
534
410
  case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
411
+ /* fallthrough */
535
412
  case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
413
+ /* fallthrough */
536
414
  case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
415
+ /* fallthrough */
537
416
  case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
417
+ /* fallthrough */
538
418
  default:;
539
419
  }
540
420
  contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
@@ -546,13 +426,6 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
546
426
  return srcSize;
547
427
  }
548
428
 
549
- /*!BIT_lookBits
550
- * Provides next n bits from local register
551
- * local register is not modified (bits are still present for next read/look)
552
- * On 32-bits, maxNbBits==25
553
- * On 64-bits, maxNbBits==57
554
- * @return : value extracted
555
- */
556
429
  MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits)
557
430
  {
558
431
  const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
@@ -572,11 +445,6 @@ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
572
445
  bitD->bitsConsumed += nbBits;
573
446
  }
574
447
 
575
- /*!BIT_readBits
576
- * Read next n bits from local register.
577
- * pay attention to not read more than nbBits contained into local register.
578
- * @return : extracted value.
579
- */
580
448
  MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
581
449
  {
582
450
  size_t value = BIT_lookBits(bitD, nbBits);
@@ -595,8 +463,8 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
595
463
 
596
464
  MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
597
465
  {
598
- if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
599
- return BIT_DStream_overflow;
466
+ if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
467
+ return BIT_DStream_overflow;
600
468
 
601
469
  if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer))
602
470
  {
@@ -814,55 +682,6 @@ static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bi
814
682
 
815
683
  static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
816
684
 
817
- /*
818
- Let's now decompose FSE_decompress_usingDTable() into its unitary components.
819
- You will decode FSE-encoded symbols from the bitStream,
820
- and also any other bitFields you put in, **in reverse order**.
821
-
822
- You will need a few variables to track your bitStream. They are :
823
-
824
- BIT_DStream_t DStream; // Stream context
825
- FSE_DState_t DState; // State context. Multiple ones are possible
826
- FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
827
-
828
- The first thing to do is to init the bitStream.
829
- errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
830
-
831
- You should then retrieve your initial state(s)
832
- (in reverse flushing order if you have several ones) :
833
- errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
834
-
835
- You can then decode your data, symbol after symbol.
836
- For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
837
- Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
838
- unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
839
-
840
- You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
841
- Note : maximum allowed nbBits is 25, for 32-bits compatibility
842
- size_t bitField = BIT_readBits(&DStream, nbBits);
843
-
844
- All above operations only read from local register (which size depends on size_t).
845
- Refueling the register from memory is manually performed by the reload method.
846
- endSignal = FSE_reloadDStream(&DStream);
847
-
848
- BIT_reloadDStream() result tells if there is still some more data to read from DStream.
849
- BIT_DStream_unfinished : there is still some data left into the DStream.
850
- BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
851
- BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
852
- BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
853
-
854
- When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
855
- to properly detect the exact end of stream.
856
- After each decoded symbol, check if DStream is fully consumed using this simple test :
857
- BIT_reloadDStream(&DStream) >= BIT_DStream_completed
858
-
859
- When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
860
- Checking if DStream has reached its end is performed by :
861
- BIT_endOfDStream(&DStream);
862
- Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
863
- FSE_endOfDState(&DState);
864
- */
865
-
866
685
 
867
686
  /******************************************
868
687
  * FSE unsafe API
@@ -1198,12 +1017,15 @@ typedef struct ZSTD_DCtx_s ZSTD_DCtx;
1198
1017
  # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
1199
1018
  # pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
1200
1019
  #else
1201
- # ifdef __GNUC__
1202
- # define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
1203
- # define FORCE_INLINE static inline __attribute__((always_inline))
1020
+ # if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
1021
+ # ifdef __GNUC__
1022
+ # define FORCE_INLINE static inline __attribute__((always_inline))
1023
+ # else
1024
+ # define FORCE_INLINE static inline
1025
+ # endif
1204
1026
  # else
1205
- # define FORCE_INLINE static inline
1206
- # endif
1027
+ # define FORCE_INLINE static
1028
+ # endif /* __STDC_VERSION__ */
1207
1029
  #endif
1208
1030
 
1209
1031
 
@@ -1350,7 +1172,7 @@ static unsigned FSE_isError(size_t code) { return ERR_isError(code); }
1350
1172
  ****************************************************************/
1351
1173
  static short FSE_abs(short a)
1352
1174
  {
1353
- return a<0 ? -a : a;
1175
+ return (short)(a<0 ? -a : a);
1354
1176
  }
1355
1177
 
1356
1178
  static size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
@@ -1451,8 +1273,8 @@ static size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsi
1451
1273
  else
1452
1274
  {
1453
1275
  bitCount -= (int)(8 * (iend - 4 - ip));
1454
- ip = iend - 4;
1455
- }
1276
+ ip = iend - 4;
1277
+ }
1456
1278
  bitStream = MEM_readLE32(ip) >> (bitCount & 31);
1457
1279
  }
1458
1280
  }
@@ -1671,15 +1493,7 @@ static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, siz
1671
1493
 
1672
1494
 
1673
1495
  #ifdef _MSC_VER /* Visual Studio */
1674
- # define FORCE_INLINE static __forceinline
1675
1496
  # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
1676
- #else
1677
- # ifdef __GNUC__
1678
- # define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
1679
- # define FORCE_INLINE static inline __attribute__((always_inline))
1680
- # else
1681
- # define FORCE_INLINE static inline
1682
- # endif
1683
1497
  #endif
1684
1498
 
1685
1499
 
@@ -1732,10 +1546,12 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
1732
1546
  U32 weightTotal;
1733
1547
  U32 tableLog;
1734
1548
  const BYTE* ip = (const BYTE*) src;
1735
- size_t iSize = ip[0];
1549
+ size_t iSize;
1736
1550
  size_t oSize;
1737
1551
  U32 n;
1738
1552
 
1553
+ if (!srcSize) return ERROR(srcSize_wrong);
1554
+ iSize = ip[0];
1739
1555
  //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
1740
1556
 
1741
1557
  if (iSize >= 128) /* special header */
@@ -1777,6 +1593,7 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
1777
1593
  rankStats[huffWeight[n]]++;
1778
1594
  weightTotal += (1 << huffWeight[n]) >> 1;
1779
1595
  }
1596
+ if (weightTotal == 0) return ERROR(corruption_detected);
1780
1597
 
1781
1598
  /* get last non-null symbol weight (implied, total must be 2^n) */
1782
1599
  tableLog = BIT_highbit32(weightTotal) + 1;
@@ -2162,7 +1979,7 @@ static size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
2162
1979
  rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
2163
1980
  }
2164
1981
 
2165
- /* Build rankVal */
1982
+ /* Build rankVal */
2166
1983
  {
2167
1984
  const U32 minBits = tableLog+1 - maxW;
2168
1985
  U32 nextRankVal = 0;
@@ -2496,7 +2313,7 @@ static size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
2496
2313
  rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
2497
2314
  }
2498
2315
 
2499
- /* Build rankVal */
2316
+ /* Build rankVal */
2500
2317
  {
2501
2318
  const U32 minBits = tableLog+1 - maxW;
2502
2319
  U32 nextRankVal = 0;
@@ -2866,17 +2683,9 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
2866
2683
  #endif
2867
2684
 
2868
2685
  #ifdef _MSC_VER /* Visual Studio */
2869
- # define FORCE_INLINE static __forceinline
2870
2686
  # include <intrin.h> /* For Visual 2005 */
2871
2687
  # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
2872
2688
  # pragma warning(disable : 4324) /* disable: C4324: padded structure */
2873
- #else
2874
- # define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
2875
- # ifdef __GNUC__
2876
- # define FORCE_INLINE static inline __attribute__((always_inline))
2877
- # else
2878
- # define FORCE_INLINE static inline
2879
- # endif
2880
2689
  #endif
2881
2690
 
2882
2691
 
@@ -2924,6 +2733,8 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
2924
2733
  #define LITERAL_NOENTROPY 63
2925
2734
  #define COMMAND_NOENTROPY 7 /* to remove */
2926
2735
 
2736
+ #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
2737
+
2927
2738
  static const size_t ZSTD_blockHeaderSize = 3;
2928
2739
  static const size_t ZSTD_frameHeaderSize = 4;
2929
2740
 
@@ -2938,7 +2749,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
2938
2749
  #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
2939
2750
 
2940
2751
  /*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
2941
- static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
2752
+ static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
2942
2753
  {
2943
2754
  const BYTE* ip = (const BYTE*)src;
2944
2755
  BYTE* op = (BYTE*)dst;
@@ -2983,98 +2794,6 @@ typedef struct {
2983
2794
  static unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
2984
2795
 
2985
2796
 
2986
- /* *************************************
2987
- * Function body to include
2988
- ***************************************/
2989
- static size_t ZSTD_read_ARCH(const void* p) { size_t r; memcpy(&r, p, sizeof(r)); return r; }
2990
-
2991
- MEM_STATIC unsigned ZSTD_NbCommonBytes (register size_t val)
2992
- {
2993
- if (MEM_isLittleEndian())
2994
- {
2995
- if (MEM_64bits())
2996
- {
2997
- # if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
2998
- unsigned long r = 0;
2999
- _BitScanForward64( &r, (U64)val );
3000
- return (int)(r>>3);
3001
- # elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
3002
- return (__builtin_ctzll((U64)val) >> 3);
3003
- # else
3004
- static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
3005
- return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
3006
- # endif
3007
- }
3008
- else /* 32 bits */
3009
- {
3010
- # if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
3011
- unsigned long r;
3012
- _BitScanForward( &r, (U32)val );
3013
- return (int)(r>>3);
3014
- # elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
3015
- return (__builtin_ctz((U32)val) >> 3);
3016
- # else
3017
- static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
3018
- return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
3019
- # endif
3020
- }
3021
- }
3022
- else /* Big Endian CPU */
3023
- {
3024
- if (MEM_32bits())
3025
- {
3026
- # if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
3027
- unsigned long r = 0;
3028
- _BitScanReverse64( &r, val );
3029
- return (unsigned)(r>>3);
3030
- # elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
3031
- return (__builtin_clzll(val) >> 3);
3032
- # else
3033
- unsigned r;
3034
- const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */
3035
- if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
3036
- if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
3037
- r += (!val);
3038
- return r;
3039
- # endif
3040
- }
3041
- else /* 32 bits */
3042
- {
3043
- # if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
3044
- unsigned long r = 0;
3045
- _BitScanReverse( &r, (unsigned long)val );
3046
- return (unsigned)(r>>3);
3047
- # elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT)
3048
- return (__builtin_clz((U32)val) >> 3);
3049
- # else
3050
- unsigned r;
3051
- if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
3052
- r += (!val);
3053
- return r;
3054
- # endif
3055
- }
3056
- }
3057
- }
3058
-
3059
-
3060
- MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
3061
- {
3062
- const BYTE* const pStart = pIn;
3063
-
3064
- while ((pIn<pInLimit-(sizeof(size_t)-1)))
3065
- {
3066
- size_t diff = ZSTD_read_ARCH(pMatch) ^ ZSTD_read_ARCH(pIn);
3067
- if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
3068
- pIn += ZSTD_NbCommonBytes(diff);
3069
- return (size_t)(pIn - pStart);
3070
- }
3071
-
3072
- if (MEM_32bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
3073
- if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
3074
- if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
3075
- return (size_t)(pIn - pStart);
3076
- }
3077
-
3078
2797
 
3079
2798
  /* *************************************************************
3080
2799
  * Decompression section
@@ -3090,7 +2809,6 @@ struct ZSTD_DCtx_s
3090
2809
  blockType_t bType;
3091
2810
  U32 phase;
3092
2811
  const BYTE* litPtr;
3093
- size_t litBufSize;
3094
2812
  size_t litSize;
3095
2813
  BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
3096
2814
  }; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
@@ -3118,7 +2836,9 @@ static size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockPropertie
3118
2836
  static size_t ZSTD_copyUncompressedBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
3119
2837
  {
3120
2838
  if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
3121
- memcpy(dst, src, srcSize);
2839
+ if (srcSize > 0) {
2840
+ memcpy(dst, src, srcSize);
2841
+ }
3122
2842
  return srcSize;
3123
2843
  }
3124
2844
 
@@ -3162,8 +2882,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
3162
2882
  size_t litSize = BLOCKSIZE;
3163
2883
  const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
3164
2884
  dctx->litPtr = dctx->litBuffer;
3165
- dctx->litBufSize = BLOCKSIZE;
3166
2885
  dctx->litSize = litSize;
2886
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
3167
2887
  return readSize; /* works if it's an error too */
3168
2888
  }
3169
2889
  case IS_RAW:
@@ -3171,16 +2891,16 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
3171
2891
  const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
3172
2892
  if (litSize > srcSize-11) /* risk of reading too far with wildcopy */
3173
2893
  {
3174
- if (litSize > srcSize-3) return ERROR(corruption_detected);
3175
- memcpy(dctx->litBuffer, istart, litSize);
3176
- dctx->litPtr = dctx->litBuffer;
3177
- dctx->litBufSize = BLOCKSIZE;
3178
- dctx->litSize = litSize;
3179
- return litSize+3;
3180
- }
3181
- /* direct reference into compressed stream */
2894
+ if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
2895
+ if (litSize > srcSize-3) return ERROR(corruption_detected);
2896
+ memcpy(dctx->litBuffer, istart, litSize);
2897
+ dctx->litPtr = dctx->litBuffer;
2898
+ dctx->litSize = litSize;
2899
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
2900
+ return litSize+3;
2901
+ }
2902
+ /* direct reference into compressed stream */
3182
2903
  dctx->litPtr = istart+3;
3183
- dctx->litBufSize = srcSize-3;
3184
2904
  dctx->litSize = litSize;
3185
2905
  return litSize+3;
3186
2906
  }
@@ -3188,9 +2908,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
3188
2908
  {
3189
2909
  const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
3190
2910
  if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
3191
- memset(dctx->litBuffer, istart[3], litSize);
2911
+ memset(dctx->litBuffer, istart[3], litSize + 8);
3192
2912
  dctx->litPtr = dctx->litBuffer;
3193
- dctx->litBufSize = BLOCKSIZE;
3194
2913
  dctx->litSize = litSize;
3195
2914
  return 4;
3196
2915
  }
@@ -3244,7 +2963,6 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
3244
2963
  /* Build DTables */
3245
2964
  switch(LLtype)
3246
2965
  {
3247
- U32 max;
3248
2966
  case bt_rle :
3249
2967
  LLlog = 0;
3250
2968
  FSE_buildDTable_rle(DTableLL, *ip++); break;
@@ -3252,17 +2970,16 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
3252
2970
  LLlog = LLbits;
3253
2971
  FSE_buildDTable_raw(DTableLL, LLbits); break;
3254
2972
  default :
3255
- max = MaxLL;
3256
- headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip);
3257
- if (FSE_isError(headerSize)) return ERROR(GENERIC);
3258
- if (LLlog > LLFSELog) return ERROR(corruption_detected);
3259
- ip += headerSize;
3260
- FSE_buildDTable(DTableLL, norm, max, LLlog);
3261
- }
2973
+ { U32 max = MaxLL;
2974
+ headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip);
2975
+ if (FSE_isError(headerSize)) return ERROR(GENERIC);
2976
+ if (LLlog > LLFSELog) return ERROR(corruption_detected);
2977
+ ip += headerSize;
2978
+ FSE_buildDTable(DTableLL, norm, max, LLlog);
2979
+ } }
3262
2980
 
3263
2981
  switch(Offtype)
3264
2982
  {
3265
- U32 max;
3266
2983
  case bt_rle :
3267
2984
  Offlog = 0;
3268
2985
  if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */
@@ -3272,17 +2989,16 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
3272
2989
  Offlog = Offbits;
3273
2990
  FSE_buildDTable_raw(DTableOffb, Offbits); break;
3274
2991
  default :
3275
- max = MaxOff;
3276
- headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip);
3277
- if (FSE_isError(headerSize)) return ERROR(GENERIC);
3278
- if (Offlog > OffFSELog) return ERROR(corruption_detected);
3279
- ip += headerSize;
3280
- FSE_buildDTable(DTableOffb, norm, max, Offlog);
3281
- }
2992
+ { U32 max = MaxOff;
2993
+ headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip);
2994
+ if (FSE_isError(headerSize)) return ERROR(GENERIC);
2995
+ if (Offlog > OffFSELog) return ERROR(corruption_detected);
2996
+ ip += headerSize;
2997
+ FSE_buildDTable(DTableOffb, norm, max, Offlog);
2998
+ } }
3282
2999
 
3283
3000
  switch(MLtype)
3284
3001
  {
3285
- U32 max;
3286
3002
  case bt_rle :
3287
3003
  MLlog = 0;
3288
3004
  if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */
@@ -3291,14 +3007,13 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
3291
3007
  MLlog = MLbits;
3292
3008
  FSE_buildDTable_raw(DTableML, MLbits); break;
3293
3009
  default :
3294
- max = MaxML;
3295
- headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip);
3296
- if (FSE_isError(headerSize)) return ERROR(GENERIC);
3297
- if (MLlog > MLFSELog) return ERROR(corruption_detected);
3298
- ip += headerSize;
3299
- FSE_buildDTable(DTableML, norm, max, MLlog);
3300
- }
3301
- }
3010
+ { U32 max = MaxML;
3011
+ headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip);
3012
+ if (FSE_isError(headerSize)) return ERROR(GENERIC);
3013
+ if (MLlog > MLFSELog) return ERROR(corruption_detected);
3014
+ ip += headerSize;
3015
+ FSE_buildDTable(DTableML, norm, max, MLlog);
3016
+ } } }
3302
3017
 
3303
3018
  return ip-istart;
3304
3019
  }
@@ -3336,11 +3051,11 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
3336
3051
  seqState->prevOffset = seq->offset;
3337
3052
  if (litLength == MaxLL)
3338
3053
  {
3339
- U32 add = *dumps++;
3054
+ const U32 add = dumps<de ? *dumps++ : 0;
3340
3055
  if (add < 255) litLength += add;
3341
- else
3056
+ else if (dumps + 3 <= de)
3342
3057
  {
3343
- litLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
3058
+ litLength = MEM_readLE24(dumps);
3344
3059
  dumps += 3;
3345
3060
  }
3346
3061
  if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
@@ -3366,11 +3081,11 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
3366
3081
  matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
3367
3082
  if (matchLength == MaxML)
3368
3083
  {
3369
- U32 add = *dumps++;
3084
+ const U32 add = dumps<de ? *dumps++ : 0;
3370
3085
  if (add < 255) matchLength += add;
3371
- else
3086
+ else if (dumps + 3 <= de)
3372
3087
  {
3373
- matchLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
3088
+ matchLength = MEM_readLE24(dumps);
3374
3089
  dumps += 3;
3375
3090
  }
3376
3091
  if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
@@ -3391,7 +3106,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3391
3106
  BYTE* const base, BYTE* const oend)
3392
3107
  {
3393
3108
  static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
3394
- static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */
3109
+ static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* subtracted */
3395
3110
  const BYTE* const ostart = op;
3396
3111
  BYTE* const oLitEnd = op + sequence.litLength;
3397
3112
  BYTE* const oMatchEnd = op + sequence.litLength + sequence.matchLength; /* risk : address space overflow (32-bits) */
@@ -3401,7 +3116,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3401
3116
  /* checks */
3402
3117
  if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
3403
3118
  if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
3404
- if (litEnd > litLimit-8) return ERROR(corruption_detected); /* overRead beyond lit buffer */
3119
+ if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
3405
3120
 
3406
3121
  /* copy Literals */
3407
3122
  ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@@ -3435,7 +3150,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3435
3150
  }
3436
3151
  op += 8; match += 8;
3437
3152
 
3438
- if (oMatchEnd > oend-12)
3153
+ if (oMatchEnd > oend-(16-MINMATCH))
3439
3154
  {
3440
3155
  if (op < oend_8)
3441
3156
  {
@@ -3447,7 +3162,7 @@ static size_t ZSTD_execSequence(BYTE* op,
3447
3162
  }
3448
3163
  else
3449
3164
  {
3450
- ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
3165
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
3451
3166
  }
3452
3167
  }
3453
3168
 
@@ -3467,7 +3182,6 @@ static size_t ZSTD_decompressSequences(
3467
3182
  BYTE* const oend = ostart + maxDstSize;
3468
3183
  size_t errorCode, dumpsLength;
3469
3184
  const BYTE* litPtr = dctx->litPtr;
3470
- const BYTE* const litMax = litPtr + dctx->litBufSize;
3471
3185
  const BYTE* const litEnd = litPtr + dctx->litSize;
3472
3186
  int nbSeq;
3473
3187
  const BYTE* dumps;
@@ -3503,7 +3217,7 @@ static size_t ZSTD_decompressSequences(
3503
3217
  size_t oneSeqSize;
3504
3218
  nbSeq--;
3505
3219
  ZSTD_decodeSequence(&sequence, &seqState);
3506
- oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litMax, base, oend);
3220
+ oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
3507
3221
  if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
3508
3222
  op += oneSeqSize;
3509
3223
  }
@@ -3517,8 +3231,10 @@ static size_t ZSTD_decompressSequences(
3517
3231
  size_t lastLLSize = litEnd - litPtr;
3518
3232
  if (litPtr > litEnd) return ERROR(corruption_detected);
3519
3233
  if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
3520
- if (op != litPtr) memmove(op, litPtr, lastLLSize);
3521
- op += lastLLSize;
3234
+ if (lastLLSize > 0) {
3235
+ if (op != litPtr) memmove(op, litPtr, lastLLSize);
3236
+ op += lastLLSize;
3237
+ }
3522
3238
  }
3523
3239
  }
3524
3240
 
@@ -3608,6 +3324,60 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
3608
3324
  return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
3609
3325
  }
3610
3326
 
3327
+ /* ZSTD_errorFrameSizeInfoLegacy() :
3328
+ assumes `cSize` and `dBound` are _not_ NULL */
3329
+ static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
3330
+ {
3331
+ *cSize = ret;
3332
+ *dBound = ZSTD_CONTENTSIZE_ERROR;
3333
+ }
3334
+
3335
+ void ZSTDv02_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
3336
+ {
3337
+ const BYTE* ip = (const BYTE*)src;
3338
+ size_t remainingSize = srcSize;
3339
+ size_t nbBlocks = 0;
3340
+ U32 magicNumber;
3341
+ blockProperties_t blockProperties;
3342
+
3343
+ /* Frame Header */
3344
+ if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) {
3345
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3346
+ return;
3347
+ }
3348
+ magicNumber = MEM_readLE32(src);
3349
+ if (magicNumber != ZSTD_magicNumber) {
3350
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(prefix_unknown));
3351
+ return;
3352
+ }
3353
+ ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
3354
+
3355
+ /* Loop on each block */
3356
+ while (1)
3357
+ {
3358
+ size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
3359
+ if (ZSTD_isError(cBlockSize)) {
3360
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, cBlockSize);
3361
+ return;
3362
+ }
3363
+
3364
+ ip += ZSTD_blockHeaderSize;
3365
+ remainingSize -= ZSTD_blockHeaderSize;
3366
+ if (cBlockSize > remainingSize) {
3367
+ ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, ERROR(srcSize_wrong));
3368
+ return;
3369
+ }
3370
+
3371
+ if (cBlockSize == 0) break; /* bt_end */
3372
+
3373
+ ip += cBlockSize;
3374
+ remainingSize -= cBlockSize;
3375
+ nbBlocks++;
3376
+ }
3377
+
3378
+ *cSize = ip - (const BYTE*)src;
3379
+ *dBound = nbBlocks * BLOCKSIZE;
3380
+ }
3611
3381
 
3612
3382
  /*******************************
3613
3383
  * Streaming Decompression API
@@ -3713,36 +3483,36 @@ static size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSi
3713
3483
 
3714
3484
  unsigned ZSTDv02_isError(size_t code)
3715
3485
  {
3716
- return ZSTD_isError(code);
3486
+ return ZSTD_isError(code);
3717
3487
  }
3718
3488
 
3719
3489
  size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
3720
3490
  const void* src, size_t compressedSize)
3721
3491
  {
3722
- return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
3492
+ return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
3723
3493
  }
3724
3494
 
3725
3495
  ZSTDv02_Dctx* ZSTDv02_createDCtx(void)
3726
3496
  {
3727
- return (ZSTDv02_Dctx*)ZSTD_createDCtx();
3497
+ return (ZSTDv02_Dctx*)ZSTD_createDCtx();
3728
3498
  }
3729
3499
 
3730
3500
  size_t ZSTDv02_freeDCtx(ZSTDv02_Dctx* dctx)
3731
3501
  {
3732
- return ZSTD_freeDCtx((ZSTD_DCtx*)dctx);
3502
+ return ZSTD_freeDCtx((ZSTD_DCtx*)dctx);
3733
3503
  }
3734
3504
 
3735
3505
  size_t ZSTDv02_resetDCtx(ZSTDv02_Dctx* dctx)
3736
3506
  {
3737
- return ZSTD_resetDCtx((ZSTD_DCtx*)dctx);
3507
+ return ZSTD_resetDCtx((ZSTD_DCtx*)dctx);
3738
3508
  }
3739
3509
 
3740
3510
  size_t ZSTDv02_nextSrcSizeToDecompress(ZSTDv02_Dctx* dctx)
3741
3511
  {
3742
- return ZSTD_nextSrcSizeToDecompress((ZSTD_DCtx*)dctx);
3512
+ return ZSTD_nextSrcSizeToDecompress((ZSTD_DCtx*)dctx);
3743
3513
  }
3744
3514
 
3745
3515
  size_t ZSTDv02_decompressContinue(ZSTDv02_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
3746
3516
  {
3747
- return ZSTD_decompressContinue((ZSTD_DCtx*)dctx, dst, maxDstSize, src, srcSize);
3517
+ return ZSTD_decompressContinue((ZSTD_DCtx*)dctx, dst, maxDstSize, src, srcSize);
3748
3518
  }