extzstd 0.0.3.CONCEPT → 0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (65) hide show
  1. checksums.yaml +4 -4
  2. data/HISTORY.ja +5 -0
  3. data/LICENSE +6 -6
  4. data/README.md +35 -22
  5. data/contrib/zstd/LICENSE +13 -9
  6. data/contrib/zstd/README.md +37 -44
  7. data/contrib/zstd/common/entropy_common.c +33 -39
  8. data/contrib/zstd/common/error_private.c +43 -0
  9. data/contrib/zstd/common/error_private.h +11 -60
  10. data/contrib/zstd/common/fse.h +11 -5
  11. data/contrib/zstd/common/fse_decompress.c +14 -16
  12. data/contrib/zstd/common/huf.h +1 -1
  13. data/contrib/zstd/common/mem.h +36 -43
  14. data/contrib/zstd/common/xxhash.c +31 -18
  15. data/contrib/zstd/common/xxhash.h +71 -35
  16. data/contrib/zstd/common/zbuff.h +29 -35
  17. data/contrib/zstd/common/zstd_common.c +24 -32
  18. data/contrib/zstd/common/zstd_errors.h +60 -0
  19. data/contrib/zstd/common/zstd_internal.h +109 -80
  20. data/contrib/zstd/compress/fse_compress.c +9 -6
  21. data/contrib/zstd/compress/huf_compress.c +30 -74
  22. data/contrib/zstd/compress/zbuff_compress.c +43 -51
  23. data/contrib/zstd/compress/zstd_compress.c +953 -763
  24. data/contrib/zstd/compress/zstd_opt.h +115 -261
  25. data/contrib/zstd/decompress/huf_decompress.c +29 -40
  26. data/contrib/zstd/decompress/zbuff_decompress.c +36 -78
  27. data/contrib/zstd/decompress/zstd_decompress.c +976 -496
  28. data/contrib/zstd/dictBuilder/divsufsort.h +5 -5
  29. data/contrib/zstd/dictBuilder/zdict.c +194 -229
  30. data/contrib/zstd/dictBuilder/zdict.h +66 -68
  31. data/contrib/zstd/legacy/zstd_legacy.h +168 -49
  32. data/contrib/zstd/legacy/zstd_v01.c +95 -178
  33. data/contrib/zstd/legacy/zstd_v01.h +12 -32
  34. data/contrib/zstd/legacy/zstd_v02.c +48 -274
  35. data/contrib/zstd/legacy/zstd_v02.h +12 -32
  36. data/contrib/zstd/legacy/zstd_v03.c +48 -274
  37. data/contrib/zstd/legacy/zstd_v03.h +12 -32
  38. data/contrib/zstd/legacy/zstd_v04.c +63 -320
  39. data/contrib/zstd/legacy/zstd_v04.h +13 -33
  40. data/contrib/zstd/legacy/zstd_v05.c +80 -345
  41. data/contrib/zstd/legacy/zstd_v05.h +9 -31
  42. data/contrib/zstd/legacy/zstd_v06.c +48 -458
  43. data/contrib/zstd/legacy/zstd_v06.h +41 -67
  44. data/contrib/zstd/legacy/zstd_v07.c +4544 -0
  45. data/contrib/zstd/legacy/zstd_v07.h +173 -0
  46. data/contrib/zstd/zstd.h +640 -0
  47. data/ext/extconf.rb +7 -3
  48. data/ext/extzstd.c +263 -106
  49. data/ext/extzstd.h +8 -6
  50. data/ext/extzstd_nogvls.h +0 -117
  51. data/ext/extzstd_stream.c +347 -0
  52. data/ext/zstd_common.c +8 -0
  53. data/ext/zstd_compress.c +6 -0
  54. data/ext/zstd_decompress.c +5 -0
  55. data/ext/zstd_dictbuilder.c +5 -0
  56. data/ext/zstd_legacy_v07.c +1 -0
  57. data/gemstub.rb +18 -16
  58. data/lib/extzstd/version.rb +1 -1
  59. data/lib/extzstd.rb +77 -43
  60. data/test/test_basic.rb +11 -6
  61. metadata +23 -10
  62. data/contrib/zstd/common/error_public.h +0 -77
  63. data/contrib/zstd/common/zstd.h +0 -475
  64. data/ext/extzstd_buffered.c +0 -265
  65. data/ext/zstd_amalgam.c +0 -18
@@ -1,42 +1,19 @@
1
- /* ******************************************************************
2
- zstd_v06.c
3
- Decompression module for ZSTD v0.6 legacy format
4
- Copyright (C) 2016, Yann Collet.
5
-
6
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
7
-
8
- Redistribution and use in source and binary forms, with or without
9
- modification, are permitted provided that the following conditions are
10
- met:
11
-
12
- * Redistributions of source code must retain the above copyright
13
- notice, this list of conditions and the following disclaimer.
14
- * Redistributions in binary form must reproduce the above
15
- copyright notice, this list of conditions and the following disclaimer
16
- in the documentation and/or other materials provided with the
17
- distribution.
18
-
19
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
1
+ /**
2
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
3
+ * All rights reserved.
4
+ *
5
+ * This source code is licensed under the BSD-style license found in the
6
+ * LICENSE file in the root directory of this source tree. An additional grant
7
+ * of patent rights can be found in the PATENTS file in the same directory.
8
+ */
30
9
 
31
- You can contact the author at :
32
- - Homepage : http://www.zstd.net/
33
- ****************************************************************** */
34
10
 
35
11
  /*- Dependencies -*/
36
12
  #include "zstd_v06.h"
37
13
  #include <stddef.h> /* size_t, ptrdiff_t */
38
14
  #include <string.h> /* memcpy */
39
15
  #include <stdlib.h> /* malloc, free, qsort */
16
+ #include "error_private.h"
40
17
 
41
18
 
42
19
 
@@ -85,6 +62,10 @@ extern "C" {
85
62
  /*-****************************************
86
63
  * Compiler specifics
87
64
  ******************************************/
65
+ #if defined(_MSC_VER) /* Visual Studio */
66
+ # include <stdlib.h> /* _byteswap_ulong */
67
+ # include <intrin.h> /* _byteswap_* */
68
+ #endif
88
69
  #if defined(__GNUC__)
89
70
  # define MEM_STATIC static __attribute__((unused))
90
71
  #elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
@@ -138,7 +119,7 @@ extern "C" {
138
119
  #ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
139
120
  # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
140
121
  # define MEM_FORCE_MEMORY_ACCESS 2
141
- # elif defined(__INTEL_COMPILER) || \
122
+ # elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
142
123
  (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
143
124
  # define MEM_FORCE_MEMORY_ACCESS 1
144
125
  # endif
@@ -160,11 +141,8 @@ Only use if no other choice to achieve best performance on target platform */
160
141
  MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
161
142
  MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
162
143
  MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
163
- MEM_STATIC U64 MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; }
164
144
 
165
145
  MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
166
- MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
167
- MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
168
146
 
169
147
  #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
170
148
 
@@ -175,11 +153,8 @@ typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed))
175
153
  MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
176
154
  MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
177
155
  MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
178
- MEM_STATIC U64 MEM_readST(const void* ptr) { return ((const unalign*)ptr)->st; }
179
156
 
180
157
  MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
181
- MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
182
- MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
183
158
 
184
159
  #else
185
160
 
@@ -201,25 +176,11 @@ MEM_STATIC U64 MEM_read64(const void* memPtr)
201
176
  U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
202
177
  }
203
178
 
204
- MEM_STATIC size_t MEM_readST(const void* memPtr)
205
- {
206
- size_t val; memcpy(&val, memPtr, sizeof(val)); return val;
207
- }
208
-
209
179
  MEM_STATIC void MEM_write16(void* memPtr, U16 value)
210
180
  {
211
181
  memcpy(memPtr, &value, sizeof(value));
212
182
  }
213
183
 
214
- MEM_STATIC void MEM_write32(void* memPtr, U32 value)
215
- {
216
- memcpy(memPtr, &value, sizeof(value));
217
- }
218
-
219
- MEM_STATIC void MEM_write64(void* memPtr, U64 value)
220
- {
221
- memcpy(memPtr, &value, sizeof(value));
222
- }
223
184
 
224
185
  #endif /* MEM_FORCE_MEMORY_ACCESS */
225
186
 
@@ -255,13 +216,6 @@ MEM_STATIC U64 MEM_swap64(U64 in)
255
216
  #endif
256
217
  }
257
218
 
258
- MEM_STATIC size_t MEM_swapST(size_t in)
259
- {
260
- if (MEM_32bits())
261
- return (size_t)MEM_swap32((U32)in);
262
- else
263
- return (size_t)MEM_swap64((U64)in);
264
- }
265
219
 
266
220
  /*=== Little endian r/w ===*/
267
221
 
@@ -294,13 +248,6 @@ MEM_STATIC U32 MEM_readLE32(const void* memPtr)
294
248
  return MEM_swap32(MEM_read32(memPtr));
295
249
  }
296
250
 
297
- MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
298
- {
299
- if (MEM_isLittleEndian())
300
- MEM_write32(memPtr, val32);
301
- else
302
- MEM_write32(memPtr, MEM_swap32(val32));
303
- }
304
251
 
305
252
  MEM_STATIC U64 MEM_readLE64(const void* memPtr)
306
253
  {
@@ -310,13 +257,6 @@ MEM_STATIC U64 MEM_readLE64(const void* memPtr)
310
257
  return MEM_swap64(MEM_read64(memPtr));
311
258
  }
312
259
 
313
- MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
314
- {
315
- if (MEM_isLittleEndian())
316
- MEM_write64(memPtr, val64);
317
- else
318
- MEM_write64(memPtr, MEM_swap64(val64));
319
- }
320
260
 
321
261
  MEM_STATIC size_t MEM_readLEST(const void* memPtr)
322
262
  {
@@ -326,156 +266,14 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
326
266
  return (size_t)MEM_readLE64(memPtr);
327
267
  }
328
268
 
329
- MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
330
- {
331
- if (MEM_32bits())
332
- MEM_writeLE32(memPtr, (U32)val);
333
- else
334
- MEM_writeLE64(memPtr, (U64)val);
335
- }
336
-
337
- /*=== Big endian r/w ===*/
338
-
339
- MEM_STATIC U32 MEM_readBE32(const void* memPtr)
340
- {
341
- if (MEM_isLittleEndian())
342
- return MEM_swap32(MEM_read32(memPtr));
343
- else
344
- return MEM_read32(memPtr);
345
- }
346
-
347
- MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32)
348
- {
349
- if (MEM_isLittleEndian())
350
- MEM_write32(memPtr, MEM_swap32(val32));
351
- else
352
- MEM_write32(memPtr, val32);
353
- }
354
-
355
- MEM_STATIC U64 MEM_readBE64(const void* memPtr)
356
- {
357
- if (MEM_isLittleEndian())
358
- return MEM_swap64(MEM_read64(memPtr));
359
- else
360
- return MEM_read64(memPtr);
361
- }
362
-
363
- MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64)
364
- {
365
- if (MEM_isLittleEndian())
366
- MEM_write64(memPtr, MEM_swap64(val64));
367
- else
368
- MEM_write64(memPtr, val64);
369
- }
370
-
371
- MEM_STATIC size_t MEM_readBEST(const void* memPtr)
372
- {
373
- if (MEM_32bits())
374
- return (size_t)MEM_readBE32(memPtr);
375
- else
376
- return (size_t)MEM_readBE64(memPtr);
377
- }
378
-
379
- MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
380
- {
381
- if (MEM_32bits())
382
- MEM_writeBE32(memPtr, (U32)val);
383
- else
384
- MEM_writeBE64(memPtr, (U64)val);
385
- }
386
269
 
387
270
 
388
- /* function safe only for comparisons */
389
- MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length)
390
- {
391
- switch (length)
392
- {
393
- default :
394
- case 4 : return MEM_read32(memPtr);
395
- case 3 : if (MEM_isLittleEndian())
396
- return MEM_read32(memPtr)<<8;
397
- else
398
- return MEM_read32(memPtr)>>8;
399
- }
400
- }
401
-
402
271
  #if defined (__cplusplus)
403
272
  }
404
273
  #endif
405
274
 
406
275
  #endif /* MEM_H_MODULE */
407
276
 
408
- /* ******************************************************************
409
- Error codes list
410
- Copyright (C) 2016, Yann Collet
411
-
412
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
413
-
414
- Redistribution and use in source and binary forms, with or without
415
- modification, are permitted provided that the following conditions are
416
- met:
417
-
418
- * Redistributions of source code must retain the above copyright
419
- notice, this list of conditions and the following disclaimer.
420
- * Redistributions in binary form must reproduce the above
421
- copyright notice, this list of conditions and the following disclaimer
422
- in the documentation and/or other materials provided with the
423
- distribution.
424
-
425
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
426
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
427
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
428
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
429
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
430
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
431
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
432
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
433
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
434
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
435
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
436
-
437
- You can contact the author at :
438
- - Homepage : http://www.zstd.net
439
- ****************************************************************** */
440
- #ifndef ERROR_PUBLIC_H_MODULE
441
- #define ERROR_PUBLIC_H_MODULE
442
-
443
- #if defined (__cplusplus)
444
- extern "C" {
445
- #endif
446
-
447
-
448
- /* ****************************************
449
- * error codes list
450
- ******************************************/
451
- typedef enum {
452
- ZSTDv06_error_no_error,
453
- ZSTDv06_error_GENERIC,
454
- ZSTDv06_error_prefix_unknown,
455
- ZSTDv06_error_frameParameter_unsupported,
456
- ZSTDv06_error_frameParameter_unsupportedBy32bits,
457
- ZSTDv06_error_compressionParameter_unsupported,
458
- ZSTDv06_error_init_missing,
459
- ZSTDv06_error_memory_allocation,
460
- ZSTDv06_error_stage_wrong,
461
- ZSTDv06_error_dstSize_tooSmall,
462
- ZSTDv06_error_srcSize_wrong,
463
- ZSTDv06_error_corruption_detected,
464
- ZSTDv06_error_tableLog_tooLarge,
465
- ZSTDv06_error_maxSymbolValue_tooLarge,
466
- ZSTDv06_error_maxSymbolValue_tooSmall,
467
- ZSTDv06_error_dictionary_corrupted,
468
- ZSTDv06_error_maxCode
469
- } ZSTDv06_ErrorCode;
470
-
471
- /* note : compare with size_t function results using ZSTDv06_getError() */
472
-
473
-
474
- #if defined (__cplusplus)
475
- }
476
- #endif
477
-
478
- #endif /* ERROR_PUBLIC_H_MODULE */
479
277
  /*
480
278
  zstd - standard compression library
481
279
  Header File for static linking only
@@ -528,7 +326,7 @@ extern "C" {
528
326
  * It avoids reloading the dictionary each time.
529
327
  * `preparedDCtx` must have been properly initialized using ZSTDv06_decompressBegin_usingDict().
530
328
  * Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */
531
- ZSTDLIB_API size_t ZSTDv06_decompress_usingPreparedDCtx(
329
+ ZSTDLIBv06_API size_t ZSTDv06_decompress_usingPreparedDCtx(
532
330
  ZSTDv06_DCtx* dctx, const ZSTDv06_DCtx* preparedDCtx,
533
331
  void* dst, size_t dstCapacity,
534
332
  const void* src, size_t srcSize);
@@ -539,7 +337,7 @@ ZSTDLIB_API size_t ZSTDv06_decompress_usingPreparedDCtx(
539
337
  static const size_t ZSTDv06_frameHeaderSize_min = 5;
540
338
  static const size_t ZSTDv06_frameHeaderSize_max = ZSTDv06_FRAMEHEADERSIZE_MAX;
541
339
 
542
- ZSTDLIB_API size_t ZSTDv06_decompressBegin(ZSTDv06_DCtx* dctx);
340
+ ZSTDLIBv06_API size_t ZSTDv06_decompressBegin(ZSTDv06_DCtx* dctx);
543
341
 
544
342
  /*
545
343
  Streaming decompression, direct mode (bufferless)
@@ -598,17 +396,8 @@ ZSTDLIB_API size_t ZSTDv06_decompressBegin(ZSTDv06_DCtx* dctx);
598
396
  */
599
397
 
600
398
  #define ZSTDv06_BLOCKSIZE_MAX (128 * 1024) /* define, for static allocation */
601
- ZSTDLIB_API size_t ZSTDv06_decompressBlock(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
602
-
399
+ ZSTDLIBv06_API size_t ZSTDv06_decompressBlock(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
603
400
 
604
- /*-*************************************
605
- * Error management
606
- ***************************************/
607
- /*! ZSTDv06_getErrorCode() :
608
- convert a `size_t` function result into a `ZSTDv06_ErrorCode` enum type,
609
- which can be used to compare directly with enum list published into "error_public.h" */
610
- ZSTDLIB_API ZSTDv06_ErrorCode ZSTDv06_getErrorCode(size_t functionResult);
611
- ZSTDLIB_API const char* ZSTDv06_getErrorString(ZSTDv06_ErrorCode code);
612
401
 
613
402
 
614
403
  #if defined (__cplusplus)
@@ -616,122 +405,6 @@ ZSTDLIB_API const char* ZSTDv06_getErrorString(ZSTDv06_ErrorCode code);
616
405
  #endif
617
406
 
618
407
  #endif /* ZSTDv06_STATIC_H */
619
- /* ******************************************************************
620
- Error codes and messages
621
- Copyright (C) 2013-2016, Yann Collet
622
-
623
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
624
-
625
- Redistribution and use in source and binary forms, with or without
626
- modification, are permitted provided that the following conditions are
627
- met:
628
-
629
- * Redistributions of source code must retain the above copyright
630
- notice, this list of conditions and the following disclaimer.
631
- * Redistributions in binary form must reproduce the above
632
- copyright notice, this list of conditions and the following disclaimer
633
- in the documentation and/or other materials provided with the
634
- distribution.
635
-
636
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
637
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
638
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
639
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
640
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
641
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
642
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
643
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
644
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
645
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
646
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
647
-
648
- You can contact the author at :
649
- - Homepage : http://www.zstd.net
650
- ****************************************************************** */
651
- /* Note : this module is expected to remain private, do not expose it */
652
-
653
- #ifndef ERROR_H_MODULE
654
- #define ERROR_H_MODULE
655
-
656
- #if defined (__cplusplus)
657
- extern "C" {
658
- #endif
659
-
660
-
661
- /* ****************************************
662
- * Compiler-specific
663
- ******************************************/
664
- #if defined(__GNUC__)
665
- # define ERR_STATIC static __attribute__((unused))
666
- #elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
667
- # define ERR_STATIC static inline
668
- #elif defined(_MSC_VER)
669
- # define ERR_STATIC static __inline
670
- #else
671
- # define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
672
- #endif
673
-
674
-
675
- /*-****************************************
676
- * Customization (error_public.h)
677
- ******************************************/
678
- typedef ZSTDv06_ErrorCode ERR_enum;
679
- #define PREFIX(name) ZSTDv06_error_##name
680
-
681
-
682
- /*-****************************************
683
- * Error codes handling
684
- ******************************************/
685
- #ifdef ERROR
686
- # undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */
687
- #endif
688
- #define ERROR(name) ((size_t)-PREFIX(name))
689
-
690
- ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
691
-
692
- ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); }
693
-
694
-
695
- /*-****************************************
696
- * Error Strings
697
- ******************************************/
698
-
699
- ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
700
- {
701
- static const char* notErrorCode = "Unspecified error code";
702
- switch( code )
703
- {
704
- case PREFIX(no_error): return "No error detected";
705
- case PREFIX(GENERIC): return "Error (generic)";
706
- case PREFIX(prefix_unknown): return "Unknown frame descriptor";
707
- case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
708
- case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
709
- case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound";
710
- case PREFIX(init_missing): return "Context should be init first";
711
- case PREFIX(memory_allocation): return "Allocation error : not enough memory";
712
- case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
713
- case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
714
- case PREFIX(srcSize_wrong): return "Src size incorrect";
715
- case PREFIX(corruption_detected): return "Corrupted block detected";
716
- case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
717
- case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
718
- case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
719
- case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
720
- case PREFIX(maxCode):
721
- default: return notErrorCode;
722
- }
723
- }
724
-
725
- ERR_STATIC const char* ERR_getErrorName(size_t code)
726
- {
727
- return ERR_getErrorString(ERR_getErrorCode(code));
728
- }
729
-
730
- #if defined (__cplusplus)
731
- }
732
- #endif
733
-
734
- #endif /* ERROR_H_MODULE */
735
408
  /*
736
409
  zstd_internal - common functions to include
737
410
  Header File for include
@@ -777,19 +450,6 @@ ERR_STATIC const char* ERR_getErrorName(size_t code)
777
450
  /*-*************************************
778
451
  * Common constants
779
452
  ***************************************/
780
- #define ZSTDv06_OPT_DEBUG 0 // 3 = compression stats; 5 = check encoded sequences; 9 = full logs
781
- #include <stdio.h>
782
- #if defined(ZSTDv06_OPT_DEBUG) && ZSTDv06_OPT_DEBUG>=9
783
- #define ZSTDv06_LOG_PARSER(...) printf(__VA_ARGS__)
784
- #define ZSTDv06_LOG_ENCODE(...) printf(__VA_ARGS__)
785
- #define ZSTDv06_LOG_BLOCK(...) printf(__VA_ARGS__)
786
- #else
787
- #define ZSTDv06_LOG_PARSER(...)
788
- #define ZSTDv06_LOG_ENCODE(...)
789
- #define ZSTDv06_LOG_BLOCK(...)
790
- #endif
791
-
792
- #define ZSTDv06_OPT_NUM (1<<12)
793
453
  #define ZSTDv06_DICT_MAGIC 0xEC30A436
794
454
 
795
455
  #define ZSTDv06_REP_NUM 3
@@ -887,27 +547,6 @@ MEM_STATIC void ZSTDv06_wildcopy(void* dst, const void* src, size_t length)
887
547
  while (op < oend);
888
548
  }
889
549
 
890
- MEM_STATIC unsigned ZSTDv06_highbit(U32 val)
891
- {
892
- # if defined(_MSC_VER) /* Visual */
893
- unsigned long r=0;
894
- _BitScanReverse(&r, val);
895
- return (unsigned)r;
896
- # elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
897
- return 31 - __builtin_clz(val);
898
- # else /* Software version */
899
- static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
900
- U32 v = val;
901
- int r;
902
- v |= v >> 1;
903
- v |= v >> 2;
904
- v |= v >> 4;
905
- v |= v >> 8;
906
- v |= v >> 16;
907
- r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
908
- return r;
909
- # endif
910
- }
911
550
 
912
551
 
913
552
  /*-*******************************************
@@ -926,15 +565,7 @@ typedef struct {
926
565
  U32 rep[ZSTDv06_REP_INIT];
927
566
  } ZSTDv06_optimal_t;
928
567
 
929
- #if ZSTDv06_OPT_DEBUG == 3
930
- #include ".debug/zstd_stats.h"
931
- #else
932
- typedef struct { U32 unused; } ZSTDv06_stats_t;
933
- MEM_STATIC void ZSTDv06_statsPrint(ZSTDv06_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; }
934
- MEM_STATIC void ZSTDv06_statsInit(ZSTDv06_stats_t* stats) { (void)stats; }
935
- MEM_STATIC void ZSTDv06_statsResetFreqs(ZSTDv06_stats_t* stats) { (void)stats; }
936
- MEM_STATIC void ZSTDv06_statsUpdatePrices(ZSTDv06_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; }
937
- #endif
568
+ typedef struct { U32 unused; } ZSTDv06_stats_t;
938
569
 
939
570
  typedef struct {
940
571
  void* buffer;
@@ -1251,9 +882,6 @@ MEM_STATIC unsigned BITv06_highbit32 (register U32 val)
1251
882
  # endif
1252
883
  }
1253
884
 
1254
- /*===== Local Constants =====*/
1255
- static const unsigned BITv06_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */
1256
-
1257
885
 
1258
886
 
1259
887
  /*-********************************************************
@@ -1299,29 +927,6 @@ MEM_STATIC size_t BITv06_initDStream(BITv06_DStream_t* bitD, const void* srcBuff
1299
927
  return srcSize;
1300
928
  }
1301
929
 
1302
- MEM_STATIC size_t BITv06_getUpperBits(size_t bitContainer, U32 const start)
1303
- {
1304
- return bitContainer >> start;
1305
- }
1306
-
1307
- MEM_STATIC size_t BITv06_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
1308
- {
1309
- #if defined(__BMI__) && defined(__GNUC__) /* experimental */
1310
- # if defined(__x86_64__)
1311
- if (sizeof(bitContainer)==8)
1312
- return _bextr_u64(bitContainer, start, nbBits);
1313
- else
1314
- # endif
1315
- return _bextr_u32(bitContainer, start, nbBits);
1316
- #else
1317
- return (bitContainer >> start) & BITv06_mask[nbBits];
1318
- #endif
1319
- }
1320
-
1321
- MEM_STATIC size_t BITv06_getLowerBits(size_t bitContainer, U32 const nbBits)
1322
- {
1323
- return bitContainer & BITv06_mask[nbBits];
1324
- }
1325
930
 
1326
931
  /*! BITv06_lookBits() :
1327
932
  * Provides next n bits from local register.
@@ -1332,12 +937,8 @@ MEM_STATIC size_t BITv06_getLowerBits(size_t bitContainer, U32 const nbBits)
1332
937
  */
1333
938
  MEM_STATIC size_t BITv06_lookBits(const BITv06_DStream_t* bitD, U32 nbBits)
1334
939
  {
1335
- #if defined(__BMI__) && defined(__GNUC__) /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */
1336
- return BITv06_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
1337
- #else
1338
940
  U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
1339
941
  return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask);
1340
- #endif
1341
942
  }
1342
943
 
1343
944
  /*! BITv06_lookBitsFast() :
@@ -1501,8 +1102,6 @@ static void FSEv06_initDState(FSEv06_DState_t* DStatePtr, BITv06_DStream_t*
1501
1102
 
1502
1103
  static unsigned char FSEv06_decodeSymbol(FSEv06_DState_t* DStatePtr, BITv06_DStream_t* bitD);
1503
1104
 
1504
- static unsigned FSEv06_endOfDState(const FSEv06_DState_t* DStatePtr);
1505
-
1506
1105
  /*!
1507
1106
  Let's now decompose FSEv06_decompress_usingDTable() into its unitary components.
1508
1107
  You will decode FSE-encoded symbols from the bitStream,
@@ -1565,7 +1164,7 @@ static unsigned char FSEv06_decodeSymbolFast(FSEv06_DState_t* DStatePtr, BITv06_
1565
1164
  *******************************************/
1566
1165
 
1567
1166
 
1568
- /*<===== Decompression =====>*/
1167
+ /* ====== Decompression ====== */
1569
1168
 
1570
1169
  typedef struct {
1571
1170
  U16 tableLog;
@@ -1626,11 +1225,6 @@ MEM_STATIC BYTE FSEv06_decodeSymbolFast(FSEv06_DState_t* DStatePtr, BITv06_DStre
1626
1225
  return symbol;
1627
1226
  }
1628
1227
 
1629
- MEM_STATIC unsigned FSEv06_endOfDState(const FSEv06_DState_t* DStatePtr)
1630
- {
1631
- return DStatePtr->state == 0;
1632
- }
1633
-
1634
1228
 
1635
1229
 
1636
1230
  #ifndef FSEv06_COMMONDEFS_ONLY
@@ -1876,12 +1470,15 @@ size_t FSEv06_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned
1876
1470
  # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
1877
1471
  # pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
1878
1472
  #else
1879
- # ifdef __GNUC__
1880
- # define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
1881
- # define FORCE_INLINE static inline __attribute__((always_inline))
1473
+ # if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
1474
+ # ifdef __GNUC__
1475
+ # define FORCE_INLINE static inline __attribute__((always_inline))
1476
+ # else
1477
+ # define FORCE_INLINE static inline
1478
+ # endif
1882
1479
  # else
1883
- # define FORCE_INLINE static inline
1884
- # endif
1480
+ # define FORCE_INLINE static
1481
+ # endif /* __STDC_VERSION__ */
1885
1482
  #endif
1886
1483
 
1887
1484
 
@@ -2335,9 +1932,11 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta
2335
1932
  {
2336
1933
  U32 weightTotal;
2337
1934
  const BYTE* ip = (const BYTE*) src;
2338
- size_t iSize = ip[0];
1935
+ size_t iSize;
2339
1936
  size_t oSize;
2340
1937
 
1938
+ if (!srcSize) return ERROR(srcSize_wrong);
1939
+ iSize = ip[0];
2341
1940
  //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
2342
1941
 
2343
1942
  if (iSize >= 128) { /* special header */
@@ -2372,6 +1971,7 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta
2372
1971
  rankStats[huffWeight[n]]++;
2373
1972
  weightTotal += (1 << huffWeight[n]) >> 1;
2374
1973
  } }
1974
+ if (weightTotal == 0) return ERROR(corruption_detected);
2375
1975
 
2376
1976
  /* get last non-null symbol weight (implied, total must be 2^n) */
2377
1977
  { U32 const tableLog = BITv06_highbit32(weightTotal) + 1;
@@ -2449,14 +2049,7 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta
2449
2049
 
2450
2050
 
2451
2051
  #ifdef _MSC_VER /* Visual Studio */
2452
- # define FORCE_INLINE static __forceinline
2453
2052
  # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
2454
- #else
2455
- # ifdef __GNUC__
2456
- # define FORCE_INLINE static inline __attribute__((always_inline))
2457
- # else
2458
- # define FORCE_INLINE static inline
2459
- # endif
2460
2053
  #endif
2461
2054
 
2462
2055
 
@@ -3201,14 +2794,6 @@ unsigned ZSTDv06_isError(size_t code) { return ERR_isError(code); }
3201
2794
  * provides error code string from function result (useful for debugging) */
3202
2795
  const char* ZSTDv06_getErrorName(size_t code) { return ERR_getErrorName(code); }
3203
2796
 
3204
- /*! ZSTDv06_getError() :
3205
- * convert a `size_t` function result into a proper ZSTDv06_errorCode enum */
3206
- ZSTDv06_ErrorCode ZSTDv06_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
3207
-
3208
- /*! ZSTDv06_getErrorString() :
3209
- * provides error code string from enum */
3210
- const char* ZSTDv06_getErrorString(ZSTDv06_ErrorCode code) { return ERR_getErrorName(code); }
3211
-
3212
2797
 
3213
2798
  /* **************************************************************
3214
2799
  * ZBUFF Error Management
@@ -3265,16 +2850,9 @@ const char* ZBUFFv06_getErrorName(size_t errorCode) { return ERR_getErrorName(er
3265
2850
  * Compiler specifics
3266
2851
  *********************************************************/
3267
2852
  #ifdef _MSC_VER /* Visual Studio */
3268
- # define FORCE_INLINE static __forceinline
3269
2853
  # include <intrin.h> /* For Visual 2005 */
3270
2854
  # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
3271
2855
  # pragma warning(disable : 4324) /* disable: C4324: padded structure */
3272
- #else
3273
- # ifdef __GNUC__
3274
- # define FORCE_INLINE static inline __attribute__((always_inline))
3275
- # else
3276
- # define FORCE_INLINE static inline
3277
- # endif
3278
2856
  #endif
3279
2857
 
3280
2858
 
@@ -3608,6 +3186,7 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
3608
3186
  lhSize=3;
3609
3187
  litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2);
3610
3188
  litCSize = ((istart[1] & 3) << 8) + istart[2];
3189
+ if (litCSize + litSize > srcSize) return ERROR(corruption_detected);
3611
3190
 
3612
3191
  { size_t const errorCode = HUFv06_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTableX4);
3613
3192
  if (HUFv06_isError(errorCode)) return ERROR(corruption_detected);
@@ -3727,10 +3306,13 @@ size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
3727
3306
  { int nbSeq = *ip++;
3728
3307
  if (!nbSeq) { *nbSeqPtr=0; return 1; }
3729
3308
  if (nbSeq > 0x7F) {
3730
- if (nbSeq == 0xFF)
3309
+ if (nbSeq == 0xFF) {
3310
+ if (ip+2 > iend) return ERROR(srcSize_wrong);
3731
3311
  nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2;
3732
- else
3312
+ } else {
3313
+ if (ip >= iend) return ERROR(srcSize_wrong);
3733
3314
  nbSeq = ((nbSeq-0x80)<<8) + *ip++;
3315
+ }
3734
3316
  }
3735
3317
  *nbSeqPtr = nbSeq;
3736
3318
  }
@@ -3891,7 +3473,12 @@ size_t ZSTDv06_execSequence(BYTE* op,
3891
3473
  op = oLitEnd + length1;
3892
3474
  sequence.matchLength -= length1;
3893
3475
  match = base;
3476
+ if (op > oend_8) {
3477
+ while (op < oMatchEnd) *op++ = *match++;
3478
+ return sequenceLength;
3479
+ }
3894
3480
  } }
3481
+ /* Requirement: op <= oend_8 */
3895
3482
 
3896
3483
  /* match within prefix */
3897
3484
  if (sequence.offset < 8) {
@@ -4247,9 +3834,10 @@ static size_t ZSTDv06_loadEntropy(ZSTDv06_DCtx* dctx, const void* dict, size_t d
4247
3834
  dictSize -= hSize;
4248
3835
 
4249
3836
  { short offcodeNCount[MaxOff+1];
4250
- U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog;
3837
+ U32 offcodeMaxValue=MaxOff, offcodeLog;
4251
3838
  offcodeHeaderSize = FSEv06_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
4252
3839
  if (FSEv06_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
3840
+ if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
4253
3841
  { size_t const errorCode = FSEv06_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog);
4254
3842
  if (FSEv06_isError(errorCode)) return ERROR(dictionary_corrupted); }
4255
3843
  dict = (const char*)dict + offcodeHeaderSize;
@@ -4257,9 +3845,10 @@ static size_t ZSTDv06_loadEntropy(ZSTDv06_DCtx* dctx, const void* dict, size_t d
4257
3845
  }
4258
3846
 
4259
3847
  { short matchlengthNCount[MaxML+1];
4260
- unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
3848
+ unsigned matchlengthMaxValue = MaxML, matchlengthLog;
4261
3849
  matchlengthHeaderSize = FSEv06_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
4262
3850
  if (FSEv06_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
3851
+ if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted);
4263
3852
  { size_t const errorCode = FSEv06_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
4264
3853
  if (FSEv06_isError(errorCode)) return ERROR(dictionary_corrupted); }
4265
3854
  dict = (const char*)dict + matchlengthHeaderSize;
@@ -4267,9 +3856,10 @@ static size_t ZSTDv06_loadEntropy(ZSTDv06_DCtx* dctx, const void* dict, size_t d
4267
3856
  }
4268
3857
 
4269
3858
  { short litlengthNCount[MaxLL+1];
4270
- unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
3859
+ unsigned litlengthMaxValue = MaxLL, litlengthLog;
4271
3860
  litlengthHeaderSize = FSEv06_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
4272
3861
  if (FSEv06_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
3862
+ if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted);
4273
3863
  { size_t const errorCode = FSEv06_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog);
4274
3864
  if (FSEv06_isError(errorCode)) return ERROR(dictionary_corrupted); }
4275
3865
  }