extzstd 0.3.1 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +28 -14
  3. data/contrib/zstd/CHANGELOG +301 -56
  4. data/contrib/zstd/CONTRIBUTING.md +169 -72
  5. data/contrib/zstd/LICENSE +4 -4
  6. data/contrib/zstd/Makefile +116 -87
  7. data/contrib/zstd/Package.swift +36 -0
  8. data/contrib/zstd/README.md +62 -32
  9. data/contrib/zstd/TESTING.md +2 -3
  10. data/contrib/zstd/appveyor.yml +52 -136
  11. data/contrib/zstd/lib/BUCK +5 -7
  12. data/contrib/zstd/lib/Makefile +225 -222
  13. data/contrib/zstd/lib/README.md +51 -6
  14. data/contrib/zstd/lib/common/allocations.h +55 -0
  15. data/contrib/zstd/lib/common/bits.h +200 -0
  16. data/contrib/zstd/lib/common/bitstream.h +45 -62
  17. data/contrib/zstd/lib/common/compiler.h +205 -22
  18. data/contrib/zstd/lib/common/cpu.h +1 -3
  19. data/contrib/zstd/lib/common/debug.c +1 -1
  20. data/contrib/zstd/lib/common/debug.h +12 -19
  21. data/contrib/zstd/lib/common/entropy_common.c +172 -48
  22. data/contrib/zstd/lib/common/error_private.c +10 -2
  23. data/contrib/zstd/lib/common/error_private.h +82 -3
  24. data/contrib/zstd/lib/common/fse.h +37 -86
  25. data/contrib/zstd/lib/common/fse_decompress.c +117 -92
  26. data/contrib/zstd/lib/common/huf.h +99 -166
  27. data/contrib/zstd/lib/common/mem.h +124 -142
  28. data/contrib/zstd/lib/common/pool.c +54 -27
  29. data/contrib/zstd/lib/common/pool.h +10 -4
  30. data/contrib/zstd/lib/common/portability_macros.h +156 -0
  31. data/contrib/zstd/lib/common/threading.c +74 -19
  32. data/contrib/zstd/lib/common/threading.h +5 -10
  33. data/contrib/zstd/lib/common/xxhash.c +7 -847
  34. data/contrib/zstd/lib/common/xxhash.h +5568 -167
  35. data/contrib/zstd/lib/common/zstd_common.c +2 -37
  36. data/contrib/zstd/lib/common/zstd_deps.h +111 -0
  37. data/contrib/zstd/lib/common/zstd_internal.h +132 -187
  38. data/contrib/zstd/lib/common/zstd_trace.h +163 -0
  39. data/contrib/zstd/lib/compress/clevels.h +134 -0
  40. data/contrib/zstd/lib/compress/fse_compress.c +83 -157
  41. data/contrib/zstd/lib/compress/hist.c +27 -29
  42. data/contrib/zstd/lib/compress/hist.h +2 -2
  43. data/contrib/zstd/lib/compress/huf_compress.c +916 -279
  44. data/contrib/zstd/lib/compress/zstd_compress.c +3773 -1019
  45. data/contrib/zstd/lib/compress/zstd_compress_internal.h +610 -203
  46. data/contrib/zstd/lib/compress/zstd_compress_literals.c +119 -42
  47. data/contrib/zstd/lib/compress/zstd_compress_literals.h +16 -6
  48. data/contrib/zstd/lib/compress/zstd_compress_sequences.c +42 -19
  49. data/contrib/zstd/lib/compress/zstd_compress_sequences.h +1 -1
  50. data/contrib/zstd/lib/compress/zstd_compress_superblock.c +49 -317
  51. data/contrib/zstd/lib/compress/zstd_compress_superblock.h +1 -1
  52. data/contrib/zstd/lib/compress/zstd_cwksp.h +320 -103
  53. data/contrib/zstd/lib/compress/zstd_double_fast.c +388 -151
  54. data/contrib/zstd/lib/compress/zstd_double_fast.h +3 -2
  55. data/contrib/zstd/lib/compress/zstd_fast.c +729 -265
  56. data/contrib/zstd/lib/compress/zstd_fast.h +3 -2
  57. data/contrib/zstd/lib/compress/zstd_lazy.c +1270 -251
  58. data/contrib/zstd/lib/compress/zstd_lazy.h +61 -1
  59. data/contrib/zstd/lib/compress/zstd_ldm.c +324 -219
  60. data/contrib/zstd/lib/compress/zstd_ldm.h +9 -2
  61. data/contrib/zstd/lib/compress/zstd_ldm_geartab.h +106 -0
  62. data/contrib/zstd/lib/compress/zstd_opt.c +481 -209
  63. data/contrib/zstd/lib/compress/zstd_opt.h +1 -1
  64. data/contrib/zstd/lib/compress/zstdmt_compress.c +181 -457
  65. data/contrib/zstd/lib/compress/zstdmt_compress.h +34 -113
  66. data/contrib/zstd/lib/decompress/huf_decompress.c +1199 -565
  67. data/contrib/zstd/lib/decompress/huf_decompress_amd64.S +576 -0
  68. data/contrib/zstd/lib/decompress/zstd_ddict.c +12 -12
  69. data/contrib/zstd/lib/decompress/zstd_ddict.h +2 -2
  70. data/contrib/zstd/lib/decompress/zstd_decompress.c +627 -157
  71. data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1086 -326
  72. data/contrib/zstd/lib/decompress/zstd_decompress_block.h +19 -5
  73. data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +62 -13
  74. data/contrib/zstd/lib/deprecated/zbuff.h +1 -1
  75. data/contrib/zstd/lib/deprecated/zbuff_common.c +1 -1
  76. data/contrib/zstd/lib/deprecated/zbuff_compress.c +24 -4
  77. data/contrib/zstd/lib/deprecated/zbuff_decompress.c +3 -1
  78. data/contrib/zstd/lib/dictBuilder/cover.c +73 -52
  79. data/contrib/zstd/lib/dictBuilder/cover.h +7 -6
  80. data/contrib/zstd/lib/dictBuilder/divsufsort.c +1 -1
  81. data/contrib/zstd/lib/dictBuilder/fastcover.c +44 -35
  82. data/contrib/zstd/lib/dictBuilder/zdict.c +103 -111
  83. data/contrib/zstd/lib/legacy/zstd_legacy.h +8 -1
  84. data/contrib/zstd/lib/legacy/zstd_v01.c +21 -54
  85. data/contrib/zstd/lib/legacy/zstd_v01.h +1 -1
  86. data/contrib/zstd/lib/legacy/zstd_v02.c +29 -70
  87. data/contrib/zstd/lib/legacy/zstd_v02.h +1 -1
  88. data/contrib/zstd/lib/legacy/zstd_v03.c +30 -73
  89. data/contrib/zstd/lib/legacy/zstd_v03.h +1 -1
  90. data/contrib/zstd/lib/legacy/zstd_v04.c +29 -71
  91. data/contrib/zstd/lib/legacy/zstd_v04.h +1 -1
  92. data/contrib/zstd/lib/legacy/zstd_v05.c +40 -86
  93. data/contrib/zstd/lib/legacy/zstd_v05.h +1 -1
  94. data/contrib/zstd/lib/legacy/zstd_v06.c +47 -88
  95. data/contrib/zstd/lib/legacy/zstd_v06.h +1 -1
  96. data/contrib/zstd/lib/legacy/zstd_v07.c +40 -83
  97. data/contrib/zstd/lib/legacy/zstd_v07.h +1 -1
  98. data/contrib/zstd/lib/libzstd.mk +214 -0
  99. data/contrib/zstd/lib/libzstd.pc.in +7 -6
  100. data/contrib/zstd/lib/module.modulemap +35 -0
  101. data/contrib/zstd/lib/{dictBuilder/zdict.h → zdict.h} +203 -34
  102. data/contrib/zstd/lib/zstd.h +1217 -287
  103. data/contrib/zstd/lib/{common/zstd_errors.h → zstd_errors.h} +28 -8
  104. data/ext/extconf.rb +7 -6
  105. data/ext/extzstd.c +19 -10
  106. data/ext/extzstd.h +6 -0
  107. data/ext/libzstd_conf.h +0 -1
  108. data/ext/zstd_decompress_asm.S +1 -0
  109. data/gemstub.rb +3 -21
  110. data/lib/extzstd/version.rb +6 -1
  111. data/lib/extzstd.rb +0 -2
  112. data/test/test_basic.rb +0 -5
  113. metadata +18 -6
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -52,7 +52,11 @@ extern "C" {
52
52
  * Basic Types
53
53
  *****************************************************************/
54
54
  #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
55
- # include <stdint.h>
55
+ # if defined(_AIX)
56
+ # include <inttypes.h>
57
+ # else
58
+ # include <stdint.h> /* intptr_t */
59
+ # endif
56
60
  typedef uint8_t BYTE;
57
61
  typedef uint16_t U16;
58
62
  typedef int16_t S16;
@@ -74,7 +78,7 @@ extern "C" {
74
78
  /*-*************************************
75
79
  * Debug
76
80
  ***************************************/
77
- #include "debug.h"
81
+ #include "../common/debug.h"
78
82
  #ifndef assert
79
83
  # define assert(condition) ((void)0)
80
84
  #endif
@@ -83,27 +87,6 @@ extern "C" {
83
87
  /****************************************************************
84
88
  * Memory I/O
85
89
  *****************************************************************/
86
- /* MEM_FORCE_MEMORY_ACCESS
87
- * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
88
- * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
89
- * The below switch allow to select different access method for improved performance.
90
- * Method 0 (default) : use `memcpy()`. Safe and portable.
91
- * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
92
- * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
93
- * Method 2 : direct access. This method is portable but violate C standard.
94
- * It can generate buggy code on targets generating assembly depending on alignment.
95
- * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
96
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
97
- * Prefer these methods in priority order (0 > 1 > 2)
98
- */
99
- #ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
100
- # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
101
- # define MEM_FORCE_MEMORY_ACCESS 2
102
- # elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
103
- (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
104
- # define MEM_FORCE_MEMORY_ACCESS 1
105
- # endif
106
- #endif
107
90
 
108
91
  MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; }
109
92
  MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; }
@@ -114,33 +97,6 @@ MEM_STATIC unsigned MEM_isLittleEndian(void)
114
97
  return one.c[0];
115
98
  }
116
99
 
117
- #if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
118
-
119
- /* violates C standard on structure alignment.
120
- Only use if no other choice to achieve best performance on target platform */
121
- MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
122
- MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
123
- MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
124
-
125
- MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
126
-
127
- #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
128
-
129
- /* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
130
- /* currently only defined for gcc and icc */
131
- typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign;
132
-
133
- MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
134
- MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
135
- MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
136
-
137
- MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
138
-
139
- #else
140
-
141
- /* default method, safe and standard.
142
- can sometimes prove slower */
143
-
144
100
  MEM_STATIC U16 MEM_read16(const void* memPtr)
145
101
  {
146
102
  U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
@@ -161,9 +117,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value)
161
117
  memcpy(memPtr, &value, sizeof(value));
162
118
  }
163
119
 
164
- #endif /* MEM_FORCE_MEMORY_ACCESS */
165
-
166
-
167
120
  MEM_STATIC U16 MEM_readLE16(const void* memPtr)
168
121
  {
169
122
  if (MEM_isLittleEndian())
@@ -541,7 +494,7 @@ If there is an error, the function will return an error code, which can be teste
541
494
  header file (to include)
542
495
  Copyright (C) 2013-2015, Yann Collet.
543
496
 
544
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
497
+ BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
545
498
 
546
499
  Redistribution and use in source and binary forms, with or without
547
500
  modification, are permitted provided that the following conditions are
@@ -623,9 +576,8 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
623
576
  MEM_STATIC unsigned BIT_highbit32 (U32 val)
624
577
  {
625
578
  # if defined(_MSC_VER) /* Visual */
626
- unsigned long r=0;
627
- _BitScanReverse ( &r, val );
628
- return (unsigned) r;
579
+ unsigned long r;
580
+ return _BitScanReverse(&r, val) ? (unsigned)r : 0;
629
581
  # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
630
582
  return __builtin_clz (val) ^ 31;
631
583
  # else /* Software version */
@@ -700,7 +652,7 @@ MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits)
700
652
  }
701
653
 
702
654
  /*! BIT_lookBitsFast :
703
- * unsafe version; only works only if nbBits >= 1 */
655
+ * unsafe version; only works if nbBits >= 1 */
704
656
  MEM_STATIC size_t BIT_lookBitsFast(BIT_DStream_t* bitD, U32 nbBits)
705
657
  {
706
658
  const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
@@ -720,7 +672,7 @@ MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
720
672
  }
721
673
 
722
674
  /*!BIT_readBitsFast :
723
- * unsafe version; only works only if nbBits >= 1 */
675
+ * unsafe version; only works if nbBits >= 1 */
724
676
  MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
725
677
  {
726
678
  size_t value = BIT_lookBitsFast(bitD, nbBits);
@@ -781,7 +733,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
781
733
  header file for static linking (only)
782
734
  Copyright (C) 2013-2015, Yann Collet
783
735
 
784
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
736
+ BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
785
737
 
786
738
  Redistribution and use in source and binary forms, with or without
787
739
  modification, are permitted provided that the following conditions are
@@ -930,7 +882,7 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
930
882
  FSE : Finite State Entropy coder
931
883
  Copyright (C) 2013-2015, Yann Collet.
932
884
 
933
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
885
+ BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
934
886
 
935
887
  Redistribution and use in source and binary forms, with or without
936
888
  modification, are permitted provided that the following conditions are
@@ -1436,7 +1388,7 @@ static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, siz
1436
1388
  header file
1437
1389
  Copyright (C) 2013-2015, Yann Collet.
1438
1390
 
1439
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
1391
+ BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
1440
1392
 
1441
1393
  Redistribution and use in source and binary forms, with or without
1442
1394
  modification, are permitted provided that the following conditions are
@@ -1514,7 +1466,7 @@ static unsigned HUF_isError(size_t code); /* tells if a return value i
1514
1466
  header file for static linking (only)
1515
1467
  Copyright (C) 2013-2015, Yann Collet
1516
1468
 
1517
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
1469
+ BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
1518
1470
 
1519
1471
  Redistribution and use in source and binary forms, with or without
1520
1472
  modification, are permitted provided that the following conditions are
@@ -1601,7 +1553,7 @@ static size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const
1601
1553
  Huff0 : Huffman coder, part of New Generation Entropy library
1602
1554
  Copyright (C) 2013-2015, Yann Collet.
1603
1555
 
1604
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
1556
+ BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
1605
1557
 
1606
1558
  Redistribution and use in source and binary forms, with or without
1607
1559
  modification, are permitted provided that the following conditions are
@@ -2401,7 +2353,7 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
2401
2353
  zstd - decompression module fo v0.4 legacy format
2402
2354
  Copyright (C) 2015-2016, Yann Collet.
2403
2355
 
2404
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
2356
+ BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
2405
2357
 
2406
2358
  Redistribution and use in source and binary forms, with or without
2407
2359
  modification, are permitted provided that the following conditions are
@@ -2876,13 +2828,19 @@ static size_t ZSTD_execSequence(BYTE* op,
2876
2828
  const BYTE* const litEnd = *litPtr + sequence.litLength;
2877
2829
  const BYTE* match = oLitEnd - sequence.offset;
2878
2830
 
2879
- /* check */
2880
- if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
2831
+ /* checks */
2832
+ size_t const seqLength = sequence.litLength + sequence.matchLength;
2833
+
2834
+ if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
2835
+ if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);
2836
+ /* Now we know there are no overflow in literal nor match lengths, can use pointer checks */
2837
+ if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);
2838
+
2881
2839
  if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
2882
- if (litEnd > litLimit) return ERROR(corruption_detected); /* risk read beyond lit buffer */
2840
+ if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
2883
2841
 
2884
2842
  /* copy Literals */
2885
- ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
2843
+ ZSTD_wildcopy(op, *litPtr, (ptrdiff_t)sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
2886
2844
  op = oLitEnd;
2887
2845
  *litPtr = litEnd; /* update for next sequence */
2888
2846
 
@@ -3283,7 +3241,7 @@ static void ZSTD_decompress_insertDictionary(ZSTD_DCtx* ctx, const void* dict, s
3283
3241
  Buffered version of Zstd compression library
3284
3242
  Copyright (C) 2015, Yann Collet.
3285
3243
 
3286
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
3244
+ BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
3287
3245
 
3288
3246
  Redistribution and use in source and binary forms, with or without
3289
3247
  modification, are permitted provided that the following conditions are
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -19,7 +19,7 @@
19
19
  low-level memory access routines
20
20
  Copyright (C) 2013-2015, Yann Collet.
21
21
 
22
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
22
+ BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
23
23
 
24
24
  Redistribution and use in source and binary forms, with or without
25
25
  modification, are permitted provided that the following conditions are
@@ -80,7 +80,11 @@ extern "C" {
80
80
  * Basic Types
81
81
  *****************************************************************/
82
82
  #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
83
- # include <stdint.h>
83
+ # if defined(_AIX)
84
+ # include <inttypes.h>
85
+ # else
86
+ # include <stdint.h> /* intptr_t */
87
+ # endif
84
88
  typedef uint8_t BYTE;
85
89
  typedef uint16_t U16;
86
90
  typedef int16_t S16;
@@ -102,27 +106,6 @@ extern "C" {
102
106
  /*-**************************************************************
103
107
  * Memory I/O
104
108
  *****************************************************************/
105
- /* MEM_FORCE_MEMORY_ACCESS :
106
- * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
107
- * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
108
- * The below switch allow to select different access method for improved performance.
109
- * Method 0 (default) : use `memcpy()`. Safe and portable.
110
- * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
111
- * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
112
- * Method 2 : direct access. This method is portable but violate C standard.
113
- * It can generate buggy code on targets depending on alignment.
114
- * In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
115
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
116
- * Prefer these methods in priority order (0 > 1 > 2)
117
- */
118
- #ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
119
- # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
120
- # define MEM_FORCE_MEMORY_ACCESS 2
121
- # elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
122
- (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
123
- # define MEM_FORCE_MEMORY_ACCESS 1
124
- # endif
125
- #endif
126
109
 
127
110
  MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; }
128
111
  MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; }
@@ -133,37 +116,6 @@ MEM_STATIC unsigned MEM_isLittleEndian(void)
133
116
  return one.c[0];
134
117
  }
135
118
 
136
- #if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
137
-
138
- /* violates C standard, by lying on structure alignment.
139
- Only use if no other choice to achieve best performance on target platform */
140
- MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
141
- MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
142
- MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
143
-
144
- MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
145
- MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
146
- MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
147
-
148
- #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
149
-
150
- /* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
151
- /* currently only defined for gcc and icc */
152
- typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign;
153
-
154
- MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
155
- MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
156
- MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
157
-
158
- MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
159
- MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
160
- MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
161
-
162
- #else
163
-
164
- /* default method, safe and standard.
165
- can sometimes prove slower */
166
-
167
119
  MEM_STATIC U16 MEM_read16(const void* memPtr)
168
120
  {
169
121
  U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
@@ -194,9 +146,6 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value)
194
146
  memcpy(memPtr, &value, sizeof(value));
195
147
  }
196
148
 
197
- #endif /* MEM_FORCE_MEMORY_ACCESS */
198
-
199
-
200
149
  MEM_STATIC U16 MEM_readLE16(const void* memPtr)
201
150
  {
202
151
  if (MEM_isLittleEndian())
@@ -261,7 +210,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
261
210
  Header File for static linking only
262
211
  Copyright (C) 2014-2016, Yann Collet.
263
212
 
264
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
213
+ BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
265
214
 
266
215
  Redistribution and use in source and binary forms, with or without
267
216
  modification, are permitted provided that the following conditions are
@@ -285,7 +234,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
285
234
  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
286
235
 
287
236
  You can contact the author at :
288
- - zstd homepage : http://www.zstd.net
237
+ - zstd homepage : https://facebook.github.io/zstd
289
238
  */
290
239
  #ifndef ZSTD_STATIC_H
291
240
  #define ZSTD_STATIC_H
@@ -397,7 +346,7 @@ size_t ZSTDv05_decompressBlock(ZSTDv05_DCtx* dctx, void* dst, size_t dstCapacity
397
346
  Header File for include
398
347
  Copyright (C) 2014-2016, Yann Collet.
399
348
 
400
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
349
+ BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
401
350
 
402
351
  Redistribution and use in source and binary forms, with or without
403
352
  modification, are permitted provided that the following conditions are
@@ -484,7 +433,7 @@ static const size_t ZSTDv05_frameHeaderSize_min = 5;
484
433
  #define FSEv05_ENCODING_DYNAMIC 3
485
434
 
486
435
 
487
- #define HufLog 12
436
+ #define ZSTD_HUFFDTABLE_CAPACITY_LOG 12
488
437
 
489
438
  #define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */
490
439
  #define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */
@@ -552,7 +501,7 @@ typedef struct {
552
501
  header file
553
502
  Copyright (C) 2013-2015, Yann Collet.
554
503
 
555
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
504
+ BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
556
505
 
557
506
  Redistribution and use in source and binary forms, with or without
558
507
  modification, are permitted provided that the following conditions are
@@ -671,7 +620,7 @@ size_t FSEv05_decompress_usingDTable(void* dst, size_t dstCapacity, const void*
671
620
  header file (to include)
672
621
  Copyright (C) 2013-2016, Yann Collet.
673
622
 
674
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
623
+ BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
675
624
 
676
625
  Redistribution and use in source and binary forms, with or without
677
626
  modification, are permitted provided that the following conditions are
@@ -752,9 +701,8 @@ MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, unsigned nbBits);
752
701
  MEM_STATIC unsigned BITv05_highbit32 (U32 val)
753
702
  {
754
703
  # if defined(_MSC_VER) /* Visual */
755
- unsigned long r=0;
756
- _BitScanReverse ( &r, val );
757
- return (unsigned) r;
704
+ unsigned long r;
705
+ return _BitScanReverse(&r, val) ? (unsigned)r : 0;
758
706
  # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
759
707
  return __builtin_clz (val) ^ 31;
760
708
  # else /* Software version */
@@ -826,7 +774,7 @@ MEM_STATIC size_t BITv05_lookBits(BITv05_DStream_t* bitD, U32 nbBits)
826
774
  }
827
775
 
828
776
  /*! BITv05_lookBitsFast :
829
- * unsafe version; only works only if nbBits >= 1 */
777
+ * unsafe version; only works if nbBits >= 1 */
830
778
  MEM_STATIC size_t BITv05_lookBitsFast(BITv05_DStream_t* bitD, U32 nbBits)
831
779
  {
832
780
  const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
@@ -846,7 +794,7 @@ MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, unsigned nbBits)
846
794
  }
847
795
 
848
796
  /*!BITv05_readBitsFast :
849
- * unsafe version; only works only if nbBits >= 1 */
797
+ * unsafe version; only works if nbBits >= 1 */
850
798
  MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, unsigned nbBits)
851
799
  {
852
800
  size_t value = BITv05_lookBitsFast(bitD, nbBits);
@@ -901,7 +849,7 @@ MEM_STATIC unsigned BITv05_endOfDStream(const BITv05_DStream_t* DStream)
901
849
  header file for static linking (only)
902
850
  Copyright (C) 2013-2015, Yann Collet
903
851
 
904
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
852
+ BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
905
853
 
906
854
  Redistribution and use in source and binary forms, with or without
907
855
  modification, are permitted provided that the following conditions are
@@ -1051,7 +999,7 @@ MEM_STATIC unsigned FSEv05_endOfDState(const FSEv05_DState_t* DStatePtr)
1051
999
  FSEv05 : Finite State Entropy coder
1052
1000
  Copyright (C) 2013-2015, Yann Collet.
1053
1001
 
1054
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
1002
+ BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
1055
1003
 
1056
1004
  Redistribution and use in source and binary forms, with or without
1057
1005
  modification, are permitted provided that the following conditions are
@@ -1537,7 +1485,7 @@ size_t FSEv05_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t
1537
1485
  header file
1538
1486
  Copyright (C) 2013-2016, Yann Collet.
1539
1487
 
1540
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
1488
+ BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
1541
1489
 
1542
1490
  Redistribution and use in source and binary forms, with or without
1543
1491
  modification, are permitted provided that the following conditions are
@@ -1610,7 +1558,7 @@ const char* HUFv05_getErrorName(size_t code); /* provides error code string (u
1610
1558
  header file, for static linking only
1611
1559
  Copyright (C) 2013-2016, Yann Collet
1612
1560
 
1613
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
1561
+ BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
1614
1562
 
1615
1563
  Redistribution and use in source and binary forms, with or without
1616
1564
  modification, are permitted provided that the following conditions are
@@ -1702,7 +1650,7 @@ size_t HUFv05_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void
1702
1650
  Huff0 : Huffman coder, part of New Generation Entropy library
1703
1651
  Copyright (C) 2013-2015, Yann Collet.
1704
1652
 
1705
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
1653
+ BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
1706
1654
 
1707
1655
  Redistribution and use in source and binary forms, with or without
1708
1656
  modification, are permitted provided that the following conditions are
@@ -2547,7 +2495,7 @@ size_t HUFv05_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
2547
2495
  zstd - standard compression library
2548
2496
  Copyright (C) 2014-2016, Yann Collet.
2549
2497
 
2550
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
2498
+ BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
2551
2499
 
2552
2500
  Redistribution and use in source and binary forms, with or without
2553
2501
  modification, are permitted provided that the following conditions are
@@ -2645,7 +2593,7 @@ struct ZSTDv05_DCtx_s
2645
2593
  FSEv05_DTable LLTable[FSEv05_DTABLE_SIZE_U32(LLFSEv05Log)];
2646
2594
  FSEv05_DTable OffTable[FSEv05_DTABLE_SIZE_U32(OffFSEv05Log)];
2647
2595
  FSEv05_DTable MLTable[FSEv05_DTABLE_SIZE_U32(MLFSEv05Log)];
2648
- unsigned hufTableX4[HUFv05_DTABLE_SIZE(HufLog)];
2596
+ unsigned hufTableX4[HUFv05_DTABLE_SIZE(ZSTD_HUFFDTABLE_CAPACITY_LOG)];
2649
2597
  const void* previousDstEnd;
2650
2598
  const void* base;
2651
2599
  const void* vBase;
@@ -2673,7 +2621,7 @@ size_t ZSTDv05_decompressBegin(ZSTDv05_DCtx* dctx)
2673
2621
  dctx->base = NULL;
2674
2622
  dctx->vBase = NULL;
2675
2623
  dctx->dictEnd = NULL;
2676
- dctx->hufTableX4[0] = HufLog;
2624
+ dctx->hufTableX4[0] = ZSTD_HUFFDTABLE_CAPACITY_LOG;
2677
2625
  dctx->flagStaticTables = 0;
2678
2626
  return 0;
2679
2627
  }
@@ -2829,7 +2777,7 @@ static size_t ZSTDv05_decodeFrameHeader_Part2(ZSTDv05_DCtx* zc, const void* src,
2829
2777
 
2830
2778
  static size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
2831
2779
  {
2832
- const BYTE* const in = (const BYTE* const)src;
2780
+ const BYTE* const in = (const BYTE*)src;
2833
2781
  BYTE headerFlags;
2834
2782
  U32 cSize;
2835
2783
 
@@ -2998,7 +2946,7 @@ static size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t
2998
2946
  FSEv05_DTable* DTableLL, FSEv05_DTable* DTableML, FSEv05_DTable* DTableOffb,
2999
2947
  const void* src, size_t srcSize, U32 flagStaticTable)
3000
2948
  {
3001
- const BYTE* const istart = (const BYTE* const)src;
2949
+ const BYTE* const istart = (const BYTE*)src;
3002
2950
  const BYTE* ip = istart;
3003
2951
  const BYTE* const iend = istart + srcSize;
3004
2952
  U32 LLtype, Offtype, MLtype;
@@ -3234,13 +3182,19 @@ static size_t ZSTDv05_execSequence(BYTE* op,
3234
3182
  const BYTE* const litEnd = *litPtr + sequence.litLength;
3235
3183
  const BYTE* match = oLitEnd - sequence.offset;
3236
3184
 
3237
- /* check */
3238
- if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
3185
+ /* checks */
3186
+ size_t const seqLength = sequence.litLength + sequence.matchLength;
3187
+
3188
+ if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall);
3189
+ if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);
3190
+ /* Now we know there are no overflow in literal nor match lengths, can use pointer checks */
3191
+ if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);
3192
+
3239
3193
  if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
3240
- if (litEnd > litLimit) return ERROR(corruption_detected); /* risk read beyond lit buffer */
3194
+ if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
3241
3195
 
3242
3196
  /* copy Literals */
3243
- ZSTDv05_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
3197
+ ZSTDv05_wildcopy(op, *litPtr, (ptrdiff_t)sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
3244
3198
  op = oLitEnd;
3245
3199
  *litPtr = litEnd; /* update for next sequence */
3246
3200
 
@@ -3306,7 +3260,7 @@ static size_t ZSTDv05_decompressSequences(
3306
3260
  {
3307
3261
  const BYTE* ip = (const BYTE*)seqStart;
3308
3262
  const BYTE* const iend = ip + seqSize;
3309
- BYTE* const ostart = (BYTE* const)dst;
3263
+ BYTE* const ostart = (BYTE*)dst;
3310
3264
  BYTE* op = ostart;
3311
3265
  BYTE* const oend = ostart + maxDstSize;
3312
3266
  size_t errorCode, dumpsLength=0;
@@ -3419,7 +3373,7 @@ static size_t ZSTDv05_decompress_continueDCtx(ZSTDv05_DCtx* dctx,
3419
3373
  {
3420
3374
  const BYTE* ip = (const BYTE*)src;
3421
3375
  const BYTE* iend = ip + srcSize;
3422
- BYTE* const ostart = (BYTE* const)dst;
3376
+ BYTE* const ostart = (BYTE*)dst;
3423
3377
  BYTE* op = ostart;
3424
3378
  BYTE* const oend = ostart + maxDstSize;
3425
3379
  size_t remainingSize = srcSize;
@@ -3746,7 +3700,7 @@ size_t ZSTDv05_decompressBegin_usingDict(ZSTDv05_DCtx* dctx, const void* dict, s
3746
3700
  Buffered version of Zstd compression library
3747
3701
  Copyright (C) 2015-2016, Yann Collet.
3748
3702
 
3749
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
3703
+ BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
3750
3704
 
3751
3705
  Redistribution and use in source and binary forms, with or without
3752
3706
  modification, are permitted provided that the following conditions are
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the