extzstd 0.3 → 0.3.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (107) hide show
  1. checksums.yaml +4 -4
  2. data/HISTORY.ja.md +8 -0
  3. data/README.md +1 -1
  4. data/contrib/zstd/CHANGELOG +94 -0
  5. data/contrib/zstd/CONTRIBUTING.md +351 -1
  6. data/contrib/zstd/Makefile +32 -10
  7. data/contrib/zstd/README.md +33 -10
  8. data/contrib/zstd/TESTING.md +2 -2
  9. data/contrib/zstd/appveyor.yml +42 -4
  10. data/contrib/zstd/lib/Makefile +128 -60
  11. data/contrib/zstd/lib/README.md +47 -16
  12. data/contrib/zstd/lib/common/bitstream.h +38 -39
  13. data/contrib/zstd/lib/common/compiler.h +40 -5
  14. data/contrib/zstd/lib/common/cpu.h +1 -1
  15. data/contrib/zstd/lib/common/debug.c +11 -31
  16. data/contrib/zstd/lib/common/debug.h +11 -31
  17. data/contrib/zstd/lib/common/entropy_common.c +13 -33
  18. data/contrib/zstd/lib/common/error_private.c +2 -1
  19. data/contrib/zstd/lib/common/error_private.h +6 -2
  20. data/contrib/zstd/lib/common/fse.h +12 -32
  21. data/contrib/zstd/lib/common/fse_decompress.c +12 -35
  22. data/contrib/zstd/lib/common/huf.h +15 -33
  23. data/contrib/zstd/lib/common/mem.h +75 -2
  24. data/contrib/zstd/lib/common/pool.c +8 -4
  25. data/contrib/zstd/lib/common/pool.h +2 -2
  26. data/contrib/zstd/lib/common/threading.c +50 -4
  27. data/contrib/zstd/lib/common/threading.h +36 -4
  28. data/contrib/zstd/lib/common/xxhash.c +23 -35
  29. data/contrib/zstd/lib/common/xxhash.h +11 -31
  30. data/contrib/zstd/lib/common/zstd_common.c +1 -1
  31. data/contrib/zstd/lib/common/zstd_errors.h +2 -1
  32. data/contrib/zstd/lib/common/zstd_internal.h +154 -26
  33. data/contrib/zstd/lib/compress/fse_compress.c +17 -40
  34. data/contrib/zstd/lib/compress/hist.c +15 -35
  35. data/contrib/zstd/lib/compress/hist.h +12 -32
  36. data/contrib/zstd/lib/compress/huf_compress.c +92 -92
  37. data/contrib/zstd/lib/compress/zstd_compress.c +1191 -1330
  38. data/contrib/zstd/lib/compress/zstd_compress_internal.h +317 -55
  39. data/contrib/zstd/lib/compress/zstd_compress_literals.c +158 -0
  40. data/contrib/zstd/lib/compress/zstd_compress_literals.h +29 -0
  41. data/contrib/zstd/lib/compress/zstd_compress_sequences.c +419 -0
  42. data/contrib/zstd/lib/compress/zstd_compress_sequences.h +54 -0
  43. data/contrib/zstd/lib/compress/zstd_compress_superblock.c +845 -0
  44. data/contrib/zstd/lib/compress/zstd_compress_superblock.h +32 -0
  45. data/contrib/zstd/lib/compress/zstd_cwksp.h +525 -0
  46. data/contrib/zstd/lib/compress/zstd_double_fast.c +65 -43
  47. data/contrib/zstd/lib/compress/zstd_double_fast.h +2 -2
  48. data/contrib/zstd/lib/compress/zstd_fast.c +92 -66
  49. data/contrib/zstd/lib/compress/zstd_fast.h +2 -2
  50. data/contrib/zstd/lib/compress/zstd_lazy.c +74 -42
  51. data/contrib/zstd/lib/compress/zstd_lazy.h +1 -1
  52. data/contrib/zstd/lib/compress/zstd_ldm.c +32 -10
  53. data/contrib/zstd/lib/compress/zstd_ldm.h +7 -2
  54. data/contrib/zstd/lib/compress/zstd_opt.c +81 -114
  55. data/contrib/zstd/lib/compress/zstd_opt.h +1 -1
  56. data/contrib/zstd/lib/compress/zstdmt_compress.c +95 -51
  57. data/contrib/zstd/lib/compress/zstdmt_compress.h +3 -2
  58. data/contrib/zstd/lib/decompress/huf_decompress.c +76 -60
  59. data/contrib/zstd/lib/decompress/zstd_ddict.c +12 -8
  60. data/contrib/zstd/lib/decompress/zstd_ddict.h +2 -2
  61. data/contrib/zstd/lib/decompress/zstd_decompress.c +292 -172
  62. data/contrib/zstd/lib/decompress/zstd_decompress_block.c +459 -338
  63. data/contrib/zstd/lib/decompress/zstd_decompress_block.h +3 -3
  64. data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +18 -4
  65. data/contrib/zstd/lib/deprecated/zbuff.h +9 -8
  66. data/contrib/zstd/lib/deprecated/zbuff_common.c +2 -2
  67. data/contrib/zstd/lib/deprecated/zbuff_compress.c +1 -1
  68. data/contrib/zstd/lib/deprecated/zbuff_decompress.c +1 -1
  69. data/contrib/zstd/lib/dictBuilder/cover.c +164 -54
  70. data/contrib/zstd/lib/dictBuilder/cover.h +52 -7
  71. data/contrib/zstd/lib/dictBuilder/fastcover.c +60 -43
  72. data/contrib/zstd/lib/dictBuilder/zdict.c +43 -19
  73. data/contrib/zstd/lib/dictBuilder/zdict.h +56 -28
  74. data/contrib/zstd/lib/legacy/zstd_legacy.h +8 -4
  75. data/contrib/zstd/lib/legacy/zstd_v01.c +110 -110
  76. data/contrib/zstd/lib/legacy/zstd_v01.h +1 -1
  77. data/contrib/zstd/lib/legacy/zstd_v02.c +23 -13
  78. data/contrib/zstd/lib/legacy/zstd_v02.h +1 -1
  79. data/contrib/zstd/lib/legacy/zstd_v03.c +23 -13
  80. data/contrib/zstd/lib/legacy/zstd_v03.h +1 -1
  81. data/contrib/zstd/lib/legacy/zstd_v04.c +30 -17
  82. data/contrib/zstd/lib/legacy/zstd_v04.h +1 -1
  83. data/contrib/zstd/lib/legacy/zstd_v05.c +113 -102
  84. data/contrib/zstd/lib/legacy/zstd_v05.h +2 -2
  85. data/contrib/zstd/lib/legacy/zstd_v06.c +20 -18
  86. data/contrib/zstd/lib/legacy/zstd_v06.h +1 -1
  87. data/contrib/zstd/lib/legacy/zstd_v07.c +25 -19
  88. data/contrib/zstd/lib/legacy/zstd_v07.h +1 -1
  89. data/contrib/zstd/lib/libzstd.pc.in +3 -2
  90. data/contrib/zstd/lib/zstd.h +265 -88
  91. data/ext/extzstd.h +1 -1
  92. data/ext/libzstd_conf.h +8 -0
  93. data/ext/zstd_common.c +1 -3
  94. data/ext/zstd_compress.c +3 -3
  95. data/ext/zstd_decompress.c +1 -5
  96. data/ext/zstd_dictbuilder.c +2 -3
  97. data/ext/zstd_dictbuilder_fastcover.c +1 -3
  98. data/ext/zstd_legacy_v01.c +2 -0
  99. data/ext/zstd_legacy_v02.c +2 -0
  100. data/ext/zstd_legacy_v03.c +2 -0
  101. data/ext/zstd_legacy_v04.c +2 -0
  102. data/ext/zstd_legacy_v05.c +2 -0
  103. data/ext/zstd_legacy_v06.c +2 -0
  104. data/ext/zstd_legacy_v07.c +2 -0
  105. data/lib/extzstd.rb +18 -10
  106. data/lib/extzstd/version.rb +1 -1
  107. metadata +15 -6
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -61,9 +61,57 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCap
61
61
  const void* samplesBuffer,
62
62
  const size_t* samplesSizes, unsigned nbSamples);
63
63
 
64
+ typedef struct {
65
+ int compressionLevel; /*< optimize for a specific zstd compression level; 0 means default */
66
+ unsigned notificationLevel; /*< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
67
+ unsigned dictID; /*< force dictID value; 0 means auto mode (32-bits random value) */
68
+ } ZDICT_params_t;
69
+
70
+ /*! ZDICT_finalizeDictionary():
71
+ * Given a custom content as a basis for dictionary, and a set of samples,
72
+ * finalize dictionary by adding headers and statistics according to the zstd
73
+ * dictionary format.
74
+ *
75
+ * Samples must be stored concatenated in a flat buffer `samplesBuffer`,
76
+ * supplied with an array of sizes `samplesSizes`, providing the size of each
77
+ * sample in order. The samples are used to construct the statistics, so they
78
+ * should be representative of what you will compress with this dictionary.
79
+ *
80
+ * The compression level can be set in `parameters`. You should pass the
81
+ * compression level you expect to use in production. The statistics for each
82
+ * compression level differ, so tuning the dictionary for the compression level
83
+ * can help quite a bit.
84
+ *
85
+ * You can set an explicit dictionary ID in `parameters`, or allow us to pick
86
+ * a random dictionary ID for you, but we can't guarantee no collisions.
87
+ *
88
+ * The dstDictBuffer and the dictContent may overlap, and the content will be
89
+ * appended to the end of the header. If the header + the content doesn't fit in
90
+ * maxDictSize the beginning of the content is truncated to make room, since it
91
+ * is presumed that the most profitable content is at the end of the dictionary,
92
+ * since that is the cheapest to reference.
93
+ *
94
+ * `dictContentSize` must be >= ZDICT_CONTENTSIZE_MIN bytes.
95
+ * `maxDictSize` must be >= max(dictContentSize, ZSTD_DICTSIZE_MIN).
96
+ *
97
+ * @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`),
98
+ * or an error code, which can be tested by ZDICT_isError().
99
+ * Note: ZDICT_finalizeDictionary() will push notifications into stderr if
100
+ * instructed to, using notificationLevel>0.
101
+ * NOTE: This function currently may fail in several edge cases including:
102
+ * * Not enough samples
103
+ * * Samples are uncompressible
104
+ * * Samples are all exactly the same
105
+ */
106
+ ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dstDictBuffer, size_t maxDictSize,
107
+ const void* dictContent, size_t dictContentSize,
108
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
109
+ ZDICT_params_t parameters);
110
+
64
111
 
65
112
  /*====== Helper functions ======*/
66
113
  ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize); /**< extracts dictID; @return zero if error (not a valid dictionary) */
114
+ ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize); /* returns dict header size; returns a ZSTD error code on failure */
67
115
  ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode);
68
116
  ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
69
117
 
@@ -78,11 +126,8 @@ ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
78
126
  * Use them only in association with static linking.
79
127
  * ==================================================================================== */
80
128
 
81
- typedef struct {
82
- int compressionLevel; /* optimize for a specific zstd compression level; 0 means default */
83
- unsigned notificationLevel; /* Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
84
- unsigned dictID; /* force dictID value; 0 means auto mode (32-bits random value) */
85
- } ZDICT_params_t;
129
+ #define ZDICT_CONTENTSIZE_MIN 128
130
+ #define ZDICT_DICTSIZE_MIN 256
86
131
 
87
132
  /*! ZDICT_cover_params_t:
88
133
  * k and d are the only required parameters.
@@ -94,6 +139,8 @@ typedef struct {
94
139
  unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */
95
140
  unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
96
141
  double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */
142
+ unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */
143
+ unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */
97
144
  ZDICT_params_t zParams;
98
145
  } ZDICT_cover_params_t;
99
146
 
@@ -105,6 +152,9 @@ typedef struct {
105
152
  unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
106
153
  double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (0.75), 1.0 when all samples are used for both training and testing */
107
154
  unsigned accel; /* Acceleration level: constraint: 0 < accel <= 10, higher means faster and less accurate, 0 means default(1) */
155
+ unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */
156
+ unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */
157
+
108
158
  ZDICT_params_t zParams;
109
159
  } ZDICT_fastCover_params_t;
110
160
 
@@ -193,28 +243,6 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
193
243
  const size_t* samplesSizes, unsigned nbSamples,
194
244
  ZDICT_fastCover_params_t* parameters);
195
245
 
196
- /*! ZDICT_finalizeDictionary():
197
- * Given a custom content as a basis for dictionary, and a set of samples,
198
- * finalize dictionary by adding headers and statistics.
199
- *
200
- * Samples must be stored concatenated in a flat buffer `samplesBuffer`,
201
- * supplied with an array of sizes `samplesSizes`, providing the size of each sample in order.
202
- *
203
- * dictContentSize must be >= ZDICT_CONTENTSIZE_MIN bytes.
204
- * maxDictSize must be >= dictContentSize, and must be >= ZDICT_DICTSIZE_MIN bytes.
205
- *
206
- * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`),
207
- * or an error code, which can be tested by ZDICT_isError().
208
- * Note: ZDICT_finalizeDictionary() will push notifications into stderr if instructed to, using notificationLevel>0.
209
- * Note 2: dictBuffer and dictContent can overlap
210
- */
211
- #define ZDICT_CONTENTSIZE_MIN 128
212
- #define ZDICT_DICTSIZE_MIN 256
213
- ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
214
- const void* dictContent, size_t dictContentSize,
215
- const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
216
- ZDICT_params_t parameters);
217
-
218
246
  typedef struct {
219
247
  unsigned selectivityLevel; /* 0 means default; larger => select more => larger dictionary */
220
248
  ZDICT_params_t zParams;
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -18,9 +18,9 @@ extern "C" {
18
18
  /* *************************************
19
19
  * Includes
20
20
  ***************************************/
21
- #include "mem.h" /* MEM_STATIC */
22
- #include "error_private.h" /* ERROR */
23
- #include "zstd_internal.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTD_frameSizeInfo */
21
+ #include "../common/mem.h" /* MEM_STATIC */
22
+ #include "../common/error_private.h" /* ERROR */
23
+ #include "../common/zstd_internal.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTD_frameSizeInfo */
24
24
 
25
25
  #if !defined (ZSTD_LEGACY_SUPPORT) || (ZSTD_LEGACY_SUPPORT == 0)
26
26
  # undef ZSTD_LEGACY_SUPPORT
@@ -238,6 +238,10 @@ MEM_STATIC ZSTD_frameSizeInfo ZSTD_findFrameSizeInfoLegacy(const void *src, size
238
238
  frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
239
239
  break;
240
240
  }
241
+ if (!ZSTD_isError(frameSizeInfo.compressedSize) && frameSizeInfo.compressedSize > srcSize) {
242
+ frameSizeInfo.compressedSize = ERROR(srcSize_wrong);
243
+ frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
244
+ }
241
245
  return frameSizeInfo;
242
246
  }
243
247
 
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -14,7 +14,7 @@
14
14
  ******************************************/
15
15
  #include <stddef.h> /* size_t, ptrdiff_t */
16
16
  #include "zstd_v01.h"
17
- #include "error_private.h"
17
+ #include "../common/error_private.h"
18
18
 
19
19
 
20
20
  /******************************************
@@ -257,7 +257,7 @@ static U64 FSE_read64(const void* memPtr)
257
257
  U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
258
258
  }
259
259
 
260
- #endif // FSE_FORCE_MEMORY_ACCESS
260
+ #endif /* FSE_FORCE_MEMORY_ACCESS */
261
261
 
262
262
  static U16 FSE_readLE16(const void* memPtr)
263
263
  {
@@ -346,7 +346,7 @@ FORCE_INLINE unsigned FSE_highbit32 (U32 val)
346
346
  _BitScanReverse ( &r, val );
347
347
  return (unsigned) r;
348
348
  # elif defined(__GNUC__) && (GCC_VERSION >= 304) /* GCC Intrinsic */
349
- return 31 - __builtin_clz (val);
349
+ return __builtin_clz (val) ^ 31;
350
350
  # else /* Software version */
351
351
  static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
352
352
  U32 v = val;
@@ -1073,99 +1073,102 @@ static size_t HUF_decompress_usingDTable( /* -3% slower when non static */
1073
1073
  const void* cSrc, size_t cSrcSize,
1074
1074
  const U16* DTable)
1075
1075
  {
1076
- BYTE* const ostart = (BYTE*) dst;
1077
- BYTE* op = ostart;
1078
- BYTE* const omax = op + maxDstSize;
1079
- BYTE* const olimit = omax-15;
1080
-
1081
- const void* ptr = DTable;
1082
- const HUF_DElt* const dt = (const HUF_DElt*)(ptr)+1;
1083
- const U32 dtLog = DTable[0];
1084
- size_t errorCode;
1085
- U32 reloadStatus;
1086
-
1087
- /* Init */
1088
-
1089
- const U16* jumpTable = (const U16*)cSrc;
1090
- const size_t length1 = FSE_readLE16(jumpTable);
1091
- const size_t length2 = FSE_readLE16(jumpTable+1);
1092
- const size_t length3 = FSE_readLE16(jumpTable+2);
1093
- const size_t length4 = cSrcSize - 6 - length1 - length2 - length3; // check coherency !!
1094
- const char* const start1 = (const char*)(cSrc) + 6;
1095
- const char* const start2 = start1 + length1;
1096
- const char* const start3 = start2 + length2;
1097
- const char* const start4 = start3 + length3;
1098
- FSE_DStream_t bitD1, bitD2, bitD3, bitD4;
1099
-
1100
- if (length1+length2+length3+6 >= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
1101
-
1102
- errorCode = FSE_initDStream(&bitD1, start1, length1);
1103
- if (FSE_isError(errorCode)) return errorCode;
1104
- errorCode = FSE_initDStream(&bitD2, start2, length2);
1105
- if (FSE_isError(errorCode)) return errorCode;
1106
- errorCode = FSE_initDStream(&bitD3, start3, length3);
1107
- if (FSE_isError(errorCode)) return errorCode;
1108
- errorCode = FSE_initDStream(&bitD4, start4, length4);
1109
- if (FSE_isError(errorCode)) return errorCode;
1110
-
1111
- reloadStatus=FSE_reloadDStream(&bitD2);
1112
-
1113
- /* 16 symbols per loop */
1114
- for ( ; (reloadStatus<FSE_DStream_completed) && (op<olimit); /* D2-3-4 are supposed to be synchronized and finish together */
1115
- op+=16, reloadStatus = FSE_reloadDStream(&bitD2) | FSE_reloadDStream(&bitD3) | FSE_reloadDStream(&bitD4), FSE_reloadDStream(&bitD1))
1076
+ if (cSrcSize < 6) return (size_t)-FSE_ERROR_srcSize_wrong;
1116
1077
  {
1117
- #define HUF_DECODE_SYMBOL_0(n, Dstream) \
1118
- op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog);
1119
-
1120
- #define HUF_DECODE_SYMBOL_1(n, Dstream) \
1121
- op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
1122
- if (FSE_32bits() && (HUF_MAX_TABLELOG>12)) FSE_reloadDStream(&Dstream)
1123
-
1124
- #define HUF_DECODE_SYMBOL_2(n, Dstream) \
1125
- op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
1126
- if (FSE_32bits()) FSE_reloadDStream(&Dstream)
1127
-
1128
- HUF_DECODE_SYMBOL_1( 0, bitD1);
1129
- HUF_DECODE_SYMBOL_1( 1, bitD2);
1130
- HUF_DECODE_SYMBOL_1( 2, bitD3);
1131
- HUF_DECODE_SYMBOL_1( 3, bitD4);
1132
- HUF_DECODE_SYMBOL_2( 4, bitD1);
1133
- HUF_DECODE_SYMBOL_2( 5, bitD2);
1134
- HUF_DECODE_SYMBOL_2( 6, bitD3);
1135
- HUF_DECODE_SYMBOL_2( 7, bitD4);
1136
- HUF_DECODE_SYMBOL_1( 8, bitD1);
1137
- HUF_DECODE_SYMBOL_1( 9, bitD2);
1138
- HUF_DECODE_SYMBOL_1(10, bitD3);
1139
- HUF_DECODE_SYMBOL_1(11, bitD4);
1140
- HUF_DECODE_SYMBOL_0(12, bitD1);
1141
- HUF_DECODE_SYMBOL_0(13, bitD2);
1142
- HUF_DECODE_SYMBOL_0(14, bitD3);
1143
- HUF_DECODE_SYMBOL_0(15, bitD4);
1144
- }
1078
+ BYTE* const ostart = (BYTE*) dst;
1079
+ BYTE* op = ostart;
1080
+ BYTE* const omax = op + maxDstSize;
1081
+ BYTE* const olimit = maxDstSize < 15 ? op : omax-15;
1082
+
1083
+ const void* ptr = DTable;
1084
+ const HUF_DElt* const dt = (const HUF_DElt*)(ptr)+1;
1085
+ const U32 dtLog = DTable[0];
1086
+ size_t errorCode;
1087
+ U32 reloadStatus;
1088
+
1089
+ /* Init */
1090
+
1091
+ const U16* jumpTable = (const U16*)cSrc;
1092
+ const size_t length1 = FSE_readLE16(jumpTable);
1093
+ const size_t length2 = FSE_readLE16(jumpTable+1);
1094
+ const size_t length3 = FSE_readLE16(jumpTable+2);
1095
+ const size_t length4 = cSrcSize - 6 - length1 - length2 - length3; /* check coherency !! */
1096
+ const char* const start1 = (const char*)(cSrc) + 6;
1097
+ const char* const start2 = start1 + length1;
1098
+ const char* const start3 = start2 + length2;
1099
+ const char* const start4 = start3 + length3;
1100
+ FSE_DStream_t bitD1, bitD2, bitD3, bitD4;
1101
+
1102
+ if (length1+length2+length3+6 >= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
1103
+
1104
+ errorCode = FSE_initDStream(&bitD1, start1, length1);
1105
+ if (FSE_isError(errorCode)) return errorCode;
1106
+ errorCode = FSE_initDStream(&bitD2, start2, length2);
1107
+ if (FSE_isError(errorCode)) return errorCode;
1108
+ errorCode = FSE_initDStream(&bitD3, start3, length3);
1109
+ if (FSE_isError(errorCode)) return errorCode;
1110
+ errorCode = FSE_initDStream(&bitD4, start4, length4);
1111
+ if (FSE_isError(errorCode)) return errorCode;
1112
+
1113
+ reloadStatus=FSE_reloadDStream(&bitD2);
1114
+
1115
+ /* 16 symbols per loop */
1116
+ for ( ; (reloadStatus<FSE_DStream_completed) && (op<olimit); /* D2-3-4 are supposed to be synchronized and finish together */
1117
+ op+=16, reloadStatus = FSE_reloadDStream(&bitD2) | FSE_reloadDStream(&bitD3) | FSE_reloadDStream(&bitD4), FSE_reloadDStream(&bitD1))
1118
+ {
1119
+ #define HUF_DECODE_SYMBOL_0(n, Dstream) \
1120
+ op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog);
1121
+
1122
+ #define HUF_DECODE_SYMBOL_1(n, Dstream) \
1123
+ op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
1124
+ if (FSE_32bits() && (HUF_MAX_TABLELOG>12)) FSE_reloadDStream(&Dstream)
1125
+
1126
+ #define HUF_DECODE_SYMBOL_2(n, Dstream) \
1127
+ op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
1128
+ if (FSE_32bits()) FSE_reloadDStream(&Dstream)
1129
+
1130
+ HUF_DECODE_SYMBOL_1( 0, bitD1);
1131
+ HUF_DECODE_SYMBOL_1( 1, bitD2);
1132
+ HUF_DECODE_SYMBOL_1( 2, bitD3);
1133
+ HUF_DECODE_SYMBOL_1( 3, bitD4);
1134
+ HUF_DECODE_SYMBOL_2( 4, bitD1);
1135
+ HUF_DECODE_SYMBOL_2( 5, bitD2);
1136
+ HUF_DECODE_SYMBOL_2( 6, bitD3);
1137
+ HUF_DECODE_SYMBOL_2( 7, bitD4);
1138
+ HUF_DECODE_SYMBOL_1( 8, bitD1);
1139
+ HUF_DECODE_SYMBOL_1( 9, bitD2);
1140
+ HUF_DECODE_SYMBOL_1(10, bitD3);
1141
+ HUF_DECODE_SYMBOL_1(11, bitD4);
1142
+ HUF_DECODE_SYMBOL_0(12, bitD1);
1143
+ HUF_DECODE_SYMBOL_0(13, bitD2);
1144
+ HUF_DECODE_SYMBOL_0(14, bitD3);
1145
+ HUF_DECODE_SYMBOL_0(15, bitD4);
1146
+ }
1145
1147
 
1146
- if (reloadStatus!=FSE_DStream_completed) /* not complete : some bitStream might be FSE_DStream_unfinished */
1147
- return (size_t)-FSE_ERROR_corruptionDetected;
1148
+ if (reloadStatus!=FSE_DStream_completed) /* not complete : some bitStream might be FSE_DStream_unfinished */
1149
+ return (size_t)-FSE_ERROR_corruptionDetected;
1148
1150
 
1149
- /* tail */
1150
- {
1151
- // bitTail = bitD1; // *much* slower : -20% !??!
1152
- FSE_DStream_t bitTail;
1153
- bitTail.ptr = bitD1.ptr;
1154
- bitTail.bitsConsumed = bitD1.bitsConsumed;
1155
- bitTail.bitContainer = bitD1.bitContainer; // required in case of FSE_DStream_endOfBuffer
1156
- bitTail.start = start1;
1157
- for ( ; (FSE_reloadDStream(&bitTail) < FSE_DStream_completed) && (op<omax) ; op++)
1151
+ /* tail */
1158
1152
  {
1159
- HUF_DECODE_SYMBOL_0(0, bitTail);
1160
- }
1153
+ /* bitTail = bitD1; */ /* *much* slower : -20% !??! */
1154
+ FSE_DStream_t bitTail;
1155
+ bitTail.ptr = bitD1.ptr;
1156
+ bitTail.bitsConsumed = bitD1.bitsConsumed;
1157
+ bitTail.bitContainer = bitD1.bitContainer; /* required in case of FSE_DStream_endOfBuffer */
1158
+ bitTail.start = start1;
1159
+ for ( ; (FSE_reloadDStream(&bitTail) < FSE_DStream_completed) && (op<omax) ; op++)
1160
+ {
1161
+ HUF_DECODE_SYMBOL_0(0, bitTail);
1162
+ }
1161
1163
 
1162
- if (FSE_endOfDStream(&bitTail))
1163
- return op-ostart;
1164
- }
1164
+ if (FSE_endOfDStream(&bitTail))
1165
+ return op-ostart;
1166
+ }
1165
1167
 
1166
- if (op==omax) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* dst buffer is full, but cSrc unfinished */
1168
+ if (op==omax) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* dst buffer is full, but cSrc unfinished */
1167
1169
 
1168
- return (size_t)-FSE_ERROR_corruptionDetected;
1170
+ return (size_t)-FSE_ERROR_corruptionDetected;
1171
+ }
1169
1172
  }
1170
1173
 
1171
1174
 
@@ -1355,8 +1358,6 @@ static unsigned ZSTD_isLittleEndian(void)
1355
1358
 
1356
1359
  static U16 ZSTD_read16(const void* p) { U16 r; memcpy(&r, p, sizeof(r)); return r; }
1357
1360
 
1358
- static U32 ZSTD_read32(const void* p) { U32 r; memcpy(&r, p, sizeof(r)); return r; }
1359
-
1360
1361
  static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
1361
1362
 
1362
1363
  static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
@@ -1381,16 +1382,9 @@ static U16 ZSTD_readLE16(const void* memPtr)
1381
1382
  }
1382
1383
  }
1383
1384
 
1384
-
1385
- static U32 ZSTD_readLE32(const void* memPtr)
1385
+ static U32 ZSTD_readLE24(const void* memPtr)
1386
1386
  {
1387
- if (ZSTD_isLittleEndian())
1388
- return ZSTD_read32(memPtr);
1389
- else
1390
- {
1391
- const BYTE* p = (const BYTE*)memPtr;
1392
- return (U32)((U32)p[0] + ((U32)p[1]<<8) + ((U32)p[2]<<16) + ((U32)p[3]<<24));
1393
- }
1387
+ return ZSTD_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
1394
1388
  }
1395
1389
 
1396
1390
  static U32 ZSTD_readBE32(const void* memPtr)
@@ -1489,7 +1483,9 @@ static size_t ZSTDv01_getcBlockSize(const void* src, size_t srcSize, blockProper
1489
1483
  static size_t ZSTD_copyUncompressedBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
1490
1484
  {
1491
1485
  if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
1492
- memcpy(dst, src, srcSize);
1486
+ if (srcSize > 0) {
1487
+ memcpy(dst, src, srcSize);
1488
+ }
1493
1489
  return srcSize;
1494
1490
  }
1495
1491
 
@@ -1508,7 +1504,7 @@ static size_t ZSTD_decompressLiterals(void* ctx,
1508
1504
  if (srcSize <= 3) return ERROR(corruption_detected);
1509
1505
 
1510
1506
  litSize = ip[1] + (ip[0]<<8);
1511
- litSize += ((ip[-3] >> 3) & 7) << 16; // mmmmh....
1507
+ litSize += ((ip[-3] >> 3) & 7) << 16; /* mmmmh.... */
1512
1508
  op = oend - litSize;
1513
1509
 
1514
1510
  (void)ctx;
@@ -1547,7 +1543,9 @@ static size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
1547
1543
  size_t rleSize = litbp.origSize;
1548
1544
  if (rleSize>maxDstSize) return ERROR(dstSize_tooSmall);
1549
1545
  if (!srcSize) return ERROR(srcSize_wrong);
1550
- memset(oend - rleSize, *ip, rleSize);
1546
+ if (rleSize > 0) {
1547
+ memset(oend - rleSize, *ip, rleSize);
1548
+ }
1551
1549
  *litStart = oend - rleSize;
1552
1550
  *litSize = rleSize;
1553
1551
  ip++;
@@ -1704,13 +1702,13 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
1704
1702
  seqState->prevOffset = seq->offset;
1705
1703
  if (litLength == MaxLL)
1706
1704
  {
1707
- U32 add = dumps<de ? *dumps++ : 0;
1705
+ const U32 add = dumps<de ? *dumps++ : 0;
1708
1706
  if (add < 255) litLength += add;
1709
1707
  else
1710
1708
  {
1711
1709
  if (dumps<=(de-3))
1712
1710
  {
1713
- litLength = ZSTD_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
1711
+ litLength = ZSTD_readLE24(dumps);
1714
1712
  dumps += 3;
1715
1713
  }
1716
1714
  }
@@ -1732,13 +1730,13 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
1732
1730
  matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
1733
1731
  if (matchLength == MaxML)
1734
1732
  {
1735
- U32 add = dumps<de ? *dumps++ : 0;
1733
+ const U32 add = dumps<de ? *dumps++ : 0;
1736
1734
  if (add < 255) matchLength += add;
1737
1735
  else
1738
1736
  {
1739
1737
  if (dumps<=(de-3))
1740
1738
  {
1741
- matchLength = ZSTD_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
1739
+ matchLength = ZSTD_readLE24(dumps);
1742
1740
  dumps += 3;
1743
1741
  }
1744
1742
  }
@@ -1907,8 +1905,10 @@ static size_t ZSTD_decompressSequences(
1907
1905
  {
1908
1906
  size_t lastLLSize = litEnd - litPtr;
1909
1907
  if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
1910
- if (op != litPtr) memmove(op, litPtr, lastLLSize);
1911
- op += lastLLSize;
1908
+ if (lastLLSize > 0) {
1909
+ if (op != litPtr) memmove(op, litPtr, lastLLSize);
1910
+ op += lastLLSize;
1911
+ }
1912
1912
  }
1913
1913
  }
1914
1914