zstdlib 0.3.0-x86-mingw32 → 0.8.0-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.
Files changed (88) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGES.md +30 -1
  3. data/README.md +2 -2
  4. data/Rakefile +1 -1
  5. data/ext/zstdlib/extconf.rb +3 -3
  6. data/ext/zstdlib/ruby/zlib-2.7/zstdlib.c +4895 -0
  7. data/ext/zstdlib/ruby/zlib-3.0/zstdlib.c +4994 -0
  8. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/bitstream.h +59 -51
  9. data/ext/zstdlib/zstd-1.5.0/lib/common/compiler.h +289 -0
  10. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/cpu.h +1 -3
  11. data/ext/zstdlib/zstd-1.5.0/lib/common/debug.c +24 -0
  12. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/debug.h +22 -49
  13. data/ext/zstdlib/zstd-1.5.0/lib/common/entropy_common.c +362 -0
  14. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/error_private.c +3 -1
  15. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/error_private.h +8 -4
  16. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/fse.h +50 -42
  17. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/fse_decompress.c +149 -55
  18. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/huf.h +43 -39
  19. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/mem.h +69 -25
  20. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/pool.c +30 -20
  21. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/pool.h +3 -3
  22. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/threading.c +51 -4
  23. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/threading.h +36 -4
  24. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/xxhash.c +40 -92
  25. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/xxhash.h +12 -32
  26. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/zstd_common.c +10 -10
  27. data/ext/zstdlib/zstd-1.5.0/lib/common/zstd_deps.h +111 -0
  28. data/ext/zstdlib/zstd-1.5.0/lib/common/zstd_internal.h +490 -0
  29. data/ext/zstdlib/zstd-1.5.0/lib/common/zstd_trace.h +154 -0
  30. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/fse_compress.c +47 -63
  31. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/hist.c +41 -63
  32. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/hist.h +13 -33
  33. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/huf_compress.c +332 -193
  34. data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_compress.c +6393 -0
  35. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_compress_internal.h +522 -86
  36. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_compress_literals.c +25 -16
  37. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_compress_literals.h +2 -2
  38. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_compress_sequences.c +50 -24
  39. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_compress_sequences.h +11 -4
  40. data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_compress_superblock.c +572 -0
  41. data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_compress_superblock.h +32 -0
  42. data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_cwksp.h +662 -0
  43. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_double_fast.c +43 -41
  44. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_double_fast.h +2 -2
  45. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_fast.c +85 -80
  46. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_fast.h +2 -2
  47. data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_lazy.c +2184 -0
  48. data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_lazy.h +125 -0
  49. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_ldm.c +333 -208
  50. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_ldm.h +15 -3
  51. data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_ldm_geartab.h +103 -0
  52. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_opt.c +228 -129
  53. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_opt.h +1 -1
  54. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstdmt_compress.c +151 -440
  55. data/ext/zstdlib/zstd-1.5.0/lib/compress/zstdmt_compress.h +110 -0
  56. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/huf_decompress.c +395 -276
  57. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_ddict.c +20 -16
  58. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_ddict.h +3 -3
  59. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_decompress.c +628 -231
  60. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_decompress_block.c +606 -380
  61. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_decompress_block.h +8 -5
  62. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_decompress_internal.h +39 -9
  63. data/ext/zstdlib/zstd-1.5.0/lib/zdict.h +452 -0
  64. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/zstd.h +740 -153
  65. data/ext/zstdlib/{zstd-1.4.2/lib/common → zstd-1.5.0/lib}/zstd_errors.h +3 -1
  66. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzclose.c +1 -1
  67. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzcompatibility.h +1 -1
  68. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzguts.h +0 -0
  69. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzlib.c +9 -9
  70. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzread.c +16 -8
  71. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzwrite.c +8 -8
  72. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/zstd_zlibwrapper.c +131 -45
  73. data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/zstd_zlibwrapper.h +1 -1
  74. data/lib/2.2/zstdlib.so +0 -0
  75. data/lib/2.3/zstdlib.so +0 -0
  76. data/lib/2.4/zstdlib.so +0 -0
  77. data/lib/2.5/zstdlib.so +0 -0
  78. data/lib/2.6/zstdlib.so +0 -0
  79. data/lib/2.7/zstdlib.so +0 -0
  80. metadata +76 -67
  81. data/ext/zstdlib/zstd-1.4.2/lib/common/compiler.h +0 -147
  82. data/ext/zstdlib/zstd-1.4.2/lib/common/debug.c +0 -44
  83. data/ext/zstdlib/zstd-1.4.2/lib/common/entropy_common.c +0 -236
  84. data/ext/zstdlib/zstd-1.4.2/lib/common/zstd_internal.h +0 -371
  85. data/ext/zstdlib/zstd-1.4.2/lib/compress/zstd_compress.c +0 -3904
  86. data/ext/zstdlib/zstd-1.4.2/lib/compress/zstd_lazy.c +0 -1111
  87. data/ext/zstdlib/zstd-1.4.2/lib/compress/zstd_lazy.h +0 -67
  88. data/ext/zstdlib/zstd-1.4.2/lib/compress/zstdmt_compress.h +0 -192
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -47,6 +47,8 @@ const char* ERR_getErrorString(ERR_enum code)
47
47
  /* following error codes are not stable and may be removed or changed in a future version */
48
48
  case PREFIX(frameIndex_tooLarge): return "Frame index is too large";
49
49
  case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
50
+ case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong";
51
+ case PREFIX(srcBuffer_wrong): return "Source buffer is wrong";
50
52
  case PREFIX(maxCode):
51
53
  default: return notErrorCode;
52
54
  }
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -21,8 +21,8 @@ extern "C" {
21
21
  /* ****************************************
22
22
  * Dependencies
23
23
  ******************************************/
24
- #include <stddef.h> /* size_t */
25
- #include "zstd_errors.h" /* enum list */
24
+ #include "../zstd_errors.h" /* enum list */
25
+ #include "zstd_deps.h" /* size_t */
26
26
 
27
27
 
28
28
  /* ****************************************
@@ -49,7 +49,7 @@ typedef ZSTD_ErrorCode ERR_enum;
49
49
  /*-****************************************
50
50
  * Error codes handling
51
51
  ******************************************/
52
- #undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */
52
+ #undef ERROR /* already defined on Visual Studio */
53
53
  #define ERROR(name) ZSTD_ERROR(name)
54
54
  #define ZSTD_ERROR(name) ((size_t)-PREFIX(name))
55
55
 
@@ -57,6 +57,10 @@ ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
57
57
 
58
58
  ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); }
59
59
 
60
+ /* check and forward error code */
61
+ #define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e
62
+ #define CHECK_F(f) { CHECK_V_F(_var_err__, f); }
63
+
60
64
 
61
65
  /*-****************************************
62
66
  * Error Strings
@@ -1,35 +1,15 @@
1
1
  /* ******************************************************************
2
- FSE : Finite State Entropy codec
3
- Public Prototypes declaration
4
- Copyright (C) 2013-2016, Yann Collet.
5
-
6
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
7
-
8
- Redistribution and use in source and binary forms, with or without
9
- modification, are permitted provided that the following conditions are
10
- met:
11
-
12
- * Redistributions of source code must retain the above copyright
13
- notice, this list of conditions and the following disclaimer.
14
- * Redistributions in binary form must reproduce the above
15
- copyright notice, this list of conditions and the following disclaimer
16
- in the documentation and/or other materials provided with the
17
- distribution.
18
-
19
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
-
31
- You can contact the author at :
32
- - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
2
+ * FSE : Finite State Entropy codec
3
+ * Public Prototypes declaration
4
+ * Copyright (c) Yann Collet, Facebook, Inc.
5
+ *
6
+ * You can contact the author at :
7
+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
8
+ *
9
+ * This source code is licensed under both the BSD-style license (found in the
10
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
11
+ * in the COPYING file in the root directory of this source tree).
12
+ * You may select, at your option, one of the above-listed licenses.
33
13
  ****************************************************************** */
34
14
 
35
15
  #if defined (__cplusplus)
@@ -43,7 +23,7 @@ extern "C" {
43
23
  /*-*****************************************
44
24
  * Dependencies
45
25
  ******************************************/
46
- #include <stddef.h> /* size_t, ptrdiff_t */
26
+ #include "zstd_deps.h" /* size_t, ptrdiff_t */
47
27
 
48
28
 
49
29
  /*-*****************************************
@@ -157,10 +137,16 @@ FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize
157
137
  /*! FSE_normalizeCount():
158
138
  normalize counts so that sum(count[]) == Power_of_2 (2^tableLog)
159
139
  'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
140
+ useLowProbCount is a boolean parameter which trades off compressed size for
141
+ faster header decoding. When it is set to 1, the compressed data will be slightly
142
+ smaller. And when it is set to 0, FSE_readNCount() and FSE_buildDTable() will be
143
+ faster. If you are compressing a small amount of data (< 2 KB) then useLowProbCount=0
144
+ is a good default, since header deserialization makes a big speed difference.
145
+ Otherwise, useLowProbCount=1 is a good default, since the speed difference is small.
160
146
  @return : tableLog,
161
147
  or an errorCode, which can be tested using FSE_isError() */
162
148
  FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog,
163
- const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
149
+ const unsigned* count, size_t srcSize, unsigned maxSymbolValue, unsigned useLowProbCount);
164
150
 
165
151
  /*! FSE_NCountWriteBound():
166
152
  Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'.
@@ -248,6 +234,13 @@ FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter,
248
234
  unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,
249
235
  const void* rBuffer, size_t rBuffSize);
250
236
 
237
+ /*! FSE_readNCount_bmi2():
238
+ * Same as FSE_readNCount() but pass bmi2=1 when your CPU supports BMI2 and 0 otherwise.
239
+ */
240
+ FSE_PUBLIC_API size_t FSE_readNCount_bmi2(short* normalizedCounter,
241
+ unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,
242
+ const void* rBuffer, size_t rBuffSize, int bmi2);
243
+
251
244
  /*! Constructor and Destructor of FSE_DTable.
252
245
  Note that its size depends on 'tableLog' */
253
246
  typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
@@ -308,12 +301,12 @@ If there is an error, the function will return an error code, which can be teste
308
301
  *******************************************/
309
302
  /* FSE buffer bounds */
310
303
  #define FSE_NCOUNTBOUND 512
311
- #define FSE_BLOCKBOUND(size) (size + (size>>7))
304
+ #define FSE_BLOCKBOUND(size) ((size) + ((size)>>7) + 4 /* fse states */ + sizeof(size_t) /* bitContainer */)
312
305
  #define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
313
306
 
314
307
  /* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */
315
- #define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2))
316
- #define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog))
308
+ #define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<((maxTableLog)-1)) + (((maxSymbolValue)+1)*2))
309
+ #define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<(maxTableLog)))
317
310
 
318
311
  /* or use the size to malloc() space directly. Pay attention to alignment restrictions though */
319
312
  #define FSE_CTABLE_SIZE(maxTableLog, maxSymbolValue) (FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(FSE_CTable))
@@ -329,9 +322,9 @@ unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsi
329
322
 
330
323
  /* FSE_compress_wksp() :
331
324
  * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
332
- * FSE_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable.
325
+ * FSE_COMPRESS_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable.
333
326
  */
334
- #define FSE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) )
327
+ #define FSE_COMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) )
335
328
  size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
336
329
 
337
330
  size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits);
@@ -342,18 +335,30 @@ size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
342
335
 
343
336
  /* FSE_buildCTable_wksp() :
344
337
  * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
345
- * `wkspSize` must be >= `(1<<tableLog)`.
338
+ * `wkspSize` must be >= `FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)` of `unsigned`.
346
339
  */
340
+ #define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (maxSymbolValue + 2 + (1ull << (tableLog - 2)))
341
+ #define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog))
347
342
  size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
348
343
 
344
+ #define FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) (sizeof(short) * (maxSymbolValue + 1) + (1ULL << maxTableLog) + 8)
345
+ #define FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ((FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) + sizeof(unsigned) - 1) / sizeof(unsigned))
346
+ FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
347
+ /**< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` */
348
+
349
349
  size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits);
350
350
  /**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */
351
351
 
352
352
  size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
353
353
  /**< build a fake FSE_DTable, designed to always generate the same symbolValue */
354
354
 
355
- size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog);
356
- /**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DTABLE_SIZE_U32(maxLog)` */
355
+ #define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1)
356
+ #define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned))
357
+ size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize);
358
+ /**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)` */
359
+
360
+ size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2);
361
+ /**< Same as FSE_decompress_wksp() but with dynamic BMI2 support. Pass 1 if your CPU supports BMI2 or 0 if it doesn't. */
357
362
 
358
363
  typedef enum {
359
364
  FSE_repeat_none, /**< Cannot use the previous table */
@@ -664,6 +669,9 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
664
669
  #ifndef FSE_DEFAULT_MEMORY_USAGE
665
670
  # define FSE_DEFAULT_MEMORY_USAGE 13
666
671
  #endif
672
+ #if (FSE_DEFAULT_MEMORY_USAGE > FSE_MAX_MEMORY_USAGE)
673
+ # error "FSE_DEFAULT_MEMORY_USAGE must be <= FSE_MAX_MEMORY_USAGE"
674
+ #endif
667
675
 
668
676
  /*!FSE_MAX_SYMBOL_VALUE :
669
677
  * Maximum symbol value authorized.
@@ -697,7 +705,7 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
697
705
  # error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported"
698
706
  #endif
699
707
 
700
- #define FSE_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3)
708
+ #define FSE_TABLESTEP(tableSize) (((tableSize)>>1) + ((tableSize)>>3) + 3)
701
709
 
702
710
 
703
711
  #endif /* FSE_STATIC_LINKING_ONLY */
@@ -1,48 +1,29 @@
1
1
  /* ******************************************************************
2
- FSE : Finite State Entropy decoder
3
- Copyright (C) 2013-2015, Yann Collet.
4
-
5
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
6
-
7
- Redistribution and use in source and binary forms, with or without
8
- modification, are permitted provided that the following conditions are
9
- met:
10
-
11
- * Redistributions of source code must retain the above copyright
12
- notice, this list of conditions and the following disclaimer.
13
- * Redistributions in binary form must reproduce the above
14
- copyright notice, this list of conditions and the following disclaimer
15
- in the documentation and/or other materials provided with the
16
- distribution.
17
-
18
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
-
30
- You can contact the author at :
31
- - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
32
- - Public forum : https://groups.google.com/forum/#!forum/lz4c
2
+ * FSE : Finite State Entropy decoder
3
+ * Copyright (c) Yann Collet, Facebook, Inc.
4
+ *
5
+ * You can contact the author at :
6
+ * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
7
+ * - Public forum : https://groups.google.com/forum/#!forum/lz4c
8
+ *
9
+ * This source code is licensed under both the BSD-style license (found in the
10
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
11
+ * in the COPYING file in the root directory of this source tree).
12
+ * You may select, at your option, one of the above-listed licenses.
33
13
  ****************************************************************** */
34
14
 
35
15
 
36
16
  /* **************************************************************
37
17
  * Includes
38
18
  ****************************************************************/
39
- #include <stdlib.h> /* malloc, free, qsort */
40
- #include <string.h> /* memcpy, memset */
19
+ #include "debug.h" /* assert */
41
20
  #include "bitstream.h"
42
21
  #include "compiler.h"
43
22
  #define FSE_STATIC_LINKING_ONLY
44
23
  #include "fse.h"
45
24
  #include "error_private.h"
25
+ #define ZSTD_DEPS_NEED_MALLOC
26
+ #include "zstd_deps.h"
46
27
 
47
28
 
48
29
  /* **************************************************************
@@ -51,9 +32,6 @@
51
32
  #define FSE_isError ERR_isError
52
33
  #define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */
53
34
 
54
- /* check and forward error code */
55
- #define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; }
56
-
57
35
 
58
36
  /* **************************************************************
59
37
  * Templates
@@ -82,25 +60,27 @@
82
60
  FSE_DTable* FSE_createDTable (unsigned tableLog)
83
61
  {
84
62
  if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
85
- return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) );
63
+ return (FSE_DTable*)ZSTD_malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) );
86
64
  }
87
65
 
88
66
  void FSE_freeDTable (FSE_DTable* dt)
89
67
  {
90
- free(dt);
68
+ ZSTD_free(dt);
91
69
  }
92
70
 
93
- size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
71
+ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
94
72
  {
95
73
  void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */
96
74
  FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr);
97
- U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1];
75
+ U16* symbolNext = (U16*)workSpace;
76
+ BYTE* spread = (BYTE*)(symbolNext + maxSymbolValue + 1);
98
77
 
99
78
  U32 const maxSV1 = maxSymbolValue + 1;
100
79
  U32 const tableSize = 1 << tableLog;
101
80
  U32 highThreshold = tableSize-1;
102
81
 
103
82
  /* Sanity Checks */
83
+ if (FSE_BUILD_DTABLE_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(maxSymbolValue_tooLarge);
104
84
  if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge);
105
85
  if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
106
86
 
@@ -118,11 +98,57 @@ size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned
118
98
  if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
119
99
  symbolNext[s] = normalizedCounter[s];
120
100
  } } }
121
- memcpy(dt, &DTableH, sizeof(DTableH));
101
+ ZSTD_memcpy(dt, &DTableH, sizeof(DTableH));
122
102
  }
123
103
 
124
104
  /* Spread symbols */
125
- { U32 const tableMask = tableSize-1;
105
+ if (highThreshold == tableSize - 1) {
106
+ size_t const tableMask = tableSize-1;
107
+ size_t const step = FSE_TABLESTEP(tableSize);
108
+ /* First lay down the symbols in order.
109
+ * We use a uint64_t to lay down 8 bytes at a time. This reduces branch
110
+ * misses since small blocks generally have small table logs, so nearly
111
+ * all symbols have counts <= 8. We ensure we have 8 bytes at the end of
112
+ * our buffer to handle the over-write.
113
+ */
114
+ {
115
+ U64 const add = 0x0101010101010101ull;
116
+ size_t pos = 0;
117
+ U64 sv = 0;
118
+ U32 s;
119
+ for (s=0; s<maxSV1; ++s, sv += add) {
120
+ int i;
121
+ int const n = normalizedCounter[s];
122
+ MEM_write64(spread + pos, sv);
123
+ for (i = 8; i < n; i += 8) {
124
+ MEM_write64(spread + pos + i, sv);
125
+ }
126
+ pos += n;
127
+ }
128
+ }
129
+ /* Now we spread those positions across the table.
130
+ * The benefit of doing it in two stages is that we avoid the the
131
+ * variable size inner loop, which caused lots of branch misses.
132
+ * Now we can run through all the positions without any branch misses.
133
+ * We unroll the loop twice, since that is what emperically worked best.
134
+ */
135
+ {
136
+ size_t position = 0;
137
+ size_t s;
138
+ size_t const unroll = 2;
139
+ assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */
140
+ for (s = 0; s < (size_t)tableSize; s += unroll) {
141
+ size_t u;
142
+ for (u = 0; u < unroll; ++u) {
143
+ size_t const uPosition = (position + (u * step)) & tableMask;
144
+ tableDecode[uPosition].symbol = spread[s + u];
145
+ }
146
+ position = (position + (unroll * step)) & tableMask;
147
+ }
148
+ assert(position == 0);
149
+ }
150
+ } else {
151
+ U32 const tableMask = tableSize-1;
126
152
  U32 const step = FSE_TABLESTEP(tableSize);
127
153
  U32 s, position = 0;
128
154
  for (s=0; s<maxSV1; s++) {
@@ -147,6 +173,11 @@ size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned
147
173
  return 0;
148
174
  }
149
175
 
176
+ size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
177
+ {
178
+ return FSE_buildDTable_internal(dt, normalizedCounter, maxSymbolValue, tableLog, workSpace, wkspSize);
179
+ }
180
+
150
181
 
151
182
  #ifndef FSE_COMMONDEFS_ONLY
152
183
 
@@ -274,36 +305,99 @@ size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
274
305
  }
275
306
 
276
307
 
277
- size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog)
308
+ size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize)
309
+ {
310
+ return FSE_decompress_wksp_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, /* bmi2 */ 0);
311
+ }
312
+
313
+ typedef struct {
314
+ short ncount[FSE_MAX_SYMBOL_VALUE + 1];
315
+ FSE_DTable dtable[1]; /* Dynamically sized */
316
+ } FSE_DecompressWksp;
317
+
318
+
319
+ FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body(
320
+ void* dst, size_t dstCapacity,
321
+ const void* cSrc, size_t cSrcSize,
322
+ unsigned maxLog, void* workSpace, size_t wkspSize,
323
+ int bmi2)
278
324
  {
279
325
  const BYTE* const istart = (const BYTE*)cSrc;
280
326
  const BYTE* ip = istart;
281
- short counting[FSE_MAX_SYMBOL_VALUE+1];
282
327
  unsigned tableLog;
283
328
  unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
329
+ FSE_DecompressWksp* const wksp = (FSE_DecompressWksp*)workSpace;
330
+
331
+ DEBUG_STATIC_ASSERT((FSE_MAX_SYMBOL_VALUE + 1) % 2 == 0);
332
+ if (wkspSize < sizeof(*wksp)) return ERROR(GENERIC);
284
333
 
285
334
  /* normal FSE decoding mode */
286
- size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
287
- if (FSE_isError(NCountLength)) return NCountLength;
288
- //if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size; supposed to be already checked in NCountLength, only remaining case : NCountLength==cSrcSize */
289
- if (tableLog > maxLog) return ERROR(tableLog_tooLarge);
290
- ip += NCountLength;
291
- cSrcSize -= NCountLength;
335
+ {
336
+ size_t const NCountLength = FSE_readNCount_bmi2(wksp->ncount, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2);
337
+ if (FSE_isError(NCountLength)) return NCountLength;
338
+ if (tableLog > maxLog) return ERROR(tableLog_tooLarge);
339
+ assert(NCountLength <= cSrcSize);
340
+ ip += NCountLength;
341
+ cSrcSize -= NCountLength;
342
+ }
343
+
344
+ if (FSE_DECOMPRESS_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(tableLog_tooLarge);
345
+ workSpace = wksp->dtable + FSE_DTABLE_SIZE_U32(tableLog);
346
+ wkspSize -= sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog);
292
347
 
293
- CHECK_F( FSE_buildDTable (workSpace, counting, maxSymbolValue, tableLog) );
348
+ CHECK_F( FSE_buildDTable_internal(wksp->dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) );
294
349
 
295
- return FSE_decompress_usingDTable (dst, dstCapacity, ip, cSrcSize, workSpace); /* always return, even if it is an error code */
350
+ {
351
+ const void* ptr = wksp->dtable;
352
+ const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;
353
+ const U32 fastMode = DTableH->fastMode;
354
+
355
+ /* select fast mode (static) */
356
+ if (fastMode) return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 1);
357
+ return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 0);
358
+ }
359
+ }
360
+
361
+ /* Avoids the FORCE_INLINE of the _body() function. */
362
+ static size_t FSE_decompress_wksp_body_default(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize)
363
+ {
364
+ return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 0);
365
+ }
366
+
367
+ #if DYNAMIC_BMI2
368
+ TARGET_ATTRIBUTE("bmi2") static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize)
369
+ {
370
+ return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 1);
371
+ }
372
+ #endif
373
+
374
+ size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2)
375
+ {
376
+ #if DYNAMIC_BMI2
377
+ if (bmi2) {
378
+ return FSE_decompress_wksp_body_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize);
379
+ }
380
+ #endif
381
+ (void)bmi2;
382
+ return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize);
296
383
  }
297
384
 
298
385
 
299
386
  typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
300
387
 
388
+ #ifndef ZSTD_NO_UNUSED_FUNCTIONS
389
+ size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) {
390
+ U32 wksp[FSE_BUILD_DTABLE_WKSP_SIZE_U32(FSE_TABLELOG_ABSOLUTE_MAX, FSE_MAX_SYMBOL_VALUE)];
391
+ return FSE_buildDTable_wksp(dt, normalizedCounter, maxSymbolValue, tableLog, wksp, sizeof(wksp));
392
+ }
393
+
301
394
  size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize)
302
395
  {
303
- DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */
304
- return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, dt, FSE_MAX_TABLELOG);
396
+ /* Static analyzer seems unable to understand this table will be properly initialized later */
397
+ U32 wksp[FSE_DECOMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
398
+ return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, FSE_MAX_TABLELOG, wksp, sizeof(wksp));
305
399
  }
306
-
400
+ #endif
307
401
 
308
402
 
309
403
  #endif /* FSE_COMMONDEFS_ONLY */