extzstd 0.2 → 0.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (88) hide show
  1. checksums.yaml +4 -4
  2. data/HISTORY.ja.md +13 -0
  3. data/README.md +17 -14
  4. data/contrib/zstd/{NEWS → CHANGELOG} +115 -2
  5. data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
  6. data/contrib/zstd/Makefile +99 -53
  7. data/contrib/zstd/README.md +59 -39
  8. data/contrib/zstd/TESTING.md +1 -1
  9. data/contrib/zstd/appveyor.yml +17 -6
  10. data/contrib/zstd/lib/BUCK +29 -2
  11. data/contrib/zstd/lib/Makefile +118 -21
  12. data/contrib/zstd/lib/README.md +84 -44
  13. data/contrib/zstd/lib/common/bitstream.h +17 -33
  14. data/contrib/zstd/lib/common/compiler.h +62 -8
  15. data/contrib/zstd/lib/common/cpu.h +215 -0
  16. data/contrib/zstd/lib/common/debug.c +44 -0
  17. data/contrib/zstd/lib/common/debug.h +134 -0
  18. data/contrib/zstd/lib/common/entropy_common.c +16 -1
  19. data/contrib/zstd/lib/common/error_private.c +7 -0
  20. data/contrib/zstd/lib/common/fse.h +48 -44
  21. data/contrib/zstd/lib/common/fse_decompress.c +3 -3
  22. data/contrib/zstd/lib/common/huf.h +169 -113
  23. data/contrib/zstd/lib/common/mem.h +20 -2
  24. data/contrib/zstd/lib/common/pool.c +135 -49
  25. data/contrib/zstd/lib/common/pool.h +40 -21
  26. data/contrib/zstd/lib/common/threading.c +2 -2
  27. data/contrib/zstd/lib/common/threading.h +12 -12
  28. data/contrib/zstd/lib/common/xxhash.c +3 -2
  29. data/contrib/zstd/lib/common/zstd_common.c +3 -6
  30. data/contrib/zstd/lib/common/zstd_errors.h +17 -7
  31. data/contrib/zstd/lib/common/zstd_internal.h +76 -48
  32. data/contrib/zstd/lib/compress/fse_compress.c +89 -209
  33. data/contrib/zstd/lib/compress/hist.c +203 -0
  34. data/contrib/zstd/lib/compress/hist.h +95 -0
  35. data/contrib/zstd/lib/compress/huf_compress.c +188 -80
  36. data/contrib/zstd/lib/compress/zstd_compress.c +2500 -1203
  37. data/contrib/zstd/lib/compress/zstd_compress_internal.h +463 -62
  38. data/contrib/zstd/lib/compress/zstd_double_fast.c +321 -131
  39. data/contrib/zstd/lib/compress/zstd_double_fast.h +13 -4
  40. data/contrib/zstd/lib/compress/zstd_fast.c +335 -108
  41. data/contrib/zstd/lib/compress/zstd_fast.h +12 -6
  42. data/contrib/zstd/lib/compress/zstd_lazy.c +654 -313
  43. data/contrib/zstd/lib/compress/zstd_lazy.h +44 -16
  44. data/contrib/zstd/lib/compress/zstd_ldm.c +310 -420
  45. data/contrib/zstd/lib/compress/zstd_ldm.h +63 -26
  46. data/contrib/zstd/lib/compress/zstd_opt.c +773 -325
  47. data/contrib/zstd/lib/compress/zstd_opt.h +31 -5
  48. data/contrib/zstd/lib/compress/zstdmt_compress.c +1468 -518
  49. data/contrib/zstd/lib/compress/zstdmt_compress.h +96 -45
  50. data/contrib/zstd/lib/decompress/huf_decompress.c +518 -282
  51. data/contrib/zstd/lib/decompress/zstd_ddict.c +240 -0
  52. data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
  53. data/contrib/zstd/lib/decompress/zstd_decompress.c +613 -1513
  54. data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1311 -0
  55. data/contrib/zstd/lib/decompress/zstd_decompress_block.h +59 -0
  56. data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +175 -0
  57. data/contrib/zstd/lib/dictBuilder/cover.c +194 -113
  58. data/contrib/zstd/lib/dictBuilder/cover.h +112 -0
  59. data/contrib/zstd/lib/dictBuilder/divsufsort.c +3 -3
  60. data/contrib/zstd/lib/dictBuilder/fastcover.c +740 -0
  61. data/contrib/zstd/lib/dictBuilder/zdict.c +142 -106
  62. data/contrib/zstd/lib/dictBuilder/zdict.h +115 -49
  63. data/contrib/zstd/lib/legacy/zstd_legacy.h +44 -12
  64. data/contrib/zstd/lib/legacy/zstd_v01.c +41 -10
  65. data/contrib/zstd/lib/legacy/zstd_v01.h +12 -7
  66. data/contrib/zstd/lib/legacy/zstd_v02.c +37 -12
  67. data/contrib/zstd/lib/legacy/zstd_v02.h +12 -7
  68. data/contrib/zstd/lib/legacy/zstd_v03.c +38 -12
  69. data/contrib/zstd/lib/legacy/zstd_v03.h +12 -7
  70. data/contrib/zstd/lib/legacy/zstd_v04.c +55 -174
  71. data/contrib/zstd/lib/legacy/zstd_v04.h +12 -7
  72. data/contrib/zstd/lib/legacy/zstd_v05.c +59 -31
  73. data/contrib/zstd/lib/legacy/zstd_v05.h +12 -7
  74. data/contrib/zstd/lib/legacy/zstd_v06.c +48 -20
  75. data/contrib/zstd/lib/legacy/zstd_v06.h +10 -5
  76. data/contrib/zstd/lib/legacy/zstd_v07.c +62 -29
  77. data/contrib/zstd/lib/legacy/zstd_v07.h +10 -5
  78. data/contrib/zstd/lib/zstd.h +1346 -832
  79. data/ext/extzstd.c +27 -19
  80. data/ext/extzstd_stream.c +20 -4
  81. data/ext/zstd_compress.c +1 -0
  82. data/ext/zstd_decompress.c +4 -0
  83. data/ext/zstd_dictbuilder.c +4 -0
  84. data/ext/zstd_dictbuilder_fastcover.c +5 -0
  85. data/lib/extzstd.rb +52 -220
  86. data/lib/extzstd/version.rb +1 -1
  87. metadata +21 -7
  88. data/contrib/zstd/circle.yml +0 -63
@@ -0,0 +1,44 @@
1
+ /* ******************************************************************
2
+ debug
3
+ Part of FSE library
4
+ Copyright (C) 2013-present, Yann Collet.
5
+
6
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
7
+
8
+ Redistribution and use in source and binary forms, with or without
9
+ modification, are permitted provided that the following conditions are
10
+ met:
11
+
12
+ * Redistributions of source code must retain the above copyright
13
+ notice, this list of conditions and the following disclaimer.
14
+ * Redistributions in binary form must reproduce the above
15
+ copyright notice, this list of conditions and the following disclaimer
16
+ in the documentation and/or other materials provided with the
17
+ distribution.
18
+
19
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
+
31
+ You can contact the author at :
32
+ - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
33
+ ****************************************************************** */
34
+
35
+
36
+ /*
37
+ * This module only hosts one global variable
38
+ * which can be used to dynamically influence the verbosity of traces,
39
+ * such as DEBUGLOG and RAWLOG
40
+ */
41
+
42
+ #include "debug.h"
43
+
44
+ int g_debuglevel = DEBUGLEVEL;
@@ -0,0 +1,134 @@
1
+ /* ******************************************************************
2
+ debug
3
+ Part of FSE library
4
+ Copyright (C) 2013-present, Yann Collet.
5
+
6
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
7
+
8
+ Redistribution and use in source and binary forms, with or without
9
+ modification, are permitted provided that the following conditions are
10
+ met:
11
+
12
+ * Redistributions of source code must retain the above copyright
13
+ notice, this list of conditions and the following disclaimer.
14
+ * Redistributions in binary form must reproduce the above
15
+ copyright notice, this list of conditions and the following disclaimer
16
+ in the documentation and/or other materials provided with the
17
+ distribution.
18
+
19
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
+
31
+ You can contact the author at :
32
+ - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
33
+ ****************************************************************** */
34
+
35
+
36
+ /*
37
+ * The purpose of this header is to enable debug functions.
38
+ * They regroup assert(), DEBUGLOG() and RAWLOG() for run-time,
39
+ * and DEBUG_STATIC_ASSERT() for compile-time.
40
+ *
41
+ * By default, DEBUGLEVEL==0, which means run-time debug is disabled.
42
+ *
43
+ * Level 1 enables assert() only.
44
+ * Starting level 2, traces can be generated and pushed to stderr.
45
+ * The higher the level, the more verbose the traces.
46
+ *
47
+ * It's possible to dynamically adjust level using variable g_debug_level,
48
+ * which is only declared if DEBUGLEVEL>=2,
49
+ * and is a global variable, not multi-thread protected (use with care)
50
+ */
51
+
52
+ #ifndef DEBUG_H_12987983217
53
+ #define DEBUG_H_12987983217
54
+
55
+ #if defined (__cplusplus)
56
+ extern "C" {
57
+ #endif
58
+
59
+
60
+ /* static assert is triggered at compile time, leaving no runtime artefact.
61
+ * static assert only works with compile-time constants.
62
+ * Also, this variant can only be used inside a function. */
63
+ #define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1])
64
+
65
+
66
+ /* DEBUGLEVEL is expected to be defined externally,
67
+ * typically through compiler command line.
68
+ * Value must be a number. */
69
+ #ifndef DEBUGLEVEL
70
+ # define DEBUGLEVEL 0
71
+ #endif
72
+
73
+
74
+ /* DEBUGFILE can be defined externally,
75
+ * typically through compiler command line.
76
+ * note : currently useless.
77
+ * Value must be stderr or stdout */
78
+ #ifndef DEBUGFILE
79
+ # define DEBUGFILE stderr
80
+ #endif
81
+
82
+
83
+ /* recommended values for DEBUGLEVEL :
84
+ * 0 : release mode, no debug, all run-time checks disabled
85
+ * 1 : enables assert() only, no display
86
+ * 2 : reserved, for currently active debug path
87
+ * 3 : events once per object lifetime (CCtx, CDict, etc.)
88
+ * 4 : events once per frame
89
+ * 5 : events once per block
90
+ * 6 : events once per sequence (verbose)
91
+ * 7+: events at every position (*very* verbose)
92
+ *
93
+ * It's generally inconvenient to output traces > 5.
94
+ * In which case, it's possible to selectively trigger high verbosity levels
95
+ * by modifying g_debug_level.
96
+ */
97
+
98
+ #if (DEBUGLEVEL>=1)
99
+ # include <assert.h>
100
+ #else
101
+ # ifndef assert /* assert may be already defined, due to prior #include <assert.h> */
102
+ # define assert(condition) ((void)0) /* disable assert (default) */
103
+ # endif
104
+ #endif
105
+
106
+ #if (DEBUGLEVEL>=2)
107
+ # include <stdio.h>
108
+ extern int g_debuglevel; /* the variable is only declared,
109
+ it actually lives in debug.c,
110
+ and is shared by the whole process.
111
+ It's not thread-safe.
112
+ It's useful when enabling very verbose levels
113
+ on selective conditions (such as position in src) */
114
+
115
+ # define RAWLOG(l, ...) { \
116
+ if (l<=g_debuglevel) { \
117
+ fprintf(stderr, __VA_ARGS__); \
118
+ } }
119
+ # define DEBUGLOG(l, ...) { \
120
+ if (l<=g_debuglevel) { \
121
+ fprintf(stderr, __FILE__ ": " __VA_ARGS__); \
122
+ fprintf(stderr, " \n"); \
123
+ } }
124
+ #else
125
+ # define RAWLOG(l, ...) {} /* disabled */
126
+ # define DEBUGLOG(l, ...) {} /* disabled */
127
+ #endif
128
+
129
+
130
+ #if defined (__cplusplus)
131
+ }
132
+ #endif
133
+
134
+ #endif /* DEBUG_H_12987983217 */
@@ -72,7 +72,21 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
72
72
  unsigned charnum = 0;
73
73
  int previous0 = 0;
74
74
 
75
- if (hbSize < 4) return ERROR(srcSize_wrong);
75
+ if (hbSize < 4) {
76
+ /* This function only works when hbSize >= 4 */
77
+ char buffer[4];
78
+ memset(buffer, 0, sizeof(buffer));
79
+ memcpy(buffer, headerBuffer, hbSize);
80
+ { size_t const countSize = FSE_readNCount(normalizedCounter, maxSVPtr, tableLogPtr,
81
+ buffer, sizeof(buffer));
82
+ if (FSE_isError(countSize)) return countSize;
83
+ if (countSize > hbSize) return ERROR(corruption_detected);
84
+ return countSize;
85
+ } }
86
+ assert(hbSize >= 4);
87
+
88
+ /* init */
89
+ memset(normalizedCounter, 0, (*maxSVPtr+1) * sizeof(normalizedCounter[0])); /* all symbols not present in NCount have a frequency of 0 */
76
90
  bitStream = MEM_readLE32(ip);
77
91
  nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */
78
92
  if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge);
@@ -105,6 +119,7 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
105
119
  if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall);
106
120
  while (charnum < n0) normalizedCounter[charnum++] = 0;
107
121
  if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
122
+ assert((bitCount >> 3) <= 3); /* For first condition to work */
108
123
  ip += bitCount>>3;
109
124
  bitCount &= 7;
110
125
  bitStream = MEM_readLE32(ip) >> bitCount;
@@ -14,6 +14,10 @@
14
14
 
15
15
  const char* ERR_getErrorString(ERR_enum code)
16
16
  {
17
+ #ifdef ZSTD_STRIP_ERROR_STRINGS
18
+ (void)code;
19
+ return "Error strings stripped";
20
+ #else
17
21
  static const char* const notErrorCode = "Unspecified error code";
18
22
  switch( code )
19
23
  {
@@ -29,6 +33,7 @@ const char* ERR_getErrorString(ERR_enum code)
29
33
  case PREFIX(parameter_outOfBound): return "Parameter is out of bound";
30
34
  case PREFIX(init_missing): return "Context should be init first";
31
35
  case PREFIX(memory_allocation): return "Allocation error : not enough memory";
36
+ case PREFIX(workSpace_tooSmall): return "workSpace buffer is not large enough";
32
37
  case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
33
38
  case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
34
39
  case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
@@ -38,10 +43,12 @@ const char* ERR_getErrorString(ERR_enum code)
38
43
  case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples";
39
44
  case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
40
45
  case PREFIX(srcSize_wrong): return "Src size is incorrect";
46
+ case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer";
41
47
  /* following error codes are not stable and may be removed or changed in a future version */
42
48
  case PREFIX(frameIndex_tooLarge): return "Frame index is too large";
43
49
  case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
44
50
  case PREFIX(maxCode):
45
51
  default: return notErrorCode;
46
52
  }
53
+ #endif
47
54
  }
@@ -72,6 +72,7 @@ extern "C" {
72
72
  #define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE)
73
73
  FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */
74
74
 
75
+
75
76
  /*-****************************************
76
77
  * FSE simple functions
77
78
  ******************************************/
@@ -129,7 +130,7 @@ FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src,
129
130
  ******************************************/
130
131
  /*!
131
132
  FSE_compress() does the following:
132
- 1. count symbol occurrence from source[] into table count[]
133
+ 1. count symbol occurrence from source[] into table count[] (see hist.h)
133
134
  2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
134
135
  3. save normalized counters to memory buffer using writeNCount()
135
136
  4. build encoding table 'CTable' from normalized counters
@@ -147,15 +148,6 @@ or to save and provide normalized distribution using external method.
147
148
 
148
149
  /* *** COMPRESSION *** */
149
150
 
150
- /*! FSE_count():
151
- Provides the precise count of each byte within a table 'count'.
152
- 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1).
153
- *maxSymbolValuePtr will be updated if detected smaller than initial value.
154
- @return : the count of the most frequent symbol (which is not identified).
155
- if return == srcSize, there is only one symbol.
156
- Can also return an error code, which can be tested with FSE_isError(). */
157
- FSE_PUBLIC_API size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
158
-
159
151
  /*! FSE_optimalTableLog():
160
152
  dynamically downsize 'tableLog' when conditions are met.
161
153
  It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
@@ -167,7 +159,8 @@ FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize
167
159
  'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
168
160
  @return : tableLog,
169
161
  or an errorCode, which can be tested using FSE_isError() */
170
- FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
162
+ FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog,
163
+ const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
171
164
 
172
165
  /*! FSE_NCountWriteBound():
173
166
  Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'.
@@ -178,8 +171,9 @@ FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tab
178
171
  Compactly save 'normalizedCounter' into 'buffer'.
179
172
  @return : size of the compressed table,
180
173
  or an errorCode, which can be tested using FSE_isError(). */
181
- FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
182
-
174
+ FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize,
175
+ const short* normalizedCounter,
176
+ unsigned maxSymbolValue, unsigned tableLog);
183
177
 
184
178
  /*! Constructor and Destructor of FSE_CTable.
185
179
  Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
@@ -250,7 +244,9 @@ If there is an error, the function will return an ErrorCode (which can be tested
250
244
  @return : size read from 'rBuffer',
251
245
  or an errorCode, which can be tested using FSE_isError().
252
246
  maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
253
- FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize);
247
+ FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter,
248
+ unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,
249
+ const void* rBuffer, size_t rBuffSize);
254
250
 
255
251
  /*! Constructor and Destructor of FSE_DTable.
256
252
  Note that its size depends on 'tableLog' */
@@ -325,33 +321,8 @@ If there is an error, the function will return an error code, which can be teste
325
321
 
326
322
 
327
323
  /* *****************************************
328
- * FSE advanced API
329
- *******************************************/
330
- /* FSE_count_wksp() :
331
- * Same as FSE_count(), but using an externally provided scratch buffer.
332
- * `workSpace` size must be table of >= `1024` unsigned
333
- */
334
- size_t FSE_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
335
- const void* source, size_t sourceSize, unsigned* workSpace);
336
-
337
- /** FSE_countFast() :
338
- * same as FSE_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr
339
- */
340
- size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
341
-
342
- /* FSE_countFast_wksp() :
343
- * Same as FSE_countFast(), but using an externally provided scratch buffer.
344
- * `workSpace` must be a table of minimum `1024` unsigned
345
- */
346
- size_t FSE_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* workSpace);
347
-
348
- /*! FSE_count_simple
349
- * Same as FSE_countFast(), but does not use any additional memory (not even on stack).
350
- * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` (presuming it's also the size of `count`).
351
- */
352
- size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
353
-
354
-
324
+ * FSE advanced API
325
+ ***************************************** */
355
326
 
356
327
  unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
357
328
  /**< same as FSE_optimalTableLog(), which used `minus==2` */
@@ -387,7 +358,7 @@ size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size
387
358
  typedef enum {
388
359
  FSE_repeat_none, /**< Cannot use the previous table */
389
360
  FSE_repeat_check, /**< Can use the previous table but it must be checked */
390
- FSE_repeat_valid /**< Can use the previous table and it is asumed to be valid */
361
+ FSE_repeat_valid /**< Can use the previous table and it is assumed to be valid */
391
362
  } FSE_repeat;
392
363
 
393
364
  /* *****************************************
@@ -541,7 +512,7 @@ MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
541
512
  const U32 tableLog = MEM_read16(ptr);
542
513
  statePtr->value = (ptrdiff_t)1<<tableLog;
543
514
  statePtr->stateTable = u16ptr+2;
544
- statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1));
515
+ statePtr->symbolTT = ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1);
545
516
  statePtr->stateLog = tableLog;
546
517
  }
547
518
 
@@ -560,7 +531,7 @@ MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U3
560
531
  }
561
532
  }
562
533
 
563
- MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol)
534
+ MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, unsigned symbol)
564
535
  {
565
536
  FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
566
537
  const U16* const stateTable = (const U16*)(statePtr->stateTable);
@@ -576,6 +547,39 @@ MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePt
576
547
  }
577
548
 
578
549
 
550
+ /* FSE_getMaxNbBits() :
551
+ * Approximate maximum cost of a symbol, in bits.
552
+ * Fractional get rounded up (i.e : a symbol with a normalized frequency of 3 gives the same result as a frequency of 2)
553
+ * note 1 : assume symbolValue is valid (<= maxSymbolValue)
554
+ * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
555
+ MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue)
556
+ {
557
+ const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr;
558
+ return (symbolTT[symbolValue].deltaNbBits + ((1<<16)-1)) >> 16;
559
+ }
560
+
561
+ /* FSE_bitCost() :
562
+ * Approximate symbol cost, as fractional value, using fixed-point format (accuracyLog fractional bits)
563
+ * note 1 : assume symbolValue is valid (<= maxSymbolValue)
564
+ * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
565
+ MEM_STATIC U32 FSE_bitCost(const void* symbolTTPtr, U32 tableLog, U32 symbolValue, U32 accuracyLog)
566
+ {
567
+ const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr;
568
+ U32 const minNbBits = symbolTT[symbolValue].deltaNbBits >> 16;
569
+ U32 const threshold = (minNbBits+1) << 16;
570
+ assert(tableLog < 16);
571
+ assert(accuracyLog < 31-tableLog); /* ensure enough room for renormalization double shift */
572
+ { U32 const tableSize = 1 << tableLog;
573
+ U32 const deltaFromThreshold = threshold - (symbolTT[symbolValue].deltaNbBits + tableSize);
574
+ U32 const normalizedDeltaFromThreshold = (deltaFromThreshold << accuracyLog) >> tableLog; /* linear interpolation (very approximate) */
575
+ U32 const bitMultiplier = 1 << accuracyLog;
576
+ assert(symbolTT[symbolValue].deltaNbBits + tableSize <= threshold);
577
+ assert(normalizedDeltaFromThreshold <= bitMultiplier);
578
+ return (minNbBits+1)*bitMultiplier - normalizedDeltaFromThreshold;
579
+ }
580
+ }
581
+
582
+
579
583
  /* ====== Decompression ====== */
580
584
 
581
585
  typedef struct {
@@ -49,7 +49,7 @@
49
49
  * Error Management
50
50
  ****************************************************************/
51
51
  #define FSE_isError ERR_isError
52
- #define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
52
+ #define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */
53
53
 
54
54
  /* check and forward error code */
55
55
  #define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; }
@@ -139,8 +139,8 @@ size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned
139
139
  { U32 u;
140
140
  for (u=0; u<tableSize; u++) {
141
141
  FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
142
- U16 nextState = symbolNext[symbol]++;
143
- tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32 ((U32)nextState) );
142
+ U32 const nextState = symbolNext[symbol]++;
143
+ tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32(nextState) );
144
144
  tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
145
145
  } }
146
146
 
@@ -1,7 +1,7 @@
1
1
  /* ******************************************************************
2
- Huffman coder, part of New Generation Entropy library
3
- header file
4
- Copyright (C) 2013-2016, Yann Collet.
2
+ huff0 huffman codec,
3
+ part of Finite State Entropy library
4
+ Copyright (C) 2013-present, Yann Collet.
5
5
 
6
6
  BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
7
7
 
@@ -58,32 +58,32 @@ extern "C" {
58
58
  #endif
59
59
 
60
60
 
61
- /* *** simple functions *** */
62
- /**
63
- HUF_compress() :
64
- Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'.
65
- 'dst' buffer must be already allocated.
66
- Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize).
67
- `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB.
68
- @return : size of compressed data (<= `dstCapacity`).
69
- Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
70
- if return == 1, srcData is a single repeated byte symbol (RLE compression).
71
- if HUF_isError(return), compression failed (more details using HUF_getErrorName())
72
- */
61
+ /* ========================== */
62
+ /* *** simple functions *** */
63
+ /* ========================== */
64
+
65
+ /** HUF_compress() :
66
+ * Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'.
67
+ * 'dst' buffer must be already allocated.
68
+ * Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize).
69
+ * `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB.
70
+ * @return : size of compressed data (<= `dstCapacity`).
71
+ * Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
72
+ * if HUF_isError(return), compression failed (more details using HUF_getErrorName())
73
+ */
73
74
  HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity,
74
75
  const void* src, size_t srcSize);
75
76
 
76
- /**
77
- HUF_decompress() :
78
- Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
79
- into already allocated buffer 'dst', of minimum size 'dstSize'.
80
- `originalSize` : **must** be the ***exact*** size of original (uncompressed) data.
81
- Note : in contrast with FSE, HUF_decompress can regenerate
82
- RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
83
- because it knows size to regenerate.
84
- @return : size of regenerated data (== originalSize),
85
- or an error code, which can be tested using HUF_isError()
86
- */
77
+ /** HUF_decompress() :
78
+ * Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
79
+ * into already allocated buffer 'dst', of minimum size 'dstSize'.
80
+ * `originalSize` : **must** be the ***exact*** size of original (uncompressed) data.
81
+ * Note : in contrast with FSE, HUF_decompress can regenerate
82
+ * RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
83
+ * because it knows size to regenerate (originalSize).
84
+ * @return : size of regenerated data (== originalSize),
85
+ * or an error code, which can be tested using HUF_isError()
86
+ */
87
87
  HUF_PUBLIC_API size_t HUF_decompress(void* dst, size_t originalSize,
88
88
  const void* cSrc, size_t cSrcSize);
89
89
 
@@ -100,39 +100,32 @@ HUF_PUBLIC_API const char* HUF_getErrorName(size_t code); /**< provides error c
100
100
  /* *** Advanced function *** */
101
101
 
102
102
  /** HUF_compress2() :
103
- * Same as HUF_compress(), but offers direct control over `maxSymbolValue` and `tableLog`.
104
- * `tableLog` must be `<= HUF_TABLELOG_MAX` . */
105
- HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
103
+ * Same as HUF_compress(), but offers control over `maxSymbolValue` and `tableLog`.
104
+ * `maxSymbolValue` must be <= HUF_SYMBOLVALUE_MAX .
105
+ * `tableLog` must be `<= HUF_TABLELOG_MAX` . */
106
+ HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity,
107
+ const void* src, size_t srcSize,
108
+ unsigned maxSymbolValue, unsigned tableLog);
106
109
 
107
110
  /** HUF_compress4X_wksp() :
108
111
  * Same as HUF_compress2(), but uses externally allocated `workSpace`.
109
- * `workspace` must have minimum alignment of 4, and be at least as large as following macro */
112
+ * `workspace` must have minimum alignment of 4, and be at least as large as HUF_WORKSPACE_SIZE */
110
113
  #define HUF_WORKSPACE_SIZE (6 << 10)
111
114
  #define HUF_WORKSPACE_SIZE_U32 (HUF_WORKSPACE_SIZE / sizeof(U32))
112
- HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
113
-
114
- /**
115
- * The minimum workspace size for the `workSpace` used in
116
- * HUF_readDTableX2_wksp() and HUF_readDTableX4_wksp().
117
- *
118
- * The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when
119
- * HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15.
120
- * Buffer overflow errors may potentially occur if code modifications result in
121
- * a required workspace size greater than that specified in the following
122
- * macro.
123
- */
124
- #define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10)
125
- #define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
115
+ HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity,
116
+ const void* src, size_t srcSize,
117
+ unsigned maxSymbolValue, unsigned tableLog,
118
+ void* workSpace, size_t wkspSize);
126
119
 
127
120
  #endif /* HUF_H_298734234 */
128
121
 
129
122
  /* ******************************************************************
130
123
  * WARNING !!
131
124
  * The following section contains advanced and experimental definitions
132
- * which shall never be used in the context of dll
125
+ * which shall never be used in the context of a dynamic library,
133
126
  * because they are not guaranteed to remain stable in the future.
134
127
  * Only consider them in association with static linking.
135
- *******************************************************************/
128
+ * *****************************************************************/
136
129
  #if defined(HUF_STATIC_LINKING_ONLY) && !defined(HUF_H_HUF_STATIC_LINKING_ONLY)
137
130
  #define HUF_H_HUF_STATIC_LINKING_ONLY
138
131
 
@@ -141,11 +134,11 @@ HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, const
141
134
 
142
135
 
143
136
  /* *** Constants *** */
144
- #define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
145
- #define HUF_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */
137
+ #define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
138
+ #define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */
146
139
  #define HUF_SYMBOLVALUE_MAX 255
147
140
 
148
- #define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
141
+ #define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
149
142
  #if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
150
143
  # error "HUF_TABLELOG_MAX is too large !"
151
144
  #endif
@@ -170,130 +163,193 @@ HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, const
170
163
  /* static allocation of HUF's DTable */
171
164
  typedef U32 HUF_DTable;
172
165
  #define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog)))
173
- #define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
166
+ #define HUF_CREATE_STATIC_DTABLEX1(DTable, maxTableLog) \
174
167
  HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) }
175
- #define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \
168
+ #define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
176
169
  HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) }
177
170
 
178
171
 
179
172
  /* ****************************************
180
173
  * Advanced decompression functions
181
174
  ******************************************/
182
- size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
183
- size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
175
+ size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
176
+ #ifndef HUF_FORCE_DECOMPRESS_X1
177
+ size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
178
+ #endif
184
179
 
185
180
  size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */
186
181
  size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
187
182
  size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */
188
- size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
189
- size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
190
- size_t HUF_decompress4X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
191
- size_t HUF_decompress4X4_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
183
+ size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
184
+ size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
185
+ #ifndef HUF_FORCE_DECOMPRESS_X1
186
+ size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
187
+ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
188
+ #endif
192
189
 
193
190
 
194
191
  /* ****************************************
195
- * HUF detailed API
196
- ******************************************/
197
- /*!
198
- HUF_compress() does the following:
199
- 1. count symbol occurrence from source[] into table count[] using FSE_count()
200
- 2. (optional) refine tableLog using HUF_optimalTableLog()
201
- 3. build Huffman table from count using HUF_buildCTable()
202
- 4. save Huffman table to memory buffer using HUF_writeCTable()
203
- 5. encode the data stream using HUF_compress4X_usingCTable()
204
-
205
- The following API allows targeting specific sub-functions for advanced tasks.
206
- For example, it's possible to compress several blocks using the same 'CTable',
207
- or to save and regenerate 'CTable' using external methods.
208
- */
209
- /* FSE_count() : find it within "fse.h" */
192
+ * HUF detailed API
193
+ * ****************************************/
194
+
195
+ /*! HUF_compress() does the following:
196
+ * 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h")
197
+ * 2. (optional) refine tableLog using HUF_optimalTableLog()
198
+ * 3. build Huffman table from count using HUF_buildCTable()
199
+ * 4. save Huffman table to memory buffer using HUF_writeCTable()
200
+ * 5. encode the data stream using HUF_compress4X_usingCTable()
201
+ *
202
+ * The following API allows targeting specific sub-functions for advanced tasks.
203
+ * For example, it's possible to compress several blocks using the same 'CTable',
204
+ * or to save and regenerate 'CTable' using external methods.
205
+ */
210
206
  unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
211
207
  typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */
212
- size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);
208
+ size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */
213
209
  size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
214
210
  size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
215
211
 
216
212
  typedef enum {
217
213
  HUF_repeat_none, /**< Cannot use the previous table */
218
214
  HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */
219
- HUF_repeat_valid /**< Can use the previous table and it is asumed to be valid */
215
+ HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */
220
216
  } HUF_repeat;
221
217
  /** HUF_compress4X_repeat() :
222
- * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
223
- * If it uses hufTable it does not modify hufTable or repeat.
224
- * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
225
- * If preferRepeat then the old table will always be used if valid. */
226
- size_t HUF_compress4X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
218
+ * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
219
+ * If it uses hufTable it does not modify hufTable or repeat.
220
+ * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
221
+ * If preferRepeat then the old table will always be used if valid. */
222
+ size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
223
+ const void* src, size_t srcSize,
224
+ unsigned maxSymbolValue, unsigned tableLog,
225
+ void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
226
+ HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2);
227
227
 
228
228
  /** HUF_buildCTable_wksp() :
229
229
  * Same as HUF_buildCTable(), but using externally allocated scratch buffer.
230
- * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned.
230
+ * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE.
231
231
  */
232
- size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize);
232
+ #define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1)
233
+ #define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned))
234
+ size_t HUF_buildCTable_wksp (HUF_CElt* tree,
235
+ const unsigned* count, U32 maxSymbolValue, U32 maxNbBits,
236
+ void* workSpace, size_t wkspSize);
233
237
 
234
238
  /*! HUF_readStats() :
235
- Read compact Huffman tree, saved by HUF_writeCTable().
236
- `huffWeight` is destination buffer.
237
- @return : size read from `src` , or an error Code .
238
- Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
239
- size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
240
- U32* nbSymbolsPtr, U32* tableLogPtr,
239
+ * Read compact Huffman tree, saved by HUF_writeCTable().
240
+ * `huffWeight` is destination buffer.
241
+ * @return : size read from `src` , or an error Code .
242
+ * Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
243
+ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize,
244
+ U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
241
245
  const void* src, size_t srcSize);
242
246
 
243
247
  /** HUF_readCTable() :
244
- * Loading a CTable saved with HUF_writeCTable() */
248
+ * Loading a CTable saved with HUF_writeCTable() */
245
249
  size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
246
250
 
251
+ /** HUF_getNbBits() :
252
+ * Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX
253
+ * Note 1 : is not inlined, as HUF_CElt definition is private
254
+ * Note 2 : const void* used, so that it can provide a statically allocated table as argument (which uses type U32) */
255
+ U32 HUF_getNbBits(const void* symbolTable, U32 symbolValue);
247
256
 
248
257
  /*
249
- HUF_decompress() does the following:
250
- 1. select the decompression algorithm (X2, X4) based on pre-computed heuristics
251
- 2. build Huffman table from save, using HUF_readDTableXn()
252
- 3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable
253
- */
258
+ * HUF_decompress() does the following:
259
+ * 1. select the decompression algorithm (X1, X2) based on pre-computed heuristics
260
+ * 2. build Huffman table from save, using HUF_readDTableX?()
261
+ * 3. decode 1 or 4 segments in parallel using HUF_decompress?X?_usingDTable()
262
+ */
254
263
 
255
264
  /** HUF_selectDecoder() :
256
- * Tells which decoder is likely to decode faster,
257
- * based on a set of pre-determined metrics.
258
- * @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
259
- * Assumption : 0 < cSrcSize < dstSize <= 128 KB */
265
+ * Tells which decoder is likely to decode faster,
266
+ * based on a set of pre-computed metrics.
267
+ * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 .
268
+ * Assumption : 0 < dstSize <= 128 KB */
260
269
  U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
261
270
 
271
+ /**
272
+ * The minimum workspace size for the `workSpace` used in
273
+ * HUF_readDTableX1_wksp() and HUF_readDTableX2_wksp().
274
+ *
275
+ * The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when
276
+ * HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15.
277
+ * Buffer overflow errors may potentially occur if code modifications result in
278
+ * a required workspace size greater than that specified in the following
279
+ * macro.
280
+ */
281
+ #define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10)
282
+ #define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
283
+
284
+ #ifndef HUF_FORCE_DECOMPRESS_X2
285
+ size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize);
286
+ size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
287
+ #endif
288
+ #ifndef HUF_FORCE_DECOMPRESS_X1
262
289
  size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize);
263
290
  size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
264
- size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize);
265
- size_t HUF_readDTableX4_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
291
+ #endif
266
292
 
267
293
  size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
294
+ #ifndef HUF_FORCE_DECOMPRESS_X2
295
+ size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
296
+ #endif
297
+ #ifndef HUF_FORCE_DECOMPRESS_X1
268
298
  size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
269
- size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
299
+ #endif
270
300
 
271
301
 
302
+ /* ====================== */
272
303
  /* single stream variants */
304
+ /* ====================== */
273
305
 
274
306
  size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
275
307
  size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
276
308
  size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
277
309
  /** HUF_compress1X_repeat() :
278
- * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
279
- * If it uses hufTable it does not modify hufTable or repeat.
280
- * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
281
- * If preferRepeat then the old table will always be used if valid. */
282
- size_t HUF_compress1X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
283
-
284
- size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
285
- size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
310
+ * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
311
+ * If it uses hufTable it does not modify hufTable or repeat.
312
+ * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
313
+ * If preferRepeat then the old table will always be used if valid. */
314
+ size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
315
+ const void* src, size_t srcSize,
316
+ unsigned maxSymbolValue, unsigned tableLog,
317
+ void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
318
+ HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2);
319
+
320
+ size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
321
+ #ifndef HUF_FORCE_DECOMPRESS_X1
322
+ size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
323
+ #endif
286
324
 
287
325
  size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
288
326
  size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);
289
- size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
290
- size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
291
- size_t HUF_decompress1X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
292
- size_t HUF_decompress1X4_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
327
+ #ifndef HUF_FORCE_DECOMPRESS_X2
328
+ size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
329
+ size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
330
+ #endif
331
+ #ifndef HUF_FORCE_DECOMPRESS_X1
332
+ size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
333
+ size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
334
+ #endif
293
335
 
294
336
  size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */
337
+ #ifndef HUF_FORCE_DECOMPRESS_X2
338
+ size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
339
+ #endif
340
+ #ifndef HUF_FORCE_DECOMPRESS_X1
295
341
  size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
296
- size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
342
+ #endif
343
+
344
+ /* BMI2 variants.
345
+ * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
346
+ */
347
+ size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
348
+ #ifndef HUF_FORCE_DECOMPRESS_X2
349
+ size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
350
+ #endif
351
+ size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
352
+ size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
297
353
 
298
354
  #endif /* HUF_STATIC_LINKING_ONLY */
299
355