extzstd 0.2 → 0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. checksums.yaml +4 -4
  2. data/HISTORY.ja.md +13 -0
  3. data/README.md +17 -14
  4. data/contrib/zstd/{NEWS → CHANGELOG} +115 -2
  5. data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
  6. data/contrib/zstd/Makefile +99 -53
  7. data/contrib/zstd/README.md +59 -39
  8. data/contrib/zstd/TESTING.md +1 -1
  9. data/contrib/zstd/appveyor.yml +17 -6
  10. data/contrib/zstd/lib/BUCK +29 -2
  11. data/contrib/zstd/lib/Makefile +118 -21
  12. data/contrib/zstd/lib/README.md +84 -44
  13. data/contrib/zstd/lib/common/bitstream.h +17 -33
  14. data/contrib/zstd/lib/common/compiler.h +62 -8
  15. data/contrib/zstd/lib/common/cpu.h +215 -0
  16. data/contrib/zstd/lib/common/debug.c +44 -0
  17. data/contrib/zstd/lib/common/debug.h +134 -0
  18. data/contrib/zstd/lib/common/entropy_common.c +16 -1
  19. data/contrib/zstd/lib/common/error_private.c +7 -0
  20. data/contrib/zstd/lib/common/fse.h +48 -44
  21. data/contrib/zstd/lib/common/fse_decompress.c +3 -3
  22. data/contrib/zstd/lib/common/huf.h +169 -113
  23. data/contrib/zstd/lib/common/mem.h +20 -2
  24. data/contrib/zstd/lib/common/pool.c +135 -49
  25. data/contrib/zstd/lib/common/pool.h +40 -21
  26. data/contrib/zstd/lib/common/threading.c +2 -2
  27. data/contrib/zstd/lib/common/threading.h +12 -12
  28. data/contrib/zstd/lib/common/xxhash.c +3 -2
  29. data/contrib/zstd/lib/common/zstd_common.c +3 -6
  30. data/contrib/zstd/lib/common/zstd_errors.h +17 -7
  31. data/contrib/zstd/lib/common/zstd_internal.h +76 -48
  32. data/contrib/zstd/lib/compress/fse_compress.c +89 -209
  33. data/contrib/zstd/lib/compress/hist.c +203 -0
  34. data/contrib/zstd/lib/compress/hist.h +95 -0
  35. data/contrib/zstd/lib/compress/huf_compress.c +188 -80
  36. data/contrib/zstd/lib/compress/zstd_compress.c +2500 -1203
  37. data/contrib/zstd/lib/compress/zstd_compress_internal.h +463 -62
  38. data/contrib/zstd/lib/compress/zstd_double_fast.c +321 -131
  39. data/contrib/zstd/lib/compress/zstd_double_fast.h +13 -4
  40. data/contrib/zstd/lib/compress/zstd_fast.c +335 -108
  41. data/contrib/zstd/lib/compress/zstd_fast.h +12 -6
  42. data/contrib/zstd/lib/compress/zstd_lazy.c +654 -313
  43. data/contrib/zstd/lib/compress/zstd_lazy.h +44 -16
  44. data/contrib/zstd/lib/compress/zstd_ldm.c +310 -420
  45. data/contrib/zstd/lib/compress/zstd_ldm.h +63 -26
  46. data/contrib/zstd/lib/compress/zstd_opt.c +773 -325
  47. data/contrib/zstd/lib/compress/zstd_opt.h +31 -5
  48. data/contrib/zstd/lib/compress/zstdmt_compress.c +1468 -518
  49. data/contrib/zstd/lib/compress/zstdmt_compress.h +96 -45
  50. data/contrib/zstd/lib/decompress/huf_decompress.c +518 -282
  51. data/contrib/zstd/lib/decompress/zstd_ddict.c +240 -0
  52. data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
  53. data/contrib/zstd/lib/decompress/zstd_decompress.c +613 -1513
  54. data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1311 -0
  55. data/contrib/zstd/lib/decompress/zstd_decompress_block.h +59 -0
  56. data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +175 -0
  57. data/contrib/zstd/lib/dictBuilder/cover.c +194 -113
  58. data/contrib/zstd/lib/dictBuilder/cover.h +112 -0
  59. data/contrib/zstd/lib/dictBuilder/divsufsort.c +3 -3
  60. data/contrib/zstd/lib/dictBuilder/fastcover.c +740 -0
  61. data/contrib/zstd/lib/dictBuilder/zdict.c +142 -106
  62. data/contrib/zstd/lib/dictBuilder/zdict.h +115 -49
  63. data/contrib/zstd/lib/legacy/zstd_legacy.h +44 -12
  64. data/contrib/zstd/lib/legacy/zstd_v01.c +41 -10
  65. data/contrib/zstd/lib/legacy/zstd_v01.h +12 -7
  66. data/contrib/zstd/lib/legacy/zstd_v02.c +37 -12
  67. data/contrib/zstd/lib/legacy/zstd_v02.h +12 -7
  68. data/contrib/zstd/lib/legacy/zstd_v03.c +38 -12
  69. data/contrib/zstd/lib/legacy/zstd_v03.h +12 -7
  70. data/contrib/zstd/lib/legacy/zstd_v04.c +55 -174
  71. data/contrib/zstd/lib/legacy/zstd_v04.h +12 -7
  72. data/contrib/zstd/lib/legacy/zstd_v05.c +59 -31
  73. data/contrib/zstd/lib/legacy/zstd_v05.h +12 -7
  74. data/contrib/zstd/lib/legacy/zstd_v06.c +48 -20
  75. data/contrib/zstd/lib/legacy/zstd_v06.h +10 -5
  76. data/contrib/zstd/lib/legacy/zstd_v07.c +62 -29
  77. data/contrib/zstd/lib/legacy/zstd_v07.h +10 -5
  78. data/contrib/zstd/lib/zstd.h +1346 -832
  79. data/ext/extzstd.c +27 -19
  80. data/ext/extzstd_stream.c +20 -4
  81. data/ext/zstd_compress.c +1 -0
  82. data/ext/zstd_decompress.c +4 -0
  83. data/ext/zstd_dictbuilder.c +4 -0
  84. data/ext/zstd_dictbuilder_fastcover.c +5 -0
  85. data/lib/extzstd.rb +52 -220
  86. data/lib/extzstd/version.rb +1 -1
  87. metadata +21 -7
  88. data/contrib/zstd/circle.yml +0 -63
@@ -0,0 +1,44 @@
1
+ /* ******************************************************************
2
+ debug
3
+ Part of FSE library
4
+ Copyright (C) 2013-present, Yann Collet.
5
+
6
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
7
+
8
+ Redistribution and use in source and binary forms, with or without
9
+ modification, are permitted provided that the following conditions are
10
+ met:
11
+
12
+ * Redistributions of source code must retain the above copyright
13
+ notice, this list of conditions and the following disclaimer.
14
+ * Redistributions in binary form must reproduce the above
15
+ copyright notice, this list of conditions and the following disclaimer
16
+ in the documentation and/or other materials provided with the
17
+ distribution.
18
+
19
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
+
31
+ You can contact the author at :
32
+ - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
33
+ ****************************************************************** */
34
+
35
+
36
+ /*
37
+ * This module only hosts one global variable
38
+ * which can be used to dynamically influence the verbosity of traces,
39
+ * such as DEBUGLOG and RAWLOG
40
+ */
41
+
42
+ #include "debug.h"
43
+
44
+ int g_debuglevel = DEBUGLEVEL;
@@ -0,0 +1,134 @@
1
+ /* ******************************************************************
2
+ debug
3
+ Part of FSE library
4
+ Copyright (C) 2013-present, Yann Collet.
5
+
6
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
7
+
8
+ Redistribution and use in source and binary forms, with or without
9
+ modification, are permitted provided that the following conditions are
10
+ met:
11
+
12
+ * Redistributions of source code must retain the above copyright
13
+ notice, this list of conditions and the following disclaimer.
14
+ * Redistributions in binary form must reproduce the above
15
+ copyright notice, this list of conditions and the following disclaimer
16
+ in the documentation and/or other materials provided with the
17
+ distribution.
18
+
19
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
+
31
+ You can contact the author at :
32
+ - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
33
+ ****************************************************************** */
34
+
35
+
36
+ /*
37
+ * The purpose of this header is to enable debug functions.
38
+ * They regroup assert(), DEBUGLOG() and RAWLOG() for run-time,
39
+ * and DEBUG_STATIC_ASSERT() for compile-time.
40
+ *
41
+ * By default, DEBUGLEVEL==0, which means run-time debug is disabled.
42
+ *
43
+ * Level 1 enables assert() only.
44
+ * Starting level 2, traces can be generated and pushed to stderr.
45
+ * The higher the level, the more verbose the traces.
46
+ *
47
+ * It's possible to dynamically adjust level using variable g_debug_level,
48
+ * which is only declared if DEBUGLEVEL>=2,
49
+ * and is a global variable, not multi-thread protected (use with care)
50
+ */
51
+
52
+ #ifndef DEBUG_H_12987983217
53
+ #define DEBUG_H_12987983217
54
+
55
+ #if defined (__cplusplus)
56
+ extern "C" {
57
+ #endif
58
+
59
+
60
+ /* static assert is triggered at compile time, leaving no runtime artefact.
61
+ * static assert only works with compile-time constants.
62
+ * Also, this variant can only be used inside a function. */
63
+ #define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1])
64
+
65
+
66
+ /* DEBUGLEVEL is expected to be defined externally,
67
+ * typically through compiler command line.
68
+ * Value must be a number. */
69
+ #ifndef DEBUGLEVEL
70
+ # define DEBUGLEVEL 0
71
+ #endif
72
+
73
+
74
+ /* DEBUGFILE can be defined externally,
75
+ * typically through compiler command line.
76
+ * note : currently useless.
77
+ * Value must be stderr or stdout */
78
+ #ifndef DEBUGFILE
79
+ # define DEBUGFILE stderr
80
+ #endif
81
+
82
+
83
+ /* recommended values for DEBUGLEVEL :
84
+ * 0 : release mode, no debug, all run-time checks disabled
85
+ * 1 : enables assert() only, no display
86
+ * 2 : reserved, for currently active debug path
87
+ * 3 : events once per object lifetime (CCtx, CDict, etc.)
88
+ * 4 : events once per frame
89
+ * 5 : events once per block
90
+ * 6 : events once per sequence (verbose)
91
+ * 7+: events at every position (*very* verbose)
92
+ *
93
+ * It's generally inconvenient to output traces > 5.
94
+ * In which case, it's possible to selectively trigger high verbosity levels
95
+ * by modifying g_debug_level.
96
+ */
97
+
98
+ #if (DEBUGLEVEL>=1)
99
+ # include <assert.h>
100
+ #else
101
+ # ifndef assert /* assert may be already defined, due to prior #include <assert.h> */
102
+ # define assert(condition) ((void)0) /* disable assert (default) */
103
+ # endif
104
+ #endif
105
+
106
+ #if (DEBUGLEVEL>=2)
107
+ # include <stdio.h>
108
+ extern int g_debuglevel; /* the variable is only declared,
109
+ it actually lives in debug.c,
110
+ and is shared by the whole process.
111
+ It's not thread-safe.
112
+ It's useful when enabling very verbose levels
113
+ on selective conditions (such as position in src) */
114
+
115
+ # define RAWLOG(l, ...) { \
116
+ if (l<=g_debuglevel) { \
117
+ fprintf(stderr, __VA_ARGS__); \
118
+ } }
119
+ # define DEBUGLOG(l, ...) { \
120
+ if (l<=g_debuglevel) { \
121
+ fprintf(stderr, __FILE__ ": " __VA_ARGS__); \
122
+ fprintf(stderr, " \n"); \
123
+ } }
124
+ #else
125
+ # define RAWLOG(l, ...) {} /* disabled */
126
+ # define DEBUGLOG(l, ...) {} /* disabled */
127
+ #endif
128
+
129
+
130
+ #if defined (__cplusplus)
131
+ }
132
+ #endif
133
+
134
+ #endif /* DEBUG_H_12987983217 */
@@ -72,7 +72,21 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
72
72
  unsigned charnum = 0;
73
73
  int previous0 = 0;
74
74
 
75
- if (hbSize < 4) return ERROR(srcSize_wrong);
75
+ if (hbSize < 4) {
76
+ /* This function only works when hbSize >= 4 */
77
+ char buffer[4];
78
+ memset(buffer, 0, sizeof(buffer));
79
+ memcpy(buffer, headerBuffer, hbSize);
80
+ { size_t const countSize = FSE_readNCount(normalizedCounter, maxSVPtr, tableLogPtr,
81
+ buffer, sizeof(buffer));
82
+ if (FSE_isError(countSize)) return countSize;
83
+ if (countSize > hbSize) return ERROR(corruption_detected);
84
+ return countSize;
85
+ } }
86
+ assert(hbSize >= 4);
87
+
88
+ /* init */
89
+ memset(normalizedCounter, 0, (*maxSVPtr+1) * sizeof(normalizedCounter[0])); /* all symbols not present in NCount have a frequency of 0 */
76
90
  bitStream = MEM_readLE32(ip);
77
91
  nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */
78
92
  if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge);
@@ -105,6 +119,7 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
105
119
  if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall);
106
120
  while (charnum < n0) normalizedCounter[charnum++] = 0;
107
121
  if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
122
+ assert((bitCount >> 3) <= 3); /* For first condition to work */
108
123
  ip += bitCount>>3;
109
124
  bitCount &= 7;
110
125
  bitStream = MEM_readLE32(ip) >> bitCount;
@@ -14,6 +14,10 @@
14
14
 
15
15
  const char* ERR_getErrorString(ERR_enum code)
16
16
  {
17
+ #ifdef ZSTD_STRIP_ERROR_STRINGS
18
+ (void)code;
19
+ return "Error strings stripped";
20
+ #else
17
21
  static const char* const notErrorCode = "Unspecified error code";
18
22
  switch( code )
19
23
  {
@@ -29,6 +33,7 @@ const char* ERR_getErrorString(ERR_enum code)
29
33
  case PREFIX(parameter_outOfBound): return "Parameter is out of bound";
30
34
  case PREFIX(init_missing): return "Context should be init first";
31
35
  case PREFIX(memory_allocation): return "Allocation error : not enough memory";
36
+ case PREFIX(workSpace_tooSmall): return "workSpace buffer is not large enough";
32
37
  case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
33
38
  case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
34
39
  case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
@@ -38,10 +43,12 @@ const char* ERR_getErrorString(ERR_enum code)
38
43
  case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples";
39
44
  case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
40
45
  case PREFIX(srcSize_wrong): return "Src size is incorrect";
46
+ case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer";
41
47
  /* following error codes are not stable and may be removed or changed in a future version */
42
48
  case PREFIX(frameIndex_tooLarge): return "Frame index is too large";
43
49
  case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
44
50
  case PREFIX(maxCode):
45
51
  default: return notErrorCode;
46
52
  }
53
+ #endif
47
54
  }
@@ -72,6 +72,7 @@ extern "C" {
72
72
  #define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE)
73
73
  FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */
74
74
 
75
+
75
76
  /*-****************************************
76
77
  * FSE simple functions
77
78
  ******************************************/
@@ -129,7 +130,7 @@ FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src,
129
130
  ******************************************/
130
131
  /*!
131
132
  FSE_compress() does the following:
132
- 1. count symbol occurrence from source[] into table count[]
133
+ 1. count symbol occurrence from source[] into table count[] (see hist.h)
133
134
  2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
134
135
  3. save normalized counters to memory buffer using writeNCount()
135
136
  4. build encoding table 'CTable' from normalized counters
@@ -147,15 +148,6 @@ or to save and provide normalized distribution using external method.
147
148
 
148
149
  /* *** COMPRESSION *** */
149
150
 
150
- /*! FSE_count():
151
- Provides the precise count of each byte within a table 'count'.
152
- 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1).
153
- *maxSymbolValuePtr will be updated if detected smaller than initial value.
154
- @return : the count of the most frequent symbol (which is not identified).
155
- if return == srcSize, there is only one symbol.
156
- Can also return an error code, which can be tested with FSE_isError(). */
157
- FSE_PUBLIC_API size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
158
-
159
151
  /*! FSE_optimalTableLog():
160
152
  dynamically downsize 'tableLog' when conditions are met.
161
153
  It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
@@ -167,7 +159,8 @@ FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize
167
159
  'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
168
160
  @return : tableLog,
169
161
  or an errorCode, which can be tested using FSE_isError() */
170
- FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
162
+ FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog,
163
+ const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
171
164
 
172
165
  /*! FSE_NCountWriteBound():
173
166
  Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'.
@@ -178,8 +171,9 @@ FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tab
178
171
  Compactly save 'normalizedCounter' into 'buffer'.
179
172
  @return : size of the compressed table,
180
173
  or an errorCode, which can be tested using FSE_isError(). */
181
- FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
182
-
174
+ FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize,
175
+ const short* normalizedCounter,
176
+ unsigned maxSymbolValue, unsigned tableLog);
183
177
 
184
178
  /*! Constructor and Destructor of FSE_CTable.
185
179
  Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
@@ -250,7 +244,9 @@ If there is an error, the function will return an ErrorCode (which can be tested
250
244
  @return : size read from 'rBuffer',
251
245
  or an errorCode, which can be tested using FSE_isError().
252
246
  maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
253
- FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize);
247
+ FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter,
248
+ unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,
249
+ const void* rBuffer, size_t rBuffSize);
254
250
 
255
251
  /*! Constructor and Destructor of FSE_DTable.
256
252
  Note that its size depends on 'tableLog' */
@@ -325,33 +321,8 @@ If there is an error, the function will return an error code, which can be teste
325
321
 
326
322
 
327
323
  /* *****************************************
328
- * FSE advanced API
329
- *******************************************/
330
- /* FSE_count_wksp() :
331
- * Same as FSE_count(), but using an externally provided scratch buffer.
332
- * `workSpace` size must be table of >= `1024` unsigned
333
- */
334
- size_t FSE_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
335
- const void* source, size_t sourceSize, unsigned* workSpace);
336
-
337
- /** FSE_countFast() :
338
- * same as FSE_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr
339
- */
340
- size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
341
-
342
- /* FSE_countFast_wksp() :
343
- * Same as FSE_countFast(), but using an externally provided scratch buffer.
344
- * `workSpace` must be a table of minimum `1024` unsigned
345
- */
346
- size_t FSE_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* workSpace);
347
-
348
- /*! FSE_count_simple
349
- * Same as FSE_countFast(), but does not use any additional memory (not even on stack).
350
- * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` (presuming it's also the size of `count`).
351
- */
352
- size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
353
-
354
-
324
+ * FSE advanced API
325
+ ***************************************** */
355
326
 
356
327
  unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
357
328
  /**< same as FSE_optimalTableLog(), which used `minus==2` */
@@ -387,7 +358,7 @@ size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size
387
358
  typedef enum {
388
359
  FSE_repeat_none, /**< Cannot use the previous table */
389
360
  FSE_repeat_check, /**< Can use the previous table but it must be checked */
390
- FSE_repeat_valid /**< Can use the previous table and it is asumed to be valid */
361
+ FSE_repeat_valid /**< Can use the previous table and it is assumed to be valid */
391
362
  } FSE_repeat;
392
363
 
393
364
  /* *****************************************
@@ -541,7 +512,7 @@ MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
541
512
  const U32 tableLog = MEM_read16(ptr);
542
513
  statePtr->value = (ptrdiff_t)1<<tableLog;
543
514
  statePtr->stateTable = u16ptr+2;
544
- statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1));
515
+ statePtr->symbolTT = ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1);
545
516
  statePtr->stateLog = tableLog;
546
517
  }
547
518
 
@@ -560,7 +531,7 @@ MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U3
560
531
  }
561
532
  }
562
533
 
563
- MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol)
534
+ MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, unsigned symbol)
564
535
  {
565
536
  FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
566
537
  const U16* const stateTable = (const U16*)(statePtr->stateTable);
@@ -576,6 +547,39 @@ MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePt
576
547
  }
577
548
 
578
549
 
550
+ /* FSE_getMaxNbBits() :
551
+ * Approximate maximum cost of a symbol, in bits.
552
+ * Fractional get rounded up (i.e : a symbol with a normalized frequency of 3 gives the same result as a frequency of 2)
553
+ * note 1 : assume symbolValue is valid (<= maxSymbolValue)
554
+ * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
555
+ MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue)
556
+ {
557
+ const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr;
558
+ return (symbolTT[symbolValue].deltaNbBits + ((1<<16)-1)) >> 16;
559
+ }
560
+
561
+ /* FSE_bitCost() :
562
+ * Approximate symbol cost, as fractional value, using fixed-point format (accuracyLog fractional bits)
563
+ * note 1 : assume symbolValue is valid (<= maxSymbolValue)
564
+ * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
565
+ MEM_STATIC U32 FSE_bitCost(const void* symbolTTPtr, U32 tableLog, U32 symbolValue, U32 accuracyLog)
566
+ {
567
+ const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr;
568
+ U32 const minNbBits = symbolTT[symbolValue].deltaNbBits >> 16;
569
+ U32 const threshold = (minNbBits+1) << 16;
570
+ assert(tableLog < 16);
571
+ assert(accuracyLog < 31-tableLog); /* ensure enough room for renormalization double shift */
572
+ { U32 const tableSize = 1 << tableLog;
573
+ U32 const deltaFromThreshold = threshold - (symbolTT[symbolValue].deltaNbBits + tableSize);
574
+ U32 const normalizedDeltaFromThreshold = (deltaFromThreshold << accuracyLog) >> tableLog; /* linear interpolation (very approximate) */
575
+ U32 const bitMultiplier = 1 << accuracyLog;
576
+ assert(symbolTT[symbolValue].deltaNbBits + tableSize <= threshold);
577
+ assert(normalizedDeltaFromThreshold <= bitMultiplier);
578
+ return (minNbBits+1)*bitMultiplier - normalizedDeltaFromThreshold;
579
+ }
580
+ }
581
+
582
+
579
583
  /* ====== Decompression ====== */
580
584
 
581
585
  typedef struct {
@@ -49,7 +49,7 @@
49
49
  * Error Management
50
50
  ****************************************************************/
51
51
  #define FSE_isError ERR_isError
52
- #define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
52
+ #define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */
53
53
 
54
54
  /* check and forward error code */
55
55
  #define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; }
@@ -139,8 +139,8 @@ size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned
139
139
  { U32 u;
140
140
  for (u=0; u<tableSize; u++) {
141
141
  FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
142
- U16 nextState = symbolNext[symbol]++;
143
- tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32 ((U32)nextState) );
142
+ U32 const nextState = symbolNext[symbol]++;
143
+ tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32(nextState) );
144
144
  tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
145
145
  } }
146
146
 
@@ -1,7 +1,7 @@
1
1
  /* ******************************************************************
2
- Huffman coder, part of New Generation Entropy library
3
- header file
4
- Copyright (C) 2013-2016, Yann Collet.
2
+ huff0 huffman codec,
3
+ part of Finite State Entropy library
4
+ Copyright (C) 2013-present, Yann Collet.
5
5
 
6
6
  BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
7
7
 
@@ -58,32 +58,32 @@ extern "C" {
58
58
  #endif
59
59
 
60
60
 
61
- /* *** simple functions *** */
62
- /**
63
- HUF_compress() :
64
- Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'.
65
- 'dst' buffer must be already allocated.
66
- Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize).
67
- `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB.
68
- @return : size of compressed data (<= `dstCapacity`).
69
- Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
70
- if return == 1, srcData is a single repeated byte symbol (RLE compression).
71
- if HUF_isError(return), compression failed (more details using HUF_getErrorName())
72
- */
61
+ /* ========================== */
62
+ /* *** simple functions *** */
63
+ /* ========================== */
64
+
65
+ /** HUF_compress() :
66
+ * Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'.
67
+ * 'dst' buffer must be already allocated.
68
+ * Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize).
69
+ * `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB.
70
+ * @return : size of compressed data (<= `dstCapacity`).
71
+ * Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
72
+ * if HUF_isError(return), compression failed (more details using HUF_getErrorName())
73
+ */
73
74
  HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity,
74
75
  const void* src, size_t srcSize);
75
76
 
76
- /**
77
- HUF_decompress() :
78
- Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
79
- into already allocated buffer 'dst', of minimum size 'dstSize'.
80
- `originalSize` : **must** be the ***exact*** size of original (uncompressed) data.
81
- Note : in contrast with FSE, HUF_decompress can regenerate
82
- RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
83
- because it knows size to regenerate.
84
- @return : size of regenerated data (== originalSize),
85
- or an error code, which can be tested using HUF_isError()
86
- */
77
+ /** HUF_decompress() :
78
+ * Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
79
+ * into already allocated buffer 'dst', of minimum size 'dstSize'.
80
+ * `originalSize` : **must** be the ***exact*** size of original (uncompressed) data.
81
+ * Note : in contrast with FSE, HUF_decompress can regenerate
82
+ * RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
83
+ * because it knows size to regenerate (originalSize).
84
+ * @return : size of regenerated data (== originalSize),
85
+ * or an error code, which can be tested using HUF_isError()
86
+ */
87
87
  HUF_PUBLIC_API size_t HUF_decompress(void* dst, size_t originalSize,
88
88
  const void* cSrc, size_t cSrcSize);
89
89
 
@@ -100,39 +100,32 @@ HUF_PUBLIC_API const char* HUF_getErrorName(size_t code); /**< provides error c
100
100
  /* *** Advanced function *** */
101
101
 
102
102
  /** HUF_compress2() :
103
- * Same as HUF_compress(), but offers direct control over `maxSymbolValue` and `tableLog`.
104
- * `tableLog` must be `<= HUF_TABLELOG_MAX` . */
105
- HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
103
+ * Same as HUF_compress(), but offers control over `maxSymbolValue` and `tableLog`.
104
+ * `maxSymbolValue` must be <= HUF_SYMBOLVALUE_MAX .
105
+ * `tableLog` must be `<= HUF_TABLELOG_MAX` . */
106
+ HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity,
107
+ const void* src, size_t srcSize,
108
+ unsigned maxSymbolValue, unsigned tableLog);
106
109
 
107
110
  /** HUF_compress4X_wksp() :
108
111
  * Same as HUF_compress2(), but uses externally allocated `workSpace`.
109
- * `workspace` must have minimum alignment of 4, and be at least as large as following macro */
112
+ * `workspace` must have minimum alignment of 4, and be at least as large as HUF_WORKSPACE_SIZE */
110
113
  #define HUF_WORKSPACE_SIZE (6 << 10)
111
114
  #define HUF_WORKSPACE_SIZE_U32 (HUF_WORKSPACE_SIZE / sizeof(U32))
112
- HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
113
-
114
- /**
115
- * The minimum workspace size for the `workSpace` used in
116
- * HUF_readDTableX2_wksp() and HUF_readDTableX4_wksp().
117
- *
118
- * The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when
119
- * HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15.
120
- * Buffer overflow errors may potentially occur if code modifications result in
121
- * a required workspace size greater than that specified in the following
122
- * macro.
123
- */
124
- #define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10)
125
- #define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
115
+ HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity,
116
+ const void* src, size_t srcSize,
117
+ unsigned maxSymbolValue, unsigned tableLog,
118
+ void* workSpace, size_t wkspSize);
126
119
 
127
120
  #endif /* HUF_H_298734234 */
128
121
 
129
122
  /* ******************************************************************
130
123
  * WARNING !!
131
124
  * The following section contains advanced and experimental definitions
132
- * which shall never be used in the context of dll
125
+ * which shall never be used in the context of a dynamic library,
133
126
  * because they are not guaranteed to remain stable in the future.
134
127
  * Only consider them in association with static linking.
135
- *******************************************************************/
128
+ * *****************************************************************/
136
129
  #if defined(HUF_STATIC_LINKING_ONLY) && !defined(HUF_H_HUF_STATIC_LINKING_ONLY)
137
130
  #define HUF_H_HUF_STATIC_LINKING_ONLY
138
131
 
@@ -141,11 +134,11 @@ HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, const
141
134
 
142
135
 
143
136
  /* *** Constants *** */
144
- #define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
145
- #define HUF_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */
137
+ #define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
138
+ #define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */
146
139
  #define HUF_SYMBOLVALUE_MAX 255
147
140
 
148
- #define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
141
+ #define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
149
142
  #if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
150
143
  # error "HUF_TABLELOG_MAX is too large !"
151
144
  #endif
@@ -170,130 +163,193 @@ HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, const
170
163
  /* static allocation of HUF's DTable */
171
164
  typedef U32 HUF_DTable;
172
165
  #define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog)))
173
- #define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
166
+ #define HUF_CREATE_STATIC_DTABLEX1(DTable, maxTableLog) \
174
167
  HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) }
175
- #define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \
168
+ #define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
176
169
  HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) }
177
170
 
178
171
 
179
172
  /* ****************************************
180
173
  * Advanced decompression functions
181
174
  ******************************************/
182
- size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
183
- size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
175
+ size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
176
+ #ifndef HUF_FORCE_DECOMPRESS_X1
177
+ size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
178
+ #endif
184
179
 
185
180
  size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */
186
181
  size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
187
182
  size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */
188
- size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
189
- size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
190
- size_t HUF_decompress4X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
191
- size_t HUF_decompress4X4_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
183
+ size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
184
+ size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
185
+ #ifndef HUF_FORCE_DECOMPRESS_X1
186
+ size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
187
+ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
188
+ #endif
192
189
 
193
190
 
194
191
  /* ****************************************
195
- * HUF detailed API
196
- ******************************************/
197
- /*!
198
- HUF_compress() does the following:
199
- 1. count symbol occurrence from source[] into table count[] using FSE_count()
200
- 2. (optional) refine tableLog using HUF_optimalTableLog()
201
- 3. build Huffman table from count using HUF_buildCTable()
202
- 4. save Huffman table to memory buffer using HUF_writeCTable()
203
- 5. encode the data stream using HUF_compress4X_usingCTable()
204
-
205
- The following API allows targeting specific sub-functions for advanced tasks.
206
- For example, it's possible to compress several blocks using the same 'CTable',
207
- or to save and regenerate 'CTable' using external methods.
208
- */
209
- /* FSE_count() : find it within "fse.h" */
192
+ * HUF detailed API
193
+ * ****************************************/
194
+
195
+ /*! HUF_compress() does the following:
196
+ * 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h")
197
+ * 2. (optional) refine tableLog using HUF_optimalTableLog()
198
+ * 3. build Huffman table from count using HUF_buildCTable()
199
+ * 4. save Huffman table to memory buffer using HUF_writeCTable()
200
+ * 5. encode the data stream using HUF_compress4X_usingCTable()
201
+ *
202
+ * The following API allows targeting specific sub-functions for advanced tasks.
203
+ * For example, it's possible to compress several blocks using the same 'CTable',
204
+ * or to save and regenerate 'CTable' using external methods.
205
+ */
210
206
  unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
211
207
  typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */
212
- size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);
208
+ size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */
213
209
  size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
214
210
  size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
215
211
 
216
212
  typedef enum {
217
213
  HUF_repeat_none, /**< Cannot use the previous table */
218
214
  HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */
219
- HUF_repeat_valid /**< Can use the previous table and it is asumed to be valid */
215
+ HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */
220
216
  } HUF_repeat;
221
217
  /** HUF_compress4X_repeat() :
222
- * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
223
- * If it uses hufTable it does not modify hufTable or repeat.
224
- * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
225
- * If preferRepeat then the old table will always be used if valid. */
226
- size_t HUF_compress4X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
218
+ * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
219
+ * If it uses hufTable it does not modify hufTable or repeat.
220
+ * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
221
+ * If preferRepeat then the old table will always be used if valid. */
222
+ size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
223
+ const void* src, size_t srcSize,
224
+ unsigned maxSymbolValue, unsigned tableLog,
225
+ void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
226
+ HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2);
227
227
 
228
228
  /** HUF_buildCTable_wksp() :
229
229
  * Same as HUF_buildCTable(), but using externally allocated scratch buffer.
230
- * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned.
230
+ * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE.
231
231
  */
232
- size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize);
232
+ #define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1)
233
+ #define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned))
234
+ size_t HUF_buildCTable_wksp (HUF_CElt* tree,
235
+ const unsigned* count, U32 maxSymbolValue, U32 maxNbBits,
236
+ void* workSpace, size_t wkspSize);
233
237
 
234
238
  /*! HUF_readStats() :
235
- Read compact Huffman tree, saved by HUF_writeCTable().
236
- `huffWeight` is destination buffer.
237
- @return : size read from `src` , or an error Code .
238
- Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
239
- size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
240
- U32* nbSymbolsPtr, U32* tableLogPtr,
239
+ * Read compact Huffman tree, saved by HUF_writeCTable().
240
+ * `huffWeight` is destination buffer.
241
+ * @return : size read from `src` , or an error Code .
242
+ * Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
243
+ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize,
244
+ U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
241
245
  const void* src, size_t srcSize);
242
246
 
243
247
  /** HUF_readCTable() :
244
- * Loading a CTable saved with HUF_writeCTable() */
248
+ * Loading a CTable saved with HUF_writeCTable() */
245
249
  size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
246
250
 
251
+ /** HUF_getNbBits() :
252
+ * Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX
253
+ * Note 1 : is not inlined, as HUF_CElt definition is private
254
+ * Note 2 : const void* used, so that it can provide a statically allocated table as argument (which uses type U32) */
255
+ U32 HUF_getNbBits(const void* symbolTable, U32 symbolValue);
247
256
 
248
257
  /*
249
- HUF_decompress() does the following:
250
- 1. select the decompression algorithm (X2, X4) based on pre-computed heuristics
251
- 2. build Huffman table from save, using HUF_readDTableXn()
252
- 3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable
253
- */
258
+ * HUF_decompress() does the following:
259
+ * 1. select the decompression algorithm (X1, X2) based on pre-computed heuristics
260
+ * 2. build Huffman table from save, using HUF_readDTableX?()
261
+ * 3. decode 1 or 4 segments in parallel using HUF_decompress?X?_usingDTable()
262
+ */
254
263
 
255
264
  /** HUF_selectDecoder() :
256
- * Tells which decoder is likely to decode faster,
257
- * based on a set of pre-determined metrics.
258
- * @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
259
- * Assumption : 0 < cSrcSize < dstSize <= 128 KB */
265
+ * Tells which decoder is likely to decode faster,
266
+ * based on a set of pre-computed metrics.
267
+ * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 .
268
+ * Assumption : 0 < dstSize <= 128 KB */
260
269
  U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
261
270
 
271
+ /**
272
+ * The minimum workspace size for the `workSpace` used in
273
+ * HUF_readDTableX1_wksp() and HUF_readDTableX2_wksp().
274
+ *
275
+ * The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when
276
+ * HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15.
277
+ * Buffer overflow errors may potentially occur if code modifications result in
278
+ * a required workspace size greater than that specified in the following
279
+ * macro.
280
+ */
281
+ #define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10)
282
+ #define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
283
+
284
+ #ifndef HUF_FORCE_DECOMPRESS_X2
285
+ size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize);
286
+ size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
287
+ #endif
288
+ #ifndef HUF_FORCE_DECOMPRESS_X1
262
289
  size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize);
263
290
  size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
264
- size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize);
265
- size_t HUF_readDTableX4_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
291
+ #endif
266
292
 
267
293
  size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
294
+ #ifndef HUF_FORCE_DECOMPRESS_X2
295
+ size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
296
+ #endif
297
+ #ifndef HUF_FORCE_DECOMPRESS_X1
268
298
  size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
269
- size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
299
+ #endif
270
300
 
271
301
 
302
+ /* ====================== */
272
303
  /* single stream variants */
304
+ /* ====================== */
273
305
 
274
306
  size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
275
307
  size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
276
308
  size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
277
309
  /** HUF_compress1X_repeat() :
278
- * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
279
- * If it uses hufTable it does not modify hufTable or repeat.
280
- * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
281
- * If preferRepeat then the old table will always be used if valid. */
282
- size_t HUF_compress1X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
283
-
284
- size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
285
- size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
310
+ * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
311
+ * If it uses hufTable it does not modify hufTable or repeat.
312
+ * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
313
+ * If preferRepeat then the old table will always be used if valid. */
314
+ size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
315
+ const void* src, size_t srcSize,
316
+ unsigned maxSymbolValue, unsigned tableLog,
317
+ void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
318
+ HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2);
319
+
320
+ size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
321
+ #ifndef HUF_FORCE_DECOMPRESS_X1
322
+ size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
323
+ #endif
286
324
 
287
325
  size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
288
326
  size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);
289
- size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
290
- size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
291
- size_t HUF_decompress1X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
292
- size_t HUF_decompress1X4_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
327
+ #ifndef HUF_FORCE_DECOMPRESS_X2
328
+ size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
329
+ size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
330
+ #endif
331
+ #ifndef HUF_FORCE_DECOMPRESS_X1
332
+ size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
333
+ size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
334
+ #endif
293
335
 
294
336
  size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */
337
+ #ifndef HUF_FORCE_DECOMPRESS_X2
338
+ size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
339
+ #endif
340
+ #ifndef HUF_FORCE_DECOMPRESS_X1
295
341
  size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
296
- size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
342
+ #endif
343
+
344
+ /* BMI2 variants.
345
+ * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
346
+ */
347
+ size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
348
+ #ifndef HUF_FORCE_DECOMPRESS_X2
349
+ size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
350
+ #endif
351
+ size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
352
+ size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
297
353
 
298
354
  #endif /* HUF_STATIC_LINKING_ONLY */
299
355