zstd-ruby 1.4.5.0 → 1.5.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (115) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/README.md +78 -5
  4. data/Rakefile +8 -2
  5. data/ext/zstdruby/common.h +15 -0
  6. data/ext/zstdruby/extconf.rb +3 -2
  7. data/ext/zstdruby/libzstd/common/allocations.h +55 -0
  8. data/ext/zstdruby/libzstd/common/bits.h +200 -0
  9. data/ext/zstdruby/libzstd/common/bitstream.h +45 -62
  10. data/ext/zstdruby/libzstd/common/compiler.h +205 -22
  11. data/ext/zstdruby/libzstd/common/cpu.h +1 -3
  12. data/ext/zstdruby/libzstd/common/debug.c +1 -1
  13. data/ext/zstdruby/libzstd/common/debug.h +12 -19
  14. data/ext/zstdruby/libzstd/common/entropy_common.c +172 -48
  15. data/ext/zstdruby/libzstd/common/error_private.c +10 -2
  16. data/ext/zstdruby/libzstd/common/error_private.h +82 -3
  17. data/ext/zstdruby/libzstd/common/fse.h +37 -86
  18. data/ext/zstdruby/libzstd/common/fse_decompress.c +117 -92
  19. data/ext/zstdruby/libzstd/common/huf.h +99 -166
  20. data/ext/zstdruby/libzstd/common/mem.h +124 -142
  21. data/ext/zstdruby/libzstd/common/pool.c +54 -27
  22. data/ext/zstdruby/libzstd/common/pool.h +10 -4
  23. data/ext/zstdruby/libzstd/common/portability_macros.h +156 -0
  24. data/ext/zstdruby/libzstd/common/threading.c +74 -19
  25. data/ext/zstdruby/libzstd/common/threading.h +5 -10
  26. data/ext/zstdruby/libzstd/common/xxhash.c +7 -847
  27. data/ext/zstdruby/libzstd/common/xxhash.h +5568 -167
  28. data/ext/zstdruby/libzstd/common/zstd_common.c +2 -37
  29. data/ext/zstdruby/libzstd/common/zstd_deps.h +111 -0
  30. data/ext/zstdruby/libzstd/common/zstd_internal.h +132 -187
  31. data/ext/zstdruby/libzstd/common/zstd_trace.h +163 -0
  32. data/ext/zstdruby/libzstd/compress/clevels.h +134 -0
  33. data/ext/zstdruby/libzstd/compress/fse_compress.c +83 -157
  34. data/ext/zstdruby/libzstd/compress/hist.c +27 -29
  35. data/ext/zstdruby/libzstd/compress/hist.h +2 -2
  36. data/ext/zstdruby/libzstd/compress/huf_compress.c +916 -279
  37. data/ext/zstdruby/libzstd/compress/zstd_compress.c +3773 -1019
  38. data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +610 -203
  39. data/ext/zstdruby/libzstd/compress/zstd_compress_literals.c +119 -42
  40. data/ext/zstdruby/libzstd/compress/zstd_compress_literals.h +16 -6
  41. data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.c +42 -19
  42. data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.h +1 -1
  43. data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.c +49 -317
  44. data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.h +1 -1
  45. data/ext/zstdruby/libzstd/compress/zstd_cwksp.h +320 -103
  46. data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +388 -151
  47. data/ext/zstdruby/libzstd/compress/zstd_double_fast.h +3 -2
  48. data/ext/zstdruby/libzstd/compress/zstd_fast.c +729 -265
  49. data/ext/zstdruby/libzstd/compress/zstd_fast.h +3 -2
  50. data/ext/zstdruby/libzstd/compress/zstd_lazy.c +1270 -251
  51. data/ext/zstdruby/libzstd/compress/zstd_lazy.h +61 -1
  52. data/ext/zstdruby/libzstd/compress/zstd_ldm.c +324 -219
  53. data/ext/zstdruby/libzstd/compress/zstd_ldm.h +9 -2
  54. data/ext/zstdruby/libzstd/compress/zstd_ldm_geartab.h +106 -0
  55. data/ext/zstdruby/libzstd/compress/zstd_opt.c +481 -209
  56. data/ext/zstdruby/libzstd/compress/zstd_opt.h +1 -1
  57. data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +181 -457
  58. data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +34 -113
  59. data/ext/zstdruby/libzstd/decompress/huf_decompress.c +1199 -565
  60. data/ext/zstdruby/libzstd/decompress/huf_decompress_amd64.S +576 -0
  61. data/ext/zstdruby/libzstd/decompress/zstd_ddict.c +12 -12
  62. data/ext/zstdruby/libzstd/decompress/zstd_ddict.h +2 -2
  63. data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +627 -157
  64. data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.c +1086 -326
  65. data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.h +19 -5
  66. data/ext/zstdruby/libzstd/decompress/zstd_decompress_internal.h +62 -13
  67. data/ext/zstdruby/libzstd/dictBuilder/cover.c +73 -52
  68. data/ext/zstdruby/libzstd/dictBuilder/cover.h +7 -6
  69. data/ext/zstdruby/libzstd/dictBuilder/divsufsort.c +1 -1
  70. data/ext/zstdruby/libzstd/dictBuilder/fastcover.c +44 -35
  71. data/ext/zstdruby/libzstd/dictBuilder/zdict.c +103 -111
  72. data/ext/zstdruby/libzstd/{dictBuilder/zdict.h → zdict.h} +203 -34
  73. data/ext/zstdruby/libzstd/zstd.h +1217 -287
  74. data/ext/zstdruby/libzstd/{common/zstd_errors.h → zstd_errors.h} +28 -8
  75. data/ext/zstdruby/main.c +20 -0
  76. data/ext/zstdruby/skippable_frame.c +63 -0
  77. data/ext/zstdruby/streaming_compress.c +177 -0
  78. data/ext/zstdruby/streaming_compress.h +5 -0
  79. data/ext/zstdruby/streaming_decompress.c +123 -0
  80. data/ext/zstdruby/zstdruby.c +114 -32
  81. data/lib/zstd-ruby/version.rb +1 -1
  82. data/lib/zstd-ruby.rb +0 -1
  83. data/zstd-ruby.gemspec +1 -1
  84. metadata +19 -36
  85. data/.travis.yml +0 -14
  86. data/ext/zstdruby/libzstd/.gitignore +0 -3
  87. data/ext/zstdruby/libzstd/BUCK +0 -234
  88. data/ext/zstdruby/libzstd/Makefile +0 -354
  89. data/ext/zstdruby/libzstd/README.md +0 -179
  90. data/ext/zstdruby/libzstd/deprecated/zbuff.h +0 -214
  91. data/ext/zstdruby/libzstd/deprecated/zbuff_common.c +0 -26
  92. data/ext/zstdruby/libzstd/deprecated/zbuff_compress.c +0 -147
  93. data/ext/zstdruby/libzstd/deprecated/zbuff_decompress.c +0 -75
  94. data/ext/zstdruby/libzstd/dll/example/Makefile +0 -48
  95. data/ext/zstdruby/libzstd/dll/example/README.md +0 -69
  96. data/ext/zstdruby/libzstd/dll/example/build_package.bat +0 -20
  97. data/ext/zstdruby/libzstd/dll/example/fullbench-dll.sln +0 -25
  98. data/ext/zstdruby/libzstd/dll/example/fullbench-dll.vcxproj +0 -181
  99. data/ext/zstdruby/libzstd/legacy/zstd_legacy.h +0 -415
  100. data/ext/zstdruby/libzstd/legacy/zstd_v01.c +0 -2158
  101. data/ext/zstdruby/libzstd/legacy/zstd_v01.h +0 -94
  102. data/ext/zstdruby/libzstd/legacy/zstd_v02.c +0 -3518
  103. data/ext/zstdruby/libzstd/legacy/zstd_v02.h +0 -93
  104. data/ext/zstdruby/libzstd/legacy/zstd_v03.c +0 -3160
  105. data/ext/zstdruby/libzstd/legacy/zstd_v03.h +0 -93
  106. data/ext/zstdruby/libzstd/legacy/zstd_v04.c +0 -3647
  107. data/ext/zstdruby/libzstd/legacy/zstd_v04.h +0 -142
  108. data/ext/zstdruby/libzstd/legacy/zstd_v05.c +0 -4050
  109. data/ext/zstdruby/libzstd/legacy/zstd_v05.h +0 -162
  110. data/ext/zstdruby/libzstd/legacy/zstd_v06.c +0 -4154
  111. data/ext/zstdruby/libzstd/legacy/zstd_v06.h +0 -172
  112. data/ext/zstdruby/libzstd/legacy/zstd_v07.c +0 -4541
  113. data/ext/zstdruby/libzstd/legacy/zstd_v07.h +0 -187
  114. data/ext/zstdruby/libzstd/libzstd.pc.in +0 -15
  115. data/ext/zstdruby/zstdruby.h +0 -6
@@ -1,7 +1,7 @@
1
1
  /* ******************************************************************
2
2
  * huff0 huffman codec,
3
3
  * part of Finite State Entropy library
4
- * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc.
4
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
5
5
  *
6
6
  * You can contact the author at :
7
7
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -20,105 +20,30 @@ extern "C" {
20
20
  #define HUF_H_298734234
21
21
 
22
22
  /* *** Dependencies *** */
23
- #include <stddef.h> /* size_t */
24
-
25
-
26
- /* *** library symbols visibility *** */
27
- /* Note : when linking with -fvisibility=hidden on gcc, or by default on Visual,
28
- * HUF symbols remain "private" (internal symbols for library only).
29
- * Set macro FSE_DLL_EXPORT to 1 if you want HUF symbols visible on DLL interface */
30
- #if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4)
31
- # define HUF_PUBLIC_API __attribute__ ((visibility ("default")))
32
- #elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */
33
- # define HUF_PUBLIC_API __declspec(dllexport)
34
- #elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1)
35
- # define HUF_PUBLIC_API __declspec(dllimport) /* not required, just to generate faster code (saves a function pointer load from IAT and an indirect jump) */
36
- #else
37
- # define HUF_PUBLIC_API
38
- #endif
39
-
40
-
41
- /* ========================== */
42
- /* *** simple functions *** */
43
- /* ========================== */
44
-
45
- /** HUF_compress() :
46
- * Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'.
47
- * 'dst' buffer must be already allocated.
48
- * Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize).
49
- * `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB.
50
- * @return : size of compressed data (<= `dstCapacity`).
51
- * Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
52
- * if HUF_isError(return), compression failed (more details using HUF_getErrorName())
53
- */
54
- HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity,
55
- const void* src, size_t srcSize);
56
-
57
- /** HUF_decompress() :
58
- * Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
59
- * into already allocated buffer 'dst', of minimum size 'dstSize'.
60
- * `originalSize` : **must** be the ***exact*** size of original (uncompressed) data.
61
- * Note : in contrast with FSE, HUF_decompress can regenerate
62
- * RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
63
- * because it knows size to regenerate (originalSize).
64
- * @return : size of regenerated data (== originalSize),
65
- * or an error code, which can be tested using HUF_isError()
66
- */
67
- HUF_PUBLIC_API size_t HUF_decompress(void* dst, size_t originalSize,
68
- const void* cSrc, size_t cSrcSize);
23
+ #include "zstd_deps.h" /* size_t */
24
+ #include "mem.h" /* U32 */
25
+ #define FSE_STATIC_LINKING_ONLY
26
+ #include "fse.h"
69
27
 
70
28
 
71
29
  /* *** Tool functions *** */
72
- #define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */
73
- HUF_PUBLIC_API size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */
30
+ #define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */
31
+ size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */
74
32
 
75
33
  /* Error Management */
76
- HUF_PUBLIC_API unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */
77
- HUF_PUBLIC_API const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */
78
-
79
-
80
- /* *** Advanced function *** */
81
-
82
- /** HUF_compress2() :
83
- * Same as HUF_compress(), but offers control over `maxSymbolValue` and `tableLog`.
84
- * `maxSymbolValue` must be <= HUF_SYMBOLVALUE_MAX .
85
- * `tableLog` must be `<= HUF_TABLELOG_MAX` . */
86
- HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity,
87
- const void* src, size_t srcSize,
88
- unsigned maxSymbolValue, unsigned tableLog);
89
-
90
- /** HUF_compress4X_wksp() :
91
- * Same as HUF_compress2(), but uses externally allocated `workSpace`.
92
- * `workspace` must have minimum alignment of 4, and be at least as large as HUF_WORKSPACE_SIZE */
93
- #define HUF_WORKSPACE_SIZE ((6 << 10) + 256)
94
- #define HUF_WORKSPACE_SIZE_U32 (HUF_WORKSPACE_SIZE / sizeof(U32))
95
- HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity,
96
- const void* src, size_t srcSize,
97
- unsigned maxSymbolValue, unsigned tableLog,
98
- void* workSpace, size_t wkspSize);
34
+ unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */
35
+ const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */
99
36
 
100
- #endif /* HUF_H_298734234 */
101
-
102
- /* ******************************************************************
103
- * WARNING !!
104
- * The following section contains advanced and experimental definitions
105
- * which shall never be used in the context of a dynamic library,
106
- * because they are not guaranteed to remain stable in the future.
107
- * Only consider them in association with static linking.
108
- * *****************************************************************/
109
- #if defined(HUF_STATIC_LINKING_ONLY) && !defined(HUF_H_HUF_STATIC_LINKING_ONLY)
110
- #define HUF_H_HUF_STATIC_LINKING_ONLY
111
-
112
- /* *** Dependencies *** */
113
- #include "mem.h" /* U32 */
114
37
 
38
+ #define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */)
39
+ #define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64))
115
40
 
116
41
  /* *** Constants *** */
117
- #define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
42
+ #define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_TABLELOG_ABSOLUTEMAX */
118
43
  #define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */
119
44
  #define HUF_SYMBOLVALUE_MAX 255
120
45
 
121
- #define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
46
+ #define HUF_TABLELOG_ABSOLUTEMAX 12 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
122
47
  #if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
123
48
  # error "HUF_TABLELOG_MAX is too large !"
124
49
  #endif
@@ -133,12 +58,12 @@ HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity,
133
58
  #define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
134
59
 
135
60
  /* static allocation of HUF's Compression Table */
136
- #define HUF_CTABLE_SIZE_U32(maxSymbolValue) ((maxSymbolValue)+1) /* Use tables of U32, for proper alignment */
137
- #define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_U32(maxSymbolValue) * sizeof(U32))
61
+ /* this is a private definition, just exposed for allocation and strict aliasing purpose. never EVER access its members directly */
62
+ typedef size_t HUF_CElt; /* consider it an incomplete type */
63
+ #define HUF_CTABLE_SIZE_ST(maxSymbolValue) ((maxSymbolValue)+2) /* Use tables of size_t, for proper alignment */
64
+ #define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_ST(maxSymbolValue) * sizeof(size_t))
138
65
  #define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \
139
- U32 name##hb[HUF_CTABLE_SIZE_U32(maxSymbolValue)]; \
140
- void* name##hv = &(name##hb); \
141
- HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */
66
+ HUF_CElt name[HUF_CTABLE_SIZE_ST(maxSymbolValue)] /* no final ; */
142
67
 
143
68
  /* static allocation of HUF's DTable */
144
69
  typedef U32 HUF_DTable;
@@ -152,25 +77,49 @@ typedef U32 HUF_DTable;
152
77
  /* ****************************************
153
78
  * Advanced decompression functions
154
79
  ******************************************/
155
- size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
156
- #ifndef HUF_FORCE_DECOMPRESS_X1
157
- size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
158
- #endif
159
80
 
160
- size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */
161
- size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
162
- size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */
163
- size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
164
- size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
165
- #ifndef HUF_FORCE_DECOMPRESS_X1
166
- size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
167
- size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
168
- #endif
81
+ /**
82
+ * Huffman flags bitset.
83
+ * For all flags, 0 is the default value.
84
+ */
85
+ typedef enum {
86
+ /**
87
+ * If compiled with DYNAMIC_BMI2: Set flag only if the CPU supports BMI2 at runtime.
88
+ * Otherwise: Ignored.
89
+ */
90
+ HUF_flags_bmi2 = (1 << 0),
91
+ /**
92
+ * If set: Test possible table depths to find the one that produces the smallest header + encoded size.
93
+ * If unset: Use heuristic to find the table depth.
94
+ */
95
+ HUF_flags_optimalDepth = (1 << 1),
96
+ /**
97
+ * If set: If the previous table can encode the input, always reuse the previous table.
98
+ * If unset: If the previous table can encode the input, reuse the previous table if it results in a smaller output.
99
+ */
100
+ HUF_flags_preferRepeat = (1 << 2),
101
+ /**
102
+ * If set: Sample the input and check if the sample is uncompressible, if it is then don't attempt to compress.
103
+ * If unset: Always histogram the entire input.
104
+ */
105
+ HUF_flags_suspectUncompressible = (1 << 3),
106
+ /**
107
+ * If set: Don't use assembly implementations
108
+ * If unset: Allow using assembly implementations
109
+ */
110
+ HUF_flags_disableAsm = (1 << 4),
111
+ /**
112
+ * If set: Don't use the fast decoding loop, always use the fallback decoding loop.
113
+ * If unset: Use the fast decoding loop when possible.
114
+ */
115
+ HUF_flags_disableFast = (1 << 5)
116
+ } HUF_flags_e;
169
117
 
170
118
 
171
119
  /* ****************************************
172
120
  * HUF detailed API
173
121
  * ****************************************/
122
+ #define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra
174
123
 
175
124
  /*! HUF_compress() does the following:
176
125
  * 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h")
@@ -183,11 +132,12 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
183
132
  * For example, it's possible to compress several blocks using the same 'CTable',
184
133
  * or to save and regenerate 'CTable' using external methods.
185
134
  */
186
- unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
187
- typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */
188
- size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */
189
- size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
190
- size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
135
+ unsigned HUF_minTableLog(unsigned symbolCardinality);
136
+ unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue);
137
+ unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace,
138
+ size_t wkspSize, HUF_CElt* table, const unsigned* count, int flags); /* table is used as scratch space for building and testing tables, not a return value */
139
+ size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize);
140
+ size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags);
191
141
  size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
192
142
  int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
193
143
 
@@ -196,22 +146,24 @@ typedef enum {
196
146
  HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */
197
147
  HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */
198
148
  } HUF_repeat;
149
+
199
150
  /** HUF_compress4X_repeat() :
200
151
  * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
201
152
  * If it uses hufTable it does not modify hufTable or repeat.
202
153
  * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
203
- * If preferRepeat then the old table will always be used if valid. */
154
+ * If preferRepeat then the old table will always be used if valid.
155
+ * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
204
156
  size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
205
157
  const void* src, size_t srcSize,
206
158
  unsigned maxSymbolValue, unsigned tableLog,
207
159
  void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
208
- HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2);
160
+ HUF_CElt* hufTable, HUF_repeat* repeat, int flags);
209
161
 
210
162
  /** HUF_buildCTable_wksp() :
211
163
  * Same as HUF_buildCTable(), but using externally allocated scratch buffer.
212
164
  * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE.
213
165
  */
214
- #define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1)
166
+ #define HUF_CTABLE_WORKSPACE_SIZE_U32 ((4 * (HUF_SYMBOLVALUE_MAX + 1)) + 192)
215
167
  #define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned))
216
168
  size_t HUF_buildCTable_wksp (HUF_CElt* tree,
217
169
  const unsigned* count, U32 maxSymbolValue, U32 maxNbBits,
@@ -226,15 +178,27 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize,
226
178
  U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
227
179
  const void* src, size_t srcSize);
228
180
 
181
+ /*! HUF_readStats_wksp() :
182
+ * Same as HUF_readStats() but takes an external workspace which must be
183
+ * 4-byte aligned and its size must be >= HUF_READ_STATS_WORKSPACE_SIZE.
184
+ * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
185
+ */
186
+ #define HUF_READ_STATS_WORKSPACE_SIZE_U32 FSE_DECOMPRESS_WKSP_SIZE_U32(6, HUF_TABLELOG_MAX-1)
187
+ #define HUF_READ_STATS_WORKSPACE_SIZE (HUF_READ_STATS_WORKSPACE_SIZE_U32 * sizeof(unsigned))
188
+ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize,
189
+ U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
190
+ const void* src, size_t srcSize,
191
+ void* workspace, size_t wkspSize,
192
+ int flags);
193
+
229
194
  /** HUF_readCTable() :
230
195
  * Loading a CTable saved with HUF_writeCTable() */
231
196
  size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned *hasZeroWeights);
232
197
 
233
- /** HUF_getNbBits() :
198
+ /** HUF_getNbBitsFromCTable() :
234
199
  * Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX
235
- * Note 1 : is not inlined, as HUF_CElt definition is private
236
- * Note 2 : const void* used, so that it can provide a statically allocated table as argument (which uses type U32) */
237
- U32 HUF_getNbBits(const void* symbolTable, U32 symbolValue);
200
+ * Note 1 : is not inlined, as HUF_CElt definition is private */
201
+ U32 HUF_getNbBitsFromCTable(const HUF_CElt* symbolTable, U32 symbolValue);
238
202
 
239
203
  /*
240
204
  * HUF_decompress() does the following:
@@ -260,80 +224,49 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
260
224
  * a required workspace size greater than that specified in the following
261
225
  * macro.
262
226
  */
263
- #define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10)
227
+ #define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9))
264
228
  #define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
265
229
 
266
- #ifndef HUF_FORCE_DECOMPRESS_X2
267
- size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize);
268
- size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
269
- #endif
270
- #ifndef HUF_FORCE_DECOMPRESS_X1
271
- size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize);
272
- size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
273
- #endif
274
-
275
- size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
276
- #ifndef HUF_FORCE_DECOMPRESS_X2
277
- size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
278
- #endif
279
- #ifndef HUF_FORCE_DECOMPRESS_X1
280
- size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
281
- #endif
282
-
283
230
 
284
231
  /* ====================== */
285
232
  /* single stream variants */
286
233
  /* ====================== */
287
234
 
288
- size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
289
- size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
290
- size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
235
+ size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags);
291
236
  /** HUF_compress1X_repeat() :
292
237
  * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
293
238
  * If it uses hufTable it does not modify hufTable or repeat.
294
239
  * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
295
- * If preferRepeat then the old table will always be used if valid. */
240
+ * If preferRepeat then the old table will always be used if valid.
241
+ * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
296
242
  size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
297
243
  const void* src, size_t srcSize,
298
244
  unsigned maxSymbolValue, unsigned tableLog,
299
245
  void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
300
- HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2);
246
+ HUF_CElt* hufTable, HUF_repeat* repeat, int flags);
301
247
 
302
- size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
248
+ size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);
303
249
  #ifndef HUF_FORCE_DECOMPRESS_X1
304
- size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
250
+ size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); /**< double-symbols decoder */
305
251
  #endif
306
252
 
307
- size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
308
- size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);
253
+ /* BMI2 variants.
254
+ * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
255
+ */
256
+ size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags);
309
257
  #ifndef HUF_FORCE_DECOMPRESS_X2
310
- size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
311
- size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
312
- #endif
313
- #ifndef HUF_FORCE_DECOMPRESS_X1
314
- size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
315
- size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
258
+ size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);
316
259
  #endif
317
-
318
- size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */
260
+ size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags);
261
+ size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);
319
262
  #ifndef HUF_FORCE_DECOMPRESS_X2
320
- size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
263
+ size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags);
321
264
  #endif
322
265
  #ifndef HUF_FORCE_DECOMPRESS_X1
323
- size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
266
+ size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags);
324
267
  #endif
325
268
 
326
- /* BMI2 variants.
327
- * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
328
- */
329
- size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
330
- #ifndef HUF_FORCE_DECOMPRESS_X2
331
- size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
332
- #endif
333
- size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
334
- size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
335
-
336
- #endif /* HUF_STATIC_LINKING_ONLY */
269
+ #endif /* HUF_H_298734234 */
337
270
 
338
271
  #if defined (__cplusplus)
339
272
  }