zstd-ruby 1.3.4.0 → 1.3.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (43) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -1
  3. data/ext/zstdruby/libzstd/Makefile +56 -10
  4. data/ext/zstdruby/libzstd/README.md +4 -0
  5. data/ext/zstdruby/libzstd/common/bitstream.h +6 -19
  6. data/ext/zstdruby/libzstd/common/compiler.h +3 -3
  7. data/ext/zstdruby/libzstd/common/cpu.h +1 -2
  8. data/ext/zstdruby/libzstd/common/debug.c +44 -0
  9. data/ext/zstdruby/libzstd/common/debug.h +123 -0
  10. data/ext/zstdruby/libzstd/common/entropy_common.c +16 -1
  11. data/ext/zstdruby/libzstd/common/fse.h +45 -41
  12. data/ext/zstdruby/libzstd/common/fse_decompress.c +1 -1
  13. data/ext/zstdruby/libzstd/common/huf.h +34 -27
  14. data/ext/zstdruby/libzstd/common/pool.c +89 -32
  15. data/ext/zstdruby/libzstd/common/pool.h +29 -19
  16. data/ext/zstdruby/libzstd/common/zstd_common.c +0 -5
  17. data/ext/zstdruby/libzstd/common/zstd_internal.h +3 -37
  18. data/ext/zstdruby/libzstd/compress/fse_compress.c +28 -163
  19. data/ext/zstdruby/libzstd/compress/hist.c +195 -0
  20. data/ext/zstdruby/libzstd/compress/hist.h +92 -0
  21. data/ext/zstdruby/libzstd/compress/huf_compress.c +14 -6
  22. data/ext/zstdruby/libzstd/compress/zstd_compress.c +798 -350
  23. data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +120 -34
  24. data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +247 -87
  25. data/ext/zstdruby/libzstd/compress/zstd_double_fast.h +4 -1
  26. data/ext/zstdruby/libzstd/compress/zstd_fast.c +177 -56
  27. data/ext/zstdruby/libzstd/compress/zstd_fast.h +4 -1
  28. data/ext/zstdruby/libzstd/compress/zstd_lazy.c +331 -65
  29. data/ext/zstdruby/libzstd/compress/zstd_lazy.h +13 -0
  30. data/ext/zstdruby/libzstd/compress/zstd_ldm.c +15 -20
  31. data/ext/zstdruby/libzstd/compress/zstd_ldm.h +1 -2
  32. data/ext/zstdruby/libzstd/compress/zstd_opt.c +503 -300
  33. data/ext/zstdruby/libzstd/compress/zstd_opt.h +7 -0
  34. data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +122 -47
  35. data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +5 -5
  36. data/ext/zstdruby/libzstd/decompress/huf_decompress.c +325 -325
  37. data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +80 -43
  38. data/ext/zstdruby/libzstd/dictBuilder/cover.c +9 -2
  39. data/ext/zstdruby/libzstd/dictBuilder/zdict.c +5 -5
  40. data/ext/zstdruby/libzstd/legacy/zstd_v04.c +12 -61
  41. data/ext/zstdruby/libzstd/zstd.h +137 -69
  42. data/lib/zstd-ruby/version.rb +1 -1
  43. metadata +7 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 100c4ca359fb8109a629f142d814ecd3fe1ff6854b6dead6c4f90a36f80f51c7
4
- data.tar.gz: 5303f715113cae2061b9b837ad7e7b8fd29b31b0b493c856a669327ef17fe773
3
+ metadata.gz: e4db2d13ebb9e0ebf43163566e394eb1aa2e5db44963cbb0b9d2863361114aa9
4
+ data.tar.gz: f2d6666a402d2ccdf4138783b696f34c3a1d5eaed2a94887a39ddf5b6892df2f
5
5
  SHA512:
6
- metadata.gz: 56af4d5c91af2fb6e70abdd970b5992751df49c234998ff127595c02c048439a3fccb98222a943581d539b98f98be3cd4804ad5b83bb20b04e4d5cfed2bac855
7
- data.tar.gz: b34865a20495d4aa35c0740bf92a340d325ce1829d52d4dca8872243e16681263cb81e6d668431fe3b657d738d7099968ca16ebb399cede9497099bbea3077a2
6
+ metadata.gz: 3e55fb48e38220cf3c6e50297f9b440b833e8061b2f76578a575d42126a5e59586ec6331522b85cb524b779a1f94cf26722d121d1019cb4f05e90968e28fa7c7
7
+ data.tar.gz: dba96481652009782745374a972ffdb3aff127b21f93d5adff3e1a2d425cf48c3396b7d12f63d0964db6dfb96715c04d92bd2794759367185b81f645d366ffdd
data/README.md CHANGED
@@ -9,7 +9,7 @@ See https://github.com/facebook/zstd
9
9
  Fork from https://github.com/jarredholman/ruby-zstd.
10
10
 
11
11
  ## Zstd version
12
- v1.3.4 (https://github.com/facebook/zstd/tree/v1.3.4)
12
+ v1.3.5 (https://github.com/facebook/zstd/tree/v1.3.5)
13
13
 
14
14
  ## Installation
15
15
 
@@ -28,10 +28,44 @@ DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
28
28
  CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
29
29
  FLAGS = $(CPPFLAGS) $(CFLAGS)
30
30
 
31
-
32
- ZSTD_FILES := $(sort $(wildcard common/*.c compress/*.c decompress/*.c dictBuilder/*.c deprecated/*.c))
31
+ ZSTDCOMMON_FILES := $(sort $(wildcard common/*.c))
32
+ ZSTDCOMP_FILES := $(sort $(wildcard compress/*.c))
33
+ ZSTDDECOMP_FILES := $(sort $(wildcard decompress/*.c))
34
+ ZDICT_FILES := $(sort $(wildcard dictBuilder/*.c))
35
+ ZDEPR_FILES := $(sort $(wildcard deprecated/*.c))
36
+ ZSTD_FILES := $(ZSTDCOMMON_FILES)
33
37
 
34
38
  ZSTD_LEGACY_SUPPORT ?= 4
39
+ ZSTD_LIB_COMPRESSION ?= 1
40
+ ZSTD_LIB_DECOMPRESSION ?= 1
41
+ ZSTD_LIB_DICTBUILDER ?= 1
42
+ ZSTD_LIB_DEPRECATED ?= 1
43
+
44
+ ifeq ($(ZSTD_LIB_COMPRESSION), 0)
45
+ ZSTD_LIB_DICTBUILDER = 0
46
+ ZSTD_LIB_DEPRECATED = 0
47
+ endif
48
+
49
+ ifeq ($(ZSTD_LIB_DECOMPRESSION), 0)
50
+ ZSTD_LEGACY_SUPPORT = 0
51
+ ZSTD_LIB_DEPRECATED = 0
52
+ endif
53
+
54
+ ifneq ($(ZSTD_LIB_COMPRESSION), 0)
55
+ ZSTD_FILES += $(ZSTDCOMP_FILES)
56
+ endif
57
+
58
+ ifneq ($(ZSTD_LIB_DECOMPRESSION), 0)
59
+ ZSTD_FILES += $(ZSTDDECOMP_FILES)
60
+ endif
61
+
62
+ ifneq ($(ZSTD_LIB_DEPRECATED), 0)
63
+ ZSTD_FILES += $(ZDEPR_FILES)
64
+ endif
65
+
66
+ ifneq ($(ZSTD_LIB_DICTBUILDER), 0)
67
+ ZSTD_FILES += $(ZDICT_FILES)
68
+ endif
35
69
 
36
70
  ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
37
71
  ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
@@ -43,7 +77,7 @@ CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
43
77
 
44
78
  ZSTD_OBJ := $(patsubst %.c,%.o,$(ZSTD_FILES))
45
79
 
46
- # OS X linker doesn't support -soname, and use different extension
80
+ # macOS linker doesn't support -soname, and use different extension
47
81
  # see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html
48
82
  ifeq ($(shell uname), Darwin)
49
83
  SHARED_EXT = dylib
@@ -111,14 +145,14 @@ libzstd-nomt: $(ZSTD_NOMT_FILES)
111
145
  @$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
112
146
 
113
147
  clean:
114
- @$(RM) -r *.dSYM # Mac OS-X specific
148
+ @$(RM) -r *.dSYM # macOS-specific
115
149
  @$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc
116
150
  @$(RM) dll/libzstd.dll dll/libzstd.lib libzstd-nomt*
117
151
  @$(RM) common/*.o compress/*.o decompress/*.o dictBuilder/*.o legacy/*.o deprecated/*.o
118
152
  @echo Cleaning library completed
119
153
 
120
154
  #-----------------------------------------------------------------------------
121
- # make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
155
+ # make install is validated only for Linux, macOS, BSD, Hurd and Solaris targets
122
156
  #-----------------------------------------------------------------------------
123
157
  ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
124
158
 
@@ -134,7 +168,7 @@ LIBDIR ?= $(libdir)
134
168
  includedir ?= $(PREFIX)/include
135
169
  INCLUDEDIR ?= $(includedir)
136
170
 
137
- ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly))
171
+ ifneq (,$(filter $(shell uname),FreeBSD NetBSD DragonFly))
138
172
  PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig
139
173
  else
140
174
  PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig
@@ -159,20 +193,32 @@ libzstd.pc: libzstd.pc.in
159
193
  -e 's|@VERSION@|$(VERSION)|' \
160
194
  $< >$@
161
195
 
162
- install: libzstd.a libzstd libzstd.pc
163
- @$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/ $(DESTDIR)$(INCLUDEDIR)/
196
+ install: install-pc install-static install-shared install-includes
197
+ @echo zstd static and shared library installed
198
+
199
+ install-pc: libzstd.pc
200
+ @$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/
164
201
  @$(INSTALL_DATA) libzstd.pc $(DESTDIR)$(PKGCONFIGDIR)/
165
- @echo Installing libraries
202
+
203
+ install-static: libzstd.a
204
+ @echo Installing static library
205
+ @$(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)/
166
206
  @$(INSTALL_DATA) libzstd.a $(DESTDIR)$(LIBDIR)
207
+
208
+ install-shared: libzstd
209
+ @echo Installing shared library
210
+ @$(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)/
167
211
  @$(INSTALL_PROGRAM) $(LIBZSTD) $(DESTDIR)$(LIBDIR)
168
212
  @ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
169
213
  @ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
214
+
215
+ install-includes:
170
216
  @echo Installing includes
217
+ @$(INSTALL) -d -m 755 $(DESTDIR)$(INCLUDEDIR)/
171
218
  @$(INSTALL_DATA) zstd.h $(DESTDIR)$(INCLUDEDIR)
172
219
  @$(INSTALL_DATA) common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)
173
220
  @$(INSTALL_DATA) deprecated/zbuff.h $(DESTDIR)$(INCLUDEDIR) # prototypes generate deprecation warnings
174
221
  @$(INSTALL_DATA) dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)
175
- @echo zstd static and shared library installed
176
222
 
177
223
  uninstall:
178
224
  @$(RM) $(DESTDIR)$(LIBDIR)/libzstd.a
@@ -61,6 +61,10 @@ It's possible to compile only a limited set of features.
61
61
  Each version also provides an additional dedicated set of advanced API.
62
62
  For example, advanced API for version `v0.4` is exposed in `lib/legacy/zstd_v04.h` .
63
63
  Note : `lib/legacy` only supports _decoding_ legacy formats.
64
+ - Similarly, you can define `ZSTD_LIB_COMPRESSION, ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`,
65
+ and `ZSTD_LIB_DEPRECATED` as 0 to forgo compilation of the corresponding features. This will
66
+ also disable compilation of all dependencies (eg. `ZSTD_LIB_COMPRESSION=0` will also disable
67
+ dictBuilder).
64
68
 
65
69
 
66
70
  #### Multithreading support
@@ -1,8 +1,7 @@
1
1
  /* ******************************************************************
2
2
  bitstream
3
3
  Part of FSE library
4
- header file (to include)
5
- Copyright (C) 2013-2017, Yann Collet.
4
+ Copyright (C) 2013-present, Yann Collet.
6
5
 
7
6
  BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
8
7
 
@@ -49,21 +48,10 @@ extern "C" {
49
48
  * Dependencies
50
49
  ******************************************/
51
50
  #include "mem.h" /* unaligned access routines */
51
+ #include "debug.h" /* assert(), DEBUGLOG(), RAWLOG() */
52
52
  #include "error_private.h" /* error codes and messages */
53
53
 
54
54
 
55
- /*-*************************************
56
- * Debug
57
- ***************************************/
58
- #if defined(BIT_DEBUG) && (BIT_DEBUG>=1)
59
- # include <assert.h>
60
- #else
61
- # ifndef assert
62
- # define assert(condition) ((void)0)
63
- # endif
64
- #endif
65
-
66
-
67
55
  /*=========================================
68
56
  * Target specific
69
57
  =========================================*/
@@ -83,8 +71,7 @@ extern "C" {
83
71
  * A critical property of these streams is that they encode and decode in **reverse** direction.
84
72
  * So the first bit sequence you add will be the last to be read, like a LIFO stack.
85
73
  */
86
- typedef struct
87
- {
74
+ typedef struct {
88
75
  size_t bitContainer;
89
76
  unsigned bitPos;
90
77
  char* startPtr;
@@ -118,8 +105,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
118
105
  /*-********************************************
119
106
  * bitStream decoding API (read backward)
120
107
  **********************************************/
121
- typedef struct
122
- {
108
+ typedef struct {
123
109
  size_t bitContainer;
124
110
  unsigned bitsConsumed;
125
111
  const char* ptr;
@@ -236,7 +222,8 @@ MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
236
222
  }
237
223
 
238
224
  /*! BIT_addBitsFast() :
239
- * works only if `value` is _clean_, meaning all high bits above nbBits are 0 */
225
+ * works only if `value` is _clean_,
226
+ * meaning all high bits above nbBits are 0 */
240
227
  MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC,
241
228
  size_t value, unsigned nbBits)
242
229
  {
@@ -77,9 +77,9 @@
77
77
  * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default.
78
78
  */
79
79
  #ifndef DYNAMIC_BMI2
80
- #if (defined(__clang__) && __has_attribute(__target__)) \
80
+ #if ((defined(__clang__) && __has_attribute(__target__)) \
81
81
  || (defined(__GNUC__) \
82
- && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8))) \
82
+ && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \
83
83
  && (defined(__x86_64__) || defined(_M_X86)) \
84
84
  && !defined(__BMI2__)
85
85
  # define DYNAMIC_BMI2 1
@@ -92,7 +92,7 @@
92
92
  #if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */
93
93
  # include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
94
94
  # define PREFETCH(ptr) _mm_prefetch((const char*)ptr, _MM_HINT_T0)
95
- #elif defined(__GNUC__)
95
+ #elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
96
96
  # define PREFETCH(ptr) __builtin_prefetch(ptr, 0, 0)
97
97
  #else
98
98
  # define PREFETCH(ptr) /* disabled */
@@ -72,8 +72,7 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
72
72
  "cpuid\n\t"
73
73
  "popl %%ebx\n\t"
74
74
  : "=a"(f1a), "=c"(f1c), "=d"(f1d)
75
- : "a"(1)
76
- :);
75
+ : "a"(1));
77
76
  }
78
77
  if (n >= 7) {
79
78
  __asm__(
@@ -0,0 +1,44 @@
1
+ /* ******************************************************************
2
+ debug
3
+ Part of FSE library
4
+ Copyright (C) 2013-present, Yann Collet.
5
+
6
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
7
+
8
+ Redistribution and use in source and binary forms, with or without
9
+ modification, are permitted provided that the following conditions are
10
+ met:
11
+
12
+ * Redistributions of source code must retain the above copyright
13
+ notice, this list of conditions and the following disclaimer.
14
+ * Redistributions in binary form must reproduce the above
15
+ copyright notice, this list of conditions and the following disclaimer
16
+ in the documentation and/or other materials provided with the
17
+ distribution.
18
+
19
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
+
31
+ You can contact the author at :
32
+ - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
33
+ ****************************************************************** */
34
+
35
+
36
+ /*
37
+ * This module only hosts one global variable
38
+ * which can be used to dynamically influence the verbosity of traces,
39
+ * such as DEBUGLOG and RAWLOG
40
+ */
41
+
42
+ #include "debug.h"
43
+
44
+ int g_debuglevel = DEBUGLEVEL;
@@ -0,0 +1,123 @@
1
+ /* ******************************************************************
2
+ debug
3
+ Part of FSE library
4
+ Copyright (C) 2013-present, Yann Collet.
5
+
6
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
7
+
8
+ Redistribution and use in source and binary forms, with or without
9
+ modification, are permitted provided that the following conditions are
10
+ met:
11
+
12
+ * Redistributions of source code must retain the above copyright
13
+ notice, this list of conditions and the following disclaimer.
14
+ * Redistributions in binary form must reproduce the above
15
+ copyright notice, this list of conditions and the following disclaimer
16
+ in the documentation and/or other materials provided with the
17
+ distribution.
18
+
19
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
+
31
+ You can contact the author at :
32
+ - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
33
+ ****************************************************************** */
34
+
35
+
36
+ /*
37
+ * The purpose of this header is to enable debug functions.
38
+ * They regroup assert(), DEBUGLOG() and RAWLOG() for run-time,
39
+ * and DEBUG_STATIC_ASSERT() for compile-time.
40
+ *
41
+ * By default, DEBUGLEVEL==0, which means run-time debug is disabled.
42
+ *
43
+ * Level 1 enables assert() only.
44
+ * Starting level 2, traces can be generated and pushed to stderr.
45
+ * The higher the level, the more verbose the traces.
46
+ *
47
+ * It's possible to dynamically adjust level using variable g_debug_level,
48
+ * which is only declared if DEBUGLEVEL>=2,
49
+ * and is a global variable, not multi-thread protected (use with care)
50
+ */
51
+
52
+ #ifndef DEBUG_H_12987983217
53
+ #define DEBUG_H_12987983217
54
+
55
+ #if defined (__cplusplus)
56
+ extern "C" {
57
+ #endif
58
+
59
+
60
+ /* static assert is triggered at compile time, leaving no runtime artefact,
61
+ * but can only work with compile-time constants.
62
+ * This variant can only be used inside a function. */
63
+ #define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1])
64
+
65
+
66
+ /* DEBUGLEVEL is expected to be defined externally,
67
+ * typically through compiler command line.
68
+ * Value must be a number. */
69
+ #ifndef DEBUGLEVEL
70
+ # define DEBUGLEVEL 0
71
+ #endif
72
+
73
+ /* recommended values for DEBUGLEVEL :
74
+ * 0 : no debug, all run-time functions disabled
75
+ * 1 : no display, enables assert() only
76
+ * 2 : reserved, for currently active debug path
77
+ * 3 : events once per object lifetime (CCtx, CDict, etc.)
78
+ * 4 : events once per frame
79
+ * 5 : events once per block
80
+ * 6 : events once per sequence (verbose)
81
+ * 7+: events at every position (*very* verbose)
82
+ *
83
+ * It's generally inconvenient to output traces > 5.
84
+ * In which case, it's possible to selectively enable higher verbosity levels
85
+ * by modifying g_debug_level.
86
+ */
87
+
88
+ #if (DEBUGLEVEL>=1)
89
+ # include <assert.h>
90
+ #else
91
+ # ifndef assert /* assert may be already defined, due to prior #include <assert.h> */
92
+ # define assert(condition) ((void)0) /* disable assert (default) */
93
+ # endif
94
+ #endif
95
+
96
+ #if (DEBUGLEVEL>=2)
97
+ # include <stdio.h>
98
+ extern int g_debuglevel; /* here, this variable is only declared,
99
+ it actually lives in debug.c,
100
+ and is shared by the whole process.
101
+ It's typically used to enable very verbose levels
102
+ on selective conditions (such as position in src) */
103
+
104
+ # define RAWLOG(l, ...) { \
105
+ if (l<=g_debuglevel) { \
106
+ fprintf(stderr, __VA_ARGS__); \
107
+ } }
108
+ # define DEBUGLOG(l, ...) { \
109
+ if (l<=g_debuglevel) { \
110
+ fprintf(stderr, __FILE__ ": " __VA_ARGS__); \
111
+ fprintf(stderr, " \n"); \
112
+ } }
113
+ #else
114
+ # define RAWLOG(l, ...) {} /* disabled */
115
+ # define DEBUGLOG(l, ...) {} /* disabled */
116
+ #endif
117
+
118
+
119
+ #if defined (__cplusplus)
120
+ }
121
+ #endif
122
+
123
+ #endif /* DEBUG_H_12987983217 */
@@ -72,7 +72,21 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
72
72
  unsigned charnum = 0;
73
73
  int previous0 = 0;
74
74
 
75
- if (hbSize < 4) return ERROR(srcSize_wrong);
75
+ if (hbSize < 4) {
76
+ /* This function only works when hbSize >= 4 */
77
+ char buffer[4];
78
+ memset(buffer, 0, sizeof(buffer));
79
+ memcpy(buffer, headerBuffer, hbSize);
80
+ { size_t const countSize = FSE_readNCount(normalizedCounter, maxSVPtr, tableLogPtr,
81
+ buffer, sizeof(buffer));
82
+ if (FSE_isError(countSize)) return countSize;
83
+ if (countSize > hbSize) return ERROR(corruption_detected);
84
+ return countSize;
85
+ } }
86
+ assert(hbSize >= 4);
87
+
88
+ /* init */
89
+ memset(normalizedCounter, 0, (*maxSVPtr+1) * sizeof(normalizedCounter[0])); /* all symbols not present in NCount have a frequency of 0 */
76
90
  bitStream = MEM_readLE32(ip);
77
91
  nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */
78
92
  if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge);
@@ -105,6 +119,7 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
105
119
  if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall);
106
120
  while (charnum < n0) normalizedCounter[charnum++] = 0;
107
121
  if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
122
+ assert((bitCount >> 3) <= 3); /* For first condition to work */
108
123
  ip += bitCount>>3;
109
124
  bitCount &= 7;
110
125
  bitStream = MEM_readLE32(ip) >> bitCount;
@@ -72,6 +72,7 @@ extern "C" {
72
72
  #define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE)
73
73
  FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */
74
74
 
75
+
75
76
  /*-****************************************
76
77
  * FSE simple functions
77
78
  ******************************************/
@@ -129,7 +130,7 @@ FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src,
129
130
  ******************************************/
130
131
  /*!
131
132
  FSE_compress() does the following:
132
- 1. count symbol occurrence from source[] into table count[]
133
+ 1. count symbol occurrence from source[] into table count[] (see hist.h)
133
134
  2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
134
135
  3. save normalized counters to memory buffer using writeNCount()
135
136
  4. build encoding table 'CTable' from normalized counters
@@ -147,15 +148,6 @@ or to save and provide normalized distribution using external method.
147
148
 
148
149
  /* *** COMPRESSION *** */
149
150
 
150
- /*! FSE_count():
151
- Provides the precise count of each byte within a table 'count'.
152
- 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1).
153
- *maxSymbolValuePtr will be updated if detected smaller than initial value.
154
- @return : the count of the most frequent symbol (which is not identified).
155
- if return == srcSize, there is only one symbol.
156
- Can also return an error code, which can be tested with FSE_isError(). */
157
- FSE_PUBLIC_API size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
158
-
159
151
  /*! FSE_optimalTableLog():
160
152
  dynamically downsize 'tableLog' when conditions are met.
161
153
  It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
@@ -167,7 +159,8 @@ FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize
167
159
  'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
168
160
  @return : tableLog,
169
161
  or an errorCode, which can be tested using FSE_isError() */
170
- FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
162
+ FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog,
163
+ const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
171
164
 
172
165
  /*! FSE_NCountWriteBound():
173
166
  Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'.
@@ -178,8 +171,9 @@ FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tab
178
171
  Compactly save 'normalizedCounter' into 'buffer'.
179
172
  @return : size of the compressed table,
180
173
  or an errorCode, which can be tested using FSE_isError(). */
181
- FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
182
-
174
+ FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize,
175
+ const short* normalizedCounter,
176
+ unsigned maxSymbolValue, unsigned tableLog);
183
177
 
184
178
  /*! Constructor and Destructor of FSE_CTable.
185
179
  Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
@@ -250,7 +244,9 @@ If there is an error, the function will return an ErrorCode (which can be tested
250
244
  @return : size read from 'rBuffer',
251
245
  or an errorCode, which can be tested using FSE_isError().
252
246
  maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
253
- FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize);
247
+ FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter,
248
+ unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,
249
+ const void* rBuffer, size_t rBuffSize);
254
250
 
255
251
  /*! Constructor and Destructor of FSE_DTable.
256
252
  Note that its size depends on 'tableLog' */
@@ -325,33 +321,8 @@ If there is an error, the function will return an error code, which can be teste
325
321
 
326
322
 
327
323
  /* *****************************************
328
- * FSE advanced API
329
- *******************************************/
330
- /* FSE_count_wksp() :
331
- * Same as FSE_count(), but using an externally provided scratch buffer.
332
- * `workSpace` size must be table of >= `1024` unsigned
333
- */
334
- size_t FSE_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
335
- const void* source, size_t sourceSize, unsigned* workSpace);
336
-
337
- /** FSE_countFast() :
338
- * same as FSE_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr
339
- */
340
- size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
341
-
342
- /* FSE_countFast_wksp() :
343
- * Same as FSE_countFast(), but using an externally provided scratch buffer.
344
- * `workSpace` must be a table of minimum `1024` unsigned
345
- */
346
- size_t FSE_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* workSpace);
347
-
348
- /*! FSE_count_simple() :
349
- * Same as FSE_countFast(), but does not use any additional memory (not even on stack).
350
- * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` (presuming it's also the size of `count`).
351
- */
352
- size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
353
-
354
-
324
+ * FSE advanced API
325
+ ***************************************** */
355
326
 
356
327
  unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
357
328
  /**< same as FSE_optimalTableLog(), which used `minus==2` */
@@ -576,6 +547,39 @@ MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePt
576
547
  }
577
548
 
578
549
 
550
+ /* FSE_getMaxNbBits() :
551
+ * Approximate maximum cost of a symbol, in bits.
552
+ * Fractional get rounded up (i.e : a symbol with a normalized frequency of 3 gives the same result as a frequency of 2)
553
+ * note 1 : assume symbolValue is valid (<= maxSymbolValue)
554
+ * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
555
+ MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue)
556
+ {
557
+ const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr;
558
+ return (symbolTT[symbolValue].deltaNbBits + ((1<<16)-1)) >> 16;
559
+ }
560
+
561
+ /* FSE_bitCost() :
562
+ * Approximate symbol cost, as fractional value, using fixed-point format (accuracyLog fractional bits)
563
+ * note 1 : assume symbolValue is valid (<= maxSymbolValue)
564
+ * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
565
+ MEM_STATIC U32 FSE_bitCost(const void* symbolTTPtr, U32 tableLog, U32 symbolValue, U32 accuracyLog)
566
+ {
567
+ const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr;
568
+ U32 const minNbBits = symbolTT[symbolValue].deltaNbBits >> 16;
569
+ U32 const threshold = (minNbBits+1) << 16;
570
+ assert(tableLog < 16);
571
+ assert(accuracyLog < 31-tableLog); /* ensure enough room for renormalization double shift */
572
+ { U32 const tableSize = 1 << tableLog;
573
+ U32 const deltaFromThreshold = threshold - (symbolTT[symbolValue].deltaNbBits + tableSize);
574
+ U32 const normalizedDeltaFromThreshold = (deltaFromThreshold << accuracyLog) >> tableLog; /* linear interpolation (very approximate) */
575
+ U32 const bitMultiplier = 1 << accuracyLog;
576
+ assert(symbolTT[symbolValue].deltaNbBits + tableSize <= threshold);
577
+ assert(normalizedDeltaFromThreshold <= bitMultiplier);
578
+ return (minNbBits+1)*bitMultiplier - normalizedDeltaFromThreshold;
579
+ }
580
+ }
581
+
582
+
579
583
  /* ====== Decompression ====== */
580
584
 
581
585
  typedef struct {