extzstd 0.0.2.CONCEPT → 0.0.3.CONCEPT

Sign up to get free protection for your applications and to get access to all the features.
Files changed (68) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +72 -22
  3. data/Rakefile +73 -19
  4. data/contrib/zstd/README.md +68 -0
  5. data/contrib/zstd/common/bitstream.h +414 -0
  6. data/contrib/zstd/common/entropy_common.c +231 -0
  7. data/contrib/zstd/common/error_private.h +125 -0
  8. data/contrib/zstd/common/error_public.h +77 -0
  9. data/contrib/zstd/common/fse.h +628 -0
  10. data/contrib/zstd/common/fse_decompress.c +331 -0
  11. data/contrib/zstd/common/huf.h +228 -0
  12. data/contrib/zstd/common/mem.h +377 -0
  13. data/contrib/zstd/common/xxhash.c +854 -0
  14. data/contrib/zstd/common/xxhash.h +273 -0
  15. data/contrib/zstd/common/zbuff.h +197 -0
  16. data/contrib/zstd/common/zstd.h +475 -0
  17. data/contrib/zstd/common/zstd_common.c +91 -0
  18. data/contrib/zstd/common/zstd_internal.h +238 -0
  19. data/contrib/zstd/compress/fse_compress.c +807 -0
  20. data/contrib/zstd/compress/huf_compress.c +577 -0
  21. data/contrib/zstd/compress/zbuff_compress.c +327 -0
  22. data/contrib/zstd/compress/zstd_compress.c +3074 -0
  23. data/contrib/zstd/compress/zstd_opt.h +1046 -0
  24. data/contrib/zstd/decompress/huf_decompress.c +894 -0
  25. data/contrib/zstd/decompress/zbuff_decompress.c +294 -0
  26. data/contrib/zstd/decompress/zstd_decompress.c +1362 -0
  27. data/contrib/zstd/dictBuilder/divsufsort.c +1913 -0
  28. data/contrib/zstd/dictBuilder/divsufsort.h +67 -0
  29. data/contrib/zstd/dictBuilder/zdict.c +1045 -0
  30. data/contrib/zstd/dictBuilder/zdict.h +113 -0
  31. data/contrib/zstd/legacy/zstd_legacy.h +140 -0
  32. data/contrib/zstd/legacy/zstd_v01.c +2178 -0
  33. data/contrib/zstd/{zstd.h → legacy/zstd_v01.h} +46 -39
  34. data/contrib/zstd/legacy/zstd_v02.c +3748 -0
  35. data/contrib/zstd/legacy/zstd_v02.h +99 -0
  36. data/contrib/zstd/legacy/zstd_v03.c +3389 -0
  37. data/contrib/zstd/legacy/zstd_v03.h +99 -0
  38. data/contrib/zstd/legacy/zstd_v04.c +4056 -0
  39. data/contrib/zstd/legacy/zstd_v04.h +148 -0
  40. data/contrib/zstd/legacy/zstd_v05.c +4325 -0
  41. data/contrib/zstd/legacy/zstd_v05.h +171 -0
  42. data/contrib/zstd/legacy/zstd_v06.c +4581 -0
  43. data/contrib/zstd/legacy/zstd_v06.h +185 -0
  44. data/ext/extconf.rb +10 -12
  45. data/ext/extzstd.c +497 -144
  46. data/ext/extzstd.h +127 -22
  47. data/ext/extzstd_buffered.c +265 -0
  48. data/ext/extzstd_nogvls.h +174 -0
  49. data/ext/zstd_amalgam.c +18 -0
  50. data/ext/zstd_legacy_v01.c +1 -0
  51. data/ext/zstd_legacy_v02.c +1 -0
  52. data/ext/zstd_legacy_v03.c +1 -0
  53. data/ext/zstd_legacy_v04.c +1 -0
  54. data/ext/zstd_legacy_v05.c +1 -0
  55. data/ext/zstd_legacy_v06.c +1 -0
  56. data/gemstub.rb +17 -1
  57. data/lib/extzstd.rb +197 -77
  58. data/lib/extzstd/version.rb +1 -1
  59. data/test/test_basic.rb +41 -0
  60. metadata +70 -20
  61. data/contrib/zstd/Makefile +0 -115
  62. data/contrib/zstd/fse.c +0 -2466
  63. data/contrib/zstd/fse.h +0 -320
  64. data/contrib/zstd/fse_static.h +0 -282
  65. data/contrib/zstd/libzstd.pc.in +0 -14
  66. data/contrib/zstd/zstd.c +0 -1768
  67. data/contrib/zstd/zstd_static.h +0 -89
  68. data/ext/extzstd-stream.c +0 -398
@@ -1,115 +0,0 @@
1
- # ################################################################
2
- # ZSTD library - Makefile
3
- # Copyright (C) Yann Collet 2015
4
- # All rights reserved.
5
- #
6
- # BSD license
7
-
8
- # Redistribution and use in source and binary forms, with or without modification,
9
- # are permitted provided that the following conditions are met:
10
- #
11
- # * Redistributions of source code must retain the above copyright notice, this
12
- # list of conditions and the following disclaimer.
13
- #
14
- # * Redistributions in binary form must reproduce the above copyright notice, this
15
- # list of conditions and the following disclaimer in the documentation and/or
16
- # other materials provided with the distribution.
17
- #
18
- # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
19
- # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20
- # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21
- # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
22
- # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23
- # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24
- # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
25
- # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26
- # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27
- # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
- #
29
- # You can contact the author at :
30
- # - ZSTD source repository : https://github.com/Cyan4973/zstd
31
- # - Public forum : https://groups.google.com/forum/#!forum/lz4c
32
- # ################################################################
33
-
34
- # Version numbers
35
- VERSION?= 0.1.2
36
- LIBVER_MAJOR=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
37
- LIBVER_MINOR=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
38
- LIBVER_PATCH=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
39
- LIBVER = $(LIBVER_MAJOR).$(LIBVER_MINOR).$(LIBVER_PATCH)
40
-
41
- DESTDIR?=
42
- PREFIX ?= /usr/local
43
- CFLAGS ?= -O3
44
- CFLAGS += -I. -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes
45
- FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
46
-
47
- LIBDIR ?= $(PREFIX)/lib
48
- INCLUDEDIR=$(PREFIX)/include
49
-
50
-
51
- # OS X linker doesn't support -soname, and use different extension
52
- # see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html
53
- ifeq ($(shell uname), Darwin)
54
- SHARED_EXT = dylib
55
- SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT)
56
- SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
57
- SONAME_FLAGS = -install_name $(PREFIX)/lib/libzstd.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
58
- else
59
- SONAME_FLAGS = -Wl,-soname=libzstd.$(SHARED_EXT).$(LIBVER_MAJOR)
60
- SHARED_EXT = so
61
- SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR)
62
- SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
63
- endif
64
-
65
- default: libzstd
66
-
67
- all: libzstd
68
-
69
- libzstd: zstd.c
70
- @echo compiling static library
71
- @$(CC) $(FLAGS) -c $^
72
- @$(AR) rcs libzstd.a zstd.o
73
- @echo compiling dynamic library $(LIBVER)
74
- @$(CC) $(FLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER)
75
- @echo creating versioned links
76
- @ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT_MAJOR)
77
- @ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT)
78
-
79
- clean:
80
- @rm -f core *.o *.a *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc
81
- @echo Cleaning library completed
82
-
83
-
84
- #------------------------------------------------------------------------
85
- #make install is validated only for Linux, OSX, kFreeBSD and Hurd targets
86
- ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU))
87
-
88
- libzstd.pc: libzstd.pc.in Makefile
89
- @echo creating pkgconfig
90
- @sed -e 's|@PREFIX@|$(PREFIX)|' \
91
- -e 's|@LIBDIR@|$(LIBDIR)|' \
92
- -e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \
93
- -e 's|@VERSION@|$(VERSION)|' \
94
- $< >$@
95
-
96
- install: libzstd libzstd.pc
97
- @install -d -m 755 $(DESTDIR)$(LIBDIR)/pkgconfig/ $(DESTDIR)$(INCLUDEDIR)/
98
- @install -m 755 libzstd.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER)
99
- @cp -a libzstd.$(SHARED_EXT_MAJOR) $(DESTDIR)$(LIBDIR)
100
- @cp -a libzstd.$(SHARED_EXT) $(DESTDIR)$(LIBDIR)
101
- @cp -a libzstd.pc $(DESTDIR)$(LIBDIR)/pkgconfig/
102
- @install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a
103
- @install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h
104
- @echo zstd static and shared library installed
105
-
106
- uninstall:
107
- @rm -f $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
108
- @rm -f $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
109
- @rm -f $(DESTDIR)$(LIBDIR)/pkgconfig/libzstd.pc
110
- @[ -x $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) ] && rm -f $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER)
111
- @[ -f $(DESTDIR)$(LIBDIR)/libzstd.a ] && rm -f $(DESTDIR)$(LIBDIR)/libzstd.a
112
- @[ -f $(DESTDIR)$(INCLUDEDIR)/zstd.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/zstd.h
113
- @echo zstd libraries successfully uninstalled
114
-
115
- endif
@@ -1,2466 +0,0 @@
1
- /* ******************************************************************
2
- FSE : Finite State Entropy coder
3
- Copyright (C) 2013-2015, Yann Collet.
4
-
5
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
6
-
7
- Redistribution and use in source and binary forms, with or without
8
- modification, are permitted provided that the following conditions are
9
- met:
10
-
11
- * Redistributions of source code must retain the above copyright
12
- notice, this list of conditions and the following disclaimer.
13
- * Redistributions in binary form must reproduce the above
14
- copyright notice, this list of conditions and the following disclaimer
15
- in the documentation and/or other materials provided with the
16
- distribution.
17
-
18
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
-
30
- You can contact the author at :
31
- - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
32
- - Public forum : https://groups.google.com/forum/#!forum/lz4c
33
- ****************************************************************** */
34
-
35
- #ifndef FSE_COMMONDEFS_ONLY
36
-
37
- /****************************************************************
38
- * Tuning parameters
39
- ****************************************************************/
40
- /* MEMORY_USAGE :
41
- * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
42
- * Increasing memory usage improves compression ratio
43
- * Reduced memory usage can improve speed, due to cache effect
44
- * Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
45
- #define FSE_MAX_MEMORY_USAGE 14
46
- #define FSE_DEFAULT_MEMORY_USAGE 13
47
-
48
- /* FSE_MAX_SYMBOL_VALUE :
49
- * Maximum symbol value authorized.
50
- * Required for proper stack allocation */
51
- #define FSE_MAX_SYMBOL_VALUE 255
52
-
53
-
54
- /****************************************************************
55
- * template functions type & suffix
56
- ****************************************************************/
57
- #define FSE_FUNCTION_TYPE BYTE
58
- #define FSE_FUNCTION_EXTENSION
59
-
60
-
61
- /****************************************************************
62
- * Byte symbol type
63
- ****************************************************************/
64
- typedef struct
65
- {
66
- unsigned short newState;
67
- unsigned char symbol;
68
- unsigned char nbBits;
69
- } FSE_decode_t; /* size == U32 */
70
-
71
- #endif /* !FSE_COMMONDEFS_ONLY */
72
-
73
-
74
- /****************************************************************
75
- * Compiler specifics
76
- ****************************************************************/
77
- #ifdef _MSC_VER /* Visual Studio */
78
- # define FORCE_INLINE static __forceinline
79
- # include <intrin.h> /* For Visual 2005 */
80
- # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
81
- # pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
82
- #else
83
- # define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
84
- # ifdef __GNUC__
85
- # define FORCE_INLINE static inline __attribute__((always_inline))
86
- # else
87
- # define FORCE_INLINE static inline
88
- # endif
89
- #endif
90
-
91
-
92
- /****************************************************************
93
- * Includes
94
- ****************************************************************/
95
- #include <stdlib.h> /* malloc, free, qsort */
96
- #include <string.h> /* memcpy, memset */
97
- #include <stdio.h> /* printf (debug) */
98
- #include "fse_static.h"
99
-
100
-
101
- #ifndef MEM_ACCESS_MODULE
102
- #define MEM_ACCESS_MODULE
103
- /****************************************************************
104
- * Basic Types
105
- *****************************************************************/
106
- #if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
107
- # include <stdint.h>
108
- typedef uint8_t BYTE;
109
- typedef uint16_t U16;
110
- typedef int16_t S16;
111
- typedef uint32_t U32;
112
- typedef int32_t S32;
113
- typedef uint64_t U64;
114
- typedef int64_t S64;
115
- #else
116
- typedef unsigned char BYTE;
117
- typedef unsigned short U16;
118
- typedef signed short S16;
119
- typedef unsigned int U32;
120
- typedef signed int S32;
121
- typedef unsigned long long U64;
122
- typedef signed long long S64;
123
- #endif
124
-
125
- #endif /* MEM_ACCESS_MODULE */
126
-
127
- /****************************************************************
128
- * Memory I/O
129
- *****************************************************************/
130
- /* FSE_FORCE_MEMORY_ACCESS
131
- * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
132
- * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
133
- * The below switch allow to select different access method for improved performance.
134
- * Method 0 (default) : use `memcpy()`. Safe and portable.
135
- * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
136
- * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
137
- * Method 2 : direct access. This method is portable but violate C standard.
138
- * It can generate buggy code on targets generating assembly depending on alignment.
139
- * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
140
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
141
- * Prefer these methods in priority order (0 > 1 > 2)
142
- */
143
- #ifndef FSE_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
144
- # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
145
- # define FSE_FORCE_MEMORY_ACCESS 2
146
- # elif defined(__INTEL_COMPILER) || \
147
- (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
148
- # define FSE_FORCE_MEMORY_ACCESS 1
149
- # endif
150
- #endif
151
-
152
-
153
- static unsigned FSE_32bits(void)
154
- {
155
- return sizeof(void*)==4;
156
- }
157
-
158
- static unsigned FSE_isLittleEndian(void)
159
- {
160
- const union { U32 i; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */
161
- return one.c[0];
162
- }
163
-
164
- #if defined(FSE_FORCE_MEMORY_ACCESS) && (FSE_FORCE_MEMORY_ACCESS==2)
165
-
166
- static U16 FSE_read16(const void* memPtr) { return *(const U16*) memPtr; }
167
- static U32 FSE_read32(const void* memPtr) { return *(const U32*) memPtr; }
168
- static U64 FSE_read64(const void* memPtr) { return *(const U64*) memPtr; }
169
-
170
- static void FSE_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
171
- static void FSE_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
172
- static void FSE_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
173
-
174
- #elif defined(FSE_FORCE_MEMORY_ACCESS) && (FSE_FORCE_MEMORY_ACCESS==1)
175
-
176
- /* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
177
- /* currently only defined for gcc and icc */
178
- typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign;
179
-
180
- static U16 FSE_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
181
- static U32 FSE_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
182
- static U64 FSE_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
183
-
184
- static void FSE_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
185
- static void FSE_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
186
- static void FSE_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
187
-
188
- #else
189
-
190
- static U16 FSE_read16(const void* memPtr)
191
- {
192
- U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
193
- }
194
-
195
- static U32 FSE_read32(const void* memPtr)
196
- {
197
- U32 val; memcpy(&val, memPtr, sizeof(val)); return val;
198
- }
199
-
200
- static U64 FSE_read64(const void* memPtr)
201
- {
202
- U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
203
- }
204
-
205
- static void FSE_write16(void* memPtr, U16 value)
206
- {
207
- memcpy(memPtr, &value, sizeof(value));
208
- }
209
-
210
- static void FSE_write32(void* memPtr, U32 value)
211
- {
212
- memcpy(memPtr, &value, sizeof(value));
213
- }
214
-
215
- static void FSE_write64(void* memPtr, U64 value)
216
- {
217
- memcpy(memPtr, &value, sizeof(value));
218
- }
219
-
220
- #endif // FSE_FORCE_MEMORY_ACCESS
221
-
222
- static U16 FSE_readLE16(const void* memPtr)
223
- {
224
- if (FSE_isLittleEndian())
225
- return FSE_read16(memPtr);
226
- else
227
- {
228
- const BYTE* p = (const BYTE*)memPtr;
229
- return (U16)(p[0] + (p[1]<<8));
230
- }
231
- }
232
-
233
- static void FSE_writeLE16(void* memPtr, U16 val)
234
- {
235
- if (FSE_isLittleEndian())
236
- {
237
- FSE_write16(memPtr, val);
238
- }
239
- else
240
- {
241
- BYTE* p = (BYTE*)memPtr;
242
- p[0] = (BYTE)val;
243
- p[1] = (BYTE)(val>>8);
244
- }
245
- }
246
-
247
- static U32 FSE_readLE32(const void* memPtr)
248
- {
249
- if (FSE_isLittleEndian())
250
- return FSE_read32(memPtr);
251
- else
252
- {
253
- const BYTE* p = (const BYTE*)memPtr;
254
- return (U32)((U32)p[0] + ((U32)p[1]<<8) + ((U32)p[2]<<16) + ((U32)p[3]<<24));
255
- }
256
- }
257
-
258
- static void FSE_writeLE32(void* memPtr, U32 val32)
259
- {
260
- if (FSE_isLittleEndian())
261
- {
262
- FSE_write32(memPtr, val32);
263
- }
264
- else
265
- {
266
- BYTE* p = (BYTE*)memPtr;
267
- p[0] = (BYTE)val32;
268
- p[1] = (BYTE)(val32>>8);
269
- p[2] = (BYTE)(val32>>16);
270
- p[3] = (BYTE)(val32>>24);
271
- }
272
- }
273
-
274
- static U64 FSE_readLE64(const void* memPtr)
275
- {
276
- if (FSE_isLittleEndian())
277
- return FSE_read64(memPtr);
278
- else
279
- {
280
- const BYTE* p = (const BYTE*)memPtr;
281
- return (U64)((U64)p[0] + ((U64)p[1]<<8) + ((U64)p[2]<<16) + ((U64)p[3]<<24)
282
- + ((U64)p[4]<<32) + ((U64)p[5]<<40) + ((U64)p[6]<<48) + ((U64)p[7]<<56));
283
- }
284
- }
285
-
286
- static void FSE_writeLE64(void* memPtr, U64 val64)
287
- {
288
- if (FSE_isLittleEndian())
289
- {
290
- FSE_write64(memPtr, val64);
291
- }
292
- else
293
- {
294
- BYTE* p = (BYTE*)memPtr;
295
- p[0] = (BYTE)val64;
296
- p[1] = (BYTE)(val64>>8);
297
- p[2] = (BYTE)(val64>>16);
298
- p[3] = (BYTE)(val64>>24);
299
- p[4] = (BYTE)(val64>>32);
300
- p[5] = (BYTE)(val64>>40);
301
- p[6] = (BYTE)(val64>>48);
302
- p[7] = (BYTE)(val64>>56);
303
- }
304
- }
305
-
306
- static size_t FSE_readLEST(const void* memPtr)
307
- {
308
- if (FSE_32bits())
309
- return (size_t)FSE_readLE32(memPtr);
310
- else
311
- return (size_t)FSE_readLE64(memPtr);
312
- }
313
-
314
- static void FSE_writeLEST(void* memPtr, size_t val)
315
- {
316
- if (FSE_32bits())
317
- FSE_writeLE32(memPtr, (U32)val);
318
- else
319
- FSE_writeLE64(memPtr, (U64)val);
320
- }
321
-
322
-
323
- /****************************************************************
324
- * Constants
325
- *****************************************************************/
326
- #define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2)
327
- #define FSE_MAX_TABLESIZE (1U<<FSE_MAX_TABLELOG)
328
- #define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE-1)
329
- #define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE-2)
330
- #define FSE_MIN_TABLELOG 5
331
-
332
- #define FSE_TABLELOG_ABSOLUTE_MAX 15
333
- #if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX
334
- #error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported"
335
- #endif
336
-
337
-
338
- /****************************************************************
339
- * Error Management
340
- ****************************************************************/
341
- #define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
342
-
343
-
344
- /****************************************************************
345
- * Complex types
346
- ****************************************************************/
347
- typedef struct
348
- {
349
- int deltaFindState;
350
- U32 deltaNbBits;
351
- } FSE_symbolCompressionTransform; /* total 8 bytes */
352
-
353
- typedef U32 CTable_max_t[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
354
- typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
355
-
356
- /****************************************************************
357
- * Internal functions
358
- ****************************************************************/
359
- FORCE_INLINE unsigned FSE_highbit32 (register U32 val)
360
- {
361
- # if defined(_MSC_VER) /* Visual */
362
- unsigned long r;
363
- _BitScanReverse ( &r, val );
364
- return (unsigned) r;
365
- # elif defined(__GNUC__) && (GCC_VERSION >= 304) /* GCC Intrinsic */
366
- return 31 - __builtin_clz (val);
367
- # else /* Software version */
368
- static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
369
- U32 v = val;
370
- unsigned r;
371
- v |= v >> 1;
372
- v |= v >> 2;
373
- v |= v >> 4;
374
- v |= v >> 8;
375
- v |= v >> 16;
376
- r = DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27];
377
- return r;
378
- # endif
379
- }
380
-
381
-
382
- /****************************************************************
383
- * Templates
384
- ****************************************************************/
385
- /*
386
- designed to be included
387
- for type-specific functions (template emulation in C)
388
- Objective is to write these functions only once, for improved maintenance
389
- */
390
-
391
- /* safety checks */
392
- #ifndef FSE_FUNCTION_EXTENSION
393
- # error "FSE_FUNCTION_EXTENSION must be defined"
394
- #endif
395
- #ifndef FSE_FUNCTION_TYPE
396
- # error "FSE_FUNCTION_TYPE must be defined"
397
- #endif
398
-
399
- /* Function names */
400
- #define FSE_CAT(X,Y) X##Y
401
- #define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y)
402
- #define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y)
403
-
404
-
405
- /* Function templates */
406
- size_t FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION)
407
- (unsigned* count, unsigned* maxSymbolValuePtr, const FSE_FUNCTION_TYPE* source, size_t sourceSize, unsigned safe)
408
- {
409
- const FSE_FUNCTION_TYPE* ip = source;
410
- const FSE_FUNCTION_TYPE* const iend = ip+sourceSize;
411
- unsigned maxSymbolValue = *maxSymbolValuePtr;
412
- unsigned max=0;
413
- int s;
414
-
415
- U32 Counting1[FSE_MAX_SYMBOL_VALUE+1] = { 0 };
416
- U32 Counting2[FSE_MAX_SYMBOL_VALUE+1] = { 0 };
417
- U32 Counting3[FSE_MAX_SYMBOL_VALUE+1] = { 0 };
418
- U32 Counting4[FSE_MAX_SYMBOL_VALUE+1] = { 0 };
419
-
420
- /* safety checks */
421
- if (!sourceSize)
422
- {
423
- memset(count, 0, (maxSymbolValue + 1) * sizeof(FSE_FUNCTION_TYPE));
424
- *maxSymbolValuePtr = 0;
425
- return 0;
426
- }
427
- if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return (size_t)-FSE_ERROR_GENERIC; /* maxSymbolValue too large : unsupported */
428
- if (!maxSymbolValue) maxSymbolValue = FSE_MAX_SYMBOL_VALUE; /* 0 == default */
429
-
430
- if ((safe) || (sizeof(FSE_FUNCTION_TYPE)>1))
431
- {
432
- /* check input values, to avoid count table overflow */
433
- while (ip < iend-3)
434
- {
435
- if (*ip>maxSymbolValue) return (size_t)-FSE_ERROR_GENERIC; Counting1[*ip++]++;
436
- if (*ip>maxSymbolValue) return (size_t)-FSE_ERROR_GENERIC; Counting2[*ip++]++;
437
- if (*ip>maxSymbolValue) return (size_t)-FSE_ERROR_GENERIC; Counting3[*ip++]++;
438
- if (*ip>maxSymbolValue) return (size_t)-FSE_ERROR_GENERIC; Counting4[*ip++]++;
439
- }
440
- }
441
- else
442
- {
443
- U32 cached = FSE_read32(ip); ip += 4;
444
- while (ip < iend-15)
445
- {
446
- U32 c = cached; cached = FSE_read32(ip); ip += 4;
447
- Counting1[(BYTE) c ]++;
448
- Counting2[(BYTE)(c>>8) ]++;
449
- Counting3[(BYTE)(c>>16)]++;
450
- Counting4[ c>>24 ]++;
451
- c = cached; cached = FSE_read32(ip); ip += 4;
452
- Counting1[(BYTE) c ]++;
453
- Counting2[(BYTE)(c>>8) ]++;
454
- Counting3[(BYTE)(c>>16)]++;
455
- Counting4[ c>>24 ]++;
456
- c = cached; cached = FSE_read32(ip); ip += 4;
457
- Counting1[(BYTE) c ]++;
458
- Counting2[(BYTE)(c>>8) ]++;
459
- Counting3[(BYTE)(c>>16)]++;
460
- Counting4[ c>>24 ]++;
461
- c = cached; cached = FSE_read32(ip); ip += 4;
462
- Counting1[(BYTE) c ]++;
463
- Counting2[(BYTE)(c>>8) ]++;
464
- Counting3[(BYTE)(c>>16)]++;
465
- Counting4[ c>>24 ]++;
466
- }
467
- ip-=4;
468
- }
469
-
470
- /* finish last symbols */
471
- while (ip<iend) { if ((safe) && (*ip>maxSymbolValue)) return (size_t)-FSE_ERROR_GENERIC; Counting1[*ip++]++; }
472
-
473
- for (s=0; s<=(int)maxSymbolValue; s++)
474
- {
475
- count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s];
476
- if (count[s] > max) max = count[s];
477
- }
478
-
479
- while (!count[maxSymbolValue]) maxSymbolValue--;
480
- *maxSymbolValuePtr = maxSymbolValue;
481
- return (size_t)max;
482
- }
483
-
484
- /* hidden fast variant (unsafe) */
485
- size_t FSE_FUNCTION_NAME(FSE_countFast, FSE_FUNCTION_EXTENSION)
486
- (unsigned* count, unsigned* maxSymbolValuePtr, const FSE_FUNCTION_TYPE* source, size_t sourceSize)
487
- {
488
- return FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) (count, maxSymbolValuePtr, source, sourceSize, 0);
489
- }
490
-
491
- size_t FSE_FUNCTION_NAME(FSE_count, FSE_FUNCTION_EXTENSION)
492
- (unsigned* count, unsigned* maxSymbolValuePtr, const FSE_FUNCTION_TYPE* source, size_t sourceSize)
493
- {
494
- if ((sizeof(FSE_FUNCTION_TYPE)==1) && (*maxSymbolValuePtr >= 255))
495
- {
496
- *maxSymbolValuePtr = 255;
497
- return FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) (count, maxSymbolValuePtr, source, sourceSize, 0);
498
- }
499
- return FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) (count, maxSymbolValuePtr, source, sourceSize, 1);
500
- }
501
-
502
-
503
- static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) + 3; }
504
-
505
- size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION)
506
- (FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
507
- {
508
- const unsigned tableSize = 1 << tableLog;
509
- const unsigned tableMask = tableSize - 1;
510
- U16* tableU16 = ( (U16*) ct) + 2;
511
- FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) (((U32*)ct) + 1 + (tableLog ? tableSize>>1 : 1) );
512
- const unsigned step = FSE_tableStep(tableSize);
513
- unsigned cumul[FSE_MAX_SYMBOL_VALUE+2];
514
- U32 position = 0;
515
- FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* init not necessary, but analyzer complain about it */
516
- U32 highThreshold = tableSize-1;
517
- unsigned symbol;
518
- unsigned i;
519
-
520
- /* header */
521
- tableU16[-2] = (U16) tableLog;
522
- tableU16[-1] = (U16) maxSymbolValue;
523
-
524
- /* For explanations on how to distribute symbol values over the table :
525
- * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
526
-
527
- /* symbol start positions */
528
- cumul[0] = 0;
529
- for (i=1; i<=maxSymbolValue+1; i++)
530
- {
531
- if (normalizedCounter[i-1]==-1) /* Low prob symbol */
532
- {
533
- cumul[i] = cumul[i-1] + 1;
534
- tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(i-1);
535
- }
536
- else
537
- cumul[i] = cumul[i-1] + normalizedCounter[i-1];
538
- }
539
- cumul[maxSymbolValue+1] = tableSize+1;
540
-
541
- /* Spread symbols */
542
- for (symbol=0; symbol<=maxSymbolValue; symbol++)
543
- {
544
- int nbOccurences;
545
- for (nbOccurences=0; nbOccurences<normalizedCounter[symbol]; nbOccurences++)
546
- {
547
- tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
548
- position = (position + step) & tableMask;
549
- while (position > highThreshold) position = (position + step) & tableMask; /* Lowprob area */
550
- }
551
- }
552
-
553
- if (position!=0) return (size_t)-FSE_ERROR_GENERIC; /* Must have gone through all positions */
554
-
555
- /* Build table */
556
- for (i=0; i<tableSize; i++)
557
- {
558
- FSE_FUNCTION_TYPE s = tableSymbol[i]; /* static analyzer doesn't understand tableSymbol is properly initialized */
559
- tableU16[cumul[s]++] = (U16) (tableSize+i); /* TableU16 : sorted by symbol order; gives next state value */
560
- }
561
-
562
- /* Build Symbol Transformation Table */
563
- {
564
- unsigned s;
565
- unsigned total = 0;
566
- for (s=0; s<=maxSymbolValue; s++)
567
- {
568
- switch (normalizedCounter[s])
569
- {
570
- case 0:
571
- break;
572
- case -1:
573
- case 1:
574
- symbolTT[s].deltaNbBits = tableLog << 16;
575
- symbolTT[s].deltaFindState = total - 1;
576
- total ++;
577
- break;
578
- default :
579
- {
580
- U32 maxBitsOut = tableLog - FSE_highbit32 (normalizedCounter[s]-1);
581
- U32 minStatePlus = normalizedCounter[s] << maxBitsOut;
582
- symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus;
583
- symbolTT[s].deltaFindState = total - normalizedCounter[s];
584
- total += normalizedCounter[s];
585
- }
586
- }
587
- }
588
- }
589
-
590
- return 0;
591
- }
592
-
593
-
594
- #define FSE_DECODE_TYPE FSE_TYPE_NAME(FSE_decode_t, FSE_FUNCTION_EXTENSION)
595
-
596
- FSE_DTable* FSE_FUNCTION_NAME(FSE_createDTable, FSE_FUNCTION_EXTENSION) (unsigned tableLog)
597
- {
598
- if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
599
- return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) );
600
- }
601
-
602
- void FSE_FUNCTION_NAME(FSE_freeDTable, FSE_FUNCTION_EXTENSION) (FSE_DTable* dt)
603
- {
604
- free(dt);
605
- }
606
-
607
- typedef struct {
608
- U16 tableLog;
609
- U16 fastMode;
610
- } FSE_DTableHeader; /* sizeof U32 */
611
-
612
- size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION)
613
- (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
614
- {
615
- FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt;
616
- FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (dt+1); /* because dt is unsigned, 32-bits aligned on 32-bits */
617
- const U32 tableSize = 1 << tableLog;
618
- const U32 tableMask = tableSize-1;
619
- const U32 step = FSE_tableStep(tableSize);
620
- U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1];
621
- U32 position = 0;
622
- U32 highThreshold = tableSize-1;
623
- const S16 largeLimit= (S16)(1 << (tableLog-1));
624
- U32 noLarge = 1;
625
- U32 s;
626
-
627
- /* Sanity Checks */
628
- if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return (size_t)-FSE_ERROR_maxSymbolValue_tooLarge;
629
- if (tableLog > FSE_MAX_TABLELOG) return (size_t)-FSE_ERROR_tableLog_tooLarge;
630
-
631
- /* Init, lay down lowprob symbols */
632
- DTableH[0].tableLog = (U16)tableLog;
633
- for (s=0; s<=maxSymbolValue; s++)
634
- {
635
- if (normalizedCounter[s]==-1)
636
- {
637
- tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s;
638
- symbolNext[s] = 1;
639
- }
640
- else
641
- {
642
- if (normalizedCounter[s] >= largeLimit) noLarge=0;
643
- symbolNext[s] = normalizedCounter[s];
644
- }
645
- }
646
-
647
- /* Spread symbols */
648
- for (s=0; s<=maxSymbolValue; s++)
649
- {
650
- int i;
651
- for (i=0; i<normalizedCounter[s]; i++)
652
- {
653
- tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s;
654
- position = (position + step) & tableMask;
655
- while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */
656
- }
657
- }
658
-
659
- if (position!=0) return (size_t)-FSE_ERROR_GENERIC; /* position must reach all cells once, otherwise normalizedCounter is incorrect */
660
-
661
- /* Build Decoding table */
662
- {
663
- U32 i;
664
- for (i=0; i<tableSize; i++)
665
- {
666
- FSE_FUNCTION_TYPE symbol = (FSE_FUNCTION_TYPE)(tableDecode[i].symbol);
667
- U16 nextState = symbolNext[symbol]++;
668
- tableDecode[i].nbBits = (BYTE) (tableLog - FSE_highbit32 ((U32)nextState) );
669
- tableDecode[i].newState = (U16) ( (nextState << tableDecode[i].nbBits) - tableSize);
670
- }
671
- }
672
-
673
- DTableH->fastMode = (U16)noLarge;
674
- return 0;
675
- }
676
-
677
-
678
- /******************************************
679
- * FSE byte symbol
680
- ******************************************/
681
- #ifndef FSE_COMMONDEFS_ONLY
682
-
683
- unsigned FSE_isError(size_t code) { return (code > (size_t)(-FSE_ERROR_maxCode)); }
684
-
685
- #define FSE_GENERATE_STRING(STRING) #STRING,
686
- static const char* FSE_errorStrings[] = { FSE_LIST_ERRORS(FSE_GENERATE_STRING) };
687
-
688
- const char* FSE_getErrorName(size_t code)
689
- {
690
- static const char* codeError = "Unspecified error code";
691
- if (FSE_isError(code)) return FSE_errorStrings[-(int)(code)];
692
- return codeError;
693
- }
694
-
695
- static short FSE_abs(short a)
696
- {
697
- return a<0? -a : a;
698
- }
699
-
700
-
701
- /****************************************************************
702
- * Header bitstream management
703
- ****************************************************************/
704
- size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
705
- {
706
- size_t maxHeaderSize = (((maxSymbolValue+1) * tableLog) >> 3) + 3;
707
- return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */
708
- }
709
-
710
- static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
711
- const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
712
- unsigned writeIsSafe)
713
- {
714
- BYTE* const ostart = (BYTE*) header;
715
- BYTE* out = ostart;
716
- BYTE* const oend = ostart + headerBufferSize;
717
- int nbBits;
718
- const int tableSize = 1 << tableLog;
719
- int remaining;
720
- int threshold;
721
- U32 bitStream;
722
- int bitCount;
723
- unsigned charnum = 0;
724
- int previous0 = 0;
725
-
726
- bitStream = 0;
727
- bitCount = 0;
728
- /* Table Size */
729
- bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount;
730
- bitCount += 4;
731
-
732
- /* Init */
733
- remaining = tableSize+1; /* +1 for extra accuracy */
734
- threshold = tableSize;
735
- nbBits = tableLog+1;
736
-
737
- while (remaining>1) /* stops at 1 */
738
- {
739
- if (previous0)
740
- {
741
- unsigned start = charnum;
742
- while (!normalizedCounter[charnum]) charnum++;
743
- while (charnum >= start+24)
744
- {
745
- start+=24;
746
- bitStream += 0xFFFFU << bitCount;
747
- if ((!writeIsSafe) && (out > oend-2)) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* Buffer overflow */
748
- out[0] = (BYTE) bitStream;
749
- out[1] = (BYTE)(bitStream>>8);
750
- out+=2;
751
- bitStream>>=16;
752
- }
753
- while (charnum >= start+3)
754
- {
755
- start+=3;
756
- bitStream += 3 << bitCount;
757
- bitCount += 2;
758
- }
759
- bitStream += (charnum-start) << bitCount;
760
- bitCount += 2;
761
- if (bitCount>16)
762
- {
763
- if ((!writeIsSafe) && (out > oend - 2)) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* Buffer overflow */
764
- out[0] = (BYTE)bitStream;
765
- out[1] = (BYTE)(bitStream>>8);
766
- out += 2;
767
- bitStream >>= 16;
768
- bitCount -= 16;
769
- }
770
- }
771
- {
772
- short count = normalizedCounter[charnum++];
773
- const short max = (short)((2*threshold-1)-remaining);
774
- remaining -= FSE_abs(count);
775
- if (remaining<1) return (size_t)-FSE_ERROR_GENERIC;
776
- count++; /* +1 for extra accuracy */
777
- if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
778
- bitStream += count << bitCount;
779
- bitCount += nbBits;
780
- bitCount -= (count<max);
781
- previous0 = (count==1);
782
- while (remaining<threshold) nbBits--, threshold>>=1;
783
- }
784
- if (bitCount>16)
785
- {
786
- if ((!writeIsSafe) && (out > oend - 2)) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* Buffer overflow */
787
- out[0] = (BYTE)bitStream;
788
- out[1] = (BYTE)(bitStream>>8);
789
- out += 2;
790
- bitStream >>= 16;
791
- bitCount -= 16;
792
- }
793
- }
794
-
795
- /* flush remaining bitStream */
796
- if ((!writeIsSafe) && (out > oend - 2)) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* Buffer overflow */
797
- out[0] = (BYTE)bitStream;
798
- out[1] = (BYTE)(bitStream>>8);
799
- out+= (bitCount+7) /8;
800
-
801
- if (charnum > maxSymbolValue + 1) return (size_t)-FSE_ERROR_GENERIC;
802
-
803
- return (out-ostart);
804
- }
805
-
806
-
807
- size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
808
- {
809
- if (tableLog > FSE_MAX_TABLELOG) return (size_t)-FSE_ERROR_GENERIC; /* Unsupported */
810
- if (tableLog < FSE_MIN_TABLELOG) return (size_t)-FSE_ERROR_GENERIC; /* Unsupported */
811
-
812
- if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog))
813
- return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0);
814
-
815
- return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1);
816
- }
817
-
818
-
819
- size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
820
- const void* headerBuffer, size_t hbSize)
821
- {
822
- const BYTE* const istart = (const BYTE*) headerBuffer;
823
- const BYTE* const iend = istart + hbSize;
824
- const BYTE* ip = istart;
825
- int nbBits;
826
- int remaining;
827
- int threshold;
828
- U32 bitStream;
829
- int bitCount;
830
- unsigned charnum = 0;
831
- int previous0 = 0;
832
-
833
- if (hbSize < 4) return (size_t)-FSE_ERROR_srcSize_wrong;
834
- bitStream = FSE_readLE32(ip);
835
- nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */
836
- if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return (size_t)-FSE_ERROR_tableLog_tooLarge;
837
- bitStream >>= 4;
838
- bitCount = 4;
839
- *tableLogPtr = nbBits;
840
- remaining = (1<<nbBits)+1;
841
- threshold = 1<<nbBits;
842
- nbBits++;
843
-
844
- while ((remaining>1) && (charnum<=*maxSVPtr))
845
- {
846
- if (previous0)
847
- {
848
- unsigned n0 = charnum;
849
- while ((bitStream & 0xFFFF) == 0xFFFF)
850
- {
851
- n0+=24;
852
- if (ip < iend-5)
853
- {
854
- ip+=2;
855
- bitStream = FSE_readLE32(ip) >> bitCount;
856
- }
857
- else
858
- {
859
- bitStream >>= 16;
860
- bitCount+=16;
861
- }
862
- }
863
- while ((bitStream & 3) == 3)
864
- {
865
- n0+=3;
866
- bitStream>>=2;
867
- bitCount+=2;
868
- }
869
- n0 += bitStream & 3;
870
- bitCount += 2;
871
- if (n0 > *maxSVPtr) return (size_t)-FSE_ERROR_maxSymbolValue_tooSmall;
872
- while (charnum < n0) normalizedCounter[charnum++] = 0;
873
- if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4))
874
- {
875
- ip += bitCount>>3;
876
- bitCount &= 7;
877
- bitStream = FSE_readLE32(ip) >> bitCount;
878
- }
879
- else
880
- bitStream >>= 2;
881
- }
882
- {
883
- const short max = (short)((2*threshold-1)-remaining);
884
- short count;
885
-
886
- if ((bitStream & (threshold-1)) < (U32)max)
887
- {
888
- count = (short)(bitStream & (threshold-1));
889
- bitCount += nbBits-1;
890
- }
891
- else
892
- {
893
- count = (short)(bitStream & (2*threshold-1));
894
- if (count >= threshold) count -= max;
895
- bitCount += nbBits;
896
- }
897
-
898
- count--; /* extra accuracy */
899
- remaining -= FSE_abs(count);
900
- normalizedCounter[charnum++] = count;
901
- previous0 = !count;
902
- while (remaining < threshold)
903
- {
904
- nbBits--;
905
- threshold >>= 1;
906
- }
907
-
908
- {
909
- if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4))
910
- {
911
- ip += bitCount>>3;
912
- bitCount &= 7;
913
- }
914
- else
915
- {
916
- bitCount -= (int)(8 * (iend - 4 - ip));
917
- ip = iend - 4;
918
- }
919
- bitStream = FSE_readLE32(ip) >> (bitCount & 31);
920
- }
921
- }
922
- }
923
- if (remaining != 1) return (size_t)-FSE_ERROR_GENERIC;
924
- *maxSVPtr = charnum-1;
925
-
926
- ip += (bitCount+7)>>3;
927
- if ((size_t)(ip-istart) > hbSize) return (size_t)-FSE_ERROR_srcSize_wrong;
928
- return ip-istart;
929
- }
930
-
931
-
932
- /****************************************************************
933
- * FSE Compression Code
934
- ****************************************************************/
935
- /*
936
- FSE_CTable[0] is a variable size structure which contains :
937
- U16 tableLog;
938
- U16 maxSymbolValue;
939
- U16 nextStateNumber[1 << tableLog]; // This size is variable
940
- FSE_symbolCompressionTransform symbolTT[maxSymbolValue+1]; // This size is variable
941
- Allocation is manual, since C standard does not support variable-size structures.
942
- */
943
-
944
- size_t FSE_sizeof_CTable (unsigned maxSymbolValue, unsigned tableLog)
945
- {
946
- size_t size;
947
- FSE_STATIC_ASSERT((size_t)FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)*4 >= sizeof(CTable_max_t)); /* A compilation error here means FSE_CTABLE_SIZE_U32 is not large enough */
948
- if (tableLog > FSE_MAX_TABLELOG) return (size_t)-FSE_ERROR_GENERIC;
949
- size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32);
950
- return size;
951
- }
952
-
953
- FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog)
954
- {
955
- size_t size;
956
- if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
957
- size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32);
958
- return (FSE_CTable*)malloc(size);
959
- }
960
-
961
- void FSE_freeCTable (FSE_CTable* ct)
962
- {
963
- free(ct);
964
- }
965
-
966
-
967
- /* provides the minimum logSize to safely represent a distribution */
968
- static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
969
- {
970
- U32 minBitsSrc = FSE_highbit32((U32)(srcSize - 1)) + 1;
971
- U32 minBitsSymbols = FSE_highbit32(maxSymbolValue) + 2;
972
- U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
973
- return minBits;
974
- }
975
-
976
- unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
977
- {
978
- U32 maxBitsSrc = FSE_highbit32((U32)(srcSize - 1)) - 2;
979
- U32 tableLog = maxTableLog;
980
- U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
981
- if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
982
- if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */
983
- if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */
984
- if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG;
985
- if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG;
986
- return tableLog;
987
- }
988
-
989
-
990
- /* Secondary normalization method.
991
- To be used when primary method fails. */
992
-
993
- static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue)
994
- {
995
- U32 s;
996
- U32 distributed = 0;
997
- U32 ToDistribute;
998
-
999
- /* Init */
1000
- U32 lowThreshold = (U32)(total >> tableLog);
1001
- U32 lowOne = (U32)((total * 3) >> (tableLog + 1));
1002
-
1003
- for (s=0; s<=maxSymbolValue; s++)
1004
- {
1005
- if (count[s] == 0)
1006
- {
1007
- norm[s]=0;
1008
- continue;
1009
- }
1010
- if (count[s] <= lowThreshold)
1011
- {
1012
- norm[s] = -1;
1013
- distributed++;
1014
- total -= count[s];
1015
- continue;
1016
- }
1017
- if (count[s] <= lowOne)
1018
- {
1019
- norm[s] = 1;
1020
- distributed++;
1021
- total -= count[s];
1022
- continue;
1023
- }
1024
- norm[s]=-2;
1025
- }
1026
- ToDistribute = (1 << tableLog) - distributed;
1027
-
1028
- if ((total / ToDistribute) > lowOne)
1029
- {
1030
- /* risk of rounding to zero */
1031
- lowOne = (U32)((total * 3) / (ToDistribute * 2));
1032
- for (s=0; s<=maxSymbolValue; s++)
1033
- {
1034
- if ((norm[s] == -2) && (count[s] <= lowOne))
1035
- {
1036
- norm[s] = 1;
1037
- distributed++;
1038
- total -= count[s];
1039
- continue;
1040
- }
1041
- }
1042
- ToDistribute = (1 << tableLog) - distributed;
1043
- }
1044
-
1045
- if (distributed == maxSymbolValue+1)
1046
- {
1047
- /* all values are pretty poor;
1048
- probably incompressible data (should have already been detected);
1049
- find max, then give all remaining points to max */
1050
- U32 maxV = 0, maxC =0;
1051
- for (s=0; s<=maxSymbolValue; s++)
1052
- if (count[s] > maxC) maxV=s, maxC=count[s];
1053
- norm[maxV] += (short)ToDistribute;
1054
- return 0;
1055
- }
1056
-
1057
- {
1058
- U64 const vStepLog = 62 - tableLog;
1059
- U64 const mid = (1ULL << (vStepLog-1)) - 1;
1060
- U64 const rStep = ((((U64)1<<vStepLog) * ToDistribute) + mid) / total; /* scale on remaining */
1061
- U64 tmpTotal = mid;
1062
- for (s=0; s<=maxSymbolValue; s++)
1063
- {
1064
- if (norm[s]==-2)
1065
- {
1066
- U64 end = tmpTotal + (count[s] * rStep);
1067
- U32 sStart = (U32)(tmpTotal >> vStepLog);
1068
- U32 sEnd = (U32)(end >> vStepLog);
1069
- U32 weight = sEnd - sStart;
1070
- if (weight < 1)
1071
- return (size_t)-FSE_ERROR_GENERIC;
1072
- norm[s] = (short)weight;
1073
- tmpTotal = end;
1074
- }
1075
- }
1076
- }
1077
-
1078
- return 0;
1079
- }
1080
-
1081
-
1082
- size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
1083
- const unsigned* count, size_t total,
1084
- unsigned maxSymbolValue)
1085
- {
1086
- /* Sanity checks */
1087
- if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
1088
- if (tableLog < FSE_MIN_TABLELOG) return (size_t)-FSE_ERROR_GENERIC; /* Unsupported size */
1089
- if (tableLog > FSE_MAX_TABLELOG) return (size_t)-FSE_ERROR_GENERIC; /* Unsupported size */
1090
- if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return (size_t)-FSE_ERROR_GENERIC; /* Too small tableLog, compression potentially impossible */
1091
-
1092
- {
1093
- U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 };
1094
- U64 const scale = 62 - tableLog;
1095
- U64 const step = ((U64)1<<62) / total; /* <== here, one division ! */
1096
- U64 const vStep = 1ULL<<(scale-20);
1097
- int stillToDistribute = 1<<tableLog;
1098
- unsigned s;
1099
- unsigned largest=0;
1100
- short largestP=0;
1101
- U32 lowThreshold = (U32)(total >> tableLog);
1102
-
1103
- for (s=0; s<=maxSymbolValue; s++)
1104
- {
1105
- if (count[s] == total) return 0;
1106
- if (count[s] == 0)
1107
- {
1108
- normalizedCounter[s]=0;
1109
- continue;
1110
- }
1111
- if (count[s] <= lowThreshold)
1112
- {
1113
- normalizedCounter[s] = -1;
1114
- stillToDistribute--;
1115
- }
1116
- else
1117
- {
1118
- short proba = (short)((count[s]*step) >> scale);
1119
- if (proba<8)
1120
- {
1121
- U64 restToBeat = vStep * rtbTable[proba];
1122
- proba += (count[s]*step) - ((U64)proba<<scale) > restToBeat;
1123
- }
1124
- if (proba > largestP)
1125
- {
1126
- largestP=proba;
1127
- largest=s;
1128
- }
1129
- normalizedCounter[s] = proba;
1130
- stillToDistribute -= proba;
1131
- }
1132
- }
1133
- if (-stillToDistribute >= (normalizedCounter[largest] >> 1))
1134
- {
1135
- /* corner case, need another normalization method */
1136
- size_t errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue);
1137
- if (FSE_isError(errorCode)) return errorCode;
1138
- }
1139
- else normalizedCounter[largest] += (short)stillToDistribute;
1140
- }
1141
-
1142
- #if 0
1143
- { /* Print Table (debug) */
1144
- U32 s;
1145
- U32 nTotal = 0;
1146
- for (s=0; s<=maxSymbolValue; s++)
1147
- printf("%3i: %4i \n", s, normalizedCounter[s]);
1148
- for (s=0; s<=maxSymbolValue; s++)
1149
- nTotal += abs(normalizedCounter[s]);
1150
- if (nTotal != (1U<<tableLog))
1151
- printf("Warning !!! Total == %u != %u !!!", nTotal, 1U<<tableLog);
1152
- getchar();
1153
- }
1154
- #endif
1155
-
1156
- return tableLog;
1157
- }
1158
-
1159
-
1160
- /* fake FSE_CTable, for raw (uncompressed) input */
1161
- size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits)
1162
- {
1163
- const unsigned tableSize = 1 << nbBits;
1164
- const unsigned tableMask = tableSize - 1;
1165
- const unsigned maxSymbolValue = tableMask;
1166
- U16* tableU16 = ( (U16*) ct) + 2;
1167
- FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) ((((U32*)ct)+1) + (tableSize>>1));
1168
- unsigned s;
1169
-
1170
- /* Sanity checks */
1171
- if (nbBits < 1) return (size_t)-FSE_ERROR_GENERIC; /* min size */
1172
-
1173
- /* header */
1174
- tableU16[-2] = (U16) nbBits;
1175
- tableU16[-1] = (U16) maxSymbolValue;
1176
-
1177
- /* Build table */
1178
- for (s=0; s<tableSize; s++)
1179
- tableU16[s] = (U16)(tableSize + s);
1180
-
1181
- /* Build Symbol Transformation Table */
1182
- for (s=0; s<=maxSymbolValue; s++)
1183
- {
1184
- symbolTT[s].deltaNbBits = nbBits << 16;
1185
- symbolTT[s].deltaFindState = s-1;
1186
- }
1187
-
1188
- return 0;
1189
- }
1190
-
1191
-
1192
- /* fake FSE_CTable, for rle (100% always same symbol) input */
1193
- size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue)
1194
- {
1195
- U16* tableU16 = ( (U16*) ct) + 2;
1196
- FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) ((U32*)ct + 2);
1197
-
1198
- /* header */
1199
- tableU16[-2] = (U16) 0;
1200
- tableU16[-1] = (U16) symbolValue;
1201
-
1202
- /* Build table */
1203
- tableU16[0] = 0;
1204
- tableU16[1] = 0; /* just in case */
1205
-
1206
- /* Build Symbol Transformation Table */
1207
- {
1208
- symbolTT[symbolValue].deltaNbBits = 0;
1209
- symbolTT[symbolValue].deltaFindState = 0;
1210
- }
1211
-
1212
- return 0;
1213
- }
1214
-
1215
-
1216
- size_t FSE_initCStream(FSE_CStream_t* bitC, void* start, size_t maxSize)
1217
- {
1218
- if (maxSize < sizeof(bitC->ptr)) return (size_t)-FSE_ERROR_dstSize_tooSmall;
1219
- bitC->bitContainer = 0;
1220
- bitC->bitPos = 0;
1221
- bitC->startPtr = (char*)start;
1222
- bitC->ptr = bitC->startPtr;
1223
- bitC->endPtr = bitC->startPtr + maxSize - sizeof(bitC->ptr);
1224
- return 0;
1225
- }
1226
-
1227
- void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
1228
- {
1229
- const U32 tableLog = ( (const U16*) ct) [0];
1230
- statePtr->value = (ptrdiff_t)1<<tableLog;
1231
- statePtr->stateTable = ((const U16*) ct) + 2;
1232
- statePtr->symbolTT = (const FSE_symbolCompressionTransform*)((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1));
1233
- statePtr->stateLog = tableLog;
1234
- }
1235
-
1236
- void FSE_addBitsFast(FSE_CStream_t* bitC, size_t value, unsigned nbBits) /* only use if upper bits are clean 0 */
1237
- {
1238
- bitC->bitContainer |= value << bitC->bitPos;
1239
- bitC->bitPos += nbBits;
1240
- }
1241
-
1242
- void FSE_addBits(FSE_CStream_t* bitC, size_t value, unsigned nbBits)
1243
- {
1244
- static const unsigned mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF }; /* up to 25 bits */
1245
- bitC->bitContainer |= (value & mask[nbBits]) << bitC->bitPos;
1246
- bitC->bitPos += nbBits;
1247
- }
1248
-
1249
- void FSE_encodeSymbol(FSE_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol)
1250
- {
1251
- const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
1252
- const U16* const stateTable = (const U16*)(statePtr->stateTable);
1253
- U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);
1254
- FSE_addBits(bitC, statePtr->value, nbBitsOut);
1255
- statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
1256
- }
1257
-
1258
- void FSE_flushBitsFast(FSE_CStream_t* bitC) /* only if dst buffer is large enough ( >= FSE_compressBound()) */
1259
- {
1260
- size_t nbBytes = bitC->bitPos >> 3;
1261
- FSE_writeLEST(bitC->ptr, bitC->bitContainer);
1262
- bitC->ptr += nbBytes;
1263
- bitC->bitPos &= 7;
1264
- bitC->bitContainer >>= nbBytes*8;
1265
- }
1266
-
1267
- void FSE_flushBits(FSE_CStream_t* bitC)
1268
- {
1269
- size_t nbBytes = bitC->bitPos >> 3;
1270
- FSE_writeLEST(bitC->ptr, bitC->bitContainer);
1271
- bitC->ptr += nbBytes;
1272
- if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr;
1273
- bitC->bitPos &= 7;
1274
- bitC->bitContainer >>= nbBytes*8;
1275
- }
1276
-
1277
- void FSE_flushCState(FSE_CStream_t* bitC, const FSE_CState_t* statePtr)
1278
- {
1279
- FSE_addBits(bitC, statePtr->value, statePtr->stateLog);
1280
- FSE_flushBits(bitC);
1281
- }
1282
-
1283
-
1284
- size_t FSE_closeCStream(FSE_CStream_t* bitC)
1285
- {
1286
- char* endPtr;
1287
-
1288
- FSE_addBitsFast(bitC, 1, 1);
1289
- FSE_flushBits(bitC);
1290
-
1291
- if (bitC->ptr >= bitC->endPtr) /* too close to buffer's end */
1292
- return 0; /* not compressible */
1293
-
1294
- endPtr = bitC->ptr;
1295
- endPtr += bitC->bitPos > 0;
1296
-
1297
- return (endPtr - bitC->startPtr);
1298
- }
1299
-
1300
-
1301
- static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize,
1302
- const void* src, size_t srcSize,
1303
- const FSE_CTable* ct, const unsigned fast)
1304
- {
1305
- const BYTE* const istart = (const BYTE*) src;
1306
- const BYTE* ip;
1307
- const BYTE* const iend = istart + srcSize;
1308
-
1309
- size_t errorCode;
1310
- FSE_CStream_t bitC;
1311
- FSE_CState_t CState1, CState2;
1312
-
1313
-
1314
- /* init */
1315
- errorCode = FSE_initCStream(&bitC, dst, dstSize);
1316
- if (FSE_isError(errorCode)) return 0;
1317
- FSE_initCState(&CState1, ct);
1318
- CState2 = CState1;
1319
-
1320
- ip=iend;
1321
-
1322
- #define FSE_FLUSHBITS(s) (fast ? FSE_flushBitsFast(s) : FSE_flushBits(s))
1323
-
1324
- /* join to even */
1325
- if (srcSize & 1)
1326
- {
1327
- FSE_encodeSymbol(&bitC, &CState1, *--ip);
1328
- FSE_FLUSHBITS(&bitC);
1329
- }
1330
-
1331
- /* join to mod 4 */
1332
- if ((sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) /* test bit 2 */
1333
- {
1334
- FSE_encodeSymbol(&bitC, &CState2, *--ip);
1335
- FSE_encodeSymbol(&bitC, &CState1, *--ip);
1336
- FSE_FLUSHBITS(&bitC);
1337
- }
1338
-
1339
- /* 2 or 4 encoding per loop */
1340
- for ( ; ip>istart ; )
1341
- {
1342
- FSE_encodeSymbol(&bitC, &CState2, *--ip);
1343
-
1344
- if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */
1345
- FSE_FLUSHBITS(&bitC);
1346
-
1347
- FSE_encodeSymbol(&bitC, &CState1, *--ip);
1348
-
1349
- if (sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) /* this test must be static */
1350
- {
1351
- FSE_encodeSymbol(&bitC, &CState2, *--ip);
1352
- FSE_encodeSymbol(&bitC, &CState1, *--ip);
1353
- }
1354
-
1355
- FSE_FLUSHBITS(&bitC);
1356
- }
1357
-
1358
- FSE_flushCState(&bitC, &CState2);
1359
- FSE_flushCState(&bitC, &CState1);
1360
- return FSE_closeCStream(&bitC);
1361
- }
1362
-
1363
- size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
1364
- const void* src, size_t srcSize,
1365
- const FSE_CTable* ct)
1366
- {
1367
- const unsigned fast = (dstSize >= FSE_BLOCKBOUND(srcSize));
1368
-
1369
- if (fast)
1370
- return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1);
1371
- else
1372
- return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0);
1373
- }
1374
-
1375
-
1376
- size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
1377
-
1378
- size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog)
1379
- {
1380
- const BYTE* const istart = (const BYTE*) src;
1381
- const BYTE* ip = istart;
1382
-
1383
- BYTE* const ostart = (BYTE*) dst;
1384
- BYTE* op = ostart;
1385
- BYTE* const oend = ostart + dstSize;
1386
-
1387
- U32 count[FSE_MAX_SYMBOL_VALUE+1];
1388
- S16 norm[FSE_MAX_SYMBOL_VALUE+1];
1389
- CTable_max_t ct;
1390
- size_t errorCode;
1391
-
1392
- /* init conditions */
1393
- if (srcSize <= 1) return 0; /* Uncompressible */
1394
- if (!maxSymbolValue) maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
1395
- if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG;
1396
-
1397
- /* Scan input and build symbol stats */
1398
- errorCode = FSE_count (count, &maxSymbolValue, ip, srcSize);
1399
- if (FSE_isError(errorCode)) return errorCode;
1400
- if (errorCode == srcSize) return 1;
1401
- if (errorCode == 1) return 0; /* each symbol only present once */
1402
- if (errorCode < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */
1403
-
1404
- tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
1405
- errorCode = FSE_normalizeCount (norm, tableLog, count, srcSize, maxSymbolValue);
1406
- if (FSE_isError(errorCode)) return errorCode;
1407
-
1408
- /* Write table description header */
1409
- errorCode = FSE_writeNCount (op, oend-op, norm, maxSymbolValue, tableLog);
1410
- if (FSE_isError(errorCode)) return errorCode;
1411
- op += errorCode;
1412
-
1413
- /* Compress */
1414
- errorCode = FSE_buildCTable (ct, norm, maxSymbolValue, tableLog);
1415
- if (FSE_isError(errorCode)) return errorCode;
1416
- errorCode = FSE_compress_usingCTable(op, oend - op, ip, srcSize, ct);
1417
- if (errorCode == 0) return 0; /* not enough space for compressed data */
1418
- op += errorCode;
1419
-
1420
- /* check compressibility */
1421
- if ( (size_t)(op-ostart) >= srcSize-1 )
1422
- return 0;
1423
-
1424
- return op-ostart;
1425
- }
1426
-
1427
- size_t FSE_compress (void* dst, size_t dstSize, const void* src, size_t srcSize)
1428
- {
1429
- return FSE_compress2(dst, dstSize, src, (U32)srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG);
1430
- }
1431
-
1432
-
1433
- /*********************************************************
1434
- * Decompression (Byte symbols)
1435
- *********************************************************/
1436
- size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
1437
- {
1438
- FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt;
1439
- FSE_decode_t* const cell = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */
1440
-
1441
- DTableH->tableLog = 0;
1442
- DTableH->fastMode = 0;
1443
-
1444
- cell->newState = 0;
1445
- cell->symbol = symbolValue;
1446
- cell->nbBits = 0;
1447
-
1448
- return 0;
1449
- }
1450
-
1451
-
1452
- size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
1453
- {
1454
- FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt;
1455
- FSE_decode_t* const dinfo = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */
1456
- const unsigned tableSize = 1 << nbBits;
1457
- const unsigned tableMask = tableSize - 1;
1458
- const unsigned maxSymbolValue = tableMask;
1459
- unsigned s;
1460
-
1461
- /* Sanity checks */
1462
- if (nbBits < 1) return (size_t)-FSE_ERROR_GENERIC; /* min size */
1463
-
1464
- /* Build Decoding Table */
1465
- DTableH->tableLog = (U16)nbBits;
1466
- DTableH->fastMode = 1;
1467
- for (s=0; s<=maxSymbolValue; s++)
1468
- {
1469
- dinfo[s].newState = 0;
1470
- dinfo[s].symbol = (BYTE)s;
1471
- dinfo[s].nbBits = (BYTE)nbBits;
1472
- }
1473
-
1474
- return 0;
1475
- }
1476
-
1477
-
1478
- /* FSE_initDStream
1479
- * Initialize a FSE_DStream_t.
1480
- * srcBuffer must point at the beginning of an FSE block.
1481
- * The function result is the size of the FSE_block (== srcSize).
1482
- * If srcSize is too small, the function will return an errorCode;
1483
- */
1484
- size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t srcSize)
1485
- {
1486
- if (srcSize < 1) return (size_t)-FSE_ERROR_srcSize_wrong;
1487
-
1488
- if (srcSize >= sizeof(size_t))
1489
- {
1490
- U32 contain32;
1491
- bitD->start = (const char*)srcBuffer;
1492
- bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(size_t);
1493
- bitD->bitContainer = FSE_readLEST(bitD->ptr);
1494
- contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
1495
- if (contain32 == 0) return (size_t)-FSE_ERROR_GENERIC; /* stop bit not present */
1496
- bitD->bitsConsumed = 8 - FSE_highbit32(contain32);
1497
- }
1498
- else
1499
- {
1500
- U32 contain32;
1501
- bitD->start = (const char*)srcBuffer;
1502
- bitD->ptr = bitD->start;
1503
- bitD->bitContainer = *(const BYTE*)(bitD->start);
1504
- switch(srcSize)
1505
- {
1506
- case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
1507
- case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
1508
- case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
1509
- case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
1510
- case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
1511
- case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
1512
- default:;
1513
- }
1514
- contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
1515
- if (contain32 == 0) return (size_t)-FSE_ERROR_GENERIC; /* stop bit not present */
1516
- bitD->bitsConsumed = 8 - FSE_highbit32(contain32);
1517
- bitD->bitsConsumed += (U32)(sizeof(size_t) - srcSize)*8;
1518
- }
1519
-
1520
- return srcSize;
1521
- }
1522
-
1523
-
1524
- /* FSE_lookBits
1525
- * Provides next n bits from the bitContainer.
1526
- * bitContainer is not modified (bits are still present for next read/look)
1527
- * On 32-bits, maxNbBits==25
1528
- * On 64-bits, maxNbBits==57
1529
- * return : value extracted.
1530
- */
1531
- static size_t FSE_lookBits(FSE_DStream_t* bitD, U32 nbBits)
1532
- {
1533
- const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
1534
- return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask);
1535
- }
1536
-
1537
- static size_t FSE_lookBitsFast(FSE_DStream_t* bitD, U32 nbBits) /* only if nbBits >= 1 !! */
1538
- {
1539
- const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
1540
- return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask);
1541
- }
1542
-
1543
- static void FSE_skipBits(FSE_DStream_t* bitD, U32 nbBits)
1544
- {
1545
- bitD->bitsConsumed += nbBits;
1546
- }
1547
-
1548
-
1549
- /* FSE_readBits
1550
- * Read next n bits from the bitContainer.
1551
- * On 32-bits, don't read more than maxNbBits==25
1552
- * On 64-bits, don't read more than maxNbBits==57
1553
- * Use the fast variant *only* if n >= 1.
1554
- * return : value extracted.
1555
- */
1556
- size_t FSE_readBits(FSE_DStream_t* bitD, U32 nbBits)
1557
- {
1558
- size_t value = FSE_lookBits(bitD, nbBits);
1559
- FSE_skipBits(bitD, nbBits);
1560
- return value;
1561
- }
1562
-
1563
- size_t FSE_readBitsFast(FSE_DStream_t* bitD, U32 nbBits) /* only if nbBits >= 1 !! */
1564
- {
1565
- size_t value = FSE_lookBitsFast(bitD, nbBits);
1566
- FSE_skipBits(bitD, nbBits);
1567
- return value;
1568
- }
1569
-
1570
- unsigned FSE_reloadDStream(FSE_DStream_t* bitD)
1571
- {
1572
- if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
1573
- return FSE_DStream_tooFar;
1574
-
1575
- if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer))
1576
- {
1577
- bitD->ptr -= bitD->bitsConsumed >> 3;
1578
- bitD->bitsConsumed &= 7;
1579
- bitD->bitContainer = FSE_readLEST(bitD->ptr);
1580
- return FSE_DStream_unfinished;
1581
- }
1582
- if (bitD->ptr == bitD->start)
1583
- {
1584
- if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return FSE_DStream_endOfBuffer;
1585
- return FSE_DStream_completed;
1586
- }
1587
- {
1588
- U32 nbBytes = bitD->bitsConsumed >> 3;
1589
- U32 result = FSE_DStream_unfinished;
1590
- if (bitD->ptr - nbBytes < bitD->start)
1591
- {
1592
- nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */
1593
- result = FSE_DStream_endOfBuffer;
1594
- }
1595
- bitD->ptr -= nbBytes;
1596
- bitD->bitsConsumed -= nbBytes*8;
1597
- bitD->bitContainer = FSE_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */
1598
- return result;
1599
- }
1600
- }
1601
-
1602
-
1603
- void FSE_initDState(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD, const FSE_DTable* dt)
1604
- {
1605
- const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)dt;
1606
- DStatePtr->state = FSE_readBits(bitD, DTableH->tableLog);
1607
- FSE_reloadDStream(bitD);
1608
- DStatePtr->table = dt + 1;
1609
- }
1610
-
1611
- BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD)
1612
- {
1613
- const FSE_decode_t DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
1614
- const U32 nbBits = DInfo.nbBits;
1615
- BYTE symbol = DInfo.symbol;
1616
- size_t lowBits = FSE_readBits(bitD, nbBits);
1617
-
1618
- DStatePtr->state = DInfo.newState + lowBits;
1619
- return symbol;
1620
- }
1621
-
1622
- BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD)
1623
- {
1624
- const FSE_decode_t DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
1625
- const U32 nbBits = DInfo.nbBits;
1626
- BYTE symbol = DInfo.symbol;
1627
- size_t lowBits = FSE_readBitsFast(bitD, nbBits);
1628
-
1629
- DStatePtr->state = DInfo.newState + lowBits;
1630
- return symbol;
1631
- }
1632
-
1633
- /* FSE_endOfDStream
1634
- Tells if bitD has reached end of bitStream or not */
1635
-
1636
- unsigned FSE_endOfDStream(const FSE_DStream_t* bitD)
1637
- {
1638
- return ((bitD->ptr == bitD->start) && (bitD->bitsConsumed == sizeof(bitD->bitContainer)*8));
1639
- }
1640
-
1641
- unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
1642
- {
1643
- return DStatePtr->state == 0;
1644
- }
1645
-
1646
-
1647
- FORCE_INLINE size_t FSE_decompress_usingDTable_generic(
1648
- void* dst, size_t maxDstSize,
1649
- const void* cSrc, size_t cSrcSize,
1650
- const FSE_DTable* dt, const unsigned fast)
1651
- {
1652
- BYTE* const ostart = (BYTE*) dst;
1653
- BYTE* op = ostart;
1654
- BYTE* const omax = op + maxDstSize;
1655
- BYTE* const olimit = omax-3;
1656
-
1657
- FSE_DStream_t bitD;
1658
- FSE_DState_t state1;
1659
- FSE_DState_t state2;
1660
- size_t errorCode;
1661
-
1662
- /* Init */
1663
- errorCode = FSE_initDStream(&bitD, cSrc, cSrcSize); /* replaced last arg by maxCompressed Size */
1664
- if (FSE_isError(errorCode)) return errorCode;
1665
-
1666
- FSE_initDState(&state1, &bitD, dt);
1667
- FSE_initDState(&state2, &bitD, dt);
1668
-
1669
- #define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD)
1670
-
1671
- /* 4 symbols per loop */
1672
- for ( ; (FSE_reloadDStream(&bitD)==FSE_DStream_unfinished) && (op<olimit) ; op+=4)
1673
- {
1674
- op[0] = FSE_GETSYMBOL(&state1);
1675
-
1676
- if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
1677
- FSE_reloadDStream(&bitD);
1678
-
1679
- op[1] = FSE_GETSYMBOL(&state2);
1680
-
1681
- if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
1682
- { if (FSE_reloadDStream(&bitD) > FSE_DStream_unfinished) { op+=2; break; } }
1683
-
1684
- op[2] = FSE_GETSYMBOL(&state1);
1685
-
1686
- if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
1687
- FSE_reloadDStream(&bitD);
1688
-
1689
- op[3] = FSE_GETSYMBOL(&state2);
1690
- }
1691
-
1692
- /* tail */
1693
- /* note : FSE_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly FSE_DStream_completed */
1694
- while (1)
1695
- {
1696
- if ( (FSE_reloadDStream(&bitD)>FSE_DStream_completed) || (op==omax) || (FSE_endOfDStream(&bitD) && (fast || FSE_endOfDState(&state1))) )
1697
- break;
1698
-
1699
- *op++ = FSE_GETSYMBOL(&state1);
1700
-
1701
- if ( (FSE_reloadDStream(&bitD)>FSE_DStream_completed) || (op==omax) || (FSE_endOfDStream(&bitD) && (fast || FSE_endOfDState(&state2))) )
1702
- break;
1703
-
1704
- *op++ = FSE_GETSYMBOL(&state2);
1705
- }
1706
-
1707
- /* end ? */
1708
- if (FSE_endOfDStream(&bitD) && FSE_endOfDState(&state1) && FSE_endOfDState(&state2))
1709
- return op-ostart;
1710
-
1711
- if (op==omax) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* dst buffer is full, but cSrc unfinished */
1712
-
1713
- return (size_t)-FSE_ERROR_corruptionDetected;
1714
- }
1715
-
1716
-
1717
- size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
1718
- const void* cSrc, size_t cSrcSize,
1719
- const FSE_DTable* dt)
1720
- {
1721
- const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)dt;
1722
- const U32 fastMode = DTableH->fastMode;
1723
-
1724
- /* select fast mode (static) */
1725
- if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
1726
- return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
1727
- }
1728
-
1729
-
1730
- size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize)
1731
- {
1732
- const BYTE* const istart = (const BYTE*)cSrc;
1733
- const BYTE* ip = istart;
1734
- short counting[FSE_MAX_SYMBOL_VALUE+1];
1735
- DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */
1736
- unsigned tableLog;
1737
- unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
1738
- size_t errorCode;
1739
-
1740
- if (cSrcSize<2) return (size_t)-FSE_ERROR_srcSize_wrong; /* too small input size */
1741
-
1742
- /* normal FSE decoding mode */
1743
- errorCode = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
1744
- if (FSE_isError(errorCode)) return errorCode;
1745
- if (errorCode >= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong; /* too small input size */
1746
- ip += errorCode;
1747
- cSrcSize -= errorCode;
1748
-
1749
- errorCode = FSE_buildDTable (dt, counting, maxSymbolValue, tableLog);
1750
- if (FSE_isError(errorCode)) return errorCode;
1751
-
1752
- /* always return, even if it is an error code */
1753
- return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt);
1754
- }
1755
-
1756
-
1757
-
1758
- /*********************************************************
1759
- * Huff0 : Huffman block compression
1760
- *********************************************************/
1761
- #define HUF_MAX_SYMBOL_VALUE 255
1762
- #define HUF_DEFAULT_TABLELOG 12 /* used by default, when not specified */
1763
- #define HUF_MAX_TABLELOG 12 /* max possible tableLog; for allocation purpose; can be modified */
1764
- #define HUF_ABSOLUTEMAX_TABLELOG 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
1765
- #if (HUF_MAX_TABLELOG > HUF_ABSOLUTEMAX_TABLELOG)
1766
- # error "HUF_MAX_TABLELOG is too large !"
1767
- #endif
1768
-
1769
- typedef struct HUF_CElt_s {
1770
- U16 val;
1771
- BYTE nbBits;
1772
- } HUF_CElt ;
1773
-
1774
- typedef struct nodeElt_s {
1775
- U32 count;
1776
- U16 parent;
1777
- BYTE byte;
1778
- BYTE nbBits;
1779
- } nodeElt;
1780
-
1781
- /* HUF_writeCTable() :
1782
- return : size of saved CTable */
1783
- size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* tree, U32 maxSymbolValue, U32 huffLog)
1784
- {
1785
- BYTE bitsToWeight[HUF_ABSOLUTEMAX_TABLELOG + 1];
1786
- BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1];
1787
- U32 n;
1788
- BYTE* op = (BYTE*)dst;
1789
- size_t size;
1790
-
1791
- /* check conditions */
1792
- if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE + 1)
1793
- return (size_t)-FSE_ERROR_GENERIC;
1794
-
1795
- /* convert to weight */
1796
- bitsToWeight[0] = 0;
1797
- for (n=1; n<=huffLog; n++)
1798
- bitsToWeight[n] = (BYTE)(huffLog + 1 - n);
1799
- for (n=0; n<maxSymbolValue; n++)
1800
- huffWeight[n] = bitsToWeight[tree[n].nbBits];
1801
-
1802
- size = FSE_compress(op+1, maxDstSize-1, huffWeight, maxSymbolValue); /* don't need last symbol stat : implied */
1803
- if (FSE_isError(size)) return size;
1804
- if (size >= 128) return (size_t)-FSE_ERROR_GENERIC; /* should never happen, since maxSymbolValue <= 255 */
1805
- if ((size <= 1) || (size >= maxSymbolValue/2))
1806
- {
1807
- if (size==1) /* RLE */
1808
- {
1809
- /* only possible case : serie of 1 (because there are at least 2) */
1810
- /* can only be 2^n or (2^n-1), otherwise not an huffman tree */
1811
- BYTE code;
1812
- switch(maxSymbolValue)
1813
- {
1814
- case 1: code = 0; break;
1815
- case 2: code = 1; break;
1816
- case 3: code = 2; break;
1817
- case 4: code = 3; break;
1818
- case 7: code = 4; break;
1819
- case 8: code = 5; break;
1820
- case 15: code = 6; break;
1821
- case 16: code = 7; break;
1822
- case 31: code = 8; break;
1823
- case 32: code = 9; break;
1824
- case 63: code = 10; break;
1825
- case 64: code = 11; break;
1826
- case 127: code = 12; break;
1827
- case 128: code = 13; break;
1828
- default : return (size_t)-FSE_ERROR_corruptionDetected;
1829
- }
1830
- op[0] = (BYTE)(255-13 + code);
1831
- return 1;
1832
- }
1833
- /* Not compressible */
1834
- if (maxSymbolValue > (241-128)) return (size_t)-FSE_ERROR_GENERIC; /* not implemented (not possible with current format) */
1835
- if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* not enough space within dst buffer */
1836
- op[0] = (BYTE)(128 /*special case*/ + 0 /* Not Compressible */ + (maxSymbolValue-1));
1837
- huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */
1838
- for (n=0; n<maxSymbolValue; n+=2)
1839
- op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]);
1840
- return ((maxSymbolValue+1)/2) + 1;
1841
- }
1842
-
1843
- /* normal header case */
1844
- op[0] = (BYTE)size;
1845
- return size+1;
1846
- }
1847
-
1848
-
1849
- static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
1850
- {
1851
- int totalCost = 0;
1852
- const U32 largestBits = huffNode[lastNonNull].nbBits;
1853
-
1854
- /* early exit : all is fine */
1855
- if (largestBits <= maxNbBits) return largestBits;
1856
-
1857
- // now we have a few too large elements (at least >= 2)
1858
- {
1859
- const U32 baseCost = 1 << (largestBits - maxNbBits);
1860
- U32 n = lastNonNull;
1861
-
1862
- while (huffNode[n].nbBits > maxNbBits)
1863
- {
1864
- totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits));
1865
- huffNode[n].nbBits = (BYTE)maxNbBits;
1866
- n --;
1867
- }
1868
-
1869
- /* renorm totalCost */
1870
- totalCost >>= (largestBits - maxNbBits); /* note : totalCost necessarily multiple of baseCost */
1871
-
1872
- // repay cost
1873
- while (huffNode[n].nbBits == maxNbBits) n--; // n at last of rank (maxNbBits-1)
1874
-
1875
- {
1876
- const U32 noOne = 0xF0F0F0F0;
1877
- // Get pos of last (smallest) symbol per rank
1878
- U32 rankLast[HUF_MAX_TABLELOG];
1879
- U32 currentNbBits = maxNbBits;
1880
- int pos;
1881
- memset(rankLast, 0xF0, sizeof(rankLast));
1882
- for (pos=n ; pos >= 0; pos--)
1883
- {
1884
- if (huffNode[pos].nbBits >= currentNbBits) continue;
1885
- currentNbBits = huffNode[pos].nbBits;
1886
- rankLast[maxNbBits-currentNbBits] = pos;
1887
- }
1888
-
1889
- while (totalCost > 0)
1890
- {
1891
- U32 nBitsToDecrease = FSE_highbit32(totalCost) + 1;
1892
- for ( ; nBitsToDecrease > 1; nBitsToDecrease--)
1893
- {
1894
- U32 highPos = rankLast[nBitsToDecrease];
1895
- U32 lowPos = rankLast[nBitsToDecrease-1];
1896
- if (highPos == noOne) continue;
1897
- if (lowPos == noOne) break;
1898
- {
1899
- U32 highTotal = huffNode[highPos].count;
1900
- U32 lowTotal = 2 * huffNode[lowPos].count;
1901
- if (highTotal <= lowTotal) break;
1902
- }
1903
- }
1904
- while (rankLast[nBitsToDecrease] == noOne)
1905
- nBitsToDecrease ++; // In some rare cases, no more rank 1 left => overshoot to closest
1906
- totalCost -= 1 << (nBitsToDecrease-1);
1907
- if (rankLast[nBitsToDecrease-1] == noOne)
1908
- rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; // now there is one elt
1909
- huffNode[rankLast[nBitsToDecrease]].nbBits ++;
1910
- if (rankLast[nBitsToDecrease] == 0)
1911
- rankLast[nBitsToDecrease] = noOne;
1912
- else
1913
- {
1914
- rankLast[nBitsToDecrease]--;
1915
- if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease)
1916
- rankLast[nBitsToDecrease] = noOne; // rank list emptied
1917
- }
1918
- }
1919
-
1920
- while (totalCost < 0) /* Sometimes, cost correction overshoot */
1921
- {
1922
- if (rankLast[1] == noOne) /* special case, no weight 1, let's find it back at n */
1923
- {
1924
- while (huffNode[n].nbBits == maxNbBits) n--;
1925
- huffNode[n+1].nbBits--;
1926
- rankLast[1] = n+1;
1927
- totalCost++;
1928
- continue;
1929
- }
1930
- huffNode[ rankLast[1] + 1 ].nbBits--;
1931
- rankLast[1]++;
1932
- totalCost ++;
1933
- }
1934
- }
1935
- }
1936
-
1937
- return maxNbBits;
1938
- }
1939
-
1940
-
1941
- typedef struct {
1942
- U32 base;
1943
- U32 current;
1944
- } rankPos;
1945
-
1946
- static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue)
1947
- {
1948
- rankPos rank[32];
1949
- U32 n;
1950
-
1951
- memset(rank, 0, sizeof(rank));
1952
- for (n=0; n<=maxSymbolValue; n++)
1953
- {
1954
- U32 r = FSE_highbit32(count[n] + 1);
1955
- rank[r].base ++;
1956
- }
1957
- for (n=30; n>0; n--) rank[n-1].base += rank[n].base;
1958
- for (n=0; n<32; n++) rank[n].current = rank[n].base;
1959
- for (n=0; n<=maxSymbolValue; n++)
1960
- {
1961
- U32 c = count[n];
1962
- U32 r = FSE_highbit32(c+1) + 1;
1963
- U32 pos = rank[r].current++;
1964
- while ((pos > rank[r].base) && (c > huffNode[pos-1].count)) huffNode[pos]=huffNode[pos-1], pos--;
1965
- huffNode[pos].count = c;
1966
- huffNode[pos].byte = (BYTE)n;
1967
- }
1968
- }
1969
-
1970
-
1971
- #define STARTNODE (HUF_MAX_SYMBOL_VALUE+1)
1972
- size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits)
1973
- {
1974
- nodeElt huffNode0[2*HUF_MAX_SYMBOL_VALUE+1 +1];
1975
- nodeElt* huffNode = huffNode0 + 1;
1976
- U32 n, nonNullRank;
1977
- int lowS, lowN;
1978
- U16 nodeNb = STARTNODE;
1979
- U32 nodeRoot;
1980
-
1981
- /* safety checks */
1982
- if (maxNbBits == 0) maxNbBits = HUF_DEFAULT_TABLELOG;
1983
- if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE) return (size_t)-FSE_ERROR_GENERIC;
1984
- memset(huffNode0, 0, sizeof(huffNode0));
1985
-
1986
- // sort, decreasing order
1987
- HUF_sort(huffNode, count, maxSymbolValue);
1988
-
1989
- // init for parents
1990
- nonNullRank = maxSymbolValue;
1991
- while(huffNode[nonNullRank].count == 0) nonNullRank--;
1992
- lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb;
1993
- huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count;
1994
- huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb;
1995
- nodeNb++; lowS-=2;
1996
- for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30);
1997
- huffNode0[0].count = (U32)(1U<<31);
1998
-
1999
- // create parents
2000
- while (nodeNb <= nodeRoot)
2001
- {
2002
- U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
2003
- U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
2004
- huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count;
2005
- huffNode[n1].parent = huffNode[n2].parent = nodeNb;
2006
- nodeNb++;
2007
- }
2008
-
2009
- // distribute weights (unlimited tree height)
2010
- huffNode[nodeRoot].nbBits = 0;
2011
- for (n=nodeRoot-1; n>=STARTNODE; n--)
2012
- huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
2013
- for (n=0; n<=nonNullRank; n++)
2014
- huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
2015
-
2016
- // enforce maxTableLog
2017
- maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits);
2018
-
2019
- // fill result into tree (val, nbBits)
2020
- {
2021
- U16 nbPerRank[HUF_ABSOLUTEMAX_TABLELOG+1] = {0};
2022
- U16 valPerRank[HUF_ABSOLUTEMAX_TABLELOG+1];
2023
- if (maxNbBits > HUF_ABSOLUTEMAX_TABLELOG) return (size_t)-FSE_ERROR_GENERIC; // check
2024
- for (n=0; n<=nonNullRank; n++)
2025
- nbPerRank[huffNode[n].nbBits]++;
2026
- {
2027
- // determine stating value per rank
2028
- U16 min = 0;
2029
- for (n=maxNbBits; n>0; n--)
2030
- {
2031
- valPerRank[n] = min; // get starting value within each rank
2032
- min += nbPerRank[n];
2033
- min >>= 1;
2034
- }
2035
- }
2036
- for (n=0; n<=maxSymbolValue; n++)
2037
- tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; // push nbBits per symbol, symbol order
2038
- for (n=0; n<=maxSymbolValue; n++)
2039
- tree[n].val = valPerRank[tree[n].nbBits]++; // assign value within rank, symbol order
2040
- }
2041
-
2042
- return maxNbBits;
2043
- }
2044
-
2045
- static void HUF_encodeSymbol(FSE_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable)
2046
- {
2047
- FSE_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits);
2048
- }
2049
-
2050
- #define FSE_FLUSHBITS_1(stream) \
2051
- if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*2+7) FSE_FLUSHBITS(stream)
2052
-
2053
- #define FSE_FLUSHBITS_2(stream) \
2054
- if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*4+7) FSE_FLUSHBITS(stream)
2055
-
2056
- size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, HUF_CElt* CTable)
2057
- {
2058
- const BYTE* ip = (const BYTE*) src;
2059
- BYTE* const ostart = (BYTE*)dst;
2060
- BYTE* op = (BYTE*) ostart;
2061
- BYTE* const oend = ostart + dstSize;
2062
- U16* jumpTable = (U16*) dst;
2063
- size_t n, streamSize;
2064
- const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize));
2065
- size_t errorCode;
2066
- FSE_CStream_t bitC;
2067
-
2068
- /* init */
2069
- if (dstSize < 8) return 0;
2070
- op += 6; /* jump Table -- could be optimized by delta / deviation */
2071
- errorCode = FSE_initCStream(&bitC, op, oend-op);
2072
- if (FSE_isError(errorCode)) return 0;
2073
-
2074
- n = srcSize & ~15; // mod 16
2075
- switch (srcSize & 15)
2076
- {
2077
- case 15: HUF_encodeSymbol(&bitC, ip[n+14], CTable);
2078
- FSE_FLUSHBITS_1(&bitC);
2079
- case 14: HUF_encodeSymbol(&bitC, ip[n+13], CTable);
2080
- FSE_FLUSHBITS_2(&bitC);
2081
- case 13: HUF_encodeSymbol(&bitC, ip[n+12], CTable);
2082
- FSE_FLUSHBITS_1(&bitC);
2083
- case 12: HUF_encodeSymbol(&bitC, ip[n+11], CTable);
2084
- FSE_FLUSHBITS(&bitC);
2085
- case 11: HUF_encodeSymbol(&bitC, ip[n+10], CTable);
2086
- FSE_FLUSHBITS_1(&bitC);
2087
- case 10: HUF_encodeSymbol(&bitC, ip[n+ 9], CTable);
2088
- FSE_FLUSHBITS_2(&bitC);
2089
- case 9 : HUF_encodeSymbol(&bitC, ip[n+ 8], CTable);
2090
- FSE_FLUSHBITS_1(&bitC);
2091
- case 8 : HUF_encodeSymbol(&bitC, ip[n+ 7], CTable);
2092
- FSE_FLUSHBITS(&bitC);
2093
- case 7 : HUF_encodeSymbol(&bitC, ip[n+ 6], CTable);
2094
- FSE_FLUSHBITS_1(&bitC);
2095
- case 6 : HUF_encodeSymbol(&bitC, ip[n+ 5], CTable);
2096
- FSE_FLUSHBITS_2(&bitC);
2097
- case 5 : HUF_encodeSymbol(&bitC, ip[n+ 4], CTable);
2098
- FSE_FLUSHBITS_1(&bitC);
2099
- case 4 : HUF_encodeSymbol(&bitC, ip[n+ 3], CTable);
2100
- FSE_FLUSHBITS(&bitC);
2101
- case 3 : HUF_encodeSymbol(&bitC, ip[n+ 2], CTable);
2102
- FSE_FLUSHBITS_2(&bitC);
2103
- case 2 : HUF_encodeSymbol(&bitC, ip[n+ 1], CTable);
2104
- FSE_FLUSHBITS_1(&bitC);
2105
- case 1 : HUF_encodeSymbol(&bitC, ip[n+ 0], CTable);
2106
- FSE_FLUSHBITS(&bitC);
2107
- case 0 :
2108
- default: ;
2109
- }
2110
-
2111
- for (; n>0; n-=16)
2112
- {
2113
- HUF_encodeSymbol(&bitC, ip[n- 4], CTable);
2114
- FSE_FLUSHBITS_1(&bitC);
2115
- HUF_encodeSymbol(&bitC, ip[n- 8], CTable);
2116
- FSE_FLUSHBITS_2(&bitC);
2117
- HUF_encodeSymbol(&bitC, ip[n-12], CTable);
2118
- FSE_FLUSHBITS_1(&bitC);
2119
- HUF_encodeSymbol(&bitC, ip[n-16], CTable);
2120
- FSE_FLUSHBITS(&bitC);
2121
- }
2122
- streamSize = FSE_closeCStream(&bitC);
2123
- if (streamSize==0) return 0; /* not enough space within dst buffer == uncompressible */
2124
- FSE_writeLE16(jumpTable, (U16)streamSize);
2125
- op += streamSize;
2126
-
2127
- errorCode = FSE_initCStream(&bitC, op, oend-op);
2128
- if (FSE_isError(errorCode)) return 0;
2129
- n = srcSize & ~15; // mod 16
2130
- for (; n>0; n-=16)
2131
- {
2132
- HUF_encodeSymbol(&bitC, ip[n- 3], CTable);
2133
- FSE_FLUSHBITS_1(&bitC);
2134
- HUF_encodeSymbol(&bitC, ip[n- 7], CTable);
2135
- FSE_FLUSHBITS_2(&bitC);
2136
- HUF_encodeSymbol(&bitC, ip[n-11], CTable);
2137
- FSE_FLUSHBITS_1(&bitC);
2138
- HUF_encodeSymbol(&bitC, ip[n-15], CTable);
2139
- FSE_FLUSHBITS(&bitC);
2140
- }
2141
- streamSize = FSE_closeCStream(&bitC);
2142
- if (streamSize==0) return 0; /* not enough space within dst buffer == uncompressible */
2143
- FSE_writeLE16(jumpTable+1, (U16)streamSize);
2144
- op += streamSize;
2145
-
2146
- errorCode = FSE_initCStream(&bitC, op, oend-op);
2147
- if (FSE_isError(errorCode)) return 0;
2148
- n = srcSize & ~15; // mod 16
2149
- for (; n>0; n-=16)
2150
- {
2151
- HUF_encodeSymbol(&bitC, ip[n- 2], CTable);
2152
- FSE_FLUSHBITS_1(&bitC);
2153
- HUF_encodeSymbol(&bitC, ip[n- 6], CTable);
2154
- FSE_FLUSHBITS_2(&bitC);
2155
- HUF_encodeSymbol(&bitC, ip[n-10], CTable);
2156
- FSE_FLUSHBITS_1(&bitC);
2157
- HUF_encodeSymbol(&bitC, ip[n-14], CTable);
2158
- FSE_FLUSHBITS(&bitC);
2159
- }
2160
- streamSize = FSE_closeCStream(&bitC);
2161
- if (streamSize==0) return 0; /* not enough space within dst buffer == uncompressible */
2162
- FSE_writeLE16(jumpTable+2, (U16)streamSize);
2163
- op += streamSize;
2164
-
2165
- errorCode = FSE_initCStream(&bitC, op, oend-op);
2166
- if (FSE_isError(errorCode)) return 0;
2167
- n = srcSize & ~15; // mod 16
2168
- for (; n>0; n-=16)
2169
- {
2170
- HUF_encodeSymbol(&bitC, ip[n- 1], CTable);
2171
- FSE_FLUSHBITS_1(&bitC);
2172
- HUF_encodeSymbol(&bitC, ip[n- 5], CTable);
2173
- FSE_FLUSHBITS_2(&bitC);
2174
- HUF_encodeSymbol(&bitC, ip[n- 9], CTable);
2175
- FSE_FLUSHBITS_1(&bitC);
2176
- HUF_encodeSymbol(&bitC, ip[n-13], CTable);
2177
- FSE_FLUSHBITS(&bitC);
2178
- }
2179
- streamSize = FSE_closeCStream(&bitC);
2180
- if (streamSize==0) return 0; /* not enough space within dst buffer == uncompressible */
2181
- op += streamSize;
2182
-
2183
- return op-ostart;
2184
- }
2185
-
2186
-
2187
- size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog)
2188
- {
2189
- BYTE* const ostart = (BYTE*)dst;
2190
- BYTE* op = ostart;
2191
- BYTE* const oend = ostart + dstSize;
2192
-
2193
- U32 count[HUF_MAX_SYMBOL_VALUE+1];
2194
- HUF_CElt CTable[HUF_MAX_SYMBOL_VALUE+1];
2195
- size_t errorCode;
2196
-
2197
- /* early out */
2198
- if (srcSize <= 1) return srcSize; /* Uncompressed or RLE */
2199
- if (!maxSymbolValue) maxSymbolValue = HUF_MAX_SYMBOL_VALUE;
2200
- if (!huffLog) huffLog = HUF_DEFAULT_TABLELOG;
2201
-
2202
- /* Scan input and build symbol stats */
2203
- errorCode = FSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize);
2204
- if (FSE_isError(errorCode)) return errorCode;
2205
- if (errorCode == srcSize) return 1;
2206
- if (errorCode < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */
2207
-
2208
- /* Build Huffman Tree */
2209
- errorCode = HUF_buildCTable (CTable, count, maxSymbolValue, huffLog);
2210
- if (FSE_isError(errorCode)) return errorCode;
2211
- huffLog = (U32)errorCode;
2212
-
2213
- /* Write table description header */
2214
- errorCode = HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog); /* don't write last symbol, implied */
2215
- if (FSE_isError(errorCode)) return errorCode;
2216
- op += errorCode;
2217
-
2218
- /* Compress */
2219
- errorCode = HUF_compress_usingCTable(op, oend - op, src, srcSize, CTable);
2220
- if (FSE_isError(errorCode)) return errorCode;
2221
- if (errorCode==0) return 0;
2222
- op += errorCode;
2223
-
2224
- /* check compressibility */
2225
- if ((size_t)(op-ostart) >= srcSize-1)
2226
- return op-ostart;
2227
-
2228
- return op-ostart;
2229
- }
2230
-
2231
- size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
2232
- {
2233
- return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_DEFAULT_TABLELOG);
2234
- }
2235
-
2236
-
2237
- /*********************************************************
2238
- * Huff0 : Huffman block decompression
2239
- *********************************************************/
2240
- typedef struct {
2241
- BYTE byte;
2242
- BYTE nbBits;
2243
- } HUF_DElt;
2244
-
2245
- size_t HUF_readDTable (U16* DTable, const void* src, size_t srcSize)
2246
- {
2247
- BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1];
2248
- U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; /* large enough for values from 0 to 16 */
2249
- U32 weightTotal;
2250
- U32 maxBits;
2251
- const BYTE* ip = (const BYTE*) src;
2252
- size_t iSize = ip[0];
2253
- size_t oSize;
2254
- U32 n;
2255
- U32 nextRankStart;
2256
- HUF_DElt* const dt = (HUF_DElt*)(DTable + 1);
2257
-
2258
- FSE_STATIC_ASSERT(sizeof(HUF_DElt) == sizeof(U16)); /* if compilation fails here, assertion is false */
2259
- //memset(huffWeight, 0, sizeof(huffWeight)); /* should not be necessary, but some analyzer complain ... */
2260
- if (iSize >= 128) /* special header */
2261
- {
2262
- if (iSize >= (242)) /* RLE */
2263
- {
2264
- static int l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 };
2265
- oSize = l[iSize-242];
2266
- memset(huffWeight, 1, oSize);
2267
- iSize = 0;
2268
- }
2269
- else /* Incompressible */
2270
- {
2271
- oSize = iSize - 127;
2272
- iSize = ((oSize+1)/2);
2273
- if (iSize+1 > srcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
2274
- ip += 1;
2275
- for (n=0; n<oSize; n+=2)
2276
- {
2277
- huffWeight[n] = ip[n/2] >> 4;
2278
- huffWeight[n+1] = ip[n/2] & 15;
2279
- }
2280
- }
2281
- }
2282
- else /* header compressed with FSE (normal case) */
2283
- {
2284
- if (iSize+1 > srcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
2285
- oSize = FSE_decompress(huffWeight, HUF_MAX_SYMBOL_VALUE, ip+1, iSize); /* max 255 values decoded, last one is implied */
2286
- if (FSE_isError(oSize)) return oSize;
2287
- }
2288
-
2289
- /* collect weight stats */
2290
- memset(rankVal, 0, sizeof(rankVal));
2291
- weightTotal = 0;
2292
- for (n=0; n<oSize; n++)
2293
- {
2294
- if (huffWeight[n] >= HUF_ABSOLUTEMAX_TABLELOG) return (size_t)-FSE_ERROR_corruptionDetected;
2295
- rankVal[huffWeight[n]]++;
2296
- weightTotal += (1 << huffWeight[n]) >> 1;
2297
- }
2298
-
2299
- /* get last non-null symbol weight (implied, total must be 2^n) */
2300
- maxBits = FSE_highbit32(weightTotal) + 1;
2301
- if (maxBits > DTable[0]) return (size_t)-FSE_ERROR_tableLog_tooLarge; /* DTable is too small */
2302
- DTable[0] = (U16)maxBits;
2303
- {
2304
- U32 total = 1 << maxBits;
2305
- U32 rest = total - weightTotal;
2306
- U32 verif = 1 << FSE_highbit32(rest);
2307
- U32 lastWeight = FSE_highbit32(rest) + 1;
2308
- if (verif != rest) return (size_t)-FSE_ERROR_corruptionDetected; /* last value must be a clean power of 2 */
2309
- huffWeight[oSize] = (BYTE)lastWeight;
2310
- rankVal[lastWeight]++;
2311
- }
2312
-
2313
- /* check tree construction validity */
2314
- if ((rankVal[1] < 2) || (rankVal[1] & 1)) return (size_t)-FSE_ERROR_corruptionDetected; /* by construction : at least 2 elts of rank 1, must be even */
2315
-
2316
- /* Prepare ranks */
2317
- nextRankStart = 0;
2318
- for (n=1; n<=maxBits; n++)
2319
- {
2320
- U32 current = nextRankStart;
2321
- nextRankStart += (rankVal[n] << (n-1));
2322
- rankVal[n] = current;
2323
- }
2324
-
2325
- /* fill DTable */
2326
- for (n=0; n<=oSize; n++)
2327
- {
2328
- const U32 w = huffWeight[n];
2329
- const U32 length = (1 << w) >> 1;
2330
- U32 i;
2331
- HUF_DElt D;
2332
- D.byte = (BYTE)n; D.nbBits = (BYTE)(maxBits + 1 - w);
2333
- for (i = rankVal[w]; i < rankVal[w] + length; i++)
2334
- dt[i] = D;
2335
- rankVal[w] += length;
2336
- }
2337
-
2338
- return iSize+1;
2339
- }
2340
-
2341
-
2342
- static BYTE HUF_decodeSymbol(FSE_DStream_t* Dstream, const HUF_DElt* dt, const U32 dtLog)
2343
- {
2344
- const size_t val = FSE_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */
2345
- const BYTE c = dt[val].byte;
2346
- FSE_skipBits(Dstream, dt[val].nbBits);
2347
- return c;
2348
- }
2349
-
2350
- static size_t HUF_decompress_usingDTable( /* -3% slower when non static */
2351
- void* dst, size_t maxDstSize,
2352
- const void* cSrc, size_t cSrcSize,
2353
- const U16* DTable)
2354
- {
2355
- BYTE* const ostart = (BYTE*) dst;
2356
- BYTE* op = ostart;
2357
- BYTE* const omax = op + maxDstSize;
2358
- BYTE* const olimit = omax-15;
2359
-
2360
- const HUF_DElt* const dt = (const HUF_DElt*)(DTable+1);
2361
- const U32 dtLog = DTable[0];
2362
- size_t errorCode;
2363
- U32 reloadStatus;
2364
-
2365
- /* Init */
2366
-
2367
- const U16* jumpTable = (const U16*)cSrc;
2368
- const size_t length1 = FSE_readLE16(jumpTable);
2369
- const size_t length2 = FSE_readLE16(jumpTable+1);
2370
- const size_t length3 = FSE_readLE16(jumpTable+2);
2371
- const size_t length4 = cSrcSize - 6 - length1 - length2 - length3; // check coherency !!
2372
- const char* const start1 = (const char*)(cSrc) + 6;
2373
- const char* const start2 = start1 + length1;
2374
- const char* const start3 = start2 + length2;
2375
- const char* const start4 = start3 + length3;
2376
- FSE_DStream_t bitD1, bitD2, bitD3, bitD4;
2377
-
2378
- if (length1+length2+length3+6 >= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
2379
-
2380
- errorCode = FSE_initDStream(&bitD1, start1, length1);
2381
- if (FSE_isError(errorCode)) return errorCode;
2382
- errorCode = FSE_initDStream(&bitD2, start2, length2);
2383
- if (FSE_isError(errorCode)) return errorCode;
2384
- errorCode = FSE_initDStream(&bitD3, start3, length3);
2385
- if (FSE_isError(errorCode)) return errorCode;
2386
- errorCode = FSE_initDStream(&bitD4, start4, length4);
2387
- if (FSE_isError(errorCode)) return errorCode;
2388
-
2389
- reloadStatus=FSE_reloadDStream(&bitD2);
2390
-
2391
- /* 16 symbols per loop */
2392
- for ( ; (reloadStatus<FSE_DStream_completed) && (op<olimit); /* D2-3-4 are supposed to be synchronized and finish together */
2393
- op+=16, reloadStatus = FSE_reloadDStream(&bitD2) | FSE_reloadDStream(&bitD3) | FSE_reloadDStream(&bitD4), FSE_reloadDStream(&bitD1))
2394
- {
2395
- #define HUF_DECODE_SYMBOL_0(n, Dstream) \
2396
- op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog);
2397
-
2398
- #define HUF_DECODE_SYMBOL_1(n, Dstream) \
2399
- op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
2400
- if (FSE_32bits() && (HUF_MAX_TABLELOG>12)) FSE_reloadDStream(&Dstream)
2401
-
2402
- #define HUF_DECODE_SYMBOL_2(n, Dstream) \
2403
- op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
2404
- if (FSE_32bits()) FSE_reloadDStream(&Dstream)
2405
-
2406
- HUF_DECODE_SYMBOL_1( 0, bitD1);
2407
- HUF_DECODE_SYMBOL_1( 1, bitD2);
2408
- HUF_DECODE_SYMBOL_1( 2, bitD3);
2409
- HUF_DECODE_SYMBOL_1( 3, bitD4);
2410
- HUF_DECODE_SYMBOL_2( 4, bitD1);
2411
- HUF_DECODE_SYMBOL_2( 5, bitD2);
2412
- HUF_DECODE_SYMBOL_2( 6, bitD3);
2413
- HUF_DECODE_SYMBOL_2( 7, bitD4);
2414
- HUF_DECODE_SYMBOL_1( 8, bitD1);
2415
- HUF_DECODE_SYMBOL_1( 9, bitD2);
2416
- HUF_DECODE_SYMBOL_1(10, bitD3);
2417
- HUF_DECODE_SYMBOL_1(11, bitD4);
2418
- HUF_DECODE_SYMBOL_0(12, bitD1);
2419
- HUF_DECODE_SYMBOL_0(13, bitD2);
2420
- HUF_DECODE_SYMBOL_0(14, bitD3);
2421
- HUF_DECODE_SYMBOL_0(15, bitD4);
2422
- }
2423
-
2424
- if (reloadStatus!=FSE_DStream_completed) /* not complete : some bitStream might be FSE_DStream_unfinished */
2425
- return (size_t)-FSE_ERROR_corruptionDetected;
2426
-
2427
- /* tail */
2428
- {
2429
- // bitTail = bitD1; // *much* slower : -20% !??!
2430
- FSE_DStream_t bitTail;
2431
- bitTail.ptr = bitD1.ptr;
2432
- bitTail.bitsConsumed = bitD1.bitsConsumed;
2433
- bitTail.bitContainer = bitD1.bitContainer; // required in case of FSE_DStream_endOfBuffer
2434
- bitTail.start = start1;
2435
- for ( ; (FSE_reloadDStream(&bitTail) < FSE_DStream_completed) && (op<omax) ; op++)
2436
- {
2437
- HUF_DECODE_SYMBOL_0(0, bitTail);
2438
- }
2439
-
2440
- if (FSE_endOfDStream(&bitTail))
2441
- return op-ostart;
2442
- }
2443
-
2444
- if (op==omax) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* dst buffer is full, but cSrc unfinished */
2445
-
2446
- return (size_t)-FSE_ERROR_corruptionDetected;
2447
- }
2448
-
2449
-
2450
- size_t HUF_decompress (void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize)
2451
- {
2452
- HUF_CREATE_STATIC_DTABLE(DTable, HUF_MAX_TABLELOG);
2453
- const BYTE* ip = (const BYTE*) cSrc;
2454
- size_t errorCode;
2455
-
2456
- errorCode = HUF_readDTable (DTable, cSrc, cSrcSize);
2457
- if (FSE_isError(errorCode)) return errorCode;
2458
- if (errorCode >= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
2459
- ip += errorCode;
2460
- cSrcSize -= errorCode;
2461
-
2462
- return HUF_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, DTable);
2463
- }
2464
-
2465
-
2466
- #endif /* FSE_COMMONDEFS_ONLY */