extzstd 0.0.2.CONCEPT-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 193b3a5ca32c1263df30b3d75266ca13981424b4
4
+ data.tar.gz: 6475585c9c01c2cce38c8c6e18e7cfa001eb0f4e
5
+ SHA512:
6
+ metadata.gz: c07f9db08869a1a74f57442d63f6997d0d0d3da0676c80eb939d9bc09a677c1e2a4cd7f507eaddddcabdbcf4663c953ca7cd688e658a23870dbfaced47351524
7
+ data.tar.gz: 260ddbc9e075cab4fd27ab73002bf89b8452b683cc88a609f6be8a5a590dd1bde87bf4b54b8a31acc0f68463d73c90030cb3bc05bd31da98ca6a1444dfda0be0
data/LICENSE ADDED
@@ -0,0 +1,24 @@
1
+ Copyright (c) 2015, dearblue. All rights reserved.
2
+
3
+ Redistribution and use in source and binary forms, with or
4
+ without modification, are permitted provided that the following
5
+ conditions are met:
6
+
7
+ 1. Redistributions of source code must retain the above copyright
8
+ notice, this list of conditions and the following disclaimer.
9
+ 2. Redistributions in binary form must reproduce the above copyright
10
+ notice, this list of conditions and the following disclaimer in
11
+ the documentation and/or other materials provided with the
12
+ distribution.
13
+
14
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
15
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
16
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
17
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
18
+ HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
19
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
20
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
24
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
data/README.md ADDED
@@ -0,0 +1,59 @@
1
+ # encoding:utf-8 ;
2
+
3
+ # extzstd - ruby bindings for Zstd (Zstandard)
4
+
5
+ This is ruby bindings for compression library
6
+ [Zstd (https://github.com/Cyan4973/zstd)](https://github.com/Cyan4973/zstd).
7
+
8
+ * package name: extzstd
9
+ * author: dearblue (mailto:dearblue@users.osdn.me)
10
+ * version: 0.0.2.CONCEPT
11
+ * software quality: EXPERIMENTAL
12
+ * license: 2-clause BSD License
13
+ * report issue to: https://osdn.jp/projects/rutsubo/ticket/
14
+ * dependency ruby: ruby-2.0+
15
+ * dependency ruby gems: (none)
16
+ * dependency library: (none)
17
+ * bundled external libraries:
18
+ * zstd-0.1.2 (https://github.com/Cyan4973/zstd/tree/zstd-0.1.2)
19
+
20
+
21
+ ## ***WARNING***
22
+
23
+ Zstd data format compatibility is not guaranteed in future versions
24
+ (There is a possibility that it becomes impossible to future use).
25
+
26
+ Written in [zstd/README.md](https://github.com/Cyan4973/zstd/blob/zstd-0.1.2/README.md):
27
+
28
+ > Zstd has not yet reached "stable" status. Specifically, it doesn't
29
+ > guarantee yet that its current compressed format will remain stable
30
+ > and supported in future versions.
31
+
32
+
33
+ ## HOW TO USE
34
+
35
+ ### basic usage (one pass encode/decode)
36
+
37
+ ``` ruby:ruby
38
+ # First, load library
39
+ require "extzstd"
40
+
41
+ # Prepair data
42
+ source = "sample data..." * 50
43
+
44
+ # Directly compression
45
+ encdata = Zstd.encode(source)
46
+ puts "encdata.bytesize=#{encdata.bytesize}"
47
+
48
+ # Directly decompression
49
+ maxdestsize = source.bytesize
50
+ decdata = Zstd.decode(encdata, maxdestsize)
51
+ puts "decdata.bytesize=#{decdata.bytesize}"
52
+
53
+ # Comparison source and decoded data
54
+ p source == decdata # => true
55
+ ```
56
+
57
+ ----
58
+
59
+ [a stub]
data/Rakefile ADDED
@@ -0,0 +1,158 @@
1
+
2
+ require "rake/clean"
3
+
4
+ DOC = FileList["{README,LICENSE,CHANGELOG,Changelog,HISTORY}{,.ja}{,.txt,.rd,.rdoc,.md,.markdown}"] +
5
+ FileList["{contrib,ext}/**/{README,LICENSE,CHANGELOG,Changelog,HISTORY}{,.ja}{,.txt,.rd,.rdoc,.md,.markdown}"] +
6
+ FileList["ext/**/*.{c,C,cc,cxx,cpp,h,H,hh}"]
7
+ #EXT = FileList["ext/**/*.{h,hh,c,cc,cpp,cxx}"] +
8
+ # FileList["ext/externals/**/*"]
9
+ EXT = FileList["ext/**/*"]
10
+ BIN = FileList["bin/*"]
11
+ LIB = FileList["lib/**/*.rb"]
12
+ SPEC = FileList["spec/**/*"]
13
+ TEST = FileList["test/**/*"]
14
+ EXAMPLE = FileList["examples/**/*"]
15
+ GEMSTUB_SRC = "gemstub.rb"
16
+ RAKEFILE = [File.basename(__FILE__), GEMSTUB_SRC]
17
+ EXTRA = []
18
+
19
+ load GEMSTUB_SRC
20
+
21
+ EXTCONF = FileList["ext/extconf.rb"]
22
+ EXTCONF.reject! { |n| !File.file?(n) }
23
+ GEMSTUB.extensions += EXTCONF
24
+ GEMSTUB.executables += FileList["bin/*"].map { |n| File.basename n }
25
+ GEMSTUB.executables.sort!
26
+
27
+ PACKAGENAME = "#{GEMSTUB.name}-#{GEMSTUB.version}"
28
+ GEMFILE = "#{PACKAGENAME}.gem"
29
+ GEMSPEC = "#{GEMSTUB.name}.gemspec"
30
+
31
+ GEMSTUB.files += DOC + EXT + EXTCONF + BIN + LIB + SPEC + TEST + EXAMPLE + RAKEFILE + EXTRA
32
+ GEMSTUB.files.sort!
33
+ if GEMSTUB.rdoc_options.nil? || GEMSTUB.rdoc_options.empty?
34
+ readme = %W(.md .markdown .rd .rdoc .txt #{""}).map { |ext| "README#{ext}" }.find { |m| DOC.find { |n| n == m } }
35
+ GEMSTUB.rdoc_options = %w(--charset UTF-8) + (readme ? %W(-m #{readme}) : [])
36
+ end
37
+ GEMSTUB.extra_rdoc_files += DOC + LIB + EXT.reject { |n| n.include?("/externals/") || !%w(.h .hh .c .cc .cpp .cxx).include?(File.extname(n)) }
38
+ GEMSTUB.extra_rdoc_files.sort!
39
+
40
+ CLEAN << GEMSPEC
41
+ CLOBBER << GEMFILE
42
+
43
+ task :default => :all
44
+
45
+
46
+ unless EXTCONF.empty?
47
+ RUBYSET ||= (ENV["RUBYSET"] || "").split(",")
48
+
49
+ if RUBYSET.nil? || RUBYSET.empty?
50
+ $stderr.puts <<-EOS
51
+ #{__FILE__}:
52
+ |
53
+ | If you want binary gem package, launch rake with ``RUBYSET`` enviroment
54
+ | variable for set ruby interpreters by comma separated.
55
+ |
56
+ | e.g.) $ rake RUBYSET=ruby
57
+ | or) $ rake RUBYSET=ruby20,ruby21,ruby22
58
+ |
59
+ EOS
60
+ else
61
+ platforms = RUBYSET.map { |ruby| `#{ruby} --disable gems -rrbconfig -e "puts RbConfig::CONFIG['arch']"`.chomp }
62
+ platforms1 = platforms.uniq
63
+ unless platforms1.size == 1 && !platforms1[0].empty?
64
+ raise "different platforms (#{Hash[*RUBYSET.zip(platforms).flatten].inspect})"
65
+ end
66
+ PLATFORM = platforms1[0]
67
+
68
+ RUBY_VERSIONS = RUBYSET.map do |ruby|
69
+ ver = `#{ruby} --disable gem -rrbconfig -e "puts RbConfig::CONFIG['ruby_version']"`.slice(/\d+\.\d+/)
70
+ raise "failed ruby checking - ``#{ruby}''" unless $?.success?
71
+ [ver, ruby]
72
+ end
73
+ SOFILES_SET = RUBY_VERSIONS.map { |(ver, ruby)| ["lib/#{ver}/#{GEMSTUB.name}.so", ruby] }
74
+ SOFILES = SOFILES_SET.map { |(lib, ruby)| lib }
75
+
76
+ GEMSTUB_NATIVE = GEMSTUB.dup
77
+ GEMSTUB_NATIVE.files += SOFILES
78
+ GEMSTUB_NATIVE.platform = Gem::Platform.new(PLATFORM).to_s
79
+ GEMSTUB_NATIVE.extensions.clear
80
+ GEMFILE_NATIVE = "#{GEMSTUB_NATIVE.name}-#{GEMSTUB_NATIVE.version}-#{GEMSTUB_NATIVE.platform}.gem"
81
+ GEMSPEC_NATIVE = "#{GEMSTUB_NATIVE.name}-#{GEMSTUB_NATIVE.platform}.gemspec"
82
+
83
+ task :all => ["native-gem", GEMFILE]
84
+
85
+ desc "build binary gem package"
86
+ task "native-gem" => GEMFILE_NATIVE
87
+
88
+ desc "generate binary gemspec"
89
+ task "native-gemspec" => GEMSPEC_NATIVE
90
+
91
+ file GEMFILE_NATIVE => DOC + EXT + EXTCONF + BIN + LIB + SPEC + TEST + EXAMPLE + SOFILES + RAKEFILE + [GEMSPEC_NATIVE] do
92
+ sh "gem build #{GEMSPEC_NATIVE}"
93
+ end
94
+
95
+ file GEMSPEC_NATIVE => RAKEFILE do
96
+ File.write(GEMSPEC_NATIVE, GEMSTUB_NATIVE.to_ruby, mode: "wb")
97
+ end
98
+
99
+ desc "build c-extension libraries"
100
+ task "sofiles" => SOFILES
101
+
102
+ SOFILES_SET.each do |(soname, ruby)|
103
+ sodir = File.dirname(soname)
104
+ makefile = File.join(sodir, "Makefile")
105
+
106
+ CLEAN << GEMSPEC_NATIVE << sodir
107
+ CLOBBER << GEMFILE_NATIVE
108
+
109
+ directory sodir
110
+
111
+ desc "generate Makefile for binary extension library"
112
+ file makefile => [sodir] + EXTCONF do
113
+ cd sodir do
114
+ sh "#{ruby} ../../#{EXTCONF[0]} \"--ruby=#{ruby}\""
115
+ end
116
+ end
117
+
118
+ desc "build binary extension library"
119
+ file soname => [makefile] + EXT do
120
+ cd sodir do
121
+ sh "make"
122
+ end
123
+ end
124
+ end
125
+ end
126
+ end
127
+
128
+
129
+ task :all => GEMFILE
130
+
131
+ desc "generate local rdoc"
132
+ task :rdoc => DOC + LIB do
133
+ sh *(%w(rdoc) + GEMSTUB.rdoc_options + DOC + LIB)
134
+ end
135
+
136
+ desc "launch rspec"
137
+ task rspec: :all do
138
+ sh "rspec"
139
+ end
140
+
141
+ desc "build gem package"
142
+ task gem: GEMFILE
143
+
144
+ desc "generate gemspec"
145
+ task gemspec: GEMSPEC
146
+
147
+ desc "print package name"
148
+ task "package-name" do
149
+ puts PACKAGENAME
150
+ end
151
+
152
+ file GEMFILE => DOC + EXT + EXTCONF + BIN + LIB + SPEC + TEST + EXAMPLE + RAKEFILE + [GEMSPEC] do
153
+ sh "gem build #{GEMSPEC}"
154
+ end
155
+
156
+ file GEMSPEC => RAKEFILE do
157
+ File.write(GEMSPEC, GEMSTUB.to_ruby, mode: "wb")
158
+ end
@@ -0,0 +1,26 @@
1
+ ZSTD Library
2
+ Copyright (c) 2014-2015, Yann Collet
3
+ All rights reserved.
4
+
5
+ BSD License
6
+
7
+ Redistribution and use in source and binary forms, with or without modification,
8
+ are permitted provided that the following conditions are met:
9
+
10
+ * Redistributions of source code must retain the above copyright notice, this
11
+ list of conditions and the following disclaimer.
12
+
13
+ * Redistributions in binary form must reproduce the above copyright notice, this
14
+ list of conditions and the following disclaimer in the documentation and/or
15
+ other materials provided with the distribution.
16
+
17
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
18
+ ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
19
+ WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
20
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
21
+ ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
22
+ (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
23
+ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
24
+ ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
26
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -0,0 +1,115 @@
1
+ # ################################################################
2
+ # ZSTD library - Makefile
3
+ # Copyright (C) Yann Collet 2015
4
+ # All rights reserved.
5
+ #
6
+ # BSD license
7
+
8
+ # Redistribution and use in source and binary forms, with or without modification,
9
+ # are permitted provided that the following conditions are met:
10
+ #
11
+ # * Redistributions of source code must retain the above copyright notice, this
12
+ # list of conditions and the following disclaimer.
13
+ #
14
+ # * Redistributions in binary form must reproduce the above copyright notice, this
15
+ # list of conditions and the following disclaimer in the documentation and/or
16
+ # other materials provided with the distribution.
17
+ #
18
+ # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
19
+ # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20
+ # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21
+ # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
22
+ # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23
+ # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24
+ # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
25
+ # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26
+ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27
+ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
+ #
29
+ # You can contact the author at :
30
+ # - ZSTD source repository : https://github.com/Cyan4973/zstd
31
+ # - Public forum : https://groups.google.com/forum/#!forum/lz4c
32
+ # ################################################################
33
+
34
+ # Version numbers
35
+ VERSION?= 0.1.2
36
+ LIBVER_MAJOR=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
37
+ LIBVER_MINOR=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
38
+ LIBVER_PATCH=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
39
+ LIBVER = $(LIBVER_MAJOR).$(LIBVER_MINOR).$(LIBVER_PATCH)
40
+
41
+ DESTDIR?=
42
+ PREFIX ?= /usr/local
43
+ CFLAGS ?= -O3
44
+ CFLAGS += -I. -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes
45
+ FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
46
+
47
+ LIBDIR ?= $(PREFIX)/lib
48
+ INCLUDEDIR=$(PREFIX)/include
49
+
50
+
51
+ # OS X linker doesn't support -soname, and use different extension
52
+ # see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html
53
+ ifeq ($(shell uname), Darwin)
54
+ SHARED_EXT = dylib
55
+ SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT)
56
+ SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
57
+ SONAME_FLAGS = -install_name $(PREFIX)/lib/libzstd.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
58
+ else
59
+ SONAME_FLAGS = -Wl,-soname=libzstd.$(SHARED_EXT).$(LIBVER_MAJOR)
60
+ SHARED_EXT = so
61
+ SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR)
62
+ SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
63
+ endif
64
+
65
+ default: libzstd
66
+
67
+ all: libzstd
68
+
69
+ libzstd: zstd.c
70
+ @echo compiling static library
71
+ @$(CC) $(FLAGS) -c $^
72
+ @$(AR) rcs libzstd.a zstd.o
73
+ @echo compiling dynamic library $(LIBVER)
74
+ @$(CC) $(FLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER)
75
+ @echo creating versioned links
76
+ @ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT_MAJOR)
77
+ @ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT)
78
+
79
+ clean:
80
+ @rm -f core *.o *.a *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc
81
+ @echo Cleaning library completed
82
+
83
+
84
+ #------------------------------------------------------------------------
85
+ #make install is validated only for Linux, OSX, kFreeBSD and Hurd targets
86
+ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU))
87
+
88
+ libzstd.pc: libzstd.pc.in Makefile
89
+ @echo creating pkgconfig
90
+ @sed -e 's|@PREFIX@|$(PREFIX)|' \
91
+ -e 's|@LIBDIR@|$(LIBDIR)|' \
92
+ -e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \
93
+ -e 's|@VERSION@|$(VERSION)|' \
94
+ $< >$@
95
+
96
+ install: libzstd libzstd.pc
97
+ @install -d -m 755 $(DESTDIR)$(LIBDIR)/pkgconfig/ $(DESTDIR)$(INCLUDEDIR)/
98
+ @install -m 755 libzstd.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER)
99
+ @cp -a libzstd.$(SHARED_EXT_MAJOR) $(DESTDIR)$(LIBDIR)
100
+ @cp -a libzstd.$(SHARED_EXT) $(DESTDIR)$(LIBDIR)
101
+ @cp -a libzstd.pc $(DESTDIR)$(LIBDIR)/pkgconfig/
102
+ @install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a
103
+ @install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h
104
+ @echo zstd static and shared library installed
105
+
106
+ uninstall:
107
+ @rm -f $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
108
+ @rm -f $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
109
+ @rm -f $(DESTDIR)$(LIBDIR)/pkgconfig/libzstd.pc
110
+ @[ -x $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) ] && rm -f $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER)
111
+ @[ -f $(DESTDIR)$(LIBDIR)/libzstd.a ] && rm -f $(DESTDIR)$(LIBDIR)/libzstd.a
112
+ @[ -f $(DESTDIR)$(INCLUDEDIR)/zstd.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/zstd.h
113
+ @echo zstd libraries successfully uninstalled
114
+
115
+ endif
@@ -0,0 +1,2466 @@
1
+ /* ******************************************************************
2
+ FSE : Finite State Entropy coder
3
+ Copyright (C) 2013-2015, Yann Collet.
4
+
5
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
6
+
7
+ Redistribution and use in source and binary forms, with or without
8
+ modification, are permitted provided that the following conditions are
9
+ met:
10
+
11
+ * Redistributions of source code must retain the above copyright
12
+ notice, this list of conditions and the following disclaimer.
13
+ * Redistributions in binary form must reproduce the above
14
+ copyright notice, this list of conditions and the following disclaimer
15
+ in the documentation and/or other materials provided with the
16
+ distribution.
17
+
18
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
+
30
+ You can contact the author at :
31
+ - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
32
+ - Public forum : https://groups.google.com/forum/#!forum/lz4c
33
+ ****************************************************************** */
34
+
35
+ #ifndef FSE_COMMONDEFS_ONLY
36
+
37
+ /****************************************************************
38
+ * Tuning parameters
39
+ ****************************************************************/
40
+ /* MEMORY_USAGE :
41
+ * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
42
+ * Increasing memory usage improves compression ratio
43
+ * Reduced memory usage can improve speed, due to cache effect
44
+ * Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
45
+ #define FSE_MAX_MEMORY_USAGE 14
46
+ #define FSE_DEFAULT_MEMORY_USAGE 13
47
+
48
+ /* FSE_MAX_SYMBOL_VALUE :
49
+ * Maximum symbol value authorized.
50
+ * Required for proper stack allocation */
51
+ #define FSE_MAX_SYMBOL_VALUE 255
52
+
53
+
54
+ /****************************************************************
55
+ * template functions type & suffix
56
+ ****************************************************************/
57
+ #define FSE_FUNCTION_TYPE BYTE
58
+ #define FSE_FUNCTION_EXTENSION
59
+
60
+
61
+ /****************************************************************
62
+ * Byte symbol type
63
+ ****************************************************************/
64
+ typedef struct
65
+ {
66
+ unsigned short newState;
67
+ unsigned char symbol;
68
+ unsigned char nbBits;
69
+ } FSE_decode_t; /* size == U32 */
70
+
71
+ #endif /* !FSE_COMMONDEFS_ONLY */
72
+
73
+
74
+ /****************************************************************
75
+ * Compiler specifics
76
+ ****************************************************************/
77
+ #ifdef _MSC_VER /* Visual Studio */
78
+ # define FORCE_INLINE static __forceinline
79
+ # include <intrin.h> /* For Visual 2005 */
80
+ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
81
+ # pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
82
+ #else
83
+ # define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
84
+ # ifdef __GNUC__
85
+ # define FORCE_INLINE static inline __attribute__((always_inline))
86
+ # else
87
+ # define FORCE_INLINE static inline
88
+ # endif
89
+ #endif
90
+
91
+
92
+ /****************************************************************
93
+ * Includes
94
+ ****************************************************************/
95
+ #include <stdlib.h> /* malloc, free, qsort */
96
+ #include <string.h> /* memcpy, memset */
97
+ #include <stdio.h> /* printf (debug) */
98
+ #include "fse_static.h"
99
+
100
+
101
+ #ifndef MEM_ACCESS_MODULE
102
+ #define MEM_ACCESS_MODULE
103
+ /****************************************************************
104
+ * Basic Types
105
+ *****************************************************************/
106
+ #if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
107
+ # include <stdint.h>
108
+ typedef uint8_t BYTE;
109
+ typedef uint16_t U16;
110
+ typedef int16_t S16;
111
+ typedef uint32_t U32;
112
+ typedef int32_t S32;
113
+ typedef uint64_t U64;
114
+ typedef int64_t S64;
115
+ #else
116
+ typedef unsigned char BYTE;
117
+ typedef unsigned short U16;
118
+ typedef signed short S16;
119
+ typedef unsigned int U32;
120
+ typedef signed int S32;
121
+ typedef unsigned long long U64;
122
+ typedef signed long long S64;
123
+ #endif
124
+
125
+ #endif /* MEM_ACCESS_MODULE */
126
+
127
+ /****************************************************************
128
+ * Memory I/O
129
+ *****************************************************************/
130
+ /* FSE_FORCE_MEMORY_ACCESS
131
+ * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
132
+ * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
133
+ * The below switch allow to select different access method for improved performance.
134
+ * Method 0 (default) : use `memcpy()`. Safe and portable.
135
+ * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
136
+ * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
137
+ * Method 2 : direct access. This method is portable but violate C standard.
138
+ * It can generate buggy code on targets generating assembly depending on alignment.
139
+ * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
140
+ * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
141
+ * Prefer these methods in priority order (0 > 1 > 2)
142
+ */
143
+ #ifndef FSE_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
144
+ # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
145
+ # define FSE_FORCE_MEMORY_ACCESS 2
146
+ # elif defined(__INTEL_COMPILER) || \
147
+ (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
148
+ # define FSE_FORCE_MEMORY_ACCESS 1
149
+ # endif
150
+ #endif
151
+
152
+
153
+ static unsigned FSE_32bits(void)
154
+ {
155
+ return sizeof(void*)==4;
156
+ }
157
+
158
+ static unsigned FSE_isLittleEndian(void)
159
+ {
160
+ const union { U32 i; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */
161
+ return one.c[0];
162
+ }
163
+
164
+ #if defined(FSE_FORCE_MEMORY_ACCESS) && (FSE_FORCE_MEMORY_ACCESS==2)
165
+
166
+ static U16 FSE_read16(const void* memPtr) { return *(const U16*) memPtr; }
167
+ static U32 FSE_read32(const void* memPtr) { return *(const U32*) memPtr; }
168
+ static U64 FSE_read64(const void* memPtr) { return *(const U64*) memPtr; }
169
+
170
+ static void FSE_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
171
+ static void FSE_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
172
+ static void FSE_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
173
+
174
+ #elif defined(FSE_FORCE_MEMORY_ACCESS) && (FSE_FORCE_MEMORY_ACCESS==1)
175
+
176
+ /* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
177
+ /* currently only defined for gcc and icc */
178
+ typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign;
179
+
180
+ static U16 FSE_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
181
+ static U32 FSE_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
182
+ static U64 FSE_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
183
+
184
+ static void FSE_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
185
+ static void FSE_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
186
+ static void FSE_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
187
+
188
+ #else
189
+
190
+ static U16 FSE_read16(const void* memPtr)
191
+ {
192
+ U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
193
+ }
194
+
195
+ static U32 FSE_read32(const void* memPtr)
196
+ {
197
+ U32 val; memcpy(&val, memPtr, sizeof(val)); return val;
198
+ }
199
+
200
+ static U64 FSE_read64(const void* memPtr)
201
+ {
202
+ U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
203
+ }
204
+
205
+ static void FSE_write16(void* memPtr, U16 value)
206
+ {
207
+ memcpy(memPtr, &value, sizeof(value));
208
+ }
209
+
210
+ static void FSE_write32(void* memPtr, U32 value)
211
+ {
212
+ memcpy(memPtr, &value, sizeof(value));
213
+ }
214
+
215
+ static void FSE_write64(void* memPtr, U64 value)
216
+ {
217
+ memcpy(memPtr, &value, sizeof(value));
218
+ }
219
+
220
+ #endif // FSE_FORCE_MEMORY_ACCESS
221
+
222
+ static U16 FSE_readLE16(const void* memPtr)
223
+ {
224
+ if (FSE_isLittleEndian())
225
+ return FSE_read16(memPtr);
226
+ else
227
+ {
228
+ const BYTE* p = (const BYTE*)memPtr;
229
+ return (U16)(p[0] + (p[1]<<8));
230
+ }
231
+ }
232
+
233
+ static void FSE_writeLE16(void* memPtr, U16 val)
234
+ {
235
+ if (FSE_isLittleEndian())
236
+ {
237
+ FSE_write16(memPtr, val);
238
+ }
239
+ else
240
+ {
241
+ BYTE* p = (BYTE*)memPtr;
242
+ p[0] = (BYTE)val;
243
+ p[1] = (BYTE)(val>>8);
244
+ }
245
+ }
246
+
247
+ static U32 FSE_readLE32(const void* memPtr)
248
+ {
249
+ if (FSE_isLittleEndian())
250
+ return FSE_read32(memPtr);
251
+ else
252
+ {
253
+ const BYTE* p = (const BYTE*)memPtr;
254
+ return (U32)((U32)p[0] + ((U32)p[1]<<8) + ((U32)p[2]<<16) + ((U32)p[3]<<24));
255
+ }
256
+ }
257
+
258
+ static void FSE_writeLE32(void* memPtr, U32 val32)
259
+ {
260
+ if (FSE_isLittleEndian())
261
+ {
262
+ FSE_write32(memPtr, val32);
263
+ }
264
+ else
265
+ {
266
+ BYTE* p = (BYTE*)memPtr;
267
+ p[0] = (BYTE)val32;
268
+ p[1] = (BYTE)(val32>>8);
269
+ p[2] = (BYTE)(val32>>16);
270
+ p[3] = (BYTE)(val32>>24);
271
+ }
272
+ }
273
+
274
+ static U64 FSE_readLE64(const void* memPtr)
275
+ {
276
+ if (FSE_isLittleEndian())
277
+ return FSE_read64(memPtr);
278
+ else
279
+ {
280
+ const BYTE* p = (const BYTE*)memPtr;
281
+ return (U64)((U64)p[0] + ((U64)p[1]<<8) + ((U64)p[2]<<16) + ((U64)p[3]<<24)
282
+ + ((U64)p[4]<<32) + ((U64)p[5]<<40) + ((U64)p[6]<<48) + ((U64)p[7]<<56));
283
+ }
284
+ }
285
+
286
+ static void FSE_writeLE64(void* memPtr, U64 val64)
287
+ {
288
+ if (FSE_isLittleEndian())
289
+ {
290
+ FSE_write64(memPtr, val64);
291
+ }
292
+ else
293
+ {
294
+ BYTE* p = (BYTE*)memPtr;
295
+ p[0] = (BYTE)val64;
296
+ p[1] = (BYTE)(val64>>8);
297
+ p[2] = (BYTE)(val64>>16);
298
+ p[3] = (BYTE)(val64>>24);
299
+ p[4] = (BYTE)(val64>>32);
300
+ p[5] = (BYTE)(val64>>40);
301
+ p[6] = (BYTE)(val64>>48);
302
+ p[7] = (BYTE)(val64>>56);
303
+ }
304
+ }
305
+
306
+ static size_t FSE_readLEST(const void* memPtr)
307
+ {
308
+ if (FSE_32bits())
309
+ return (size_t)FSE_readLE32(memPtr);
310
+ else
311
+ return (size_t)FSE_readLE64(memPtr);
312
+ }
313
+
314
+ static void FSE_writeLEST(void* memPtr, size_t val)
315
+ {
316
+ if (FSE_32bits())
317
+ FSE_writeLE32(memPtr, (U32)val);
318
+ else
319
+ FSE_writeLE64(memPtr, (U64)val);
320
+ }
321
+
322
+
323
+ /****************************************************************
324
+ * Constants
325
+ *****************************************************************/
326
+ #define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2)
327
+ #define FSE_MAX_TABLESIZE (1U<<FSE_MAX_TABLELOG)
328
+ #define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE-1)
329
+ #define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE-2)
330
+ #define FSE_MIN_TABLELOG 5
331
+
332
+ #define FSE_TABLELOG_ABSOLUTE_MAX 15
333
+ #if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX
334
+ #error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported"
335
+ #endif
336
+
337
+
338
+ /****************************************************************
339
+ * Error Management
340
+ ****************************************************************/
341
+ #define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
342
+
343
+
344
+ /****************************************************************
345
+ * Complex types
346
+ ****************************************************************/
347
+ typedef struct
348
+ {
349
+ int deltaFindState;
350
+ U32 deltaNbBits;
351
+ } FSE_symbolCompressionTransform; /* total 8 bytes */
352
+
353
+ typedef U32 CTable_max_t[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
354
+ typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
355
+
356
+ /****************************************************************
357
+ * Internal functions
358
+ ****************************************************************/
359
+ FORCE_INLINE unsigned FSE_highbit32 (register U32 val)
360
+ {
361
+ # if defined(_MSC_VER) /* Visual */
362
+ unsigned long r;
363
+ _BitScanReverse ( &r, val );
364
+ return (unsigned) r;
365
+ # elif defined(__GNUC__) && (GCC_VERSION >= 304) /* GCC Intrinsic */
366
+ return 31 - __builtin_clz (val);
367
+ # else /* Software version */
368
+ static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
369
+ U32 v = val;
370
+ unsigned r;
371
+ v |= v >> 1;
372
+ v |= v >> 2;
373
+ v |= v >> 4;
374
+ v |= v >> 8;
375
+ v |= v >> 16;
376
+ r = DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27];
377
+ return r;
378
+ # endif
379
+ }
380
+
381
+
382
+ /****************************************************************
383
+ * Templates
384
+ ****************************************************************/
385
+ /*
386
+ designed to be included
387
+ for type-specific functions (template emulation in C)
388
+ Objective is to write these functions only once, for improved maintenance
389
+ */
390
+
391
+ /* safety checks */
392
+ #ifndef FSE_FUNCTION_EXTENSION
393
+ # error "FSE_FUNCTION_EXTENSION must be defined"
394
+ #endif
395
+ #ifndef FSE_FUNCTION_TYPE
396
+ # error "FSE_FUNCTION_TYPE must be defined"
397
+ #endif
398
+
399
+ /* Function names */
400
+ #define FSE_CAT(X,Y) X##Y
401
+ #define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y)
402
+ #define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y)
403
+
404
+
405
+ /* Function templates */
406
+ size_t FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION)
407
+ (unsigned* count, unsigned* maxSymbolValuePtr, const FSE_FUNCTION_TYPE* source, size_t sourceSize, unsigned safe)
408
+ {
409
+ const FSE_FUNCTION_TYPE* ip = source;
410
+ const FSE_FUNCTION_TYPE* const iend = ip+sourceSize;
411
+ unsigned maxSymbolValue = *maxSymbolValuePtr;
412
+ unsigned max=0;
413
+ int s;
414
+
415
+ U32 Counting1[FSE_MAX_SYMBOL_VALUE+1] = { 0 };
416
+ U32 Counting2[FSE_MAX_SYMBOL_VALUE+1] = { 0 };
417
+ U32 Counting3[FSE_MAX_SYMBOL_VALUE+1] = { 0 };
418
+ U32 Counting4[FSE_MAX_SYMBOL_VALUE+1] = { 0 };
419
+
420
+ /* safety checks */
421
+ if (!sourceSize)
422
+ {
423
+ memset(count, 0, (maxSymbolValue + 1) * sizeof(FSE_FUNCTION_TYPE));
424
+ *maxSymbolValuePtr = 0;
425
+ return 0;
426
+ }
427
+ if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return (size_t)-FSE_ERROR_GENERIC; /* maxSymbolValue too large : unsupported */
428
+ if (!maxSymbolValue) maxSymbolValue = FSE_MAX_SYMBOL_VALUE; /* 0 == default */
429
+
430
+ if ((safe) || (sizeof(FSE_FUNCTION_TYPE)>1))
431
+ {
432
+ /* check input values, to avoid count table overflow */
433
+ while (ip < iend-3)
434
+ {
435
+ if (*ip>maxSymbolValue) return (size_t)-FSE_ERROR_GENERIC; Counting1[*ip++]++;
436
+ if (*ip>maxSymbolValue) return (size_t)-FSE_ERROR_GENERIC; Counting2[*ip++]++;
437
+ if (*ip>maxSymbolValue) return (size_t)-FSE_ERROR_GENERIC; Counting3[*ip++]++;
438
+ if (*ip>maxSymbolValue) return (size_t)-FSE_ERROR_GENERIC; Counting4[*ip++]++;
439
+ }
440
+ }
441
+ else
442
+ {
443
+ U32 cached = FSE_read32(ip); ip += 4;
444
+ while (ip < iend-15)
445
+ {
446
+ U32 c = cached; cached = FSE_read32(ip); ip += 4;
447
+ Counting1[(BYTE) c ]++;
448
+ Counting2[(BYTE)(c>>8) ]++;
449
+ Counting3[(BYTE)(c>>16)]++;
450
+ Counting4[ c>>24 ]++;
451
+ c = cached; cached = FSE_read32(ip); ip += 4;
452
+ Counting1[(BYTE) c ]++;
453
+ Counting2[(BYTE)(c>>8) ]++;
454
+ Counting3[(BYTE)(c>>16)]++;
455
+ Counting4[ c>>24 ]++;
456
+ c = cached; cached = FSE_read32(ip); ip += 4;
457
+ Counting1[(BYTE) c ]++;
458
+ Counting2[(BYTE)(c>>8) ]++;
459
+ Counting3[(BYTE)(c>>16)]++;
460
+ Counting4[ c>>24 ]++;
461
+ c = cached; cached = FSE_read32(ip); ip += 4;
462
+ Counting1[(BYTE) c ]++;
463
+ Counting2[(BYTE)(c>>8) ]++;
464
+ Counting3[(BYTE)(c>>16)]++;
465
+ Counting4[ c>>24 ]++;
466
+ }
467
+ ip-=4;
468
+ }
469
+
470
+ /* finish last symbols */
471
+ while (ip<iend) { if ((safe) && (*ip>maxSymbolValue)) return (size_t)-FSE_ERROR_GENERIC; Counting1[*ip++]++; }
472
+
473
+ for (s=0; s<=(int)maxSymbolValue; s++)
474
+ {
475
+ count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s];
476
+ if (count[s] > max) max = count[s];
477
+ }
478
+
479
+ while (!count[maxSymbolValue]) maxSymbolValue--;
480
+ *maxSymbolValuePtr = maxSymbolValue;
481
+ return (size_t)max;
482
+ }
483
+
484
+ /* hidden fast variant (unsafe) */
485
+ size_t FSE_FUNCTION_NAME(FSE_countFast, FSE_FUNCTION_EXTENSION)
486
+ (unsigned* count, unsigned* maxSymbolValuePtr, const FSE_FUNCTION_TYPE* source, size_t sourceSize)
487
+ {
488
+ return FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) (count, maxSymbolValuePtr, source, sourceSize, 0);
489
+ }
490
+
491
+ size_t FSE_FUNCTION_NAME(FSE_count, FSE_FUNCTION_EXTENSION)
492
+ (unsigned* count, unsigned* maxSymbolValuePtr, const FSE_FUNCTION_TYPE* source, size_t sourceSize)
493
+ {
494
+ if ((sizeof(FSE_FUNCTION_TYPE)==1) && (*maxSymbolValuePtr >= 255))
495
+ {
496
+ *maxSymbolValuePtr = 255;
497
+ return FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) (count, maxSymbolValuePtr, source, sourceSize, 0);
498
+ }
499
+ return FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) (count, maxSymbolValuePtr, source, sourceSize, 1);
500
+ }
501
+
502
+
503
+ static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) + 3; }
504
+
505
+ size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION)
506
+ (FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
507
+ {
508
+ const unsigned tableSize = 1 << tableLog;
509
+ const unsigned tableMask = tableSize - 1;
510
+ U16* tableU16 = ( (U16*) ct) + 2;
511
+ FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) (((U32*)ct) + 1 + (tableLog ? tableSize>>1 : 1) );
512
+ const unsigned step = FSE_tableStep(tableSize);
513
+ unsigned cumul[FSE_MAX_SYMBOL_VALUE+2];
514
+ U32 position = 0;
515
+ FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* init not necessary, but analyzer complain about it */
516
+ U32 highThreshold = tableSize-1;
517
+ unsigned symbol;
518
+ unsigned i;
519
+
520
+ /* header */
521
+ tableU16[-2] = (U16) tableLog;
522
+ tableU16[-1] = (U16) maxSymbolValue;
523
+
524
+ /* For explanations on how to distribute symbol values over the table :
525
+ * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
526
+
527
+ /* symbol start positions */
528
+ cumul[0] = 0;
529
+ for (i=1; i<=maxSymbolValue+1; i++)
530
+ {
531
+ if (normalizedCounter[i-1]==-1) /* Low prob symbol */
532
+ {
533
+ cumul[i] = cumul[i-1] + 1;
534
+ tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(i-1);
535
+ }
536
+ else
537
+ cumul[i] = cumul[i-1] + normalizedCounter[i-1];
538
+ }
539
+ cumul[maxSymbolValue+1] = tableSize+1;
540
+
541
+ /* Spread symbols */
542
+ for (symbol=0; symbol<=maxSymbolValue; symbol++)
543
+ {
544
+ int nbOccurences;
545
+ for (nbOccurences=0; nbOccurences<normalizedCounter[symbol]; nbOccurences++)
546
+ {
547
+ tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
548
+ position = (position + step) & tableMask;
549
+ while (position > highThreshold) position = (position + step) & tableMask; /* Lowprob area */
550
+ }
551
+ }
552
+
553
+ if (position!=0) return (size_t)-FSE_ERROR_GENERIC; /* Must have gone through all positions */
554
+
555
+ /* Build table */
556
+ for (i=0; i<tableSize; i++)
557
+ {
558
+ FSE_FUNCTION_TYPE s = tableSymbol[i]; /* static analyzer doesn't understand tableSymbol is properly initialized */
559
+ tableU16[cumul[s]++] = (U16) (tableSize+i); /* TableU16 : sorted by symbol order; gives next state value */
560
+ }
561
+
562
+ /* Build Symbol Transformation Table */
563
+ {
564
+ unsigned s;
565
+ unsigned total = 0;
566
+ for (s=0; s<=maxSymbolValue; s++)
567
+ {
568
+ switch (normalizedCounter[s])
569
+ {
570
+ case 0:
571
+ break;
572
+ case -1:
573
+ case 1:
574
+ symbolTT[s].deltaNbBits = tableLog << 16;
575
+ symbolTT[s].deltaFindState = total - 1;
576
+ total ++;
577
+ break;
578
+ default :
579
+ {
580
+ U32 maxBitsOut = tableLog - FSE_highbit32 (normalizedCounter[s]-1);
581
+ U32 minStatePlus = normalizedCounter[s] << maxBitsOut;
582
+ symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus;
583
+ symbolTT[s].deltaFindState = total - normalizedCounter[s];
584
+ total += normalizedCounter[s];
585
+ }
586
+ }
587
+ }
588
+ }
589
+
590
+ return 0;
591
+ }
592
+
593
+
594
+ #define FSE_DECODE_TYPE FSE_TYPE_NAME(FSE_decode_t, FSE_FUNCTION_EXTENSION)
595
+
596
+ FSE_DTable* FSE_FUNCTION_NAME(FSE_createDTable, FSE_FUNCTION_EXTENSION) (unsigned tableLog)
597
+ {
598
+ if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
599
+ return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) );
600
+ }
601
+
602
+ void FSE_FUNCTION_NAME(FSE_freeDTable, FSE_FUNCTION_EXTENSION) (FSE_DTable* dt)
603
+ {
604
+ free(dt);
605
+ }
606
+
607
+ typedef struct {
608
+ U16 tableLog;
609
+ U16 fastMode;
610
+ } FSE_DTableHeader; /* sizeof U32 */
611
+
612
+ size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION)
613
+ (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
614
+ {
615
+ FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt;
616
+ FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (dt+1); /* because dt is unsigned, 32-bits aligned on 32-bits */
617
+ const U32 tableSize = 1 << tableLog;
618
+ const U32 tableMask = tableSize-1;
619
+ const U32 step = FSE_tableStep(tableSize);
620
+ U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1];
621
+ U32 position = 0;
622
+ U32 highThreshold = tableSize-1;
623
+ const S16 largeLimit= (S16)(1 << (tableLog-1));
624
+ U32 noLarge = 1;
625
+ U32 s;
626
+
627
+ /* Sanity Checks */
628
+ if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return (size_t)-FSE_ERROR_maxSymbolValue_tooLarge;
629
+ if (tableLog > FSE_MAX_TABLELOG) return (size_t)-FSE_ERROR_tableLog_tooLarge;
630
+
631
+ /* Init, lay down lowprob symbols */
632
+ DTableH[0].tableLog = (U16)tableLog;
633
+ for (s=0; s<=maxSymbolValue; s++)
634
+ {
635
+ if (normalizedCounter[s]==-1)
636
+ {
637
+ tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s;
638
+ symbolNext[s] = 1;
639
+ }
640
+ else
641
+ {
642
+ if (normalizedCounter[s] >= largeLimit) noLarge=0;
643
+ symbolNext[s] = normalizedCounter[s];
644
+ }
645
+ }
646
+
647
+ /* Spread symbols */
648
+ for (s=0; s<=maxSymbolValue; s++)
649
+ {
650
+ int i;
651
+ for (i=0; i<normalizedCounter[s]; i++)
652
+ {
653
+ tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s;
654
+ position = (position + step) & tableMask;
655
+ while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */
656
+ }
657
+ }
658
+
659
+ if (position!=0) return (size_t)-FSE_ERROR_GENERIC; /* position must reach all cells once, otherwise normalizedCounter is incorrect */
660
+
661
+ /* Build Decoding table */
662
+ {
663
+ U32 i;
664
+ for (i=0; i<tableSize; i++)
665
+ {
666
+ FSE_FUNCTION_TYPE symbol = (FSE_FUNCTION_TYPE)(tableDecode[i].symbol);
667
+ U16 nextState = symbolNext[symbol]++;
668
+ tableDecode[i].nbBits = (BYTE) (tableLog - FSE_highbit32 ((U32)nextState) );
669
+ tableDecode[i].newState = (U16) ( (nextState << tableDecode[i].nbBits) - tableSize);
670
+ }
671
+ }
672
+
673
+ DTableH->fastMode = (U16)noLarge;
674
+ return 0;
675
+ }
676
+
677
+
678
+ /******************************************
679
+ * FSE byte symbol
680
+ ******************************************/
681
+ #ifndef FSE_COMMONDEFS_ONLY
682
+
683
+ unsigned FSE_isError(size_t code) { return (code > (size_t)(-FSE_ERROR_maxCode)); }
684
+
685
+ #define FSE_GENERATE_STRING(STRING) #STRING,
686
+ static const char* FSE_errorStrings[] = { FSE_LIST_ERRORS(FSE_GENERATE_STRING) };
687
+
688
+ const char* FSE_getErrorName(size_t code)
689
+ {
690
+ static const char* codeError = "Unspecified error code";
691
+ if (FSE_isError(code)) return FSE_errorStrings[-(int)(code)];
692
+ return codeError;
693
+ }
694
+
695
+ static short FSE_abs(short a)
696
+ {
697
+ return a<0? -a : a;
698
+ }
699
+
700
+
701
+ /****************************************************************
702
+ * Header bitstream management
703
+ ****************************************************************/
704
+ size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
705
+ {
706
+ size_t maxHeaderSize = (((maxSymbolValue+1) * tableLog) >> 3) + 3;
707
+ return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */
708
+ }
709
+
710
+ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
711
+ const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
712
+ unsigned writeIsSafe)
713
+ {
714
+ BYTE* const ostart = (BYTE*) header;
715
+ BYTE* out = ostart;
716
+ BYTE* const oend = ostart + headerBufferSize;
717
+ int nbBits;
718
+ const int tableSize = 1 << tableLog;
719
+ int remaining;
720
+ int threshold;
721
+ U32 bitStream;
722
+ int bitCount;
723
+ unsigned charnum = 0;
724
+ int previous0 = 0;
725
+
726
+ bitStream = 0;
727
+ bitCount = 0;
728
+ /* Table Size */
729
+ bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount;
730
+ bitCount += 4;
731
+
732
+ /* Init */
733
+ remaining = tableSize+1; /* +1 for extra accuracy */
734
+ threshold = tableSize;
735
+ nbBits = tableLog+1;
736
+
737
+ while (remaining>1) /* stops at 1 */
738
+ {
739
+ if (previous0)
740
+ {
741
+ unsigned start = charnum;
742
+ while (!normalizedCounter[charnum]) charnum++;
743
+ while (charnum >= start+24)
744
+ {
745
+ start+=24;
746
+ bitStream += 0xFFFFU << bitCount;
747
+ if ((!writeIsSafe) && (out > oend-2)) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* Buffer overflow */
748
+ out[0] = (BYTE) bitStream;
749
+ out[1] = (BYTE)(bitStream>>8);
750
+ out+=2;
751
+ bitStream>>=16;
752
+ }
753
+ while (charnum >= start+3)
754
+ {
755
+ start+=3;
756
+ bitStream += 3 << bitCount;
757
+ bitCount += 2;
758
+ }
759
+ bitStream += (charnum-start) << bitCount;
760
+ bitCount += 2;
761
+ if (bitCount>16)
762
+ {
763
+ if ((!writeIsSafe) && (out > oend - 2)) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* Buffer overflow */
764
+ out[0] = (BYTE)bitStream;
765
+ out[1] = (BYTE)(bitStream>>8);
766
+ out += 2;
767
+ bitStream >>= 16;
768
+ bitCount -= 16;
769
+ }
770
+ }
771
+ {
772
+ short count = normalizedCounter[charnum++];
773
+ const short max = (short)((2*threshold-1)-remaining);
774
+ remaining -= FSE_abs(count);
775
+ if (remaining<1) return (size_t)-FSE_ERROR_GENERIC;
776
+ count++; /* +1 for extra accuracy */
777
+ if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
778
+ bitStream += count << bitCount;
779
+ bitCount += nbBits;
780
+ bitCount -= (count<max);
781
+ previous0 = (count==1);
782
+ while (remaining<threshold) nbBits--, threshold>>=1;
783
+ }
784
+ if (bitCount>16)
785
+ {
786
+ if ((!writeIsSafe) && (out > oend - 2)) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* Buffer overflow */
787
+ out[0] = (BYTE)bitStream;
788
+ out[1] = (BYTE)(bitStream>>8);
789
+ out += 2;
790
+ bitStream >>= 16;
791
+ bitCount -= 16;
792
+ }
793
+ }
794
+
795
+ /* flush remaining bitStream */
796
+ if ((!writeIsSafe) && (out > oend - 2)) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* Buffer overflow */
797
+ out[0] = (BYTE)bitStream;
798
+ out[1] = (BYTE)(bitStream>>8);
799
+ out+= (bitCount+7) /8;
800
+
801
+ if (charnum > maxSymbolValue + 1) return (size_t)-FSE_ERROR_GENERIC;
802
+
803
+ return (out-ostart);
804
+ }
805
+
806
+
807
+ size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
808
+ {
809
+ if (tableLog > FSE_MAX_TABLELOG) return (size_t)-FSE_ERROR_GENERIC; /* Unsupported */
810
+ if (tableLog < FSE_MIN_TABLELOG) return (size_t)-FSE_ERROR_GENERIC; /* Unsupported */
811
+
812
+ if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog))
813
+ return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0);
814
+
815
+ return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1);
816
+ }
817
+
818
+
819
+ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
820
+ const void* headerBuffer, size_t hbSize)
821
+ {
822
+ const BYTE* const istart = (const BYTE*) headerBuffer;
823
+ const BYTE* const iend = istart + hbSize;
824
+ const BYTE* ip = istart;
825
+ int nbBits;
826
+ int remaining;
827
+ int threshold;
828
+ U32 bitStream;
829
+ int bitCount;
830
+ unsigned charnum = 0;
831
+ int previous0 = 0;
832
+
833
+ if (hbSize < 4) return (size_t)-FSE_ERROR_srcSize_wrong;
834
+ bitStream = FSE_readLE32(ip);
835
+ nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */
836
+ if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return (size_t)-FSE_ERROR_tableLog_tooLarge;
837
+ bitStream >>= 4;
838
+ bitCount = 4;
839
+ *tableLogPtr = nbBits;
840
+ remaining = (1<<nbBits)+1;
841
+ threshold = 1<<nbBits;
842
+ nbBits++;
843
+
844
+ while ((remaining>1) && (charnum<=*maxSVPtr))
845
+ {
846
+ if (previous0)
847
+ {
848
+ unsigned n0 = charnum;
849
+ while ((bitStream & 0xFFFF) == 0xFFFF)
850
+ {
851
+ n0+=24;
852
+ if (ip < iend-5)
853
+ {
854
+ ip+=2;
855
+ bitStream = FSE_readLE32(ip) >> bitCount;
856
+ }
857
+ else
858
+ {
859
+ bitStream >>= 16;
860
+ bitCount+=16;
861
+ }
862
+ }
863
+ while ((bitStream & 3) == 3)
864
+ {
865
+ n0+=3;
866
+ bitStream>>=2;
867
+ bitCount+=2;
868
+ }
869
+ n0 += bitStream & 3;
870
+ bitCount += 2;
871
+ if (n0 > *maxSVPtr) return (size_t)-FSE_ERROR_maxSymbolValue_tooSmall;
872
+ while (charnum < n0) normalizedCounter[charnum++] = 0;
873
+ if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4))
874
+ {
875
+ ip += bitCount>>3;
876
+ bitCount &= 7;
877
+ bitStream = FSE_readLE32(ip) >> bitCount;
878
+ }
879
+ else
880
+ bitStream >>= 2;
881
+ }
882
+ {
883
+ const short max = (short)((2*threshold-1)-remaining);
884
+ short count;
885
+
886
+ if ((bitStream & (threshold-1)) < (U32)max)
887
+ {
888
+ count = (short)(bitStream & (threshold-1));
889
+ bitCount += nbBits-1;
890
+ }
891
+ else
892
+ {
893
+ count = (short)(bitStream & (2*threshold-1));
894
+ if (count >= threshold) count -= max;
895
+ bitCount += nbBits;
896
+ }
897
+
898
+ count--; /* extra accuracy */
899
+ remaining -= FSE_abs(count);
900
+ normalizedCounter[charnum++] = count;
901
+ previous0 = !count;
902
+ while (remaining < threshold)
903
+ {
904
+ nbBits--;
905
+ threshold >>= 1;
906
+ }
907
+
908
+ {
909
+ if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4))
910
+ {
911
+ ip += bitCount>>3;
912
+ bitCount &= 7;
913
+ }
914
+ else
915
+ {
916
+ bitCount -= (int)(8 * (iend - 4 - ip));
917
+ ip = iend - 4;
918
+ }
919
+ bitStream = FSE_readLE32(ip) >> (bitCount & 31);
920
+ }
921
+ }
922
+ }
923
+ if (remaining != 1) return (size_t)-FSE_ERROR_GENERIC;
924
+ *maxSVPtr = charnum-1;
925
+
926
+ ip += (bitCount+7)>>3;
927
+ if ((size_t)(ip-istart) > hbSize) return (size_t)-FSE_ERROR_srcSize_wrong;
928
+ return ip-istart;
929
+ }
930
+
931
+
932
+ /****************************************************************
933
+ * FSE Compression Code
934
+ ****************************************************************/
935
+ /*
936
+ FSE_CTable[0] is a variable size structure which contains :
937
+ U16 tableLog;
938
+ U16 maxSymbolValue;
939
+ U16 nextStateNumber[1 << tableLog]; // This size is variable
940
+ FSE_symbolCompressionTransform symbolTT[maxSymbolValue+1]; // This size is variable
941
+ Allocation is manual, since C standard does not support variable-size structures.
942
+ */
943
+
944
+ size_t FSE_sizeof_CTable (unsigned maxSymbolValue, unsigned tableLog)
945
+ {
946
+ size_t size;
947
+ FSE_STATIC_ASSERT((size_t)FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)*4 >= sizeof(CTable_max_t)); /* A compilation error here means FSE_CTABLE_SIZE_U32 is not large enough */
948
+ if (tableLog > FSE_MAX_TABLELOG) return (size_t)-FSE_ERROR_GENERIC;
949
+ size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32);
950
+ return size;
951
+ }
952
+
953
+ FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog)
954
+ {
955
+ size_t size;
956
+ if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
957
+ size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32);
958
+ return (FSE_CTable*)malloc(size);
959
+ }
960
+
961
+ void FSE_freeCTable (FSE_CTable* ct)
962
+ {
963
+ free(ct);
964
+ }
965
+
966
+
967
+ /* provides the minimum logSize to safely represent a distribution */
968
+ static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
969
+ {
970
+ U32 minBitsSrc = FSE_highbit32((U32)(srcSize - 1)) + 1;
971
+ U32 minBitsSymbols = FSE_highbit32(maxSymbolValue) + 2;
972
+ U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
973
+ return minBits;
974
+ }
975
+
976
+ unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
977
+ {
978
+ U32 maxBitsSrc = FSE_highbit32((U32)(srcSize - 1)) - 2;
979
+ U32 tableLog = maxTableLog;
980
+ U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
981
+ if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
982
+ if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */
983
+ if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */
984
+ if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG;
985
+ if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG;
986
+ return tableLog;
987
+ }
988
+
989
+
990
+ /* Secondary normalization method.
991
+ To be used when primary method fails. */
992
+
993
+ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue)
994
+ {
995
+ U32 s;
996
+ U32 distributed = 0;
997
+ U32 ToDistribute;
998
+
999
+ /* Init */
1000
+ U32 lowThreshold = (U32)(total >> tableLog);
1001
+ U32 lowOne = (U32)((total * 3) >> (tableLog + 1));
1002
+
1003
+ for (s=0; s<=maxSymbolValue; s++)
1004
+ {
1005
+ if (count[s] == 0)
1006
+ {
1007
+ norm[s]=0;
1008
+ continue;
1009
+ }
1010
+ if (count[s] <= lowThreshold)
1011
+ {
1012
+ norm[s] = -1;
1013
+ distributed++;
1014
+ total -= count[s];
1015
+ continue;
1016
+ }
1017
+ if (count[s] <= lowOne)
1018
+ {
1019
+ norm[s] = 1;
1020
+ distributed++;
1021
+ total -= count[s];
1022
+ continue;
1023
+ }
1024
+ norm[s]=-2;
1025
+ }
1026
+ ToDistribute = (1 << tableLog) - distributed;
1027
+
1028
+ if ((total / ToDistribute) > lowOne)
1029
+ {
1030
+ /* risk of rounding to zero */
1031
+ lowOne = (U32)((total * 3) / (ToDistribute * 2));
1032
+ for (s=0; s<=maxSymbolValue; s++)
1033
+ {
1034
+ if ((norm[s] == -2) && (count[s] <= lowOne))
1035
+ {
1036
+ norm[s] = 1;
1037
+ distributed++;
1038
+ total -= count[s];
1039
+ continue;
1040
+ }
1041
+ }
1042
+ ToDistribute = (1 << tableLog) - distributed;
1043
+ }
1044
+
1045
+ if (distributed == maxSymbolValue+1)
1046
+ {
1047
+ /* all values are pretty poor;
1048
+ probably incompressible data (should have already been detected);
1049
+ find max, then give all remaining points to max */
1050
+ U32 maxV = 0, maxC =0;
1051
+ for (s=0; s<=maxSymbolValue; s++)
1052
+ if (count[s] > maxC) maxV=s, maxC=count[s];
1053
+ norm[maxV] += (short)ToDistribute;
1054
+ return 0;
1055
+ }
1056
+
1057
+ {
1058
+ U64 const vStepLog = 62 - tableLog;
1059
+ U64 const mid = (1ULL << (vStepLog-1)) - 1;
1060
+ U64 const rStep = ((((U64)1<<vStepLog) * ToDistribute) + mid) / total; /* scale on remaining */
1061
+ U64 tmpTotal = mid;
1062
+ for (s=0; s<=maxSymbolValue; s++)
1063
+ {
1064
+ if (norm[s]==-2)
1065
+ {
1066
+ U64 end = tmpTotal + (count[s] * rStep);
1067
+ U32 sStart = (U32)(tmpTotal >> vStepLog);
1068
+ U32 sEnd = (U32)(end >> vStepLog);
1069
+ U32 weight = sEnd - sStart;
1070
+ if (weight < 1)
1071
+ return (size_t)-FSE_ERROR_GENERIC;
1072
+ norm[s] = (short)weight;
1073
+ tmpTotal = end;
1074
+ }
1075
+ }
1076
+ }
1077
+
1078
+ return 0;
1079
+ }
1080
+
1081
+
1082
+ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
1083
+ const unsigned* count, size_t total,
1084
+ unsigned maxSymbolValue)
1085
+ {
1086
+ /* Sanity checks */
1087
+ if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
1088
+ if (tableLog < FSE_MIN_TABLELOG) return (size_t)-FSE_ERROR_GENERIC; /* Unsupported size */
1089
+ if (tableLog > FSE_MAX_TABLELOG) return (size_t)-FSE_ERROR_GENERIC; /* Unsupported size */
1090
+ if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return (size_t)-FSE_ERROR_GENERIC; /* Too small tableLog, compression potentially impossible */
1091
+
1092
+ {
1093
+ U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 };
1094
+ U64 const scale = 62 - tableLog;
1095
+ U64 const step = ((U64)1<<62) / total; /* <== here, one division ! */
1096
+ U64 const vStep = 1ULL<<(scale-20);
1097
+ int stillToDistribute = 1<<tableLog;
1098
+ unsigned s;
1099
+ unsigned largest=0;
1100
+ short largestP=0;
1101
+ U32 lowThreshold = (U32)(total >> tableLog);
1102
+
1103
+ for (s=0; s<=maxSymbolValue; s++)
1104
+ {
1105
+ if (count[s] == total) return 0;
1106
+ if (count[s] == 0)
1107
+ {
1108
+ normalizedCounter[s]=0;
1109
+ continue;
1110
+ }
1111
+ if (count[s] <= lowThreshold)
1112
+ {
1113
+ normalizedCounter[s] = -1;
1114
+ stillToDistribute--;
1115
+ }
1116
+ else
1117
+ {
1118
+ short proba = (short)((count[s]*step) >> scale);
1119
+ if (proba<8)
1120
+ {
1121
+ U64 restToBeat = vStep * rtbTable[proba];
1122
+ proba += (count[s]*step) - ((U64)proba<<scale) > restToBeat;
1123
+ }
1124
+ if (proba > largestP)
1125
+ {
1126
+ largestP=proba;
1127
+ largest=s;
1128
+ }
1129
+ normalizedCounter[s] = proba;
1130
+ stillToDistribute -= proba;
1131
+ }
1132
+ }
1133
+ if (-stillToDistribute >= (normalizedCounter[largest] >> 1))
1134
+ {
1135
+ /* corner case, need another normalization method */
1136
+ size_t errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue);
1137
+ if (FSE_isError(errorCode)) return errorCode;
1138
+ }
1139
+ else normalizedCounter[largest] += (short)stillToDistribute;
1140
+ }
1141
+
1142
+ #if 0
1143
+ { /* Print Table (debug) */
1144
+ U32 s;
1145
+ U32 nTotal = 0;
1146
+ for (s=0; s<=maxSymbolValue; s++)
1147
+ printf("%3i: %4i \n", s, normalizedCounter[s]);
1148
+ for (s=0; s<=maxSymbolValue; s++)
1149
+ nTotal += abs(normalizedCounter[s]);
1150
+ if (nTotal != (1U<<tableLog))
1151
+ printf("Warning !!! Total == %u != %u !!!", nTotal, 1U<<tableLog);
1152
+ getchar();
1153
+ }
1154
+ #endif
1155
+
1156
+ return tableLog;
1157
+ }
1158
+
1159
+
1160
+ /* fake FSE_CTable, for raw (uncompressed) input */
1161
+ size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits)
1162
+ {
1163
+ const unsigned tableSize = 1 << nbBits;
1164
+ const unsigned tableMask = tableSize - 1;
1165
+ const unsigned maxSymbolValue = tableMask;
1166
+ U16* tableU16 = ( (U16*) ct) + 2;
1167
+ FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) ((((U32*)ct)+1) + (tableSize>>1));
1168
+ unsigned s;
1169
+
1170
+ /* Sanity checks */
1171
+ if (nbBits < 1) return (size_t)-FSE_ERROR_GENERIC; /* min size */
1172
+
1173
+ /* header */
1174
+ tableU16[-2] = (U16) nbBits;
1175
+ tableU16[-1] = (U16) maxSymbolValue;
1176
+
1177
+ /* Build table */
1178
+ for (s=0; s<tableSize; s++)
1179
+ tableU16[s] = (U16)(tableSize + s);
1180
+
1181
+ /* Build Symbol Transformation Table */
1182
+ for (s=0; s<=maxSymbolValue; s++)
1183
+ {
1184
+ symbolTT[s].deltaNbBits = nbBits << 16;
1185
+ symbolTT[s].deltaFindState = s-1;
1186
+ }
1187
+
1188
+ return 0;
1189
+ }
1190
+
1191
+
1192
+ /* fake FSE_CTable, for rle (100% always same symbol) input */
1193
+ size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue)
1194
+ {
1195
+ U16* tableU16 = ( (U16*) ct) + 2;
1196
+ FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) ((U32*)ct + 2);
1197
+
1198
+ /* header */
1199
+ tableU16[-2] = (U16) 0;
1200
+ tableU16[-1] = (U16) symbolValue;
1201
+
1202
+ /* Build table */
1203
+ tableU16[0] = 0;
1204
+ tableU16[1] = 0; /* just in case */
1205
+
1206
+ /* Build Symbol Transformation Table */
1207
+ {
1208
+ symbolTT[symbolValue].deltaNbBits = 0;
1209
+ symbolTT[symbolValue].deltaFindState = 0;
1210
+ }
1211
+
1212
+ return 0;
1213
+ }
1214
+
1215
+
1216
+ size_t FSE_initCStream(FSE_CStream_t* bitC, void* start, size_t maxSize)
1217
+ {
1218
+ if (maxSize < sizeof(bitC->ptr)) return (size_t)-FSE_ERROR_dstSize_tooSmall;
1219
+ bitC->bitContainer = 0;
1220
+ bitC->bitPos = 0;
1221
+ bitC->startPtr = (char*)start;
1222
+ bitC->ptr = bitC->startPtr;
1223
+ bitC->endPtr = bitC->startPtr + maxSize - sizeof(bitC->ptr);
1224
+ return 0;
1225
+ }
1226
+
1227
+ void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
1228
+ {
1229
+ const U32 tableLog = ( (const U16*) ct) [0];
1230
+ statePtr->value = (ptrdiff_t)1<<tableLog;
1231
+ statePtr->stateTable = ((const U16*) ct) + 2;
1232
+ statePtr->symbolTT = (const FSE_symbolCompressionTransform*)((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1));
1233
+ statePtr->stateLog = tableLog;
1234
+ }
1235
+
1236
+ void FSE_addBitsFast(FSE_CStream_t* bitC, size_t value, unsigned nbBits) /* only use if upper bits are clean 0 */
1237
+ {
1238
+ bitC->bitContainer |= value << bitC->bitPos;
1239
+ bitC->bitPos += nbBits;
1240
+ }
1241
+
1242
+ void FSE_addBits(FSE_CStream_t* bitC, size_t value, unsigned nbBits)
1243
+ {
1244
+ static const unsigned mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF }; /* up to 25 bits */
1245
+ bitC->bitContainer |= (value & mask[nbBits]) << bitC->bitPos;
1246
+ bitC->bitPos += nbBits;
1247
+ }
1248
+
1249
+ void FSE_encodeSymbol(FSE_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol)
1250
+ {
1251
+ const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
1252
+ const U16* const stateTable = (const U16*)(statePtr->stateTable);
1253
+ U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);
1254
+ FSE_addBits(bitC, statePtr->value, nbBitsOut);
1255
+ statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
1256
+ }
1257
+
1258
+ void FSE_flushBitsFast(FSE_CStream_t* bitC) /* only if dst buffer is large enough ( >= FSE_compressBound()) */
1259
+ {
1260
+ size_t nbBytes = bitC->bitPos >> 3;
1261
+ FSE_writeLEST(bitC->ptr, bitC->bitContainer);
1262
+ bitC->ptr += nbBytes;
1263
+ bitC->bitPos &= 7;
1264
+ bitC->bitContainer >>= nbBytes*8;
1265
+ }
1266
+
1267
+ void FSE_flushBits(FSE_CStream_t* bitC)
1268
+ {
1269
+ size_t nbBytes = bitC->bitPos >> 3;
1270
+ FSE_writeLEST(bitC->ptr, bitC->bitContainer);
1271
+ bitC->ptr += nbBytes;
1272
+ if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr;
1273
+ bitC->bitPos &= 7;
1274
+ bitC->bitContainer >>= nbBytes*8;
1275
+ }
1276
+
1277
+ void FSE_flushCState(FSE_CStream_t* bitC, const FSE_CState_t* statePtr)
1278
+ {
1279
+ FSE_addBits(bitC, statePtr->value, statePtr->stateLog);
1280
+ FSE_flushBits(bitC);
1281
+ }
1282
+
1283
+
1284
+ size_t FSE_closeCStream(FSE_CStream_t* bitC)
1285
+ {
1286
+ char* endPtr;
1287
+
1288
+ FSE_addBitsFast(bitC, 1, 1);
1289
+ FSE_flushBits(bitC);
1290
+
1291
+ if (bitC->ptr >= bitC->endPtr) /* too close to buffer's end */
1292
+ return 0; /* not compressible */
1293
+
1294
+ endPtr = bitC->ptr;
1295
+ endPtr += bitC->bitPos > 0;
1296
+
1297
+ return (endPtr - bitC->startPtr);
1298
+ }
1299
+
1300
+
1301
+ static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize,
1302
+ const void* src, size_t srcSize,
1303
+ const FSE_CTable* ct, const unsigned fast)
1304
+ {
1305
+ const BYTE* const istart = (const BYTE*) src;
1306
+ const BYTE* ip;
1307
+ const BYTE* const iend = istart + srcSize;
1308
+
1309
+ size_t errorCode;
1310
+ FSE_CStream_t bitC;
1311
+ FSE_CState_t CState1, CState2;
1312
+
1313
+
1314
+ /* init */
1315
+ errorCode = FSE_initCStream(&bitC, dst, dstSize);
1316
+ if (FSE_isError(errorCode)) return 0;
1317
+ FSE_initCState(&CState1, ct);
1318
+ CState2 = CState1;
1319
+
1320
+ ip=iend;
1321
+
1322
+ #define FSE_FLUSHBITS(s) (fast ? FSE_flushBitsFast(s) : FSE_flushBits(s))
1323
+
1324
+ /* join to even */
1325
+ if (srcSize & 1)
1326
+ {
1327
+ FSE_encodeSymbol(&bitC, &CState1, *--ip);
1328
+ FSE_FLUSHBITS(&bitC);
1329
+ }
1330
+
1331
+ /* join to mod 4 */
1332
+ if ((sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) /* test bit 2 */
1333
+ {
1334
+ FSE_encodeSymbol(&bitC, &CState2, *--ip);
1335
+ FSE_encodeSymbol(&bitC, &CState1, *--ip);
1336
+ FSE_FLUSHBITS(&bitC);
1337
+ }
1338
+
1339
+ /* 2 or 4 encoding per loop */
1340
+ for ( ; ip>istart ; )
1341
+ {
1342
+ FSE_encodeSymbol(&bitC, &CState2, *--ip);
1343
+
1344
+ if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */
1345
+ FSE_FLUSHBITS(&bitC);
1346
+
1347
+ FSE_encodeSymbol(&bitC, &CState1, *--ip);
1348
+
1349
+ if (sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) /* this test must be static */
1350
+ {
1351
+ FSE_encodeSymbol(&bitC, &CState2, *--ip);
1352
+ FSE_encodeSymbol(&bitC, &CState1, *--ip);
1353
+ }
1354
+
1355
+ FSE_FLUSHBITS(&bitC);
1356
+ }
1357
+
1358
+ FSE_flushCState(&bitC, &CState2);
1359
+ FSE_flushCState(&bitC, &CState1);
1360
+ return FSE_closeCStream(&bitC);
1361
+ }
1362
+
1363
+ size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
1364
+ const void* src, size_t srcSize,
1365
+ const FSE_CTable* ct)
1366
+ {
1367
+ const unsigned fast = (dstSize >= FSE_BLOCKBOUND(srcSize));
1368
+
1369
+ if (fast)
1370
+ return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1);
1371
+ else
1372
+ return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0);
1373
+ }
1374
+
1375
+
1376
+ size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
1377
+
1378
+ size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog)
1379
+ {
1380
+ const BYTE* const istart = (const BYTE*) src;
1381
+ const BYTE* ip = istart;
1382
+
1383
+ BYTE* const ostart = (BYTE*) dst;
1384
+ BYTE* op = ostart;
1385
+ BYTE* const oend = ostart + dstSize;
1386
+
1387
+ U32 count[FSE_MAX_SYMBOL_VALUE+1];
1388
+ S16 norm[FSE_MAX_SYMBOL_VALUE+1];
1389
+ CTable_max_t ct;
1390
+ size_t errorCode;
1391
+
1392
+ /* init conditions */
1393
+ if (srcSize <= 1) return 0; /* Uncompressible */
1394
+ if (!maxSymbolValue) maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
1395
+ if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG;
1396
+
1397
+ /* Scan input and build symbol stats */
1398
+ errorCode = FSE_count (count, &maxSymbolValue, ip, srcSize);
1399
+ if (FSE_isError(errorCode)) return errorCode;
1400
+ if (errorCode == srcSize) return 1;
1401
+ if (errorCode == 1) return 0; /* each symbol only present once */
1402
+ if (errorCode < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */
1403
+
1404
+ tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
1405
+ errorCode = FSE_normalizeCount (norm, tableLog, count, srcSize, maxSymbolValue);
1406
+ if (FSE_isError(errorCode)) return errorCode;
1407
+
1408
+ /* Write table description header */
1409
+ errorCode = FSE_writeNCount (op, oend-op, norm, maxSymbolValue, tableLog);
1410
+ if (FSE_isError(errorCode)) return errorCode;
1411
+ op += errorCode;
1412
+
1413
+ /* Compress */
1414
+ errorCode = FSE_buildCTable (ct, norm, maxSymbolValue, tableLog);
1415
+ if (FSE_isError(errorCode)) return errorCode;
1416
+ errorCode = FSE_compress_usingCTable(op, oend - op, ip, srcSize, ct);
1417
+ if (errorCode == 0) return 0; /* not enough space for compressed data */
1418
+ op += errorCode;
1419
+
1420
+ /* check compressibility */
1421
+ if ( (size_t)(op-ostart) >= srcSize-1 )
1422
+ return 0;
1423
+
1424
+ return op-ostart;
1425
+ }
1426
+
1427
+ size_t FSE_compress (void* dst, size_t dstSize, const void* src, size_t srcSize)
1428
+ {
1429
+ return FSE_compress2(dst, dstSize, src, (U32)srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG);
1430
+ }
1431
+
1432
+
1433
+ /*********************************************************
1434
+ * Decompression (Byte symbols)
1435
+ *********************************************************/
1436
+ size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
1437
+ {
1438
+ FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt;
1439
+ FSE_decode_t* const cell = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */
1440
+
1441
+ DTableH->tableLog = 0;
1442
+ DTableH->fastMode = 0;
1443
+
1444
+ cell->newState = 0;
1445
+ cell->symbol = symbolValue;
1446
+ cell->nbBits = 0;
1447
+
1448
+ return 0;
1449
+ }
1450
+
1451
+
1452
+ size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
1453
+ {
1454
+ FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt;
1455
+ FSE_decode_t* const dinfo = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */
1456
+ const unsigned tableSize = 1 << nbBits;
1457
+ const unsigned tableMask = tableSize - 1;
1458
+ const unsigned maxSymbolValue = tableMask;
1459
+ unsigned s;
1460
+
1461
+ /* Sanity checks */
1462
+ if (nbBits < 1) return (size_t)-FSE_ERROR_GENERIC; /* min size */
1463
+
1464
+ /* Build Decoding Table */
1465
+ DTableH->tableLog = (U16)nbBits;
1466
+ DTableH->fastMode = 1;
1467
+ for (s=0; s<=maxSymbolValue; s++)
1468
+ {
1469
+ dinfo[s].newState = 0;
1470
+ dinfo[s].symbol = (BYTE)s;
1471
+ dinfo[s].nbBits = (BYTE)nbBits;
1472
+ }
1473
+
1474
+ return 0;
1475
+ }
1476
+
1477
+
1478
+ /* FSE_initDStream
1479
+ * Initialize a FSE_DStream_t.
1480
+ * srcBuffer must point at the beginning of an FSE block.
1481
+ * The function result is the size of the FSE_block (== srcSize).
1482
+ * If srcSize is too small, the function will return an errorCode;
1483
+ */
1484
+ size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t srcSize)
1485
+ {
1486
+ if (srcSize < 1) return (size_t)-FSE_ERROR_srcSize_wrong;
1487
+
1488
+ if (srcSize >= sizeof(size_t))
1489
+ {
1490
+ U32 contain32;
1491
+ bitD->start = (const char*)srcBuffer;
1492
+ bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(size_t);
1493
+ bitD->bitContainer = FSE_readLEST(bitD->ptr);
1494
+ contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
1495
+ if (contain32 == 0) return (size_t)-FSE_ERROR_GENERIC; /* stop bit not present */
1496
+ bitD->bitsConsumed = 8 - FSE_highbit32(contain32);
1497
+ }
1498
+ else
1499
+ {
1500
+ U32 contain32;
1501
+ bitD->start = (const char*)srcBuffer;
1502
+ bitD->ptr = bitD->start;
1503
+ bitD->bitContainer = *(const BYTE*)(bitD->start);
1504
+ switch(srcSize)
1505
+ {
1506
+ case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
1507
+ case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
1508
+ case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
1509
+ case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
1510
+ case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
1511
+ case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
1512
+ default:;
1513
+ }
1514
+ contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
1515
+ if (contain32 == 0) return (size_t)-FSE_ERROR_GENERIC; /* stop bit not present */
1516
+ bitD->bitsConsumed = 8 - FSE_highbit32(contain32);
1517
+ bitD->bitsConsumed += (U32)(sizeof(size_t) - srcSize)*8;
1518
+ }
1519
+
1520
+ return srcSize;
1521
+ }
1522
+
1523
+
1524
+ /* FSE_lookBits
1525
+ * Provides next n bits from the bitContainer.
1526
+ * bitContainer is not modified (bits are still present for next read/look)
1527
+ * On 32-bits, maxNbBits==25
1528
+ * On 64-bits, maxNbBits==57
1529
+ * return : value extracted.
1530
+ */
1531
+ static size_t FSE_lookBits(FSE_DStream_t* bitD, U32 nbBits)
1532
+ {
1533
+ const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
1534
+ return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask);
1535
+ }
1536
+
1537
+ static size_t FSE_lookBitsFast(FSE_DStream_t* bitD, U32 nbBits) /* only if nbBits >= 1 !! */
1538
+ {
1539
+ const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
1540
+ return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask);
1541
+ }
1542
+
1543
+ static void FSE_skipBits(FSE_DStream_t* bitD, U32 nbBits)
1544
+ {
1545
+ bitD->bitsConsumed += nbBits;
1546
+ }
1547
+
1548
+
1549
+ /* FSE_readBits
1550
+ * Read next n bits from the bitContainer.
1551
+ * On 32-bits, don't read more than maxNbBits==25
1552
+ * On 64-bits, don't read more than maxNbBits==57
1553
+ * Use the fast variant *only* if n >= 1.
1554
+ * return : value extracted.
1555
+ */
1556
+ size_t FSE_readBits(FSE_DStream_t* bitD, U32 nbBits)
1557
+ {
1558
+ size_t value = FSE_lookBits(bitD, nbBits);
1559
+ FSE_skipBits(bitD, nbBits);
1560
+ return value;
1561
+ }
1562
+
1563
+ size_t FSE_readBitsFast(FSE_DStream_t* bitD, U32 nbBits) /* only if nbBits >= 1 !! */
1564
+ {
1565
+ size_t value = FSE_lookBitsFast(bitD, nbBits);
1566
+ FSE_skipBits(bitD, nbBits);
1567
+ return value;
1568
+ }
1569
+
1570
+ unsigned FSE_reloadDStream(FSE_DStream_t* bitD)
1571
+ {
1572
+ if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
1573
+ return FSE_DStream_tooFar;
1574
+
1575
+ if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer))
1576
+ {
1577
+ bitD->ptr -= bitD->bitsConsumed >> 3;
1578
+ bitD->bitsConsumed &= 7;
1579
+ bitD->bitContainer = FSE_readLEST(bitD->ptr);
1580
+ return FSE_DStream_unfinished;
1581
+ }
1582
+ if (bitD->ptr == bitD->start)
1583
+ {
1584
+ if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return FSE_DStream_endOfBuffer;
1585
+ return FSE_DStream_completed;
1586
+ }
1587
+ {
1588
+ U32 nbBytes = bitD->bitsConsumed >> 3;
1589
+ U32 result = FSE_DStream_unfinished;
1590
+ if (bitD->ptr - nbBytes < bitD->start)
1591
+ {
1592
+ nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */
1593
+ result = FSE_DStream_endOfBuffer;
1594
+ }
1595
+ bitD->ptr -= nbBytes;
1596
+ bitD->bitsConsumed -= nbBytes*8;
1597
+ bitD->bitContainer = FSE_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */
1598
+ return result;
1599
+ }
1600
+ }
1601
+
1602
+
1603
+ void FSE_initDState(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD, const FSE_DTable* dt)
1604
+ {
1605
+ const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)dt;
1606
+ DStatePtr->state = FSE_readBits(bitD, DTableH->tableLog);
1607
+ FSE_reloadDStream(bitD);
1608
+ DStatePtr->table = dt + 1;
1609
+ }
1610
+
1611
+ BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD)
1612
+ {
1613
+ const FSE_decode_t DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
1614
+ const U32 nbBits = DInfo.nbBits;
1615
+ BYTE symbol = DInfo.symbol;
1616
+ size_t lowBits = FSE_readBits(bitD, nbBits);
1617
+
1618
+ DStatePtr->state = DInfo.newState + lowBits;
1619
+ return symbol;
1620
+ }
1621
+
1622
+ BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD)
1623
+ {
1624
+ const FSE_decode_t DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
1625
+ const U32 nbBits = DInfo.nbBits;
1626
+ BYTE symbol = DInfo.symbol;
1627
+ size_t lowBits = FSE_readBitsFast(bitD, nbBits);
1628
+
1629
+ DStatePtr->state = DInfo.newState + lowBits;
1630
+ return symbol;
1631
+ }
1632
+
1633
+ /* FSE_endOfDStream
1634
+ Tells if bitD has reached end of bitStream or not */
1635
+
1636
+ unsigned FSE_endOfDStream(const FSE_DStream_t* bitD)
1637
+ {
1638
+ return ((bitD->ptr == bitD->start) && (bitD->bitsConsumed == sizeof(bitD->bitContainer)*8));
1639
+ }
1640
+
1641
+ unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
1642
+ {
1643
+ return DStatePtr->state == 0;
1644
+ }
1645
+
1646
+
1647
+ FORCE_INLINE size_t FSE_decompress_usingDTable_generic(
1648
+ void* dst, size_t maxDstSize,
1649
+ const void* cSrc, size_t cSrcSize,
1650
+ const FSE_DTable* dt, const unsigned fast)
1651
+ {
1652
+ BYTE* const ostart = (BYTE*) dst;
1653
+ BYTE* op = ostart;
1654
+ BYTE* const omax = op + maxDstSize;
1655
+ BYTE* const olimit = omax-3;
1656
+
1657
+ FSE_DStream_t bitD;
1658
+ FSE_DState_t state1;
1659
+ FSE_DState_t state2;
1660
+ size_t errorCode;
1661
+
1662
+ /* Init */
1663
+ errorCode = FSE_initDStream(&bitD, cSrc, cSrcSize); /* replaced last arg by maxCompressed Size */
1664
+ if (FSE_isError(errorCode)) return errorCode;
1665
+
1666
+ FSE_initDState(&state1, &bitD, dt);
1667
+ FSE_initDState(&state2, &bitD, dt);
1668
+
1669
+ #define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD)
1670
+
1671
+ /* 4 symbols per loop */
1672
+ for ( ; (FSE_reloadDStream(&bitD)==FSE_DStream_unfinished) && (op<olimit) ; op+=4)
1673
+ {
1674
+ op[0] = FSE_GETSYMBOL(&state1);
1675
+
1676
+ if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
1677
+ FSE_reloadDStream(&bitD);
1678
+
1679
+ op[1] = FSE_GETSYMBOL(&state2);
1680
+
1681
+ if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
1682
+ { if (FSE_reloadDStream(&bitD) > FSE_DStream_unfinished) { op+=2; break; } }
1683
+
1684
+ op[2] = FSE_GETSYMBOL(&state1);
1685
+
1686
+ if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
1687
+ FSE_reloadDStream(&bitD);
1688
+
1689
+ op[3] = FSE_GETSYMBOL(&state2);
1690
+ }
1691
+
1692
+ /* tail */
1693
+ /* note : FSE_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly FSE_DStream_completed */
1694
+ while (1)
1695
+ {
1696
+ if ( (FSE_reloadDStream(&bitD)>FSE_DStream_completed) || (op==omax) || (FSE_endOfDStream(&bitD) && (fast || FSE_endOfDState(&state1))) )
1697
+ break;
1698
+
1699
+ *op++ = FSE_GETSYMBOL(&state1);
1700
+
1701
+ if ( (FSE_reloadDStream(&bitD)>FSE_DStream_completed) || (op==omax) || (FSE_endOfDStream(&bitD) && (fast || FSE_endOfDState(&state2))) )
1702
+ break;
1703
+
1704
+ *op++ = FSE_GETSYMBOL(&state2);
1705
+ }
1706
+
1707
+ /* end ? */
1708
+ if (FSE_endOfDStream(&bitD) && FSE_endOfDState(&state1) && FSE_endOfDState(&state2))
1709
+ return op-ostart;
1710
+
1711
+ if (op==omax) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* dst buffer is full, but cSrc unfinished */
1712
+
1713
+ return (size_t)-FSE_ERROR_corruptionDetected;
1714
+ }
1715
+
1716
+
1717
+ size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
1718
+ const void* cSrc, size_t cSrcSize,
1719
+ const FSE_DTable* dt)
1720
+ {
1721
+ const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)dt;
1722
+ const U32 fastMode = DTableH->fastMode;
1723
+
1724
+ /* select fast mode (static) */
1725
+ if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
1726
+ return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
1727
+ }
1728
+
1729
+
1730
+ size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize)
1731
+ {
1732
+ const BYTE* const istart = (const BYTE*)cSrc;
1733
+ const BYTE* ip = istart;
1734
+ short counting[FSE_MAX_SYMBOL_VALUE+1];
1735
+ DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */
1736
+ unsigned tableLog;
1737
+ unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
1738
+ size_t errorCode;
1739
+
1740
+ if (cSrcSize<2) return (size_t)-FSE_ERROR_srcSize_wrong; /* too small input size */
1741
+
1742
+ /* normal FSE decoding mode */
1743
+ errorCode = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
1744
+ if (FSE_isError(errorCode)) return errorCode;
1745
+ if (errorCode >= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong; /* too small input size */
1746
+ ip += errorCode;
1747
+ cSrcSize -= errorCode;
1748
+
1749
+ errorCode = FSE_buildDTable (dt, counting, maxSymbolValue, tableLog);
1750
+ if (FSE_isError(errorCode)) return errorCode;
1751
+
1752
+ /* always return, even if it is an error code */
1753
+ return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt);
1754
+ }
1755
+
1756
+
1757
+
1758
+ /*********************************************************
1759
+ * Huff0 : Huffman block compression
1760
+ *********************************************************/
1761
+ #define HUF_MAX_SYMBOL_VALUE 255
1762
+ #define HUF_DEFAULT_TABLELOG 12 /* used by default, when not specified */
1763
+ #define HUF_MAX_TABLELOG 12 /* max possible tableLog; for allocation purpose; can be modified */
1764
+ #define HUF_ABSOLUTEMAX_TABLELOG 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
1765
+ #if (HUF_MAX_TABLELOG > HUF_ABSOLUTEMAX_TABLELOG)
1766
+ # error "HUF_MAX_TABLELOG is too large !"
1767
+ #endif
1768
+
1769
+ typedef struct HUF_CElt_s {
1770
+ U16 val;
1771
+ BYTE nbBits;
1772
+ } HUF_CElt ;
1773
+
1774
+ typedef struct nodeElt_s {
1775
+ U32 count;
1776
+ U16 parent;
1777
+ BYTE byte;
1778
+ BYTE nbBits;
1779
+ } nodeElt;
1780
+
1781
+ /* HUF_writeCTable() :
1782
+ return : size of saved CTable */
1783
+ size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* tree, U32 maxSymbolValue, U32 huffLog)
1784
+ {
1785
+ BYTE bitsToWeight[HUF_ABSOLUTEMAX_TABLELOG + 1];
1786
+ BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1];
1787
+ U32 n;
1788
+ BYTE* op = (BYTE*)dst;
1789
+ size_t size;
1790
+
1791
+ /* check conditions */
1792
+ if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE + 1)
1793
+ return (size_t)-FSE_ERROR_GENERIC;
1794
+
1795
+ /* convert to weight */
1796
+ bitsToWeight[0] = 0;
1797
+ for (n=1; n<=huffLog; n++)
1798
+ bitsToWeight[n] = (BYTE)(huffLog + 1 - n);
1799
+ for (n=0; n<maxSymbolValue; n++)
1800
+ huffWeight[n] = bitsToWeight[tree[n].nbBits];
1801
+
1802
+ size = FSE_compress(op+1, maxDstSize-1, huffWeight, maxSymbolValue); /* don't need last symbol stat : implied */
1803
+ if (FSE_isError(size)) return size;
1804
+ if (size >= 128) return (size_t)-FSE_ERROR_GENERIC; /* should never happen, since maxSymbolValue <= 255 */
1805
+ if ((size <= 1) || (size >= maxSymbolValue/2))
1806
+ {
1807
+ if (size==1) /* RLE */
1808
+ {
1809
+ /* only possible case : serie of 1 (because there are at least 2) */
1810
+ /* can only be 2^n or (2^n-1), otherwise not an huffman tree */
1811
+ BYTE code;
1812
+ switch(maxSymbolValue)
1813
+ {
1814
+ case 1: code = 0; break;
1815
+ case 2: code = 1; break;
1816
+ case 3: code = 2; break;
1817
+ case 4: code = 3; break;
1818
+ case 7: code = 4; break;
1819
+ case 8: code = 5; break;
1820
+ case 15: code = 6; break;
1821
+ case 16: code = 7; break;
1822
+ case 31: code = 8; break;
1823
+ case 32: code = 9; break;
1824
+ case 63: code = 10; break;
1825
+ case 64: code = 11; break;
1826
+ case 127: code = 12; break;
1827
+ case 128: code = 13; break;
1828
+ default : return (size_t)-FSE_ERROR_corruptionDetected;
1829
+ }
1830
+ op[0] = (BYTE)(255-13 + code);
1831
+ return 1;
1832
+ }
1833
+ /* Not compressible */
1834
+ if (maxSymbolValue > (241-128)) return (size_t)-FSE_ERROR_GENERIC; /* not implemented (not possible with current format) */
1835
+ if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* not enough space within dst buffer */
1836
+ op[0] = (BYTE)(128 /*special case*/ + 0 /* Not Compressible */ + (maxSymbolValue-1));
1837
+ huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */
1838
+ for (n=0; n<maxSymbolValue; n+=2)
1839
+ op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]);
1840
+ return ((maxSymbolValue+1)/2) + 1;
1841
+ }
1842
+
1843
+ /* normal header case */
1844
+ op[0] = (BYTE)size;
1845
+ return size+1;
1846
+ }
1847
+
1848
+
1849
+ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
1850
+ {
1851
+ int totalCost = 0;
1852
+ const U32 largestBits = huffNode[lastNonNull].nbBits;
1853
+
1854
+ /* early exit : all is fine */
1855
+ if (largestBits <= maxNbBits) return largestBits;
1856
+
1857
+ // now we have a few too large elements (at least >= 2)
1858
+ {
1859
+ const U32 baseCost = 1 << (largestBits - maxNbBits);
1860
+ U32 n = lastNonNull;
1861
+
1862
+ while (huffNode[n].nbBits > maxNbBits)
1863
+ {
1864
+ totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits));
1865
+ huffNode[n].nbBits = (BYTE)maxNbBits;
1866
+ n --;
1867
+ }
1868
+
1869
+ /* renorm totalCost */
1870
+ totalCost >>= (largestBits - maxNbBits); /* note : totalCost necessarily multiple of baseCost */
1871
+
1872
+ // repay cost
1873
+ while (huffNode[n].nbBits == maxNbBits) n--; // n at last of rank (maxNbBits-1)
1874
+
1875
+ {
1876
+ const U32 noOne = 0xF0F0F0F0;
1877
+ // Get pos of last (smallest) symbol per rank
1878
+ U32 rankLast[HUF_MAX_TABLELOG];
1879
+ U32 currentNbBits = maxNbBits;
1880
+ int pos;
1881
+ memset(rankLast, 0xF0, sizeof(rankLast));
1882
+ for (pos=n ; pos >= 0; pos--)
1883
+ {
1884
+ if (huffNode[pos].nbBits >= currentNbBits) continue;
1885
+ currentNbBits = huffNode[pos].nbBits;
1886
+ rankLast[maxNbBits-currentNbBits] = pos;
1887
+ }
1888
+
1889
+ while (totalCost > 0)
1890
+ {
1891
+ U32 nBitsToDecrease = FSE_highbit32(totalCost) + 1;
1892
+ for ( ; nBitsToDecrease > 1; nBitsToDecrease--)
1893
+ {
1894
+ U32 highPos = rankLast[nBitsToDecrease];
1895
+ U32 lowPos = rankLast[nBitsToDecrease-1];
1896
+ if (highPos == noOne) continue;
1897
+ if (lowPos == noOne) break;
1898
+ {
1899
+ U32 highTotal = huffNode[highPos].count;
1900
+ U32 lowTotal = 2 * huffNode[lowPos].count;
1901
+ if (highTotal <= lowTotal) break;
1902
+ }
1903
+ }
1904
+ while (rankLast[nBitsToDecrease] == noOne)
1905
+ nBitsToDecrease ++; // In some rare cases, no more rank 1 left => overshoot to closest
1906
+ totalCost -= 1 << (nBitsToDecrease-1);
1907
+ if (rankLast[nBitsToDecrease-1] == noOne)
1908
+ rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; // now there is one elt
1909
+ huffNode[rankLast[nBitsToDecrease]].nbBits ++;
1910
+ if (rankLast[nBitsToDecrease] == 0)
1911
+ rankLast[nBitsToDecrease] = noOne;
1912
+ else
1913
+ {
1914
+ rankLast[nBitsToDecrease]--;
1915
+ if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease)
1916
+ rankLast[nBitsToDecrease] = noOne; // rank list emptied
1917
+ }
1918
+ }
1919
+
1920
+ while (totalCost < 0) /* Sometimes, cost correction overshoot */
1921
+ {
1922
+ if (rankLast[1] == noOne) /* special case, no weight 1, let's find it back at n */
1923
+ {
1924
+ while (huffNode[n].nbBits == maxNbBits) n--;
1925
+ huffNode[n+1].nbBits--;
1926
+ rankLast[1] = n+1;
1927
+ totalCost++;
1928
+ continue;
1929
+ }
1930
+ huffNode[ rankLast[1] + 1 ].nbBits--;
1931
+ rankLast[1]++;
1932
+ totalCost ++;
1933
+ }
1934
+ }
1935
+ }
1936
+
1937
+ return maxNbBits;
1938
+ }
1939
+
1940
+
1941
+ typedef struct {
1942
+ U32 base;
1943
+ U32 current;
1944
+ } rankPos;
1945
+
1946
+ static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue)
1947
+ {
1948
+ rankPos rank[32];
1949
+ U32 n;
1950
+
1951
+ memset(rank, 0, sizeof(rank));
1952
+ for (n=0; n<=maxSymbolValue; n++)
1953
+ {
1954
+ U32 r = FSE_highbit32(count[n] + 1);
1955
+ rank[r].base ++;
1956
+ }
1957
+ for (n=30; n>0; n--) rank[n-1].base += rank[n].base;
1958
+ for (n=0; n<32; n++) rank[n].current = rank[n].base;
1959
+ for (n=0; n<=maxSymbolValue; n++)
1960
+ {
1961
+ U32 c = count[n];
1962
+ U32 r = FSE_highbit32(c+1) + 1;
1963
+ U32 pos = rank[r].current++;
1964
+ while ((pos > rank[r].base) && (c > huffNode[pos-1].count)) huffNode[pos]=huffNode[pos-1], pos--;
1965
+ huffNode[pos].count = c;
1966
+ huffNode[pos].byte = (BYTE)n;
1967
+ }
1968
+ }
1969
+
1970
+
1971
+ #define STARTNODE (HUF_MAX_SYMBOL_VALUE+1)
1972
+ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits)
1973
+ {
1974
+ nodeElt huffNode0[2*HUF_MAX_SYMBOL_VALUE+1 +1];
1975
+ nodeElt* huffNode = huffNode0 + 1;
1976
+ U32 n, nonNullRank;
1977
+ int lowS, lowN;
1978
+ U16 nodeNb = STARTNODE;
1979
+ U32 nodeRoot;
1980
+
1981
+ /* safety checks */
1982
+ if (maxNbBits == 0) maxNbBits = HUF_DEFAULT_TABLELOG;
1983
+ if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE) return (size_t)-FSE_ERROR_GENERIC;
1984
+ memset(huffNode0, 0, sizeof(huffNode0));
1985
+
1986
+ // sort, decreasing order
1987
+ HUF_sort(huffNode, count, maxSymbolValue);
1988
+
1989
+ // init for parents
1990
+ nonNullRank = maxSymbolValue;
1991
+ while(huffNode[nonNullRank].count == 0) nonNullRank--;
1992
+ lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb;
1993
+ huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count;
1994
+ huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb;
1995
+ nodeNb++; lowS-=2;
1996
+ for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30);
1997
+ huffNode0[0].count = (U32)(1U<<31);
1998
+
1999
+ // create parents
2000
+ while (nodeNb <= nodeRoot)
2001
+ {
2002
+ U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
2003
+ U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
2004
+ huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count;
2005
+ huffNode[n1].parent = huffNode[n2].parent = nodeNb;
2006
+ nodeNb++;
2007
+ }
2008
+
2009
+ // distribute weights (unlimited tree height)
2010
+ huffNode[nodeRoot].nbBits = 0;
2011
+ for (n=nodeRoot-1; n>=STARTNODE; n--)
2012
+ huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
2013
+ for (n=0; n<=nonNullRank; n++)
2014
+ huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
2015
+
2016
+ // enforce maxTableLog
2017
+ maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits);
2018
+
2019
+ // fill result into tree (val, nbBits)
2020
+ {
2021
+ U16 nbPerRank[HUF_ABSOLUTEMAX_TABLELOG+1] = {0};
2022
+ U16 valPerRank[HUF_ABSOLUTEMAX_TABLELOG+1];
2023
+ if (maxNbBits > HUF_ABSOLUTEMAX_TABLELOG) return (size_t)-FSE_ERROR_GENERIC; // check
2024
+ for (n=0; n<=nonNullRank; n++)
2025
+ nbPerRank[huffNode[n].nbBits]++;
2026
+ {
2027
+ // determine stating value per rank
2028
+ U16 min = 0;
2029
+ for (n=maxNbBits; n>0; n--)
2030
+ {
2031
+ valPerRank[n] = min; // get starting value within each rank
2032
+ min += nbPerRank[n];
2033
+ min >>= 1;
2034
+ }
2035
+ }
2036
+ for (n=0; n<=maxSymbolValue; n++)
2037
+ tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; // push nbBits per symbol, symbol order
2038
+ for (n=0; n<=maxSymbolValue; n++)
2039
+ tree[n].val = valPerRank[tree[n].nbBits]++; // assign value within rank, symbol order
2040
+ }
2041
+
2042
+ return maxNbBits;
2043
+ }
2044
+
2045
+ static void HUF_encodeSymbol(FSE_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable)
2046
+ {
2047
+ FSE_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits);
2048
+ }
2049
+
2050
+ #define FSE_FLUSHBITS_1(stream) \
2051
+ if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*2+7) FSE_FLUSHBITS(stream)
2052
+
2053
+ #define FSE_FLUSHBITS_2(stream) \
2054
+ if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*4+7) FSE_FLUSHBITS(stream)
2055
+
2056
+ size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, HUF_CElt* CTable)
2057
+ {
2058
+ const BYTE* ip = (const BYTE*) src;
2059
+ BYTE* const ostart = (BYTE*)dst;
2060
+ BYTE* op = (BYTE*) ostart;
2061
+ BYTE* const oend = ostart + dstSize;
2062
+ U16* jumpTable = (U16*) dst;
2063
+ size_t n, streamSize;
2064
+ const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize));
2065
+ size_t errorCode;
2066
+ FSE_CStream_t bitC;
2067
+
2068
+ /* init */
2069
+ if (dstSize < 8) return 0;
2070
+ op += 6; /* jump Table -- could be optimized by delta / deviation */
2071
+ errorCode = FSE_initCStream(&bitC, op, oend-op);
2072
+ if (FSE_isError(errorCode)) return 0;
2073
+
2074
+ n = srcSize & ~15; // mod 16
2075
+ switch (srcSize & 15)
2076
+ {
2077
+ case 15: HUF_encodeSymbol(&bitC, ip[n+14], CTable);
2078
+ FSE_FLUSHBITS_1(&bitC);
2079
+ case 14: HUF_encodeSymbol(&bitC, ip[n+13], CTable);
2080
+ FSE_FLUSHBITS_2(&bitC);
2081
+ case 13: HUF_encodeSymbol(&bitC, ip[n+12], CTable);
2082
+ FSE_FLUSHBITS_1(&bitC);
2083
+ case 12: HUF_encodeSymbol(&bitC, ip[n+11], CTable);
2084
+ FSE_FLUSHBITS(&bitC);
2085
+ case 11: HUF_encodeSymbol(&bitC, ip[n+10], CTable);
2086
+ FSE_FLUSHBITS_1(&bitC);
2087
+ case 10: HUF_encodeSymbol(&bitC, ip[n+ 9], CTable);
2088
+ FSE_FLUSHBITS_2(&bitC);
2089
+ case 9 : HUF_encodeSymbol(&bitC, ip[n+ 8], CTable);
2090
+ FSE_FLUSHBITS_1(&bitC);
2091
+ case 8 : HUF_encodeSymbol(&bitC, ip[n+ 7], CTable);
2092
+ FSE_FLUSHBITS(&bitC);
2093
+ case 7 : HUF_encodeSymbol(&bitC, ip[n+ 6], CTable);
2094
+ FSE_FLUSHBITS_1(&bitC);
2095
+ case 6 : HUF_encodeSymbol(&bitC, ip[n+ 5], CTable);
2096
+ FSE_FLUSHBITS_2(&bitC);
2097
+ case 5 : HUF_encodeSymbol(&bitC, ip[n+ 4], CTable);
2098
+ FSE_FLUSHBITS_1(&bitC);
2099
+ case 4 : HUF_encodeSymbol(&bitC, ip[n+ 3], CTable);
2100
+ FSE_FLUSHBITS(&bitC);
2101
+ case 3 : HUF_encodeSymbol(&bitC, ip[n+ 2], CTable);
2102
+ FSE_FLUSHBITS_2(&bitC);
2103
+ case 2 : HUF_encodeSymbol(&bitC, ip[n+ 1], CTable);
2104
+ FSE_FLUSHBITS_1(&bitC);
2105
+ case 1 : HUF_encodeSymbol(&bitC, ip[n+ 0], CTable);
2106
+ FSE_FLUSHBITS(&bitC);
2107
+ case 0 :
2108
+ default: ;
2109
+ }
2110
+
2111
+ for (; n>0; n-=16)
2112
+ {
2113
+ HUF_encodeSymbol(&bitC, ip[n- 4], CTable);
2114
+ FSE_FLUSHBITS_1(&bitC);
2115
+ HUF_encodeSymbol(&bitC, ip[n- 8], CTable);
2116
+ FSE_FLUSHBITS_2(&bitC);
2117
+ HUF_encodeSymbol(&bitC, ip[n-12], CTable);
2118
+ FSE_FLUSHBITS_1(&bitC);
2119
+ HUF_encodeSymbol(&bitC, ip[n-16], CTable);
2120
+ FSE_FLUSHBITS(&bitC);
2121
+ }
2122
+ streamSize = FSE_closeCStream(&bitC);
2123
+ if (streamSize==0) return 0; /* not enough space within dst buffer == uncompressible */
2124
+ FSE_writeLE16(jumpTable, (U16)streamSize);
2125
+ op += streamSize;
2126
+
2127
+ errorCode = FSE_initCStream(&bitC, op, oend-op);
2128
+ if (FSE_isError(errorCode)) return 0;
2129
+ n = srcSize & ~15; // mod 16
2130
+ for (; n>0; n-=16)
2131
+ {
2132
+ HUF_encodeSymbol(&bitC, ip[n- 3], CTable);
2133
+ FSE_FLUSHBITS_1(&bitC);
2134
+ HUF_encodeSymbol(&bitC, ip[n- 7], CTable);
2135
+ FSE_FLUSHBITS_2(&bitC);
2136
+ HUF_encodeSymbol(&bitC, ip[n-11], CTable);
2137
+ FSE_FLUSHBITS_1(&bitC);
2138
+ HUF_encodeSymbol(&bitC, ip[n-15], CTable);
2139
+ FSE_FLUSHBITS(&bitC);
2140
+ }
2141
+ streamSize = FSE_closeCStream(&bitC);
2142
+ if (streamSize==0) return 0; /* not enough space within dst buffer == uncompressible */
2143
+ FSE_writeLE16(jumpTable+1, (U16)streamSize);
2144
+ op += streamSize;
2145
+
2146
+ errorCode = FSE_initCStream(&bitC, op, oend-op);
2147
+ if (FSE_isError(errorCode)) return 0;
2148
+ n = srcSize & ~15; // mod 16
2149
+ for (; n>0; n-=16)
2150
+ {
2151
+ HUF_encodeSymbol(&bitC, ip[n- 2], CTable);
2152
+ FSE_FLUSHBITS_1(&bitC);
2153
+ HUF_encodeSymbol(&bitC, ip[n- 6], CTable);
2154
+ FSE_FLUSHBITS_2(&bitC);
2155
+ HUF_encodeSymbol(&bitC, ip[n-10], CTable);
2156
+ FSE_FLUSHBITS_1(&bitC);
2157
+ HUF_encodeSymbol(&bitC, ip[n-14], CTable);
2158
+ FSE_FLUSHBITS(&bitC);
2159
+ }
2160
+ streamSize = FSE_closeCStream(&bitC);
2161
+ if (streamSize==0) return 0; /* not enough space within dst buffer == uncompressible */
2162
+ FSE_writeLE16(jumpTable+2, (U16)streamSize);
2163
+ op += streamSize;
2164
+
2165
+ errorCode = FSE_initCStream(&bitC, op, oend-op);
2166
+ if (FSE_isError(errorCode)) return 0;
2167
+ n = srcSize & ~15; // mod 16
2168
+ for (; n>0; n-=16)
2169
+ {
2170
+ HUF_encodeSymbol(&bitC, ip[n- 1], CTable);
2171
+ FSE_FLUSHBITS_1(&bitC);
2172
+ HUF_encodeSymbol(&bitC, ip[n- 5], CTable);
2173
+ FSE_FLUSHBITS_2(&bitC);
2174
+ HUF_encodeSymbol(&bitC, ip[n- 9], CTable);
2175
+ FSE_FLUSHBITS_1(&bitC);
2176
+ HUF_encodeSymbol(&bitC, ip[n-13], CTable);
2177
+ FSE_FLUSHBITS(&bitC);
2178
+ }
2179
+ streamSize = FSE_closeCStream(&bitC);
2180
+ if (streamSize==0) return 0; /* not enough space within dst buffer == uncompressible */
2181
+ op += streamSize;
2182
+
2183
+ return op-ostart;
2184
+ }
2185
+
2186
+
2187
+ size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog)
2188
+ {
2189
+ BYTE* const ostart = (BYTE*)dst;
2190
+ BYTE* op = ostart;
2191
+ BYTE* const oend = ostart + dstSize;
2192
+
2193
+ U32 count[HUF_MAX_SYMBOL_VALUE+1];
2194
+ HUF_CElt CTable[HUF_MAX_SYMBOL_VALUE+1];
2195
+ size_t errorCode;
2196
+
2197
+ /* early out */
2198
+ if (srcSize <= 1) return srcSize; /* Uncompressed or RLE */
2199
+ if (!maxSymbolValue) maxSymbolValue = HUF_MAX_SYMBOL_VALUE;
2200
+ if (!huffLog) huffLog = HUF_DEFAULT_TABLELOG;
2201
+
2202
+ /* Scan input and build symbol stats */
2203
+ errorCode = FSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize);
2204
+ if (FSE_isError(errorCode)) return errorCode;
2205
+ if (errorCode == srcSize) return 1;
2206
+ if (errorCode < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */
2207
+
2208
+ /* Build Huffman Tree */
2209
+ errorCode = HUF_buildCTable (CTable, count, maxSymbolValue, huffLog);
2210
+ if (FSE_isError(errorCode)) return errorCode;
2211
+ huffLog = (U32)errorCode;
2212
+
2213
+ /* Write table description header */
2214
+ errorCode = HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog); /* don't write last symbol, implied */
2215
+ if (FSE_isError(errorCode)) return errorCode;
2216
+ op += errorCode;
2217
+
2218
+ /* Compress */
2219
+ errorCode = HUF_compress_usingCTable(op, oend - op, src, srcSize, CTable);
2220
+ if (FSE_isError(errorCode)) return errorCode;
2221
+ if (errorCode==0) return 0;
2222
+ op += errorCode;
2223
+
2224
+ /* check compressibility */
2225
+ if ((size_t)(op-ostart) >= srcSize-1)
2226
+ return op-ostart;
2227
+
2228
+ return op-ostart;
2229
+ }
2230
+
2231
+ size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
2232
+ {
2233
+ return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_DEFAULT_TABLELOG);
2234
+ }
2235
+
2236
+
2237
+ /*********************************************************
2238
+ * Huff0 : Huffman block decompression
2239
+ *********************************************************/
2240
+ typedef struct {
2241
+ BYTE byte;
2242
+ BYTE nbBits;
2243
+ } HUF_DElt;
2244
+
2245
+ size_t HUF_readDTable (U16* DTable, const void* src, size_t srcSize)
2246
+ {
2247
+ BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1];
2248
+ U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; /* large enough for values from 0 to 16 */
2249
+ U32 weightTotal;
2250
+ U32 maxBits;
2251
+ const BYTE* ip = (const BYTE*) src;
2252
+ size_t iSize = ip[0];
2253
+ size_t oSize;
2254
+ U32 n;
2255
+ U32 nextRankStart;
2256
+ HUF_DElt* const dt = (HUF_DElt*)(DTable + 1);
2257
+
2258
+ FSE_STATIC_ASSERT(sizeof(HUF_DElt) == sizeof(U16)); /* if compilation fails here, assertion is false */
2259
+ //memset(huffWeight, 0, sizeof(huffWeight)); /* should not be necessary, but some analyzer complain ... */
2260
+ if (iSize >= 128) /* special header */
2261
+ {
2262
+ if (iSize >= (242)) /* RLE */
2263
+ {
2264
+ static int l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 };
2265
+ oSize = l[iSize-242];
2266
+ memset(huffWeight, 1, oSize);
2267
+ iSize = 0;
2268
+ }
2269
+ else /* Incompressible */
2270
+ {
2271
+ oSize = iSize - 127;
2272
+ iSize = ((oSize+1)/2);
2273
+ if (iSize+1 > srcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
2274
+ ip += 1;
2275
+ for (n=0; n<oSize; n+=2)
2276
+ {
2277
+ huffWeight[n] = ip[n/2] >> 4;
2278
+ huffWeight[n+1] = ip[n/2] & 15;
2279
+ }
2280
+ }
2281
+ }
2282
+ else /* header compressed with FSE (normal case) */
2283
+ {
2284
+ if (iSize+1 > srcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
2285
+ oSize = FSE_decompress(huffWeight, HUF_MAX_SYMBOL_VALUE, ip+1, iSize); /* max 255 values decoded, last one is implied */
2286
+ if (FSE_isError(oSize)) return oSize;
2287
+ }
2288
+
2289
+ /* collect weight stats */
2290
+ memset(rankVal, 0, sizeof(rankVal));
2291
+ weightTotal = 0;
2292
+ for (n=0; n<oSize; n++)
2293
+ {
2294
+ if (huffWeight[n] >= HUF_ABSOLUTEMAX_TABLELOG) return (size_t)-FSE_ERROR_corruptionDetected;
2295
+ rankVal[huffWeight[n]]++;
2296
+ weightTotal += (1 << huffWeight[n]) >> 1;
2297
+ }
2298
+
2299
+ /* get last non-null symbol weight (implied, total must be 2^n) */
2300
+ maxBits = FSE_highbit32(weightTotal) + 1;
2301
+ if (maxBits > DTable[0]) return (size_t)-FSE_ERROR_tableLog_tooLarge; /* DTable is too small */
2302
+ DTable[0] = (U16)maxBits;
2303
+ {
2304
+ U32 total = 1 << maxBits;
2305
+ U32 rest = total - weightTotal;
2306
+ U32 verif = 1 << FSE_highbit32(rest);
2307
+ U32 lastWeight = FSE_highbit32(rest) + 1;
2308
+ if (verif != rest) return (size_t)-FSE_ERROR_corruptionDetected; /* last value must be a clean power of 2 */
2309
+ huffWeight[oSize] = (BYTE)lastWeight;
2310
+ rankVal[lastWeight]++;
2311
+ }
2312
+
2313
+ /* check tree construction validity */
2314
+ if ((rankVal[1] < 2) || (rankVal[1] & 1)) return (size_t)-FSE_ERROR_corruptionDetected; /* by construction : at least 2 elts of rank 1, must be even */
2315
+
2316
+ /* Prepare ranks */
2317
+ nextRankStart = 0;
2318
+ for (n=1; n<=maxBits; n++)
2319
+ {
2320
+ U32 current = nextRankStart;
2321
+ nextRankStart += (rankVal[n] << (n-1));
2322
+ rankVal[n] = current;
2323
+ }
2324
+
2325
+ /* fill DTable */
2326
+ for (n=0; n<=oSize; n++)
2327
+ {
2328
+ const U32 w = huffWeight[n];
2329
+ const U32 length = (1 << w) >> 1;
2330
+ U32 i;
2331
+ HUF_DElt D;
2332
+ D.byte = (BYTE)n; D.nbBits = (BYTE)(maxBits + 1 - w);
2333
+ for (i = rankVal[w]; i < rankVal[w] + length; i++)
2334
+ dt[i] = D;
2335
+ rankVal[w] += length;
2336
+ }
2337
+
2338
+ return iSize+1;
2339
+ }
2340
+
2341
+
2342
+ static BYTE HUF_decodeSymbol(FSE_DStream_t* Dstream, const HUF_DElt* dt, const U32 dtLog)
2343
+ {
2344
+ const size_t val = FSE_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */
2345
+ const BYTE c = dt[val].byte;
2346
+ FSE_skipBits(Dstream, dt[val].nbBits);
2347
+ return c;
2348
+ }
2349
+
2350
+ static size_t HUF_decompress_usingDTable( /* -3% slower when non static */
2351
+ void* dst, size_t maxDstSize,
2352
+ const void* cSrc, size_t cSrcSize,
2353
+ const U16* DTable)
2354
+ {
2355
+ BYTE* const ostart = (BYTE*) dst;
2356
+ BYTE* op = ostart;
2357
+ BYTE* const omax = op + maxDstSize;
2358
+ BYTE* const olimit = omax-15;
2359
+
2360
+ const HUF_DElt* const dt = (const HUF_DElt*)(DTable+1);
2361
+ const U32 dtLog = DTable[0];
2362
+ size_t errorCode;
2363
+ U32 reloadStatus;
2364
+
2365
+ /* Init */
2366
+
2367
+ const U16* jumpTable = (const U16*)cSrc;
2368
+ const size_t length1 = FSE_readLE16(jumpTable);
2369
+ const size_t length2 = FSE_readLE16(jumpTable+1);
2370
+ const size_t length3 = FSE_readLE16(jumpTable+2);
2371
+ const size_t length4 = cSrcSize - 6 - length1 - length2 - length3; // check coherency !!
2372
+ const char* const start1 = (const char*)(cSrc) + 6;
2373
+ const char* const start2 = start1 + length1;
2374
+ const char* const start3 = start2 + length2;
2375
+ const char* const start4 = start3 + length3;
2376
+ FSE_DStream_t bitD1, bitD2, bitD3, bitD4;
2377
+
2378
+ if (length1+length2+length3+6 >= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
2379
+
2380
+ errorCode = FSE_initDStream(&bitD1, start1, length1);
2381
+ if (FSE_isError(errorCode)) return errorCode;
2382
+ errorCode = FSE_initDStream(&bitD2, start2, length2);
2383
+ if (FSE_isError(errorCode)) return errorCode;
2384
+ errorCode = FSE_initDStream(&bitD3, start3, length3);
2385
+ if (FSE_isError(errorCode)) return errorCode;
2386
+ errorCode = FSE_initDStream(&bitD4, start4, length4);
2387
+ if (FSE_isError(errorCode)) return errorCode;
2388
+
2389
+ reloadStatus=FSE_reloadDStream(&bitD2);
2390
+
2391
+ /* 16 symbols per loop */
2392
+ for ( ; (reloadStatus<FSE_DStream_completed) && (op<olimit); /* D2-3-4 are supposed to be synchronized and finish together */
2393
+ op+=16, reloadStatus = FSE_reloadDStream(&bitD2) | FSE_reloadDStream(&bitD3) | FSE_reloadDStream(&bitD4), FSE_reloadDStream(&bitD1))
2394
+ {
2395
+ #define HUF_DECODE_SYMBOL_0(n, Dstream) \
2396
+ op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog);
2397
+
2398
+ #define HUF_DECODE_SYMBOL_1(n, Dstream) \
2399
+ op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
2400
+ if (FSE_32bits() && (HUF_MAX_TABLELOG>12)) FSE_reloadDStream(&Dstream)
2401
+
2402
+ #define HUF_DECODE_SYMBOL_2(n, Dstream) \
2403
+ op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
2404
+ if (FSE_32bits()) FSE_reloadDStream(&Dstream)
2405
+
2406
+ HUF_DECODE_SYMBOL_1( 0, bitD1);
2407
+ HUF_DECODE_SYMBOL_1( 1, bitD2);
2408
+ HUF_DECODE_SYMBOL_1( 2, bitD3);
2409
+ HUF_DECODE_SYMBOL_1( 3, bitD4);
2410
+ HUF_DECODE_SYMBOL_2( 4, bitD1);
2411
+ HUF_DECODE_SYMBOL_2( 5, bitD2);
2412
+ HUF_DECODE_SYMBOL_2( 6, bitD3);
2413
+ HUF_DECODE_SYMBOL_2( 7, bitD4);
2414
+ HUF_DECODE_SYMBOL_1( 8, bitD1);
2415
+ HUF_DECODE_SYMBOL_1( 9, bitD2);
2416
+ HUF_DECODE_SYMBOL_1(10, bitD3);
2417
+ HUF_DECODE_SYMBOL_1(11, bitD4);
2418
+ HUF_DECODE_SYMBOL_0(12, bitD1);
2419
+ HUF_DECODE_SYMBOL_0(13, bitD2);
2420
+ HUF_DECODE_SYMBOL_0(14, bitD3);
2421
+ HUF_DECODE_SYMBOL_0(15, bitD4);
2422
+ }
2423
+
2424
+ if (reloadStatus!=FSE_DStream_completed) /* not complete : some bitStream might be FSE_DStream_unfinished */
2425
+ return (size_t)-FSE_ERROR_corruptionDetected;
2426
+
2427
+ /* tail */
2428
+ {
2429
+ // bitTail = bitD1; // *much* slower : -20% !??!
2430
+ FSE_DStream_t bitTail;
2431
+ bitTail.ptr = bitD1.ptr;
2432
+ bitTail.bitsConsumed = bitD1.bitsConsumed;
2433
+ bitTail.bitContainer = bitD1.bitContainer; // required in case of FSE_DStream_endOfBuffer
2434
+ bitTail.start = start1;
2435
+ for ( ; (FSE_reloadDStream(&bitTail) < FSE_DStream_completed) && (op<omax) ; op++)
2436
+ {
2437
+ HUF_DECODE_SYMBOL_0(0, bitTail);
2438
+ }
2439
+
2440
+ if (FSE_endOfDStream(&bitTail))
2441
+ return op-ostart;
2442
+ }
2443
+
2444
+ if (op==omax) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* dst buffer is full, but cSrc unfinished */
2445
+
2446
+ return (size_t)-FSE_ERROR_corruptionDetected;
2447
+ }
2448
+
2449
+
2450
+ size_t HUF_decompress (void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize)
2451
+ {
2452
+ HUF_CREATE_STATIC_DTABLE(DTable, HUF_MAX_TABLELOG);
2453
+ const BYTE* ip = (const BYTE*) cSrc;
2454
+ size_t errorCode;
2455
+
2456
+ errorCode = HUF_readDTable (DTable, cSrc, cSrcSize);
2457
+ if (FSE_isError(errorCode)) return errorCode;
2458
+ if (errorCode >= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
2459
+ ip += errorCode;
2460
+ cSrcSize -= errorCode;
2461
+
2462
+ return HUF_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, DTable);
2463
+ }
2464
+
2465
+
2466
+ #endif /* FSE_COMMONDEFS_ONLY */