zstdlib 0.3.0-x64-mingw32 → 0.8.0-x64-mingw32
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGES.md +30 -1
- data/README.md +2 -2
- data/Rakefile +1 -1
- data/ext/zstdlib/extconf.rb +3 -3
- data/ext/zstdlib/ruby/zlib-2.7/zstdlib.c +4895 -0
- data/ext/zstdlib/ruby/zlib-3.0/zstdlib.c +4994 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/bitstream.h +59 -51
- data/ext/zstdlib/zstd-1.5.0/lib/common/compiler.h +289 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/cpu.h +1 -3
- data/ext/zstdlib/zstd-1.5.0/lib/common/debug.c +24 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/debug.h +22 -49
- data/ext/zstdlib/zstd-1.5.0/lib/common/entropy_common.c +362 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/error_private.c +3 -1
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/error_private.h +8 -4
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/fse.h +50 -42
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/fse_decompress.c +149 -55
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/huf.h +43 -39
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/mem.h +69 -25
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/pool.c +30 -20
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/pool.h +3 -3
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/threading.c +51 -4
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/threading.h +36 -4
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/xxhash.c +40 -92
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/xxhash.h +12 -32
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/zstd_common.c +10 -10
- data/ext/zstdlib/zstd-1.5.0/lib/common/zstd_deps.h +111 -0
- data/ext/zstdlib/zstd-1.5.0/lib/common/zstd_internal.h +490 -0
- data/ext/zstdlib/zstd-1.5.0/lib/common/zstd_trace.h +154 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/fse_compress.c +47 -63
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/hist.c +41 -63
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/hist.h +13 -33
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/huf_compress.c +332 -193
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_compress.c +6393 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_compress_internal.h +522 -86
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_compress_literals.c +25 -16
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_compress_literals.h +2 -2
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_compress_sequences.c +50 -24
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_compress_sequences.h +11 -4
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_compress_superblock.c +572 -0
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_compress_superblock.h +32 -0
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_cwksp.h +662 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_double_fast.c +43 -41
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_double_fast.h +2 -2
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_fast.c +85 -80
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_fast.h +2 -2
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_lazy.c +2184 -0
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_lazy.h +125 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_ldm.c +333 -208
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_ldm.h +15 -3
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_ldm_geartab.h +103 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_opt.c +228 -129
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_opt.h +1 -1
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstdmt_compress.c +151 -440
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstdmt_compress.h +110 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/huf_decompress.c +395 -276
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_ddict.c +20 -16
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_ddict.h +3 -3
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_decompress.c +628 -231
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_decompress_block.c +606 -380
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_decompress_block.h +8 -5
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_decompress_internal.h +39 -9
- data/ext/zstdlib/zstd-1.5.0/lib/zdict.h +452 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/zstd.h +740 -153
- data/ext/zstdlib/{zstd-1.4.2/lib/common → zstd-1.5.0/lib}/zstd_errors.h +3 -1
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzclose.c +1 -1
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzcompatibility.h +1 -1
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzguts.h +0 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzlib.c +9 -9
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzread.c +16 -8
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzwrite.c +8 -8
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/zstd_zlibwrapper.c +131 -45
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/zstd_zlibwrapper.h +1 -1
- data/lib/2.2/zstdlib.so +0 -0
- data/lib/2.3/zstdlib.so +0 -0
- data/lib/2.4/zstdlib.so +0 -0
- data/lib/2.5/zstdlib.so +0 -0
- data/lib/2.6/zstdlib.so +0 -0
- data/lib/2.7/zstdlib.so +0 -0
- metadata +76 -67
- data/ext/zstdlib/zstd-1.4.2/lib/common/compiler.h +0 -147
- data/ext/zstdlib/zstd-1.4.2/lib/common/debug.c +0 -44
- data/ext/zstdlib/zstd-1.4.2/lib/common/entropy_common.c +0 -236
- data/ext/zstdlib/zstd-1.4.2/lib/common/zstd_internal.h +0 -371
- data/ext/zstdlib/zstd-1.4.2/lib/compress/zstd_compress.c +0 -3904
- data/ext/zstdlib/zstd-1.4.2/lib/compress/zstd_lazy.c +0 -1111
- data/ext/zstdlib/zstd-1.4.2/lib/compress/zstd_lazy.h +0 -67
- data/ext/zstdlib/zstd-1.4.2/lib/compress/zstdmt_compress.h +0 -192
@@ -0,0 +1,154 @@
|
|
1
|
+
/*
|
2
|
+
* Copyright (c) Facebook, Inc.
|
3
|
+
* All rights reserved.
|
4
|
+
*
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
9
|
+
*/
|
10
|
+
|
11
|
+
#ifndef ZSTD_TRACE_H
|
12
|
+
#define ZSTD_TRACE_H
|
13
|
+
|
14
|
+
#if defined (__cplusplus)
|
15
|
+
extern "C" {
|
16
|
+
#endif
|
17
|
+
|
18
|
+
#include <stddef.h>
|
19
|
+
|
20
|
+
/* weak symbol support */
|
21
|
+
#if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && defined(__GNUC__) && \
|
22
|
+
!defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \
|
23
|
+
!defined(__CYGWIN__)
|
24
|
+
# define ZSTD_HAVE_WEAK_SYMBOLS 1
|
25
|
+
#else
|
26
|
+
# define ZSTD_HAVE_WEAK_SYMBOLS 0
|
27
|
+
#endif
|
28
|
+
#if ZSTD_HAVE_WEAK_SYMBOLS
|
29
|
+
# define ZSTD_WEAK_ATTR __attribute__((__weak__))
|
30
|
+
#else
|
31
|
+
# define ZSTD_WEAK_ATTR
|
32
|
+
#endif
|
33
|
+
|
34
|
+
/* Only enable tracing when weak symbols are available. */
|
35
|
+
#ifndef ZSTD_TRACE
|
36
|
+
# define ZSTD_TRACE ZSTD_HAVE_WEAK_SYMBOLS
|
37
|
+
#endif
|
38
|
+
|
39
|
+
#if ZSTD_TRACE
|
40
|
+
|
41
|
+
struct ZSTD_CCtx_s;
|
42
|
+
struct ZSTD_DCtx_s;
|
43
|
+
struct ZSTD_CCtx_params_s;
|
44
|
+
|
45
|
+
typedef struct {
|
46
|
+
/**
|
47
|
+
* ZSTD_VERSION_NUMBER
|
48
|
+
*
|
49
|
+
* This is guaranteed to be the first member of ZSTD_trace.
|
50
|
+
* Otherwise, this struct is not stable between versions. If
|
51
|
+
* the version number does not match your expectation, you
|
52
|
+
* should not interpret the rest of the struct.
|
53
|
+
*/
|
54
|
+
unsigned version;
|
55
|
+
/**
|
56
|
+
* Non-zero if streaming (de)compression is used.
|
57
|
+
*/
|
58
|
+
unsigned streaming;
|
59
|
+
/**
|
60
|
+
* The dictionary ID.
|
61
|
+
*/
|
62
|
+
unsigned dictionaryID;
|
63
|
+
/**
|
64
|
+
* Is the dictionary cold?
|
65
|
+
* Only set on decompression.
|
66
|
+
*/
|
67
|
+
unsigned dictionaryIsCold;
|
68
|
+
/**
|
69
|
+
* The dictionary size or zero if no dictionary.
|
70
|
+
*/
|
71
|
+
size_t dictionarySize;
|
72
|
+
/**
|
73
|
+
* The uncompressed size of the data.
|
74
|
+
*/
|
75
|
+
size_t uncompressedSize;
|
76
|
+
/**
|
77
|
+
* The compressed size of the data.
|
78
|
+
*/
|
79
|
+
size_t compressedSize;
|
80
|
+
/**
|
81
|
+
* The fully resolved CCtx parameters (NULL on decompression).
|
82
|
+
*/
|
83
|
+
struct ZSTD_CCtx_params_s const* params;
|
84
|
+
/**
|
85
|
+
* The ZSTD_CCtx pointer (NULL on decompression).
|
86
|
+
*/
|
87
|
+
struct ZSTD_CCtx_s const* cctx;
|
88
|
+
/**
|
89
|
+
* The ZSTD_DCtx pointer (NULL on compression).
|
90
|
+
*/
|
91
|
+
struct ZSTD_DCtx_s const* dctx;
|
92
|
+
} ZSTD_Trace;
|
93
|
+
|
94
|
+
/**
|
95
|
+
* A tracing context. It must be 0 when tracing is disabled.
|
96
|
+
* Otherwise, any non-zero value returned by a tracing begin()
|
97
|
+
* function is presented to any subsequent calls to end().
|
98
|
+
*
|
99
|
+
* Any non-zero value is treated as tracing is enabled and not
|
100
|
+
* interpreted by the library.
|
101
|
+
*
|
102
|
+
* Two possible uses are:
|
103
|
+
* * A timestamp for when the begin() function was called.
|
104
|
+
* * A unique key identifying the (de)compression, like the
|
105
|
+
* address of the [dc]ctx pointer if you need to track
|
106
|
+
* more information than just a timestamp.
|
107
|
+
*/
|
108
|
+
typedef unsigned long long ZSTD_TraceCtx;
|
109
|
+
|
110
|
+
/**
|
111
|
+
* Trace the beginning of a compression call.
|
112
|
+
* @param cctx The dctx pointer for the compression.
|
113
|
+
* It can be used as a key to map begin() to end().
|
114
|
+
* @returns Non-zero if tracing is enabled. The return value is
|
115
|
+
* passed to ZSTD_trace_compress_end().
|
116
|
+
*/
|
117
|
+
ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_compress_begin(
|
118
|
+
struct ZSTD_CCtx_s const* cctx);
|
119
|
+
|
120
|
+
/**
|
121
|
+
* Trace the end of a compression call.
|
122
|
+
* @param ctx The return value of ZSTD_trace_compress_begin().
|
123
|
+
* @param trace The zstd tracing info.
|
124
|
+
*/
|
125
|
+
ZSTD_WEAK_ATTR void ZSTD_trace_compress_end(
|
126
|
+
ZSTD_TraceCtx ctx,
|
127
|
+
ZSTD_Trace const* trace);
|
128
|
+
|
129
|
+
/**
|
130
|
+
* Trace the beginning of a decompression call.
|
131
|
+
* @param dctx The dctx pointer for the decompression.
|
132
|
+
* It can be used as a key to map begin() to end().
|
133
|
+
* @returns Non-zero if tracing is enabled. The return value is
|
134
|
+
* passed to ZSTD_trace_compress_end().
|
135
|
+
*/
|
136
|
+
ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_decompress_begin(
|
137
|
+
struct ZSTD_DCtx_s const* dctx);
|
138
|
+
|
139
|
+
/**
|
140
|
+
* Trace the end of a decompression call.
|
141
|
+
* @param ctx The return value of ZSTD_trace_decompress_begin().
|
142
|
+
* @param trace The zstd tracing info.
|
143
|
+
*/
|
144
|
+
ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end(
|
145
|
+
ZSTD_TraceCtx ctx,
|
146
|
+
ZSTD_Trace const* trace);
|
147
|
+
|
148
|
+
#endif /* ZSTD_TRACE */
|
149
|
+
|
150
|
+
#if defined (__cplusplus)
|
151
|
+
}
|
152
|
+
#endif
|
153
|
+
|
154
|
+
#endif /* ZSTD_TRACE_H */
|
@@ -1,50 +1,31 @@
|
|
1
1
|
/* ******************************************************************
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
* Redistributions in binary form must reproduce the above
|
14
|
-
copyright notice, this list of conditions and the following disclaimer
|
15
|
-
in the documentation and/or other materials provided with the
|
16
|
-
distribution.
|
17
|
-
|
18
|
-
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
19
|
-
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
20
|
-
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
21
|
-
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
22
|
-
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
23
|
-
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
24
|
-
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
25
|
-
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
26
|
-
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
27
|
-
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
28
|
-
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
29
|
-
|
30
|
-
You can contact the author at :
|
31
|
-
- FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
32
|
-
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
2
|
+
* FSE : Finite State Entropy encoder
|
3
|
+
* Copyright (c) Yann Collet, Facebook, Inc.
|
4
|
+
*
|
5
|
+
* You can contact the author at :
|
6
|
+
* - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
7
|
+
* - Public forum : https://groups.google.com/forum/#!forum/lz4c
|
8
|
+
*
|
9
|
+
* This source code is licensed under both the BSD-style license (found in the
|
10
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
11
|
+
* in the COPYING file in the root directory of this source tree).
|
12
|
+
* You may select, at your option, one of the above-listed licenses.
|
33
13
|
****************************************************************** */
|
34
14
|
|
35
15
|
/* **************************************************************
|
36
16
|
* Includes
|
37
17
|
****************************************************************/
|
38
|
-
#include
|
39
|
-
#include
|
40
|
-
#include "
|
41
|
-
#include "mem.h" /* U32, U16, etc. */
|
42
|
-
#include "debug.h" /* assert, DEBUGLOG */
|
18
|
+
#include "../common/compiler.h"
|
19
|
+
#include "../common/mem.h" /* U32, U16, etc. */
|
20
|
+
#include "../common/debug.h" /* assert, DEBUGLOG */
|
43
21
|
#include "hist.h" /* HIST_count_wksp */
|
44
|
-
#include "bitstream.h"
|
22
|
+
#include "../common/bitstream.h"
|
45
23
|
#define FSE_STATIC_LINKING_ONLY
|
46
|
-
#include "fse.h"
|
47
|
-
#include "error_private.h"
|
24
|
+
#include "../common/fse.h"
|
25
|
+
#include "../common/error_private.h"
|
26
|
+
#define ZSTD_DEPS_NEED_MALLOC
|
27
|
+
#define ZSTD_DEPS_NEED_MATH64
|
28
|
+
#include "../common/zstd_deps.h" /* ZSTD_malloc, ZSTD_free, ZSTD_memcpy, ZSTD_memset */
|
48
29
|
|
49
30
|
|
50
31
|
/* **************************************************************
|
@@ -94,13 +75,15 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
|
|
94
75
|
void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableLog ? tableSize>>1 : 1) ;
|
95
76
|
FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
|
96
77
|
U32 const step = FSE_TABLESTEP(tableSize);
|
97
|
-
U32 cumul[FSE_MAX_SYMBOL_VALUE+2];
|
98
78
|
|
99
|
-
|
79
|
+
U32* cumul = (U32*)workSpace;
|
80
|
+
FSE_FUNCTION_TYPE* tableSymbol = (FSE_FUNCTION_TYPE*)(cumul + (maxSymbolValue + 2));
|
81
|
+
|
100
82
|
U32 highThreshold = tableSize-1;
|
101
83
|
|
84
|
+
if ((size_t)workSpace & 3) return ERROR(GENERIC); /* Must be 4 byte aligned */
|
85
|
+
if (FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) > wkspSize) return ERROR(tableLog_tooLarge);
|
102
86
|
/* CTable header */
|
103
|
-
if (((size_t)1 << tableLog) * sizeof(FSE_FUNCTION_TYPE) > wkspSize) return ERROR(tableLog_tooLarge);
|
104
87
|
tableU16[-2] = (U16) tableLog;
|
105
88
|
tableU16[-1] = (U16) maxSymbolValue;
|
106
89
|
assert(tableLog < 16); /* required for threshold strategy to work */
|
@@ -109,7 +92,7 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
|
|
109
92
|
* http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
|
110
93
|
|
111
94
|
#ifdef __clang_analyzer__
|
112
|
-
|
95
|
+
ZSTD_memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */
|
113
96
|
#endif
|
114
97
|
|
115
98
|
/* symbol start positions */
|
@@ -188,12 +171,13 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
|
|
188
171
|
return 0;
|
189
172
|
}
|
190
173
|
|
191
|
-
|
174
|
+
#ifndef ZSTD_NO_UNUSED_FUNCTIONS
|
192
175
|
size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
|
193
176
|
{
|
194
177
|
FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* memset() is not necessary, even if static analyzer complain about it */
|
195
178
|
return FSE_buildCTable_wksp(ct, normalizedCounter, maxSymbolValue, tableLog, tableSymbol, sizeof(tableSymbol));
|
196
179
|
}
|
180
|
+
#endif
|
197
181
|
|
198
182
|
|
199
183
|
|
@@ -327,10 +311,10 @@ FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog)
|
|
327
311
|
size_t size;
|
328
312
|
if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
|
329
313
|
size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32);
|
330
|
-
return (FSE_CTable*)
|
314
|
+
return (FSE_CTable*)ZSTD_malloc(size);
|
331
315
|
}
|
332
316
|
|
333
|
-
void FSE_freeCTable (FSE_CTable* ct) {
|
317
|
+
void FSE_freeCTable (FSE_CTable* ct) { ZSTD_free(ct); }
|
334
318
|
|
335
319
|
/* provides the minimum logSize to safely represent a distribution */
|
336
320
|
static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
|
@@ -361,11 +345,10 @@ unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
|
|
361
345
|
return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2);
|
362
346
|
}
|
363
347
|
|
364
|
-
|
365
348
|
/* Secondary normalization method.
|
366
349
|
To be used when primary method fails. */
|
367
350
|
|
368
|
-
static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue)
|
351
|
+
static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue, short lowProbCount)
|
369
352
|
{
|
370
353
|
short const NOT_YET_ASSIGNED = -2;
|
371
354
|
U32 s;
|
@@ -382,7 +365,7 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
|
|
382
365
|
continue;
|
383
366
|
}
|
384
367
|
if (count[s] <= lowThreshold) {
|
385
|
-
norm[s] =
|
368
|
+
norm[s] = lowProbCount;
|
386
369
|
distributed++;
|
387
370
|
total -= count[s];
|
388
371
|
continue;
|
@@ -434,7 +417,7 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
|
|
434
417
|
|
435
418
|
{ U64 const vStepLog = 62 - tableLog;
|
436
419
|
U64 const mid = (1ULL << (vStepLog-1)) - 1;
|
437
|
-
U64 const rStep = ((((U64)1<<vStepLog) * ToDistribute) + mid)
|
420
|
+
U64 const rStep = ZSTD_div64((((U64)1<<vStepLog) * ToDistribute) + mid, (U32)total); /* scale on remaining */
|
438
421
|
U64 tmpTotal = mid;
|
439
422
|
for (s=0; s<=maxSymbolValue; s++) {
|
440
423
|
if (norm[s]==NOT_YET_ASSIGNED) {
|
@@ -451,10 +434,9 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
|
|
451
434
|
return 0;
|
452
435
|
}
|
453
436
|
|
454
|
-
|
455
437
|
size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
|
456
438
|
const unsigned* count, size_t total,
|
457
|
-
unsigned maxSymbolValue)
|
439
|
+
unsigned maxSymbolValue, unsigned useLowProbCount)
|
458
440
|
{
|
459
441
|
/* Sanity checks */
|
460
442
|
if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
|
@@ -463,8 +445,9 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
|
|
463
445
|
if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */
|
464
446
|
|
465
447
|
{ static U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 };
|
448
|
+
short const lowProbCount = useLowProbCount ? -1 : 1;
|
466
449
|
U64 const scale = 62 - tableLog;
|
467
|
-
U64 const step = ((U64)1<<62)
|
450
|
+
U64 const step = ZSTD_div64((U64)1<<62, (U32)total); /* <== here, one division ! */
|
468
451
|
U64 const vStep = 1ULL<<(scale-20);
|
469
452
|
int stillToDistribute = 1<<tableLog;
|
470
453
|
unsigned s;
|
@@ -476,7 +459,7 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
|
|
476
459
|
if (count[s] == total) return 0; /* rle special case */
|
477
460
|
if (count[s] == 0) { normalizedCounter[s]=0; continue; }
|
478
461
|
if (count[s] <= lowThreshold) {
|
479
|
-
normalizedCounter[s] =
|
462
|
+
normalizedCounter[s] = lowProbCount;
|
480
463
|
stillToDistribute--;
|
481
464
|
} else {
|
482
465
|
short proba = (short)((count[s]*step) >> scale);
|
@@ -490,7 +473,7 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
|
|
490
473
|
} }
|
491
474
|
if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) {
|
492
475
|
/* corner case, need another normalization method */
|
493
|
-
size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue);
|
476
|
+
size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue, lowProbCount);
|
494
477
|
if (FSE_isError(errorCode)) return errorCode;
|
495
478
|
}
|
496
479
|
else normalizedCounter[largest] += (short)stillToDistribute;
|
@@ -645,9 +628,7 @@ size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
|
|
645
628
|
|
646
629
|
size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
|
647
630
|
|
648
|
-
#
|
649
|
-
#define CHECK_F(f) { CHECK_V_F(_var_err__, f); }
|
650
|
-
|
631
|
+
#ifndef ZSTD_NO_UNUSED_FUNCTIONS
|
651
632
|
/* FSE_compress_wksp() :
|
652
633
|
* Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
|
653
634
|
* `wkspSize` size must be `(1<<tableLog)`.
|
@@ -666,7 +647,7 @@ size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t src
|
|
666
647
|
size_t const scratchBufferSize = wkspSize - (CTableSize * sizeof(FSE_CTable));
|
667
648
|
|
668
649
|
/* init conditions */
|
669
|
-
if (wkspSize <
|
650
|
+
if (wkspSize < FSE_COMPRESS_WKSP_SIZE_U32(tableLog, maxSymbolValue)) return ERROR(tableLog_tooLarge);
|
670
651
|
if (srcSize <= 1) return 0; /* Not compressible */
|
671
652
|
if (!maxSymbolValue) maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
|
672
653
|
if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG;
|
@@ -679,7 +660,7 @@ size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t src
|
|
679
660
|
}
|
680
661
|
|
681
662
|
tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
|
682
|
-
CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue) );
|
663
|
+
CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue, /* useLowProbCount */ srcSize >= 2048) );
|
683
664
|
|
684
665
|
/* Write table description header */
|
685
666
|
{ CHECK_V_F(nc_err, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) );
|
@@ -701,13 +682,16 @@ size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t src
|
|
701
682
|
|
702
683
|
typedef struct {
|
703
684
|
FSE_CTable CTable_max[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
|
704
|
-
|
685
|
+
union {
|
686
|
+
U32 hist_wksp[HIST_WKSP_SIZE_U32];
|
687
|
+
BYTE scratchBuffer[1 << FSE_MAX_TABLELOG];
|
688
|
+
} workspace;
|
705
689
|
} fseWkspMax_t;
|
706
690
|
|
707
691
|
size_t FSE_compress2 (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog)
|
708
692
|
{
|
709
693
|
fseWkspMax_t scratchBuffer;
|
710
|
-
DEBUG_STATIC_ASSERT(sizeof(scratchBuffer) >=
|
694
|
+
DEBUG_STATIC_ASSERT(sizeof(scratchBuffer) >= FSE_COMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)); /* compilation failures here means scratchBuffer is not large enough */
|
711
695
|
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
712
696
|
return FSE_compress_wksp(dst, dstCapacity, src, srcSize, maxSymbolValue, tableLog, &scratchBuffer, sizeof(scratchBuffer));
|
713
697
|
}
|
@@ -716,6 +700,6 @@ size_t FSE_compress (void* dst, size_t dstCapacity, const void* src, size_t srcS
|
|
716
700
|
{
|
717
701
|
return FSE_compress2(dst, dstCapacity, src, srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG);
|
718
702
|
}
|
719
|
-
|
703
|
+
#endif
|
720
704
|
|
721
705
|
#endif /* FSE_COMMONDEFS_ONLY */
|
@@ -1,42 +1,22 @@
|
|
1
1
|
/* ******************************************************************
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
* Redistributions in binary form must reproduce the above
|
15
|
-
copyright notice, this list of conditions and the following disclaimer
|
16
|
-
in the documentation and/or other materials provided with the
|
17
|
-
distribution.
|
18
|
-
|
19
|
-
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
20
|
-
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
21
|
-
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
22
|
-
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
23
|
-
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
24
|
-
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
25
|
-
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
26
|
-
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
27
|
-
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
28
|
-
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
29
|
-
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
30
|
-
|
31
|
-
You can contact the author at :
|
32
|
-
- FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
33
|
-
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
2
|
+
* hist : Histogram functions
|
3
|
+
* part of Finite State Entropy project
|
4
|
+
* Copyright (c) Yann Collet, Facebook, Inc.
|
5
|
+
*
|
6
|
+
* You can contact the author at :
|
7
|
+
* - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
8
|
+
* - Public forum : https://groups.google.com/forum/#!forum/lz4c
|
9
|
+
*
|
10
|
+
* This source code is licensed under both the BSD-style license (found in the
|
11
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
12
|
+
* in the COPYING file in the root directory of this source tree).
|
13
|
+
* You may select, at your option, one of the above-listed licenses.
|
34
14
|
****************************************************************** */
|
35
15
|
|
36
16
|
/* --- dependencies --- */
|
37
|
-
#include "mem.h" /* U32, BYTE, etc. */
|
38
|
-
#include "debug.h" /* assert, DEBUGLOG */
|
39
|
-
#include "error_private.h" /* ERROR */
|
17
|
+
#include "../common/mem.h" /* U32, BYTE, etc. */
|
18
|
+
#include "../common/debug.h" /* assert, DEBUGLOG */
|
19
|
+
#include "../common/error_private.h" /* ERROR */
|
40
20
|
#include "hist.h"
|
41
21
|
|
42
22
|
|
@@ -54,7 +34,7 @@ unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
|
|
54
34
|
unsigned maxSymbolValue = *maxSymbolValuePtr;
|
55
35
|
unsigned largestCount=0;
|
56
36
|
|
57
|
-
|
37
|
+
ZSTD_memset(count, 0, (maxSymbolValue+1) * sizeof(*count));
|
58
38
|
if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; }
|
59
39
|
|
60
40
|
while (ip<end) {
|
@@ -80,9 +60,9 @@ typedef enum { trustInput, checkMaxSymbolValue } HIST_checkInput_e;
|
|
80
60
|
* this design makes better use of OoO cpus,
|
81
61
|
* and is noticeably faster when some values are heavily repeated.
|
82
62
|
* But it needs some additional workspace for intermediate tables.
|
83
|
-
* `workSpace`
|
63
|
+
* `workSpace` must be a U32 table of size >= HIST_WKSP_SIZE_U32.
|
84
64
|
* @return : largest histogram frequency,
|
85
|
-
* or an error code (notably when histogram
|
65
|
+
* or an error code (notably when histogram's alphabet is larger than *maxSymbolValuePtr) */
|
86
66
|
static size_t HIST_count_parallel_wksp(
|
87
67
|
unsigned* count, unsigned* maxSymbolValuePtr,
|
88
68
|
const void* source, size_t sourceSize,
|
@@ -91,22 +71,21 @@ static size_t HIST_count_parallel_wksp(
|
|
91
71
|
{
|
92
72
|
const BYTE* ip = (const BYTE*)source;
|
93
73
|
const BYTE* const iend = ip+sourceSize;
|
94
|
-
|
74
|
+
size_t const countSize = (*maxSymbolValuePtr + 1) * sizeof(*count);
|
95
75
|
unsigned max=0;
|
96
76
|
U32* const Counting1 = workSpace;
|
97
77
|
U32* const Counting2 = Counting1 + 256;
|
98
78
|
U32* const Counting3 = Counting2 + 256;
|
99
79
|
U32* const Counting4 = Counting3 + 256;
|
100
80
|
|
101
|
-
memset(workSpace, 0, 4*256*sizeof(unsigned));
|
102
|
-
|
103
81
|
/* safety checks */
|
82
|
+
assert(*maxSymbolValuePtr <= 255);
|
104
83
|
if (!sourceSize) {
|
105
|
-
|
84
|
+
ZSTD_memset(count, 0, countSize);
|
106
85
|
*maxSymbolValuePtr = 0;
|
107
86
|
return 0;
|
108
87
|
}
|
109
|
-
|
88
|
+
ZSTD_memset(workSpace, 0, 4*256*sizeof(unsigned));
|
110
89
|
|
111
90
|
/* by stripes of 16 bytes */
|
112
91
|
{ U32 cached = MEM_read32(ip); ip += 4;
|
@@ -138,21 +117,18 @@ static size_t HIST_count_parallel_wksp(
|
|
138
117
|
/* finish last symbols */
|
139
118
|
while (ip<iend) Counting1[*ip++]++;
|
140
119
|
|
141
|
-
if (check) { /* verify stats will fit into destination table */
|
142
|
-
U32 s; for (s=255; s>maxSymbolValue; s--) {
|
143
|
-
Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s];
|
144
|
-
if (Counting1[s]) return ERROR(maxSymbolValue_tooSmall);
|
145
|
-
} }
|
146
|
-
|
147
120
|
{ U32 s;
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
if (count[s] > max) max = count[s];
|
121
|
+
for (s=0; s<256; s++) {
|
122
|
+
Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s];
|
123
|
+
if (Counting1[s] > max) max = Counting1[s];
|
152
124
|
} }
|
153
125
|
|
154
|
-
|
155
|
-
|
126
|
+
{ unsigned maxSymbolValue = 255;
|
127
|
+
while (!Counting1[maxSymbolValue]) maxSymbolValue--;
|
128
|
+
if (check && maxSymbolValue > *maxSymbolValuePtr) return ERROR(maxSymbolValue_tooSmall);
|
129
|
+
*maxSymbolValuePtr = maxSymbolValue;
|
130
|
+
ZSTD_memmove(count, Counting1, countSize); /* in case count & Counting1 are overlapping */
|
131
|
+
}
|
156
132
|
return (size_t)max;
|
157
133
|
}
|
158
134
|
|
@@ -172,14 +148,6 @@ size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
|
|
172
148
|
return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, trustInput, (U32*)workSpace);
|
173
149
|
}
|
174
150
|
|
175
|
-
/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */
|
176
|
-
size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr,
|
177
|
-
const void* source, size_t sourceSize)
|
178
|
-
{
|
179
|
-
unsigned tmpCounters[HIST_WKSP_SIZE_U32];
|
180
|
-
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters, sizeof(tmpCounters));
|
181
|
-
}
|
182
|
-
|
183
151
|
/* HIST_count_wksp() :
|
184
152
|
* Same as HIST_count(), but using an externally provided scratch buffer.
|
185
153
|
* `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */
|
@@ -195,9 +163,19 @@ size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
|
|
195
163
|
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace, workSpaceSize);
|
196
164
|
}
|
197
165
|
|
166
|
+
#ifndef ZSTD_NO_UNUSED_FUNCTIONS
|
167
|
+
/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */
|
168
|
+
size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr,
|
169
|
+
const void* source, size_t sourceSize)
|
170
|
+
{
|
171
|
+
unsigned tmpCounters[HIST_WKSP_SIZE_U32];
|
172
|
+
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters, sizeof(tmpCounters));
|
173
|
+
}
|
174
|
+
|
198
175
|
size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr,
|
199
176
|
const void* src, size_t srcSize)
|
200
177
|
{
|
201
178
|
unsigned tmpCounters[HIST_WKSP_SIZE_U32];
|
202
179
|
return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters, sizeof(tmpCounters));
|
203
180
|
}
|
181
|
+
#endif
|