zstdlib 0.3.0-x86-mingw32 → 0.8.0-x86-mingw32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGES.md +30 -1
- data/README.md +2 -2
- data/Rakefile +1 -1
- data/ext/zstdlib/extconf.rb +3 -3
- data/ext/zstdlib/ruby/zlib-2.7/zstdlib.c +4895 -0
- data/ext/zstdlib/ruby/zlib-3.0/zstdlib.c +4994 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/bitstream.h +59 -51
- data/ext/zstdlib/zstd-1.5.0/lib/common/compiler.h +289 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/cpu.h +1 -3
- data/ext/zstdlib/zstd-1.5.0/lib/common/debug.c +24 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/debug.h +22 -49
- data/ext/zstdlib/zstd-1.5.0/lib/common/entropy_common.c +362 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/error_private.c +3 -1
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/error_private.h +8 -4
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/fse.h +50 -42
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/fse_decompress.c +149 -55
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/huf.h +43 -39
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/mem.h +69 -25
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/pool.c +30 -20
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/pool.h +3 -3
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/threading.c +51 -4
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/threading.h +36 -4
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/xxhash.c +40 -92
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/xxhash.h +12 -32
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/common/zstd_common.c +10 -10
- data/ext/zstdlib/zstd-1.5.0/lib/common/zstd_deps.h +111 -0
- data/ext/zstdlib/zstd-1.5.0/lib/common/zstd_internal.h +490 -0
- data/ext/zstdlib/zstd-1.5.0/lib/common/zstd_trace.h +154 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/fse_compress.c +47 -63
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/hist.c +41 -63
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/hist.h +13 -33
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/huf_compress.c +332 -193
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_compress.c +6393 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_compress_internal.h +522 -86
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_compress_literals.c +25 -16
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_compress_literals.h +2 -2
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_compress_sequences.c +50 -24
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_compress_sequences.h +11 -4
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_compress_superblock.c +572 -0
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_compress_superblock.h +32 -0
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_cwksp.h +662 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_double_fast.c +43 -41
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_double_fast.h +2 -2
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_fast.c +85 -80
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_fast.h +2 -2
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_lazy.c +2184 -0
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_lazy.h +125 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_ldm.c +333 -208
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_ldm.h +15 -3
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_ldm_geartab.h +103 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_opt.c +228 -129
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstd_opt.h +1 -1
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/compress/zstdmt_compress.c +151 -440
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstdmt_compress.h +110 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/huf_decompress.c +395 -276
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_ddict.c +20 -16
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_ddict.h +3 -3
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_decompress.c +628 -231
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_decompress_block.c +606 -380
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_decompress_block.h +8 -5
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/decompress/zstd_decompress_internal.h +39 -9
- data/ext/zstdlib/zstd-1.5.0/lib/zdict.h +452 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/lib/zstd.h +740 -153
- data/ext/zstdlib/{zstd-1.4.2/lib/common → zstd-1.5.0/lib}/zstd_errors.h +3 -1
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzclose.c +1 -1
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzcompatibility.h +1 -1
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzguts.h +0 -0
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzlib.c +9 -9
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzread.c +16 -8
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/gzwrite.c +8 -8
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/zstd_zlibwrapper.c +131 -45
- data/ext/zstdlib/{zstd-1.4.2 → zstd-1.5.0}/zlibWrapper/zstd_zlibwrapper.h +1 -1
- data/lib/2.2/zstdlib.so +0 -0
- data/lib/2.3/zstdlib.so +0 -0
- data/lib/2.4/zstdlib.so +0 -0
- data/lib/2.5/zstdlib.so +0 -0
- data/lib/2.6/zstdlib.so +0 -0
- data/lib/2.7/zstdlib.so +0 -0
- metadata +76 -67
- data/ext/zstdlib/zstd-1.4.2/lib/common/compiler.h +0 -147
- data/ext/zstdlib/zstd-1.4.2/lib/common/debug.c +0 -44
- data/ext/zstdlib/zstd-1.4.2/lib/common/entropy_common.c +0 -236
- data/ext/zstdlib/zstd-1.4.2/lib/common/zstd_internal.h +0 -371
- data/ext/zstdlib/zstd-1.4.2/lib/compress/zstd_compress.c +0 -3904
- data/ext/zstdlib/zstd-1.4.2/lib/compress/zstd_lazy.c +0 -1111
- data/ext/zstdlib/zstd-1.4.2/lib/compress/zstd_lazy.h +0 -67
- data/ext/zstdlib/zstd-1.4.2/lib/compress/zstdmt_compress.h +0 -192
@@ -0,0 +1,154 @@
|
|
1
|
+
/*
|
2
|
+
* Copyright (c) Facebook, Inc.
|
3
|
+
* All rights reserved.
|
4
|
+
*
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
9
|
+
*/
|
10
|
+
|
11
|
+
#ifndef ZSTD_TRACE_H
|
12
|
+
#define ZSTD_TRACE_H
|
13
|
+
|
14
|
+
#if defined (__cplusplus)
|
15
|
+
extern "C" {
|
16
|
+
#endif
|
17
|
+
|
18
|
+
#include <stddef.h>
|
19
|
+
|
20
|
+
/* weak symbol support */
|
21
|
+
#if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && defined(__GNUC__) && \
|
22
|
+
!defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \
|
23
|
+
!defined(__CYGWIN__)
|
24
|
+
# define ZSTD_HAVE_WEAK_SYMBOLS 1
|
25
|
+
#else
|
26
|
+
# define ZSTD_HAVE_WEAK_SYMBOLS 0
|
27
|
+
#endif
|
28
|
+
#if ZSTD_HAVE_WEAK_SYMBOLS
|
29
|
+
# define ZSTD_WEAK_ATTR __attribute__((__weak__))
|
30
|
+
#else
|
31
|
+
# define ZSTD_WEAK_ATTR
|
32
|
+
#endif
|
33
|
+
|
34
|
+
/* Only enable tracing when weak symbols are available. */
|
35
|
+
#ifndef ZSTD_TRACE
|
36
|
+
# define ZSTD_TRACE ZSTD_HAVE_WEAK_SYMBOLS
|
37
|
+
#endif
|
38
|
+
|
39
|
+
#if ZSTD_TRACE
|
40
|
+
|
41
|
+
struct ZSTD_CCtx_s;
|
42
|
+
struct ZSTD_DCtx_s;
|
43
|
+
struct ZSTD_CCtx_params_s;
|
44
|
+
|
45
|
+
typedef struct {
|
46
|
+
/**
|
47
|
+
* ZSTD_VERSION_NUMBER
|
48
|
+
*
|
49
|
+
* This is guaranteed to be the first member of ZSTD_trace.
|
50
|
+
* Otherwise, this struct is not stable between versions. If
|
51
|
+
* the version number does not match your expectation, you
|
52
|
+
* should not interpret the rest of the struct.
|
53
|
+
*/
|
54
|
+
unsigned version;
|
55
|
+
/**
|
56
|
+
* Non-zero if streaming (de)compression is used.
|
57
|
+
*/
|
58
|
+
unsigned streaming;
|
59
|
+
/**
|
60
|
+
* The dictionary ID.
|
61
|
+
*/
|
62
|
+
unsigned dictionaryID;
|
63
|
+
/**
|
64
|
+
* Is the dictionary cold?
|
65
|
+
* Only set on decompression.
|
66
|
+
*/
|
67
|
+
unsigned dictionaryIsCold;
|
68
|
+
/**
|
69
|
+
* The dictionary size or zero if no dictionary.
|
70
|
+
*/
|
71
|
+
size_t dictionarySize;
|
72
|
+
/**
|
73
|
+
* The uncompressed size of the data.
|
74
|
+
*/
|
75
|
+
size_t uncompressedSize;
|
76
|
+
/**
|
77
|
+
* The compressed size of the data.
|
78
|
+
*/
|
79
|
+
size_t compressedSize;
|
80
|
+
/**
|
81
|
+
* The fully resolved CCtx parameters (NULL on decompression).
|
82
|
+
*/
|
83
|
+
struct ZSTD_CCtx_params_s const* params;
|
84
|
+
/**
|
85
|
+
* The ZSTD_CCtx pointer (NULL on decompression).
|
86
|
+
*/
|
87
|
+
struct ZSTD_CCtx_s const* cctx;
|
88
|
+
/**
|
89
|
+
* The ZSTD_DCtx pointer (NULL on compression).
|
90
|
+
*/
|
91
|
+
struct ZSTD_DCtx_s const* dctx;
|
92
|
+
} ZSTD_Trace;
|
93
|
+
|
94
|
+
/**
|
95
|
+
* A tracing context. It must be 0 when tracing is disabled.
|
96
|
+
* Otherwise, any non-zero value returned by a tracing begin()
|
97
|
+
* function is presented to any subsequent calls to end().
|
98
|
+
*
|
99
|
+
* Any non-zero value is treated as tracing is enabled and not
|
100
|
+
* interpreted by the library.
|
101
|
+
*
|
102
|
+
* Two possible uses are:
|
103
|
+
* * A timestamp for when the begin() function was called.
|
104
|
+
* * A unique key identifying the (de)compression, like the
|
105
|
+
* address of the [dc]ctx pointer if you need to track
|
106
|
+
* more information than just a timestamp.
|
107
|
+
*/
|
108
|
+
typedef unsigned long long ZSTD_TraceCtx;
|
109
|
+
|
110
|
+
/**
|
111
|
+
* Trace the beginning of a compression call.
|
112
|
+
* @param cctx The dctx pointer for the compression.
|
113
|
+
* It can be used as a key to map begin() to end().
|
114
|
+
* @returns Non-zero if tracing is enabled. The return value is
|
115
|
+
* passed to ZSTD_trace_compress_end().
|
116
|
+
*/
|
117
|
+
ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_compress_begin(
|
118
|
+
struct ZSTD_CCtx_s const* cctx);
|
119
|
+
|
120
|
+
/**
|
121
|
+
* Trace the end of a compression call.
|
122
|
+
* @param ctx The return value of ZSTD_trace_compress_begin().
|
123
|
+
* @param trace The zstd tracing info.
|
124
|
+
*/
|
125
|
+
ZSTD_WEAK_ATTR void ZSTD_trace_compress_end(
|
126
|
+
ZSTD_TraceCtx ctx,
|
127
|
+
ZSTD_Trace const* trace);
|
128
|
+
|
129
|
+
/**
|
130
|
+
* Trace the beginning of a decompression call.
|
131
|
+
* @param dctx The dctx pointer for the decompression.
|
132
|
+
* It can be used as a key to map begin() to end().
|
133
|
+
* @returns Non-zero if tracing is enabled. The return value is
|
134
|
+
* passed to ZSTD_trace_compress_end().
|
135
|
+
*/
|
136
|
+
ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_decompress_begin(
|
137
|
+
struct ZSTD_DCtx_s const* dctx);
|
138
|
+
|
139
|
+
/**
|
140
|
+
* Trace the end of a decompression call.
|
141
|
+
* @param ctx The return value of ZSTD_trace_decompress_begin().
|
142
|
+
* @param trace The zstd tracing info.
|
143
|
+
*/
|
144
|
+
ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end(
|
145
|
+
ZSTD_TraceCtx ctx,
|
146
|
+
ZSTD_Trace const* trace);
|
147
|
+
|
148
|
+
#endif /* ZSTD_TRACE */
|
149
|
+
|
150
|
+
#if defined (__cplusplus)
|
151
|
+
}
|
152
|
+
#endif
|
153
|
+
|
154
|
+
#endif /* ZSTD_TRACE_H */
|
@@ -1,50 +1,31 @@
|
|
1
1
|
/* ******************************************************************
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
* Redistributions in binary form must reproduce the above
|
14
|
-
copyright notice, this list of conditions and the following disclaimer
|
15
|
-
in the documentation and/or other materials provided with the
|
16
|
-
distribution.
|
17
|
-
|
18
|
-
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
19
|
-
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
20
|
-
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
21
|
-
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
22
|
-
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
23
|
-
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
24
|
-
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
25
|
-
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
26
|
-
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
27
|
-
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
28
|
-
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
29
|
-
|
30
|
-
You can contact the author at :
|
31
|
-
- FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
32
|
-
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
2
|
+
* FSE : Finite State Entropy encoder
|
3
|
+
* Copyright (c) Yann Collet, Facebook, Inc.
|
4
|
+
*
|
5
|
+
* You can contact the author at :
|
6
|
+
* - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
7
|
+
* - Public forum : https://groups.google.com/forum/#!forum/lz4c
|
8
|
+
*
|
9
|
+
* This source code is licensed under both the BSD-style license (found in the
|
10
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
11
|
+
* in the COPYING file in the root directory of this source tree).
|
12
|
+
* You may select, at your option, one of the above-listed licenses.
|
33
13
|
****************************************************************** */
|
34
14
|
|
35
15
|
/* **************************************************************
|
36
16
|
* Includes
|
37
17
|
****************************************************************/
|
38
|
-
#include
|
39
|
-
#include
|
40
|
-
#include "
|
41
|
-
#include "mem.h" /* U32, U16, etc. */
|
42
|
-
#include "debug.h" /* assert, DEBUGLOG */
|
18
|
+
#include "../common/compiler.h"
|
19
|
+
#include "../common/mem.h" /* U32, U16, etc. */
|
20
|
+
#include "../common/debug.h" /* assert, DEBUGLOG */
|
43
21
|
#include "hist.h" /* HIST_count_wksp */
|
44
|
-
#include "bitstream.h"
|
22
|
+
#include "../common/bitstream.h"
|
45
23
|
#define FSE_STATIC_LINKING_ONLY
|
46
|
-
#include "fse.h"
|
47
|
-
#include "error_private.h"
|
24
|
+
#include "../common/fse.h"
|
25
|
+
#include "../common/error_private.h"
|
26
|
+
#define ZSTD_DEPS_NEED_MALLOC
|
27
|
+
#define ZSTD_DEPS_NEED_MATH64
|
28
|
+
#include "../common/zstd_deps.h" /* ZSTD_malloc, ZSTD_free, ZSTD_memcpy, ZSTD_memset */
|
48
29
|
|
49
30
|
|
50
31
|
/* **************************************************************
|
@@ -94,13 +75,15 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
|
|
94
75
|
void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableLog ? tableSize>>1 : 1) ;
|
95
76
|
FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
|
96
77
|
U32 const step = FSE_TABLESTEP(tableSize);
|
97
|
-
U32 cumul[FSE_MAX_SYMBOL_VALUE+2];
|
98
78
|
|
99
|
-
|
79
|
+
U32* cumul = (U32*)workSpace;
|
80
|
+
FSE_FUNCTION_TYPE* tableSymbol = (FSE_FUNCTION_TYPE*)(cumul + (maxSymbolValue + 2));
|
81
|
+
|
100
82
|
U32 highThreshold = tableSize-1;
|
101
83
|
|
84
|
+
if ((size_t)workSpace & 3) return ERROR(GENERIC); /* Must be 4 byte aligned */
|
85
|
+
if (FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) > wkspSize) return ERROR(tableLog_tooLarge);
|
102
86
|
/* CTable header */
|
103
|
-
if (((size_t)1 << tableLog) * sizeof(FSE_FUNCTION_TYPE) > wkspSize) return ERROR(tableLog_tooLarge);
|
104
87
|
tableU16[-2] = (U16) tableLog;
|
105
88
|
tableU16[-1] = (U16) maxSymbolValue;
|
106
89
|
assert(tableLog < 16); /* required for threshold strategy to work */
|
@@ -109,7 +92,7 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
|
|
109
92
|
* http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
|
110
93
|
|
111
94
|
#ifdef __clang_analyzer__
|
112
|
-
|
95
|
+
ZSTD_memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */
|
113
96
|
#endif
|
114
97
|
|
115
98
|
/* symbol start positions */
|
@@ -188,12 +171,13 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
|
|
188
171
|
return 0;
|
189
172
|
}
|
190
173
|
|
191
|
-
|
174
|
+
#ifndef ZSTD_NO_UNUSED_FUNCTIONS
|
192
175
|
size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
|
193
176
|
{
|
194
177
|
FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* memset() is not necessary, even if static analyzer complain about it */
|
195
178
|
return FSE_buildCTable_wksp(ct, normalizedCounter, maxSymbolValue, tableLog, tableSymbol, sizeof(tableSymbol));
|
196
179
|
}
|
180
|
+
#endif
|
197
181
|
|
198
182
|
|
199
183
|
|
@@ -327,10 +311,10 @@ FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog)
|
|
327
311
|
size_t size;
|
328
312
|
if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
|
329
313
|
size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32);
|
330
|
-
return (FSE_CTable*)
|
314
|
+
return (FSE_CTable*)ZSTD_malloc(size);
|
331
315
|
}
|
332
316
|
|
333
|
-
void FSE_freeCTable (FSE_CTable* ct) {
|
317
|
+
void FSE_freeCTable (FSE_CTable* ct) { ZSTD_free(ct); }
|
334
318
|
|
335
319
|
/* provides the minimum logSize to safely represent a distribution */
|
336
320
|
static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
|
@@ -361,11 +345,10 @@ unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
|
|
361
345
|
return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2);
|
362
346
|
}
|
363
347
|
|
364
|
-
|
365
348
|
/* Secondary normalization method.
|
366
349
|
To be used when primary method fails. */
|
367
350
|
|
368
|
-
static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue)
|
351
|
+
static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue, short lowProbCount)
|
369
352
|
{
|
370
353
|
short const NOT_YET_ASSIGNED = -2;
|
371
354
|
U32 s;
|
@@ -382,7 +365,7 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
|
|
382
365
|
continue;
|
383
366
|
}
|
384
367
|
if (count[s] <= lowThreshold) {
|
385
|
-
norm[s] =
|
368
|
+
norm[s] = lowProbCount;
|
386
369
|
distributed++;
|
387
370
|
total -= count[s];
|
388
371
|
continue;
|
@@ -434,7 +417,7 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
|
|
434
417
|
|
435
418
|
{ U64 const vStepLog = 62 - tableLog;
|
436
419
|
U64 const mid = (1ULL << (vStepLog-1)) - 1;
|
437
|
-
U64 const rStep = ((((U64)1<<vStepLog) * ToDistribute) + mid)
|
420
|
+
U64 const rStep = ZSTD_div64((((U64)1<<vStepLog) * ToDistribute) + mid, (U32)total); /* scale on remaining */
|
438
421
|
U64 tmpTotal = mid;
|
439
422
|
for (s=0; s<=maxSymbolValue; s++) {
|
440
423
|
if (norm[s]==NOT_YET_ASSIGNED) {
|
@@ -451,10 +434,9 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
|
|
451
434
|
return 0;
|
452
435
|
}
|
453
436
|
|
454
|
-
|
455
437
|
size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
|
456
438
|
const unsigned* count, size_t total,
|
457
|
-
unsigned maxSymbolValue)
|
439
|
+
unsigned maxSymbolValue, unsigned useLowProbCount)
|
458
440
|
{
|
459
441
|
/* Sanity checks */
|
460
442
|
if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
|
@@ -463,8 +445,9 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
|
|
463
445
|
if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */
|
464
446
|
|
465
447
|
{ static U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 };
|
448
|
+
short const lowProbCount = useLowProbCount ? -1 : 1;
|
466
449
|
U64 const scale = 62 - tableLog;
|
467
|
-
U64 const step = ((U64)1<<62)
|
450
|
+
U64 const step = ZSTD_div64((U64)1<<62, (U32)total); /* <== here, one division ! */
|
468
451
|
U64 const vStep = 1ULL<<(scale-20);
|
469
452
|
int stillToDistribute = 1<<tableLog;
|
470
453
|
unsigned s;
|
@@ -476,7 +459,7 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
|
|
476
459
|
if (count[s] == total) return 0; /* rle special case */
|
477
460
|
if (count[s] == 0) { normalizedCounter[s]=0; continue; }
|
478
461
|
if (count[s] <= lowThreshold) {
|
479
|
-
normalizedCounter[s] =
|
462
|
+
normalizedCounter[s] = lowProbCount;
|
480
463
|
stillToDistribute--;
|
481
464
|
} else {
|
482
465
|
short proba = (short)((count[s]*step) >> scale);
|
@@ -490,7 +473,7 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
|
|
490
473
|
} }
|
491
474
|
if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) {
|
492
475
|
/* corner case, need another normalization method */
|
493
|
-
size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue);
|
476
|
+
size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue, lowProbCount);
|
494
477
|
if (FSE_isError(errorCode)) return errorCode;
|
495
478
|
}
|
496
479
|
else normalizedCounter[largest] += (short)stillToDistribute;
|
@@ -645,9 +628,7 @@ size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
|
|
645
628
|
|
646
629
|
size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
|
647
630
|
|
648
|
-
#
|
649
|
-
#define CHECK_F(f) { CHECK_V_F(_var_err__, f); }
|
650
|
-
|
631
|
+
#ifndef ZSTD_NO_UNUSED_FUNCTIONS
|
651
632
|
/* FSE_compress_wksp() :
|
652
633
|
* Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
|
653
634
|
* `wkspSize` size must be `(1<<tableLog)`.
|
@@ -666,7 +647,7 @@ size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t src
|
|
666
647
|
size_t const scratchBufferSize = wkspSize - (CTableSize * sizeof(FSE_CTable));
|
667
648
|
|
668
649
|
/* init conditions */
|
669
|
-
if (wkspSize <
|
650
|
+
if (wkspSize < FSE_COMPRESS_WKSP_SIZE_U32(tableLog, maxSymbolValue)) return ERROR(tableLog_tooLarge);
|
670
651
|
if (srcSize <= 1) return 0; /* Not compressible */
|
671
652
|
if (!maxSymbolValue) maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
|
672
653
|
if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG;
|
@@ -679,7 +660,7 @@ size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t src
|
|
679
660
|
}
|
680
661
|
|
681
662
|
tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
|
682
|
-
CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue) );
|
663
|
+
CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue, /* useLowProbCount */ srcSize >= 2048) );
|
683
664
|
|
684
665
|
/* Write table description header */
|
685
666
|
{ CHECK_V_F(nc_err, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) );
|
@@ -701,13 +682,16 @@ size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t src
|
|
701
682
|
|
702
683
|
typedef struct {
|
703
684
|
FSE_CTable CTable_max[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
|
704
|
-
|
685
|
+
union {
|
686
|
+
U32 hist_wksp[HIST_WKSP_SIZE_U32];
|
687
|
+
BYTE scratchBuffer[1 << FSE_MAX_TABLELOG];
|
688
|
+
} workspace;
|
705
689
|
} fseWkspMax_t;
|
706
690
|
|
707
691
|
size_t FSE_compress2 (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog)
|
708
692
|
{
|
709
693
|
fseWkspMax_t scratchBuffer;
|
710
|
-
DEBUG_STATIC_ASSERT(sizeof(scratchBuffer) >=
|
694
|
+
DEBUG_STATIC_ASSERT(sizeof(scratchBuffer) >= FSE_COMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)); /* compilation failures here means scratchBuffer is not large enough */
|
711
695
|
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
712
696
|
return FSE_compress_wksp(dst, dstCapacity, src, srcSize, maxSymbolValue, tableLog, &scratchBuffer, sizeof(scratchBuffer));
|
713
697
|
}
|
@@ -716,6 +700,6 @@ size_t FSE_compress (void* dst, size_t dstCapacity, const void* src, size_t srcS
|
|
716
700
|
{
|
717
701
|
return FSE_compress2(dst, dstCapacity, src, srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG);
|
718
702
|
}
|
719
|
-
|
703
|
+
#endif
|
720
704
|
|
721
705
|
#endif /* FSE_COMMONDEFS_ONLY */
|
@@ -1,42 +1,22 @@
|
|
1
1
|
/* ******************************************************************
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
* Redistributions in binary form must reproduce the above
|
15
|
-
copyright notice, this list of conditions and the following disclaimer
|
16
|
-
in the documentation and/or other materials provided with the
|
17
|
-
distribution.
|
18
|
-
|
19
|
-
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
20
|
-
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
21
|
-
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
22
|
-
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
23
|
-
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
24
|
-
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
25
|
-
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
26
|
-
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
27
|
-
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
28
|
-
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
29
|
-
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
30
|
-
|
31
|
-
You can contact the author at :
|
32
|
-
- FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
33
|
-
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
2
|
+
* hist : Histogram functions
|
3
|
+
* part of Finite State Entropy project
|
4
|
+
* Copyright (c) Yann Collet, Facebook, Inc.
|
5
|
+
*
|
6
|
+
* You can contact the author at :
|
7
|
+
* - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
8
|
+
* - Public forum : https://groups.google.com/forum/#!forum/lz4c
|
9
|
+
*
|
10
|
+
* This source code is licensed under both the BSD-style license (found in the
|
11
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
12
|
+
* in the COPYING file in the root directory of this source tree).
|
13
|
+
* You may select, at your option, one of the above-listed licenses.
|
34
14
|
****************************************************************** */
|
35
15
|
|
36
16
|
/* --- dependencies --- */
|
37
|
-
#include "mem.h" /* U32, BYTE, etc. */
|
38
|
-
#include "debug.h" /* assert, DEBUGLOG */
|
39
|
-
#include "error_private.h" /* ERROR */
|
17
|
+
#include "../common/mem.h" /* U32, BYTE, etc. */
|
18
|
+
#include "../common/debug.h" /* assert, DEBUGLOG */
|
19
|
+
#include "../common/error_private.h" /* ERROR */
|
40
20
|
#include "hist.h"
|
41
21
|
|
42
22
|
|
@@ -54,7 +34,7 @@ unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
|
|
54
34
|
unsigned maxSymbolValue = *maxSymbolValuePtr;
|
55
35
|
unsigned largestCount=0;
|
56
36
|
|
57
|
-
|
37
|
+
ZSTD_memset(count, 0, (maxSymbolValue+1) * sizeof(*count));
|
58
38
|
if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; }
|
59
39
|
|
60
40
|
while (ip<end) {
|
@@ -80,9 +60,9 @@ typedef enum { trustInput, checkMaxSymbolValue } HIST_checkInput_e;
|
|
80
60
|
* this design makes better use of OoO cpus,
|
81
61
|
* and is noticeably faster when some values are heavily repeated.
|
82
62
|
* But it needs some additional workspace for intermediate tables.
|
83
|
-
* `workSpace`
|
63
|
+
* `workSpace` must be a U32 table of size >= HIST_WKSP_SIZE_U32.
|
84
64
|
* @return : largest histogram frequency,
|
85
|
-
* or an error code (notably when histogram
|
65
|
+
* or an error code (notably when histogram's alphabet is larger than *maxSymbolValuePtr) */
|
86
66
|
static size_t HIST_count_parallel_wksp(
|
87
67
|
unsigned* count, unsigned* maxSymbolValuePtr,
|
88
68
|
const void* source, size_t sourceSize,
|
@@ -91,22 +71,21 @@ static size_t HIST_count_parallel_wksp(
|
|
91
71
|
{
|
92
72
|
const BYTE* ip = (const BYTE*)source;
|
93
73
|
const BYTE* const iend = ip+sourceSize;
|
94
|
-
|
74
|
+
size_t const countSize = (*maxSymbolValuePtr + 1) * sizeof(*count);
|
95
75
|
unsigned max=0;
|
96
76
|
U32* const Counting1 = workSpace;
|
97
77
|
U32* const Counting2 = Counting1 + 256;
|
98
78
|
U32* const Counting3 = Counting2 + 256;
|
99
79
|
U32* const Counting4 = Counting3 + 256;
|
100
80
|
|
101
|
-
memset(workSpace, 0, 4*256*sizeof(unsigned));
|
102
|
-
|
103
81
|
/* safety checks */
|
82
|
+
assert(*maxSymbolValuePtr <= 255);
|
104
83
|
if (!sourceSize) {
|
105
|
-
|
84
|
+
ZSTD_memset(count, 0, countSize);
|
106
85
|
*maxSymbolValuePtr = 0;
|
107
86
|
return 0;
|
108
87
|
}
|
109
|
-
|
88
|
+
ZSTD_memset(workSpace, 0, 4*256*sizeof(unsigned));
|
110
89
|
|
111
90
|
/* by stripes of 16 bytes */
|
112
91
|
{ U32 cached = MEM_read32(ip); ip += 4;
|
@@ -138,21 +117,18 @@ static size_t HIST_count_parallel_wksp(
|
|
138
117
|
/* finish last symbols */
|
139
118
|
while (ip<iend) Counting1[*ip++]++;
|
140
119
|
|
141
|
-
if (check) { /* verify stats will fit into destination table */
|
142
|
-
U32 s; for (s=255; s>maxSymbolValue; s--) {
|
143
|
-
Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s];
|
144
|
-
if (Counting1[s]) return ERROR(maxSymbolValue_tooSmall);
|
145
|
-
} }
|
146
|
-
|
147
120
|
{ U32 s;
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
if (count[s] > max) max = count[s];
|
121
|
+
for (s=0; s<256; s++) {
|
122
|
+
Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s];
|
123
|
+
if (Counting1[s] > max) max = Counting1[s];
|
152
124
|
} }
|
153
125
|
|
154
|
-
|
155
|
-
|
126
|
+
{ unsigned maxSymbolValue = 255;
|
127
|
+
while (!Counting1[maxSymbolValue]) maxSymbolValue--;
|
128
|
+
if (check && maxSymbolValue > *maxSymbolValuePtr) return ERROR(maxSymbolValue_tooSmall);
|
129
|
+
*maxSymbolValuePtr = maxSymbolValue;
|
130
|
+
ZSTD_memmove(count, Counting1, countSize); /* in case count & Counting1 are overlapping */
|
131
|
+
}
|
156
132
|
return (size_t)max;
|
157
133
|
}
|
158
134
|
|
@@ -172,14 +148,6 @@ size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
|
|
172
148
|
return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, trustInput, (U32*)workSpace);
|
173
149
|
}
|
174
150
|
|
175
|
-
/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */
|
176
|
-
size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr,
|
177
|
-
const void* source, size_t sourceSize)
|
178
|
-
{
|
179
|
-
unsigned tmpCounters[HIST_WKSP_SIZE_U32];
|
180
|
-
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters, sizeof(tmpCounters));
|
181
|
-
}
|
182
|
-
|
183
151
|
/* HIST_count_wksp() :
|
184
152
|
* Same as HIST_count(), but using an externally provided scratch buffer.
|
185
153
|
* `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */
|
@@ -195,9 +163,19 @@ size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
|
|
195
163
|
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace, workSpaceSize);
|
196
164
|
}
|
197
165
|
|
166
|
+
#ifndef ZSTD_NO_UNUSED_FUNCTIONS
|
167
|
+
/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */
|
168
|
+
size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr,
|
169
|
+
const void* source, size_t sourceSize)
|
170
|
+
{
|
171
|
+
unsigned tmpCounters[HIST_WKSP_SIZE_U32];
|
172
|
+
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters, sizeof(tmpCounters));
|
173
|
+
}
|
174
|
+
|
198
175
|
size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr,
|
199
176
|
const void* src, size_t srcSize)
|
200
177
|
{
|
201
178
|
unsigned tmpCounters[HIST_WKSP_SIZE_U32];
|
202
179
|
return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters, sizeof(tmpCounters));
|
203
180
|
}
|
181
|
+
#endif
|