zstdlib 0.7.0-x86-mingw32 → 0.10.0-x86-mingw32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGES.md +20 -0
- data/README.md +7 -1
- data/Rakefile +38 -8
- data/ext/{zstdlib → zstdlib_c}/extconf.rb +11 -6
- data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.2/zstdlib.c +2 -2
- data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.3/zstdlib.c +2 -2
- data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.4/zstdlib.c +2 -2
- data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.5/zstdlib.c +2 -2
- data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.6/zstdlib.c +2 -2
- data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.7/zstdlib.c +2 -2
- data/ext/zstdlib_c/ruby/zlib-3.0/zstdlib.c +4994 -0
- data/ext/zstdlib_c/ruby/zlib-3.1/zstdlib.c +5076 -0
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/adler32.c +0 -0
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/compress.c +0 -0
- data/ext/zstdlib_c/zlib-1.2.12/crc32.c +1116 -0
- data/ext/zstdlib_c/zlib-1.2.12/crc32.h +9446 -0
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/deflate.c +78 -30
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/deflate.h +12 -15
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/gzclose.c +0 -0
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/gzguts.h +3 -2
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/gzlib.c +5 -3
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/gzread.c +5 -7
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/gzwrite.c +25 -13
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/infback.c +2 -1
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inffast.c +14 -14
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inffast.h +0 -0
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inffixed.h +0 -0
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inflate.c +39 -8
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inflate.h +3 -2
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inftrees.c +3 -3
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inftrees.h +0 -0
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/trees.c +27 -48
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/trees.h +0 -0
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/uncompr.c +0 -0
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/zconf.h +0 -0
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/zlib.h +123 -100
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/zutil.c +2 -2
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/zutil.h +12 -9
- data/ext/{zstdlib → zstdlib_c}/zlib.mk +0 -0
- data/ext/{zstdlib → zstdlib_c}/zlibwrapper/zlibwrapper.c +1 -5
- data/ext/{zstdlib → zstdlib_c}/zlibwrapper.mk +0 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/bitstream.h +46 -22
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/compiler.h +335 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/cpu.h +1 -3
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/debug.c +1 -1
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/debug.h +12 -19
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/entropy_common.c +368 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/error_private.c +2 -1
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/error_private.h +159 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/fse.h +41 -12
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/fse_decompress.c +139 -22
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/huf.h +47 -23
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/mem.h +87 -98
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/pool.c +34 -23
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/pool.h +4 -4
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/portability_macros.h +137 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/threading.c +6 -5
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/threading.h +0 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/xxhash.c +24 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/xxhash.h +5686 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/zstd_common.c +10 -10
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/zstd_deps.h +111 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/zstd_internal.h +191 -145
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/zstd_trace.h +163 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/clevels.h +134 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/fse_compress.c +89 -46
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/hist.c +27 -29
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/hist.h +2 -2
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/huf_compress.c +1370 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress.c +2917 -868
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_internal.h +458 -125
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_literals.c +12 -11
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_literals.h +4 -2
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_sequences.c +41 -18
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_sequences.h +1 -1
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_superblock.c +26 -298
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_superblock.h +1 -1
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_cwksp.h +234 -83
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_double_fast.c +313 -138
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_double_fast.h +1 -1
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_fast.c +329 -150
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_fast.h +1 -1
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_lazy.c +2104 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_lazy.h +125 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_ldm.c +321 -216
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_ldm.h +9 -2
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_ldm_geartab.h +106 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_opt.c +412 -166
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_opt.h +1 -1
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstdmt_compress.c +169 -453
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstdmt_compress.h +113 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/huf_decompress.c +1044 -403
- data/ext/zstdlib_c/zstd-1.5.2/lib/decompress/huf_decompress_amd64.S +585 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_ddict.c +9 -9
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_ddict.h +2 -2
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress.c +450 -105
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress_block.c +913 -273
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress_block.h +14 -5
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress_internal.h +59 -12
- data/ext/zstdlib_c/zstd-1.5.2/lib/zdict.h +452 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/zstd.h +699 -214
- data/ext/{zstdlib/zstd-1.4.5/lib/common → zstdlib_c/zstd-1.5.2/lib}/zstd_errors.h +2 -1
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzclose.c +0 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzcompatibility.h +1 -1
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzguts.h +0 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzlib.c +0 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzread.c +0 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzwrite.c +0 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/zstd_zlibwrapper.c +133 -44
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/zstd_zlibwrapper.h +1 -1
- data/ext/zstdlib_c/zstd.mk +15 -0
- data/lib/2.4/zstdlib_c.so +0 -0
- data/lib/2.5/zstdlib_c.so +0 -0
- data/lib/2.6/zstdlib_c.so +0 -0
- data/lib/2.7/zstdlib_c.so +0 -0
- data/lib/3.0/zstdlib_c.so +0 -0
- data/lib/3.1/zstdlib_c.so +0 -0
- data/lib/zstdlib.rb +2 -2
- metadata +125 -116
- data/ext/zstdlib/zlib-1.2.11/crc32.c +0 -442
- data/ext/zstdlib/zlib-1.2.11/crc32.h +0 -441
- data/ext/zstdlib/zstd-1.4.5/lib/common/compiler.h +0 -175
- data/ext/zstdlib/zstd-1.4.5/lib/common/entropy_common.c +0 -216
- data/ext/zstdlib/zstd-1.4.5/lib/common/error_private.h +0 -80
- data/ext/zstdlib/zstd-1.4.5/lib/common/xxhash.c +0 -864
- data/ext/zstdlib/zstd-1.4.5/lib/common/xxhash.h +0 -285
- data/ext/zstdlib/zstd-1.4.5/lib/compress/huf_compress.c +0 -798
- data/ext/zstdlib/zstd-1.4.5/lib/compress/zstd_lazy.c +0 -1138
- data/ext/zstdlib/zstd-1.4.5/lib/compress/zstd_lazy.h +0 -67
- data/ext/zstdlib/zstd-1.4.5/lib/compress/zstdmt_compress.h +0 -192
- data/ext/zstdlib/zstd.mk +0 -14
- data/lib/2.2/zstdlib.so +0 -0
- data/lib/2.3/zstdlib.so +0 -0
- data/lib/2.4/zstdlib.so +0 -0
- data/lib/2.5/zstdlib.so +0 -0
- data/lib/2.6/zstdlib.so +0 -0
- data/lib/2.7/zstdlib.so +0 -0
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c)
|
2
|
+
* Copyright (c) Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -15,7 +15,7 @@
|
|
15
15
|
|
16
16
|
size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
17
17
|
{
|
18
|
-
BYTE* const ostart = (BYTE*
|
18
|
+
BYTE* const ostart = (BYTE*)dst;
|
19
19
|
U32 const flSize = 1 + (srcSize>31) + (srcSize>4095);
|
20
20
|
|
21
21
|
RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall, "");
|
@@ -35,14 +35,14 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src,
|
|
35
35
|
assert(0);
|
36
36
|
}
|
37
37
|
|
38
|
-
|
38
|
+
ZSTD_memcpy(ostart + flSize, src, srcSize);
|
39
39
|
DEBUGLOG(5, "Raw literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize));
|
40
40
|
return srcSize + flSize;
|
41
41
|
}
|
42
42
|
|
43
43
|
size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
44
44
|
{
|
45
|
-
BYTE* const ostart = (BYTE*
|
45
|
+
BYTE* const ostart = (BYTE*)dst;
|
46
46
|
U32 const flSize = 1 + (srcSize>31) + (srcSize>4095);
|
47
47
|
|
48
48
|
(void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */
|
@@ -73,7 +73,8 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
|
|
73
73
|
void* dst, size_t dstCapacity,
|
74
74
|
const void* src, size_t srcSize,
|
75
75
|
void* entropyWorkspace, size_t entropyWorkspaceSize,
|
76
|
-
const int bmi2
|
76
|
+
const int bmi2,
|
77
|
+
unsigned suspectUncompressible)
|
77
78
|
{
|
78
79
|
size_t const minGain = ZSTD_minGain(srcSize, strategy);
|
79
80
|
size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
|
@@ -86,7 +87,7 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
|
|
86
87
|
disableLiteralCompression, (U32)srcSize);
|
87
88
|
|
88
89
|
/* Prepare nextEntropy assuming reusing the existing table */
|
89
|
-
|
90
|
+
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
|
90
91
|
|
91
92
|
if (disableLiteralCompression)
|
92
93
|
return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
|
@@ -105,11 +106,11 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
|
|
105
106
|
HUF_compress1X_repeat(
|
106
107
|
ostart+lhSize, dstCapacity-lhSize, src, srcSize,
|
107
108
|
HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize,
|
108
|
-
(HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2) :
|
109
|
+
(HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2, suspectUncompressible) :
|
109
110
|
HUF_compress4X_repeat(
|
110
111
|
ostart+lhSize, dstCapacity-lhSize, src, srcSize,
|
111
112
|
HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize,
|
112
|
-
(HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2);
|
113
|
+
(HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2, suspectUncompressible);
|
113
114
|
if (repeat != HUF_repeat_none) {
|
114
115
|
/* reused the existing table */
|
115
116
|
DEBUGLOG(5, "Reusing previous huffman table");
|
@@ -117,12 +118,12 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
|
|
117
118
|
}
|
118
119
|
}
|
119
120
|
|
120
|
-
if ((cLitSize==0)
|
121
|
-
|
121
|
+
if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) {
|
122
|
+
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
|
122
123
|
return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
|
123
124
|
}
|
124
125
|
if (cLitSize==1) {
|
125
|
-
|
126
|
+
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
|
126
127
|
return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
|
127
128
|
}
|
128
129
|
|
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c)
|
2
|
+
* Copyright (c) Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -18,12 +18,14 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src,
|
|
18
18
|
|
19
19
|
size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
20
20
|
|
21
|
+
/* If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
|
21
22
|
size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
|
22
23
|
ZSTD_hufCTables_t* nextHuf,
|
23
24
|
ZSTD_strategy strategy, int disableLiteralCompression,
|
24
25
|
void* dst, size_t dstCapacity,
|
25
26
|
const void* src, size_t srcSize,
|
26
27
|
void* entropyWorkspace, size_t entropyWorkspaceSize,
|
27
|
-
const int bmi2
|
28
|
+
const int bmi2,
|
29
|
+
unsigned suspectUncompressible);
|
28
30
|
|
29
31
|
#endif /* ZSTD_COMPRESS_LITERALS_H */
|
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c)
|
2
|
+
* Copyright (c) Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -50,6 +50,19 @@ static unsigned ZSTD_getFSEMaxSymbolValue(FSE_CTable const* ctable) {
|
|
50
50
|
return maxSymbolValue;
|
51
51
|
}
|
52
52
|
|
53
|
+
/**
|
54
|
+
* Returns true if we should use ncount=-1 else we should
|
55
|
+
* use ncount=1 for low probability symbols instead.
|
56
|
+
*/
|
57
|
+
static unsigned ZSTD_useLowProbCount(size_t const nbSeq)
|
58
|
+
{
|
59
|
+
/* Heuristic: This should cover most blocks <= 16K and
|
60
|
+
* start to fade out after 16K to about 32K depending on
|
61
|
+
* comprssibility.
|
62
|
+
*/
|
63
|
+
return nbSeq >= 2048;
|
64
|
+
}
|
65
|
+
|
53
66
|
/**
|
54
67
|
* Returns the cost in bytes of encoding the normalized count header.
|
55
68
|
* Returns an error if any of the helper functions return an error.
|
@@ -60,7 +73,7 @@ static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max,
|
|
60
73
|
BYTE wksp[FSE_NCOUNTBOUND];
|
61
74
|
S16 norm[MaxSeq + 1];
|
62
75
|
const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max);
|
63
|
-
FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max), "");
|
76
|
+
FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max, ZSTD_useLowProbCount(nbSeq)), "");
|
64
77
|
return FSE_writeNCount(wksp, sizeof(wksp), norm, max, tableLog);
|
65
78
|
}
|
66
79
|
|
@@ -72,6 +85,8 @@ static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t
|
|
72
85
|
{
|
73
86
|
unsigned cost = 0;
|
74
87
|
unsigned s;
|
88
|
+
|
89
|
+
assert(total > 0);
|
75
90
|
for (s = 0; s <= max; ++s) {
|
76
91
|
unsigned norm = (unsigned)((256 * count[s]) / total);
|
77
92
|
if (count[s] != 0 && norm == 0)
|
@@ -219,6 +234,11 @@ ZSTD_selectEncodingType(
|
|
219
234
|
return set_compressed;
|
220
235
|
}
|
221
236
|
|
237
|
+
typedef struct {
|
238
|
+
S16 norm[MaxSeq + 1];
|
239
|
+
U32 wksp[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(MaxSeq, MaxFSELog)];
|
240
|
+
} ZSTD_BuildCTableWksp;
|
241
|
+
|
222
242
|
size_t
|
223
243
|
ZSTD_buildCTable(void* dst, size_t dstCapacity,
|
224
244
|
FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type,
|
@@ -239,13 +259,13 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
|
|
239
259
|
*op = codeTable[0];
|
240
260
|
return 1;
|
241
261
|
case set_repeat:
|
242
|
-
|
262
|
+
ZSTD_memcpy(nextCTable, prevCTable, prevCTableSize);
|
243
263
|
return 0;
|
244
264
|
case set_basic:
|
245
265
|
FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize), ""); /* note : could be pre-calculated */
|
246
266
|
return 0;
|
247
267
|
case set_compressed: {
|
248
|
-
|
268
|
+
ZSTD_BuildCTableWksp* wksp = (ZSTD_BuildCTableWksp*)entropyWorkspace;
|
249
269
|
size_t nbSeq_1 = nbSeq;
|
250
270
|
const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max);
|
251
271
|
if (count[codeTable[nbSeq-1]] > 1) {
|
@@ -253,10 +273,13 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
|
|
253
273
|
nbSeq_1--;
|
254
274
|
}
|
255
275
|
assert(nbSeq_1 > 1);
|
256
|
-
|
257
|
-
|
276
|
+
assert(entropyWorkspaceSize >= sizeof(ZSTD_BuildCTableWksp));
|
277
|
+
(void)entropyWorkspaceSize;
|
278
|
+
FORWARD_IF_ERROR(FSE_normalizeCount(wksp->norm, tableLog, count, nbSeq_1, max, ZSTD_useLowProbCount(nbSeq_1)), "FSE_normalizeCount failed");
|
279
|
+
assert(oend >= op);
|
280
|
+
{ size_t const NCountSize = FSE_writeNCount(op, (size_t)(oend - op), wksp->norm, max, tableLog); /* overflow protected */
|
258
281
|
FORWARD_IF_ERROR(NCountSize, "FSE_writeNCount failed");
|
259
|
-
FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog,
|
282
|
+
FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, wksp->norm, max, tableLog, wksp->wksp, sizeof(wksp->wksp)), "FSE_buildCTable_wksp failed");
|
260
283
|
return NCountSize;
|
261
284
|
}
|
262
285
|
}
|
@@ -290,19 +313,19 @@ ZSTD_encodeSequences_body(
|
|
290
313
|
FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]);
|
291
314
|
BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]);
|
292
315
|
if (MEM_32bits()) BIT_flushBits(&blockStream);
|
293
|
-
BIT_addBits(&blockStream, sequences[nbSeq-1].
|
316
|
+
BIT_addBits(&blockStream, sequences[nbSeq-1].mlBase, ML_bits[mlCodeTable[nbSeq-1]]);
|
294
317
|
if (MEM_32bits()) BIT_flushBits(&blockStream);
|
295
318
|
if (longOffsets) {
|
296
319
|
U32 const ofBits = ofCodeTable[nbSeq-1];
|
297
320
|
unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
|
298
321
|
if (extraBits) {
|
299
|
-
BIT_addBits(&blockStream, sequences[nbSeq-1].
|
322
|
+
BIT_addBits(&blockStream, sequences[nbSeq-1].offBase, extraBits);
|
300
323
|
BIT_flushBits(&blockStream);
|
301
324
|
}
|
302
|
-
BIT_addBits(&blockStream, sequences[nbSeq-1].
|
325
|
+
BIT_addBits(&blockStream, sequences[nbSeq-1].offBase >> extraBits,
|
303
326
|
ofBits - extraBits);
|
304
327
|
} else {
|
305
|
-
BIT_addBits(&blockStream, sequences[nbSeq-1].
|
328
|
+
BIT_addBits(&blockStream, sequences[nbSeq-1].offBase, ofCodeTable[nbSeq-1]);
|
306
329
|
}
|
307
330
|
BIT_flushBits(&blockStream);
|
308
331
|
|
@@ -316,8 +339,8 @@ ZSTD_encodeSequences_body(
|
|
316
339
|
U32 const mlBits = ML_bits[mlCode];
|
317
340
|
DEBUGLOG(6, "encoding: litlen:%2u - matchlen:%2u - offCode:%7u",
|
318
341
|
(unsigned)sequences[n].litLength,
|
319
|
-
(unsigned)sequences[n].
|
320
|
-
(unsigned)sequences[n].
|
342
|
+
(unsigned)sequences[n].mlBase + MINMATCH,
|
343
|
+
(unsigned)sequences[n].offBase);
|
321
344
|
/* 32b*/ /* 64b*/
|
322
345
|
/* (7)*/ /* (7)*/
|
323
346
|
FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 15 */ /* 15 */
|
@@ -328,18 +351,18 @@ ZSTD_encodeSequences_body(
|
|
328
351
|
BIT_flushBits(&blockStream); /* (7)*/
|
329
352
|
BIT_addBits(&blockStream, sequences[n].litLength, llBits);
|
330
353
|
if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream);
|
331
|
-
BIT_addBits(&blockStream, sequences[n].
|
354
|
+
BIT_addBits(&blockStream, sequences[n].mlBase, mlBits);
|
332
355
|
if (MEM_32bits() || (ofBits+mlBits+llBits > 56)) BIT_flushBits(&blockStream);
|
333
356
|
if (longOffsets) {
|
334
357
|
unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
|
335
358
|
if (extraBits) {
|
336
|
-
BIT_addBits(&blockStream, sequences[n].
|
359
|
+
BIT_addBits(&blockStream, sequences[n].offBase, extraBits);
|
337
360
|
BIT_flushBits(&blockStream); /* (7)*/
|
338
361
|
}
|
339
|
-
BIT_addBits(&blockStream, sequences[n].
|
362
|
+
BIT_addBits(&blockStream, sequences[n].offBase >> extraBits,
|
340
363
|
ofBits - extraBits); /* 31 */
|
341
364
|
} else {
|
342
|
-
BIT_addBits(&blockStream, sequences[n].
|
365
|
+
BIT_addBits(&blockStream, sequences[n].offBase, ofBits); /* 31 */
|
343
366
|
}
|
344
367
|
BIT_flushBits(&blockStream); /* (7)*/
|
345
368
|
DEBUGLOG(7, "remaining space : %i", (int)(blockStream.endPtr - blockStream.ptr));
|
@@ -376,7 +399,7 @@ ZSTD_encodeSequences_default(
|
|
376
399
|
|
377
400
|
#if DYNAMIC_BMI2
|
378
401
|
|
379
|
-
static
|
402
|
+
static BMI2_TARGET_ATTRIBUTE size_t
|
380
403
|
ZSTD_encodeSequences_bmi2(
|
381
404
|
void* dst, size_t dstCapacity,
|
382
405
|
FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
|