zstdlib 0.7.0-x86-mingw32 → 0.10.0-x86-mingw32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGES.md +20 -0
- data/README.md +7 -1
- data/Rakefile +38 -8
- data/ext/{zstdlib → zstdlib_c}/extconf.rb +11 -6
- data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.2/zstdlib.c +2 -2
- data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.3/zstdlib.c +2 -2
- data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.4/zstdlib.c +2 -2
- data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.5/zstdlib.c +2 -2
- data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.6/zstdlib.c +2 -2
- data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.7/zstdlib.c +2 -2
- data/ext/zstdlib_c/ruby/zlib-3.0/zstdlib.c +4994 -0
- data/ext/zstdlib_c/ruby/zlib-3.1/zstdlib.c +5076 -0
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/adler32.c +0 -0
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/compress.c +0 -0
- data/ext/zstdlib_c/zlib-1.2.12/crc32.c +1116 -0
- data/ext/zstdlib_c/zlib-1.2.12/crc32.h +9446 -0
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/deflate.c +78 -30
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/deflate.h +12 -15
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/gzclose.c +0 -0
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/gzguts.h +3 -2
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/gzlib.c +5 -3
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/gzread.c +5 -7
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/gzwrite.c +25 -13
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/infback.c +2 -1
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inffast.c +14 -14
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inffast.h +0 -0
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inffixed.h +0 -0
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inflate.c +39 -8
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inflate.h +3 -2
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inftrees.c +3 -3
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inftrees.h +0 -0
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/trees.c +27 -48
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/trees.h +0 -0
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/uncompr.c +0 -0
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/zconf.h +0 -0
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/zlib.h +123 -100
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/zutil.c +2 -2
- data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/zutil.h +12 -9
- data/ext/{zstdlib → zstdlib_c}/zlib.mk +0 -0
- data/ext/{zstdlib → zstdlib_c}/zlibwrapper/zlibwrapper.c +1 -5
- data/ext/{zstdlib → zstdlib_c}/zlibwrapper.mk +0 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/bitstream.h +46 -22
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/compiler.h +335 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/cpu.h +1 -3
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/debug.c +1 -1
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/debug.h +12 -19
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/entropy_common.c +368 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/error_private.c +2 -1
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/error_private.h +159 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/fse.h +41 -12
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/fse_decompress.c +139 -22
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/huf.h +47 -23
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/mem.h +87 -98
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/pool.c +34 -23
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/pool.h +4 -4
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/portability_macros.h +137 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/threading.c +6 -5
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/threading.h +0 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/xxhash.c +24 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/xxhash.h +5686 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/zstd_common.c +10 -10
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/zstd_deps.h +111 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/zstd_internal.h +191 -145
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/zstd_trace.h +163 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/clevels.h +134 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/fse_compress.c +89 -46
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/hist.c +27 -29
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/hist.h +2 -2
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/huf_compress.c +1370 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress.c +2917 -868
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_internal.h +458 -125
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_literals.c +12 -11
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_literals.h +4 -2
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_sequences.c +41 -18
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_sequences.h +1 -1
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_superblock.c +26 -298
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_superblock.h +1 -1
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_cwksp.h +234 -83
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_double_fast.c +313 -138
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_double_fast.h +1 -1
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_fast.c +329 -150
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_fast.h +1 -1
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_lazy.c +2104 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_lazy.h +125 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_ldm.c +321 -216
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_ldm.h +9 -2
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_ldm_geartab.h +106 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_opt.c +412 -166
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_opt.h +1 -1
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstdmt_compress.c +169 -453
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstdmt_compress.h +113 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/huf_decompress.c +1044 -403
- data/ext/zstdlib_c/zstd-1.5.2/lib/decompress/huf_decompress_amd64.S +585 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_ddict.c +9 -9
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_ddict.h +2 -2
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress.c +450 -105
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress_block.c +913 -273
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress_block.h +14 -5
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress_internal.h +59 -12
- data/ext/zstdlib_c/zstd-1.5.2/lib/zdict.h +452 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/zstd.h +699 -214
- data/ext/{zstdlib/zstd-1.4.5/lib/common → zstdlib_c/zstd-1.5.2/lib}/zstd_errors.h +2 -1
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzclose.c +0 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzcompatibility.h +1 -1
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzguts.h +0 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzlib.c +0 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzread.c +0 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzwrite.c +0 -0
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/zstd_zlibwrapper.c +133 -44
- data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/zstd_zlibwrapper.h +1 -1
- data/ext/zstdlib_c/zstd.mk +15 -0
- data/lib/2.4/zstdlib_c.so +0 -0
- data/lib/2.5/zstdlib_c.so +0 -0
- data/lib/2.6/zstdlib_c.so +0 -0
- data/lib/2.7/zstdlib_c.so +0 -0
- data/lib/3.0/zstdlib_c.so +0 -0
- data/lib/3.1/zstdlib_c.so +0 -0
- data/lib/zstdlib.rb +2 -2
- metadata +125 -116
- data/ext/zstdlib/zlib-1.2.11/crc32.c +0 -442
- data/ext/zstdlib/zlib-1.2.11/crc32.h +0 -441
- data/ext/zstdlib/zstd-1.4.5/lib/common/compiler.h +0 -175
- data/ext/zstdlib/zstd-1.4.5/lib/common/entropy_common.c +0 -216
- data/ext/zstdlib/zstd-1.4.5/lib/common/error_private.h +0 -80
- data/ext/zstdlib/zstd-1.4.5/lib/common/xxhash.c +0 -864
- data/ext/zstdlib/zstd-1.4.5/lib/common/xxhash.h +0 -285
- data/ext/zstdlib/zstd-1.4.5/lib/compress/huf_compress.c +0 -798
- data/ext/zstdlib/zstd-1.4.5/lib/compress/zstd_lazy.c +0 -1138
- data/ext/zstdlib/zstd-1.4.5/lib/compress/zstd_lazy.h +0 -67
- data/ext/zstdlib/zstd-1.4.5/lib/compress/zstdmt_compress.h +0 -192
- data/ext/zstdlib/zstd.mk +0 -14
- data/lib/2.2/zstdlib.so +0 -0
- data/lib/2.3/zstdlib.so +0 -0
- data/lib/2.4/zstdlib.so +0 -0
- data/lib/2.5/zstdlib.so +0 -0
- data/lib/2.6/zstdlib.so +0 -0
- data/lib/2.7/zstdlib.so +0 -0
data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_superblock.c
RENAMED
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c)
|
2
|
+
* Copyright (c) Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -15,288 +15,10 @@
|
|
15
15
|
|
16
16
|
#include "../common/zstd_internal.h" /* ZSTD_getSequenceLength */
|
17
17
|
#include "hist.h" /* HIST_countFast_wksp */
|
18
|
-
#include "zstd_compress_internal.h"
|
18
|
+
#include "zstd_compress_internal.h" /* ZSTD_[huf|fse|entropy]CTablesMetadata_t */
|
19
19
|
#include "zstd_compress_sequences.h"
|
20
20
|
#include "zstd_compress_literals.h"
|
21
21
|
|
22
|
-
/*-*************************************
|
23
|
-
* Superblock entropy buffer structs
|
24
|
-
***************************************/
|
25
|
-
/** ZSTD_hufCTablesMetadata_t :
|
26
|
-
* Stores Literals Block Type for a super-block in hType, and
|
27
|
-
* huffman tree description in hufDesBuffer.
|
28
|
-
* hufDesSize refers to the size of huffman tree description in bytes.
|
29
|
-
* This metadata is populated in ZSTD_buildSuperBlockEntropy_literal() */
|
30
|
-
typedef struct {
|
31
|
-
symbolEncodingType_e hType;
|
32
|
-
BYTE hufDesBuffer[500]; /* TODO give name to this value */
|
33
|
-
size_t hufDesSize;
|
34
|
-
} ZSTD_hufCTablesMetadata_t;
|
35
|
-
|
36
|
-
/** ZSTD_fseCTablesMetadata_t :
|
37
|
-
* Stores symbol compression modes for a super-block in {ll, ol, ml}Type, and
|
38
|
-
* fse tables in fseTablesBuffer.
|
39
|
-
* fseTablesSize refers to the size of fse tables in bytes.
|
40
|
-
* This metadata is populated in ZSTD_buildSuperBlockEntropy_sequences() */
|
41
|
-
typedef struct {
|
42
|
-
symbolEncodingType_e llType;
|
43
|
-
symbolEncodingType_e ofType;
|
44
|
-
symbolEncodingType_e mlType;
|
45
|
-
BYTE fseTablesBuffer[500]; /* TODO give name to this value */
|
46
|
-
size_t fseTablesSize;
|
47
|
-
size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_compressSubBlock_sequences() */
|
48
|
-
} ZSTD_fseCTablesMetadata_t;
|
49
|
-
|
50
|
-
typedef struct {
|
51
|
-
ZSTD_hufCTablesMetadata_t hufMetadata;
|
52
|
-
ZSTD_fseCTablesMetadata_t fseMetadata;
|
53
|
-
} ZSTD_entropyCTablesMetadata_t;
|
54
|
-
|
55
|
-
|
56
|
-
/** ZSTD_buildSuperBlockEntropy_literal() :
|
57
|
-
* Builds entropy for the super-block literals.
|
58
|
-
* Stores literals block type (raw, rle, compressed, repeat) and
|
59
|
-
* huffman description table to hufMetadata.
|
60
|
-
* @return : size of huffman description table or error code */
|
61
|
-
static size_t ZSTD_buildSuperBlockEntropy_literal(void* const src, size_t srcSize,
|
62
|
-
const ZSTD_hufCTables_t* prevHuf,
|
63
|
-
ZSTD_hufCTables_t* nextHuf,
|
64
|
-
ZSTD_hufCTablesMetadata_t* hufMetadata,
|
65
|
-
const int disableLiteralsCompression,
|
66
|
-
void* workspace, size_t wkspSize)
|
67
|
-
{
|
68
|
-
BYTE* const wkspStart = (BYTE*)workspace;
|
69
|
-
BYTE* const wkspEnd = wkspStart + wkspSize;
|
70
|
-
BYTE* const countWkspStart = wkspStart;
|
71
|
-
unsigned* const countWksp = (unsigned*)workspace;
|
72
|
-
const size_t countWkspSize = (HUF_SYMBOLVALUE_MAX + 1) * sizeof(unsigned);
|
73
|
-
BYTE* const nodeWksp = countWkspStart + countWkspSize;
|
74
|
-
const size_t nodeWkspSize = wkspEnd-nodeWksp;
|
75
|
-
unsigned maxSymbolValue = 255;
|
76
|
-
unsigned huffLog = HUF_TABLELOG_DEFAULT;
|
77
|
-
HUF_repeat repeat = prevHuf->repeatMode;
|
78
|
-
|
79
|
-
DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy_literal (srcSize=%zu)", srcSize);
|
80
|
-
|
81
|
-
/* Prepare nextEntropy assuming reusing the existing table */
|
82
|
-
memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
|
83
|
-
|
84
|
-
if (disableLiteralsCompression) {
|
85
|
-
DEBUGLOG(5, "set_basic - disabled");
|
86
|
-
hufMetadata->hType = set_basic;
|
87
|
-
return 0;
|
88
|
-
}
|
89
|
-
|
90
|
-
/* small ? don't even attempt compression (speed opt) */
|
91
|
-
# define COMPRESS_LITERALS_SIZE_MIN 63
|
92
|
-
{ size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN;
|
93
|
-
if (srcSize <= minLitSize) {
|
94
|
-
DEBUGLOG(5, "set_basic - too small");
|
95
|
-
hufMetadata->hType = set_basic;
|
96
|
-
return 0;
|
97
|
-
}
|
98
|
-
}
|
99
|
-
|
100
|
-
/* Scan input and build symbol stats */
|
101
|
-
{ size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)src, srcSize, workspace, wkspSize);
|
102
|
-
FORWARD_IF_ERROR(largest, "HIST_count_wksp failed");
|
103
|
-
if (largest == srcSize) {
|
104
|
-
DEBUGLOG(5, "set_rle");
|
105
|
-
hufMetadata->hType = set_rle;
|
106
|
-
return 0;
|
107
|
-
}
|
108
|
-
if (largest <= (srcSize >> 7)+4) {
|
109
|
-
DEBUGLOG(5, "set_basic - no gain");
|
110
|
-
hufMetadata->hType = set_basic;
|
111
|
-
return 0;
|
112
|
-
}
|
113
|
-
}
|
114
|
-
|
115
|
-
/* Validate the previous Huffman table */
|
116
|
-
if (repeat == HUF_repeat_check && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) {
|
117
|
-
repeat = HUF_repeat_none;
|
118
|
-
}
|
119
|
-
|
120
|
-
/* Build Huffman Tree */
|
121
|
-
memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable));
|
122
|
-
huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
|
123
|
-
{ size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp,
|
124
|
-
maxSymbolValue, huffLog,
|
125
|
-
nodeWksp, nodeWkspSize);
|
126
|
-
FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp");
|
127
|
-
huffLog = (U32)maxBits;
|
128
|
-
{ /* Build and write the CTable */
|
129
|
-
size_t const newCSize = HUF_estimateCompressedSize(
|
130
|
-
(HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue);
|
131
|
-
size_t const hSize = HUF_writeCTable(
|
132
|
-
hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer),
|
133
|
-
(HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog);
|
134
|
-
/* Check against repeating the previous CTable */
|
135
|
-
if (repeat != HUF_repeat_none) {
|
136
|
-
size_t const oldCSize = HUF_estimateCompressedSize(
|
137
|
-
(HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue);
|
138
|
-
if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) {
|
139
|
-
DEBUGLOG(5, "set_repeat - smaller");
|
140
|
-
memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
|
141
|
-
hufMetadata->hType = set_repeat;
|
142
|
-
return 0;
|
143
|
-
}
|
144
|
-
}
|
145
|
-
if (newCSize + hSize >= srcSize) {
|
146
|
-
DEBUGLOG(5, "set_basic - no gains");
|
147
|
-
memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
|
148
|
-
hufMetadata->hType = set_basic;
|
149
|
-
return 0;
|
150
|
-
}
|
151
|
-
DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize);
|
152
|
-
hufMetadata->hType = set_compressed;
|
153
|
-
nextHuf->repeatMode = HUF_repeat_check;
|
154
|
-
return hSize;
|
155
|
-
}
|
156
|
-
}
|
157
|
-
}
|
158
|
-
|
159
|
-
/** ZSTD_buildSuperBlockEntropy_sequences() :
|
160
|
-
* Builds entropy for the super-block sequences.
|
161
|
-
* Stores symbol compression modes and fse table to fseMetadata.
|
162
|
-
* @return : size of fse tables or error code */
|
163
|
-
static size_t ZSTD_buildSuperBlockEntropy_sequences(seqStore_t* seqStorePtr,
|
164
|
-
const ZSTD_fseCTables_t* prevEntropy,
|
165
|
-
ZSTD_fseCTables_t* nextEntropy,
|
166
|
-
const ZSTD_CCtx_params* cctxParams,
|
167
|
-
ZSTD_fseCTablesMetadata_t* fseMetadata,
|
168
|
-
void* workspace, size_t wkspSize)
|
169
|
-
{
|
170
|
-
BYTE* const wkspStart = (BYTE*)workspace;
|
171
|
-
BYTE* const wkspEnd = wkspStart + wkspSize;
|
172
|
-
BYTE* const countWkspStart = wkspStart;
|
173
|
-
unsigned* const countWksp = (unsigned*)workspace;
|
174
|
-
const size_t countWkspSize = (MaxSeq + 1) * sizeof(unsigned);
|
175
|
-
BYTE* const cTableWksp = countWkspStart + countWkspSize;
|
176
|
-
const size_t cTableWkspSize = wkspEnd-cTableWksp;
|
177
|
-
ZSTD_strategy const strategy = cctxParams->cParams.strategy;
|
178
|
-
FSE_CTable* CTable_LitLength = nextEntropy->litlengthCTable;
|
179
|
-
FSE_CTable* CTable_OffsetBits = nextEntropy->offcodeCTable;
|
180
|
-
FSE_CTable* CTable_MatchLength = nextEntropy->matchlengthCTable;
|
181
|
-
const BYTE* const ofCodeTable = seqStorePtr->ofCode;
|
182
|
-
const BYTE* const llCodeTable = seqStorePtr->llCode;
|
183
|
-
const BYTE* const mlCodeTable = seqStorePtr->mlCode;
|
184
|
-
size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart;
|
185
|
-
BYTE* const ostart = fseMetadata->fseTablesBuffer;
|
186
|
-
BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer);
|
187
|
-
BYTE* op = ostart;
|
188
|
-
|
189
|
-
assert(cTableWkspSize >= (1 << MaxFSELog) * sizeof(FSE_FUNCTION_TYPE));
|
190
|
-
DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy_sequences (nbSeq=%zu)", nbSeq);
|
191
|
-
memset(workspace, 0, wkspSize);
|
192
|
-
|
193
|
-
fseMetadata->lastCountSize = 0;
|
194
|
-
/* convert length/distances into codes */
|
195
|
-
ZSTD_seqToCodes(seqStorePtr);
|
196
|
-
/* build CTable for Literal Lengths */
|
197
|
-
{ U32 LLtype;
|
198
|
-
unsigned max = MaxLL;
|
199
|
-
size_t const mostFrequent = HIST_countFast_wksp(countWksp, &max, llCodeTable, nbSeq, workspace, wkspSize); /* can't fail */
|
200
|
-
DEBUGLOG(5, "Building LL table");
|
201
|
-
nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode;
|
202
|
-
LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode,
|
203
|
-
countWksp, max, mostFrequent, nbSeq,
|
204
|
-
LLFSELog, prevEntropy->litlengthCTable,
|
205
|
-
LL_defaultNorm, LL_defaultNormLog,
|
206
|
-
ZSTD_defaultAllowed, strategy);
|
207
|
-
assert(set_basic < set_compressed && set_rle < set_compressed);
|
208
|
-
assert(!(LLtype < set_compressed && nextEntropy->litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
|
209
|
-
{ size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype,
|
210
|
-
countWksp, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL,
|
211
|
-
prevEntropy->litlengthCTable, sizeof(prevEntropy->litlengthCTable),
|
212
|
-
cTableWksp, cTableWkspSize);
|
213
|
-
FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for LitLens failed");
|
214
|
-
if (LLtype == set_compressed)
|
215
|
-
fseMetadata->lastCountSize = countSize;
|
216
|
-
op += countSize;
|
217
|
-
fseMetadata->llType = (symbolEncodingType_e) LLtype;
|
218
|
-
} }
|
219
|
-
/* build CTable for Offsets */
|
220
|
-
{ U32 Offtype;
|
221
|
-
unsigned max = MaxOff;
|
222
|
-
size_t const mostFrequent = HIST_countFast_wksp(countWksp, &max, ofCodeTable, nbSeq, workspace, wkspSize); /* can't fail */
|
223
|
-
/* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */
|
224
|
-
ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed;
|
225
|
-
DEBUGLOG(5, "Building OF table");
|
226
|
-
nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode;
|
227
|
-
Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode,
|
228
|
-
countWksp, max, mostFrequent, nbSeq,
|
229
|
-
OffFSELog, prevEntropy->offcodeCTable,
|
230
|
-
OF_defaultNorm, OF_defaultNormLog,
|
231
|
-
defaultPolicy, strategy);
|
232
|
-
assert(!(Offtype < set_compressed && nextEntropy->offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */
|
233
|
-
{ size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype,
|
234
|
-
countWksp, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
|
235
|
-
prevEntropy->offcodeCTable, sizeof(prevEntropy->offcodeCTable),
|
236
|
-
cTableWksp, cTableWkspSize);
|
237
|
-
FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for Offsets failed");
|
238
|
-
if (Offtype == set_compressed)
|
239
|
-
fseMetadata->lastCountSize = countSize;
|
240
|
-
op += countSize;
|
241
|
-
fseMetadata->ofType = (symbolEncodingType_e) Offtype;
|
242
|
-
} }
|
243
|
-
/* build CTable for MatchLengths */
|
244
|
-
{ U32 MLtype;
|
245
|
-
unsigned max = MaxML;
|
246
|
-
size_t const mostFrequent = HIST_countFast_wksp(countWksp, &max, mlCodeTable, nbSeq, workspace, wkspSize); /* can't fail */
|
247
|
-
DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op));
|
248
|
-
nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode;
|
249
|
-
MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode,
|
250
|
-
countWksp, max, mostFrequent, nbSeq,
|
251
|
-
MLFSELog, prevEntropy->matchlengthCTable,
|
252
|
-
ML_defaultNorm, ML_defaultNormLog,
|
253
|
-
ZSTD_defaultAllowed, strategy);
|
254
|
-
assert(!(MLtype < set_compressed && nextEntropy->matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
|
255
|
-
{ size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype,
|
256
|
-
countWksp, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML,
|
257
|
-
prevEntropy->matchlengthCTable, sizeof(prevEntropy->matchlengthCTable),
|
258
|
-
cTableWksp, cTableWkspSize);
|
259
|
-
FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for MatchLengths failed");
|
260
|
-
if (MLtype == set_compressed)
|
261
|
-
fseMetadata->lastCountSize = countSize;
|
262
|
-
op += countSize;
|
263
|
-
fseMetadata->mlType = (symbolEncodingType_e) MLtype;
|
264
|
-
} }
|
265
|
-
assert((size_t) (op-ostart) <= sizeof(fseMetadata->fseTablesBuffer));
|
266
|
-
return op-ostart;
|
267
|
-
}
|
268
|
-
|
269
|
-
|
270
|
-
/** ZSTD_buildSuperBlockEntropy() :
|
271
|
-
* Builds entropy for the super-block.
|
272
|
-
* @return : 0 on success or error code */
|
273
|
-
static size_t
|
274
|
-
ZSTD_buildSuperBlockEntropy(seqStore_t* seqStorePtr,
|
275
|
-
const ZSTD_entropyCTables_t* prevEntropy,
|
276
|
-
ZSTD_entropyCTables_t* nextEntropy,
|
277
|
-
const ZSTD_CCtx_params* cctxParams,
|
278
|
-
ZSTD_entropyCTablesMetadata_t* entropyMetadata,
|
279
|
-
void* workspace, size_t wkspSize)
|
280
|
-
{
|
281
|
-
size_t const litSize = seqStorePtr->lit - seqStorePtr->litStart;
|
282
|
-
DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy");
|
283
|
-
entropyMetadata->hufMetadata.hufDesSize =
|
284
|
-
ZSTD_buildSuperBlockEntropy_literal(seqStorePtr->litStart, litSize,
|
285
|
-
&prevEntropy->huf, &nextEntropy->huf,
|
286
|
-
&entropyMetadata->hufMetadata,
|
287
|
-
ZSTD_disableLiteralsCompression(cctxParams),
|
288
|
-
workspace, wkspSize);
|
289
|
-
FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildSuperBlockEntropy_literal failed");
|
290
|
-
entropyMetadata->fseMetadata.fseTablesSize =
|
291
|
-
ZSTD_buildSuperBlockEntropy_sequences(seqStorePtr,
|
292
|
-
&prevEntropy->fse, &nextEntropy->fse,
|
293
|
-
cctxParams,
|
294
|
-
&entropyMetadata->fseMetadata,
|
295
|
-
workspace, wkspSize);
|
296
|
-
FORWARD_IF_ERROR(entropyMetadata->fseMetadata.fseTablesSize, "ZSTD_buildSuperBlockEntropy_sequences failed");
|
297
|
-
return 0;
|
298
|
-
}
|
299
|
-
|
300
22
|
/** ZSTD_compressSubBlock_literal() :
|
301
23
|
* Compresses literals section for a sub-block.
|
302
24
|
* When we have to write the Huffman table we will sometimes choose a header
|
@@ -304,7 +26,7 @@ ZSTD_buildSuperBlockEntropy(seqStore_t* seqStorePtr,
|
|
304
26
|
* before we know the table size + compressed size, so we have a bound on the
|
305
27
|
* table size. If we guessed incorrectly, we fall back to uncompressed literals.
|
306
28
|
*
|
307
|
-
* We write the header when writeEntropy=1 and set
|
29
|
+
* We write the header when writeEntropy=1 and set entropyWritten=1 when we succeeded
|
308
30
|
* in writing the header, otherwise it is set to 0.
|
309
31
|
*
|
310
32
|
* hufMetadata->hType has literals block type info.
|
@@ -348,7 +70,7 @@ static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
|
|
348
70
|
assert(hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat);
|
349
71
|
|
350
72
|
if (writeEntropy && hufMetadata->hType == set_compressed) {
|
351
|
-
|
73
|
+
ZSTD_memcpy(op, hufMetadata->hufDesBuffer, hufMetadata->hufDesSize);
|
352
74
|
op += hufMetadata->hufDesSize;
|
353
75
|
cLitSize += hufMetadata->hufDesSize;
|
354
76
|
DEBUGLOG(5, "ZSTD_compressSubBlock_literal (hSize=%zu)", hufMetadata->hufDesSize);
|
@@ -410,6 +132,7 @@ static size_t ZSTD_seqDecompressedSize(seqStore_t const* seqStore, const seqDef*
|
|
410
132
|
const seqDef* sp = sstart;
|
411
133
|
size_t matchLengthSum = 0;
|
412
134
|
size_t litLengthSum = 0;
|
135
|
+
(void)(litLengthSum); /* suppress unused variable warning on some environments */
|
413
136
|
while (send-sp > 0) {
|
414
137
|
ZSTD_sequenceLength const seqLen = ZSTD_getSequenceLength(seqStore, sp);
|
415
138
|
litLengthSum += seqLen.litLength;
|
@@ -474,7 +197,7 @@ static size_t ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables
|
|
474
197
|
const U32 MLtype = fseMetadata->mlType;
|
475
198
|
DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (fseTablesSize=%zu)", fseMetadata->fseTablesSize);
|
476
199
|
*seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2));
|
477
|
-
|
200
|
+
ZSTD_memcpy(op, fseMetadata->fseTablesBuffer, fseMetadata->fseTablesSize);
|
478
201
|
op += fseMetadata->fseTablesSize;
|
479
202
|
} else {
|
480
203
|
const U32 repeat = set_repeat;
|
@@ -602,8 +325,8 @@ static size_t ZSTD_estimateSubBlockSize_literal(const BYTE* literals, size_t lit
|
|
602
325
|
static size_t ZSTD_estimateSubBlockSize_symbolType(symbolEncodingType_e type,
|
603
326
|
const BYTE* codeTable, unsigned maxCode,
|
604
327
|
size_t nbSeq, const FSE_CTable* fseCTable,
|
605
|
-
const
|
606
|
-
short const* defaultNorm, U32 defaultNormLog,
|
328
|
+
const U8* additionalBits,
|
329
|
+
short const* defaultNorm, U32 defaultNormLog, U32 defaultMax,
|
607
330
|
void* workspace, size_t wkspSize)
|
608
331
|
{
|
609
332
|
unsigned* const countWksp = (unsigned*)workspace;
|
@@ -615,7 +338,11 @@ static size_t ZSTD_estimateSubBlockSize_symbolType(symbolEncodingType_e type,
|
|
615
338
|
|
616
339
|
HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */
|
617
340
|
if (type == set_basic) {
|
618
|
-
|
341
|
+
/* We selected this encoding type, so it must be valid. */
|
342
|
+
assert(max <= defaultMax);
|
343
|
+
cSymbolTypeSizeEstimateInBits = max <= defaultMax
|
344
|
+
? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max)
|
345
|
+
: ERROR(GENERIC);
|
619
346
|
} else if (type == set_rle) {
|
620
347
|
cSymbolTypeSizeEstimateInBits = 0;
|
621
348
|
} else if (type == set_compressed || type == set_repeat) {
|
@@ -639,19 +366,20 @@ static size_t ZSTD_estimateSubBlockSize_sequences(const BYTE* ofCodeTable,
|
|
639
366
|
void* workspace, size_t wkspSize,
|
640
367
|
int writeEntropy)
|
641
368
|
{
|
642
|
-
size_t sequencesSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */
|
369
|
+
size_t const sequencesSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */
|
643
370
|
size_t cSeqSizeEstimate = 0;
|
371
|
+
if (nbSeq == 0) return sequencesSectionHeaderSize;
|
644
372
|
cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, MaxOff,
|
645
373
|
nbSeq, fseTables->offcodeCTable, NULL,
|
646
|
-
OF_defaultNorm, OF_defaultNormLog,
|
374
|
+
OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
|
647
375
|
workspace, wkspSize);
|
648
376
|
cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->llType, llCodeTable, MaxLL,
|
649
377
|
nbSeq, fseTables->litlengthCTable, LL_bits,
|
650
|
-
LL_defaultNorm, LL_defaultNormLog,
|
378
|
+
LL_defaultNorm, LL_defaultNormLog, MaxLL,
|
651
379
|
workspace, wkspSize);
|
652
380
|
cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, MaxML,
|
653
381
|
nbSeq, fseTables->matchlengthCTable, ML_bits,
|
654
|
-
ML_defaultNorm, ML_defaultNormLog,
|
382
|
+
ML_defaultNorm, ML_defaultNormLog, MaxML,
|
655
383
|
workspace, wkspSize);
|
656
384
|
if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize;
|
657
385
|
return cSeqSizeEstimate + sequencesSectionHeaderSize;
|
@@ -747,7 +475,7 @@ static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr,
|
|
747
475
|
/* I think there is an optimization opportunity here.
|
748
476
|
* Calling ZSTD_estimateSubBlockSize for every sequence can be wasteful
|
749
477
|
* since it recalculates estimate from scratch.
|
750
|
-
* For example, it would recount literal distribution and symbol codes
|
478
|
+
* For example, it would recount literal distribution and symbol codes every time.
|
751
479
|
*/
|
752
480
|
cBlockSizeEstimate = ZSTD_estimateSubBlockSize(lp, litSize, ofCodePtr, llCodePtr, mlCodePtr, seqCount,
|
753
481
|
&nextCBlock->entropy, entropyMetadata,
|
@@ -790,7 +518,7 @@ static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr,
|
|
790
518
|
} while (!lastSequence);
|
791
519
|
if (writeLitEntropy) {
|
792
520
|
DEBUGLOG(5, "ZSTD_compressSubBlock_multi has literal entropy tables unwritten");
|
793
|
-
|
521
|
+
ZSTD_memcpy(&nextCBlock->entropy.huf, &prevCBlock->entropy.huf, sizeof(prevCBlock->entropy.huf));
|
794
522
|
}
|
795
523
|
if (writeSeqEntropy && ZSTD_needSequenceEntropyTables(&entropyMetadata->fseMetadata)) {
|
796
524
|
/* If we haven't written our entropy tables, then we've violated our contract and
|
@@ -809,11 +537,11 @@ static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr,
|
|
809
537
|
if (sp < send) {
|
810
538
|
seqDef const* seq;
|
811
539
|
repcodes_t rep;
|
812
|
-
|
540
|
+
ZSTD_memcpy(&rep, prevCBlock->rep, sizeof(rep));
|
813
541
|
for (seq = sstart; seq < sp; ++seq) {
|
814
|
-
|
542
|
+
ZSTD_updateRep(rep.rep, seq->offBase - 1, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0);
|
815
543
|
}
|
816
|
-
|
544
|
+
ZSTD_memcpy(nextCBlock->rep, &rep, sizeof(rep));
|
817
545
|
}
|
818
546
|
}
|
819
547
|
DEBUGLOG(5, "ZSTD_compressSubBlock_multi compressed");
|
@@ -826,12 +554,12 @@ size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc,
|
|
826
554
|
unsigned lastBlock) {
|
827
555
|
ZSTD_entropyCTablesMetadata_t entropyMetadata;
|
828
556
|
|
829
|
-
FORWARD_IF_ERROR(
|
557
|
+
FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(&zc->seqStore,
|
830
558
|
&zc->blockState.prevCBlock->entropy,
|
831
559
|
&zc->blockState.nextCBlock->entropy,
|
832
560
|
&zc->appliedParams,
|
833
561
|
&entropyMetadata,
|
834
|
-
zc->entropyWorkspace,
|
562
|
+
zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), "");
|
835
563
|
|
836
564
|
return ZSTD_compressSubBlock_multi(&zc->seqStore,
|
837
565
|
zc->blockState.prevCBlock,
|
@@ -841,5 +569,5 @@ size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc,
|
|
841
569
|
dst, dstCapacity,
|
842
570
|
src, srcSize,
|
843
571
|
zc->bmi2, lastBlock,
|
844
|
-
zc->entropyWorkspace,
|
572
|
+
zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */);
|
845
573
|
}
|