vibe_zstd 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.standard.yml +3 -0
- data/CHANGELOG.md +22 -0
- data/LICENSE.txt +21 -0
- data/README.md +978 -0
- data/Rakefile +20 -0
- data/benchmark/README.md +198 -0
- data/benchmark/compression_levels.rb +99 -0
- data/benchmark/context_reuse.rb +174 -0
- data/benchmark/decompression_speed_by_level.rb +65 -0
- data/benchmark/dictionary_training.rb +182 -0
- data/benchmark/dictionary_usage.rb +121 -0
- data/benchmark/for_readme.rb +157 -0
- data/benchmark/generate_fixture.rb +82 -0
- data/benchmark/helpers.rb +237 -0
- data/benchmark/multithreading.rb +105 -0
- data/benchmark/run_all.rb +150 -0
- data/benchmark/streaming.rb +154 -0
- data/ext/vibe_zstd/Makefile +270 -0
- data/ext/vibe_zstd/cctx.c +565 -0
- data/ext/vibe_zstd/dctx.c +493 -0
- data/ext/vibe_zstd/dict.c +587 -0
- data/ext/vibe_zstd/extconf.rb +52 -0
- data/ext/vibe_zstd/frames.c +132 -0
- data/ext/vibe_zstd/libzstd/LICENSE +30 -0
- data/ext/vibe_zstd/libzstd/common/allocations.h +55 -0
- data/ext/vibe_zstd/libzstd/common/bits.h +205 -0
- data/ext/vibe_zstd/libzstd/common/bitstream.h +454 -0
- data/ext/vibe_zstd/libzstd/common/compiler.h +464 -0
- data/ext/vibe_zstd/libzstd/common/cpu.h +249 -0
- data/ext/vibe_zstd/libzstd/common/debug.c +30 -0
- data/ext/vibe_zstd/libzstd/common/debug.h +107 -0
- data/ext/vibe_zstd/libzstd/common/entropy_common.c +340 -0
- data/ext/vibe_zstd/libzstd/common/error_private.c +64 -0
- data/ext/vibe_zstd/libzstd/common/error_private.h +158 -0
- data/ext/vibe_zstd/libzstd/common/fse.h +625 -0
- data/ext/vibe_zstd/libzstd/common/fse_decompress.c +315 -0
- data/ext/vibe_zstd/libzstd/common/huf.h +277 -0
- data/ext/vibe_zstd/libzstd/common/mem.h +422 -0
- data/ext/vibe_zstd/libzstd/common/pool.c +371 -0
- data/ext/vibe_zstd/libzstd/common/pool.h +81 -0
- data/ext/vibe_zstd/libzstd/common/portability_macros.h +171 -0
- data/ext/vibe_zstd/libzstd/common/threading.c +182 -0
- data/ext/vibe_zstd/libzstd/common/threading.h +142 -0
- data/ext/vibe_zstd/libzstd/common/xxhash.c +18 -0
- data/ext/vibe_zstd/libzstd/common/xxhash.h +7094 -0
- data/ext/vibe_zstd/libzstd/common/zstd_common.c +48 -0
- data/ext/vibe_zstd/libzstd/common/zstd_deps.h +123 -0
- data/ext/vibe_zstd/libzstd/common/zstd_internal.h +324 -0
- data/ext/vibe_zstd/libzstd/common/zstd_trace.h +156 -0
- data/ext/vibe_zstd/libzstd/compress/clevels.h +134 -0
- data/ext/vibe_zstd/libzstd/compress/fse_compress.c +625 -0
- data/ext/vibe_zstd/libzstd/compress/hist.c +191 -0
- data/ext/vibe_zstd/libzstd/compress/hist.h +82 -0
- data/ext/vibe_zstd/libzstd/compress/huf_compress.c +1464 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_compress.c +7843 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_compress_internal.h +1636 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_compress_literals.c +235 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_compress_literals.h +39 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_compress_sequences.c +442 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_compress_sequences.h +55 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_compress_superblock.c +688 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_compress_superblock.h +32 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_cwksp.h +765 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_double_fast.c +778 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_double_fast.h +42 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_fast.c +985 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_fast.h +30 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_lazy.c +2199 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_lazy.h +193 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_ldm.c +745 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_ldm.h +109 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_ldm_geartab.h +106 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_opt.c +1580 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_opt.h +72 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_preSplit.c +238 -0
- data/ext/vibe_zstd/libzstd/compress/zstd_preSplit.h +33 -0
- data/ext/vibe_zstd/libzstd/compress/zstdmt_compress.c +1923 -0
- data/ext/vibe_zstd/libzstd/compress/zstdmt_compress.h +102 -0
- data/ext/vibe_zstd/libzstd/decompress/huf_decompress.c +1944 -0
- data/ext/vibe_zstd/libzstd/decompress/huf_decompress_amd64.S +602 -0
- data/ext/vibe_zstd/libzstd/decompress/zstd_ddict.c +244 -0
- data/ext/vibe_zstd/libzstd/decompress/zstd_ddict.h +44 -0
- data/ext/vibe_zstd/libzstd/decompress/zstd_decompress.c +2410 -0
- data/ext/vibe_zstd/libzstd/decompress/zstd_decompress_block.c +2209 -0
- data/ext/vibe_zstd/libzstd/decompress/zstd_decompress_block.h +73 -0
- data/ext/vibe_zstd/libzstd/decompress/zstd_decompress_internal.h +240 -0
- data/ext/vibe_zstd/libzstd/deprecated/zbuff.h +214 -0
- data/ext/vibe_zstd/libzstd/deprecated/zbuff_common.c +26 -0
- data/ext/vibe_zstd/libzstd/deprecated/zbuff_compress.c +167 -0
- data/ext/vibe_zstd/libzstd/deprecated/zbuff_decompress.c +77 -0
- data/ext/vibe_zstd/libzstd/dictBuilder/cover.c +1302 -0
- data/ext/vibe_zstd/libzstd/dictBuilder/cover.h +152 -0
- data/ext/vibe_zstd/libzstd/dictBuilder/divsufsort.c +1913 -0
- data/ext/vibe_zstd/libzstd/dictBuilder/divsufsort.h +57 -0
- data/ext/vibe_zstd/libzstd/dictBuilder/fastcover.c +766 -0
- data/ext/vibe_zstd/libzstd/dictBuilder/zdict.c +1133 -0
- data/ext/vibe_zstd/libzstd/zdict.h +481 -0
- data/ext/vibe_zstd/libzstd/zstd.h +3198 -0
- data/ext/vibe_zstd/libzstd/zstd_errors.h +107 -0
- data/ext/vibe_zstd/streaming.c +410 -0
- data/ext/vibe_zstd/vibe_zstd.c +293 -0
- data/ext/vibe_zstd/vibe_zstd.h +56 -0
- data/ext/vibe_zstd/vibe_zstd_internal.h +27 -0
- data/lib/vibe_zstd/constants.rb +67 -0
- data/lib/vibe_zstd/version.rb +5 -0
- data/lib/vibe_zstd.rb +255 -0
- data/sig/vibe_zstd.rbs +76 -0
- metadata +179 -0
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
3
|
+
* All rights reserved.
|
|
4
|
+
*
|
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
#ifndef ZSTD_DEC_BLOCK_H
|
|
13
|
+
#define ZSTD_DEC_BLOCK_H
|
|
14
|
+
|
|
15
|
+
/*-*******************************************************
|
|
16
|
+
* Dependencies
|
|
17
|
+
*********************************************************/
|
|
18
|
+
#include "../common/zstd_deps.h" /* size_t */
|
|
19
|
+
#include "../zstd.h" /* DCtx, and some public functions */
|
|
20
|
+
#include "../common/zstd_internal.h" /* blockProperties_t, and some public functions */
|
|
21
|
+
#include "zstd_decompress_internal.h" /* ZSTD_seqSymbol */
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
/* === Prototypes === */
|
|
25
|
+
|
|
26
|
+
/* note: prototypes already published within `zstd.h` :
|
|
27
|
+
* ZSTD_decompressBlock()
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
/* note: prototypes already published within `zstd_internal.h` :
|
|
31
|
+
* ZSTD_getcBlockSize()
|
|
32
|
+
* ZSTD_decodeSeqHeaders()
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
/* Streaming state is used to inform allocation of the literal buffer */
|
|
37
|
+
typedef enum {
|
|
38
|
+
not_streaming = 0,
|
|
39
|
+
is_streaming = 1
|
|
40
|
+
} streaming_operation;
|
|
41
|
+
|
|
42
|
+
/* ZSTD_decompressBlock_internal() :
|
|
43
|
+
* decompress block, starting at `src`,
|
|
44
|
+
* into destination buffer `dst`.
|
|
45
|
+
* @return : decompressed block size,
|
|
46
|
+
* or an error code (which can be tested using ZSTD_isError())
|
|
47
|
+
*/
|
|
48
|
+
size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
|
|
49
|
+
void* dst, size_t dstCapacity,
|
|
50
|
+
const void* src, size_t srcSize, const streaming_operation streaming);
|
|
51
|
+
|
|
52
|
+
/* ZSTD_buildFSETable() :
|
|
53
|
+
* generate FSE decoding table for one symbol (ll, ml or off)
|
|
54
|
+
* this function must be called with valid parameters only
|
|
55
|
+
* (dt is large enough, normalizedCounter distribution total is a power of 2, max is within range, etc.)
|
|
56
|
+
* in which case it cannot fail.
|
|
57
|
+
* The workspace must be 4-byte aligned and at least ZSTD_BUILD_FSE_TABLE_WKSP_SIZE bytes, which is
|
|
58
|
+
* defined in zstd_decompress_internal.h.
|
|
59
|
+
* Internal use only.
|
|
60
|
+
*/
|
|
61
|
+
void ZSTD_buildFSETable(ZSTD_seqSymbol* dt,
|
|
62
|
+
const short* normalizedCounter, unsigned maxSymbolValue,
|
|
63
|
+
const U32* baseValue, const U8* nbAdditionalBits,
|
|
64
|
+
unsigned tableLog, void* wksp, size_t wkspSize,
|
|
65
|
+
int bmi2);
|
|
66
|
+
|
|
67
|
+
/* Internal definition of ZSTD_decompressBlock() to avoid deprecation warnings. */
|
|
68
|
+
size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx,
|
|
69
|
+
void* dst, size_t dstCapacity,
|
|
70
|
+
const void* src, size_t srcSize);
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
#endif /* ZSTD_DEC_BLOCK_H */
|
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
3
|
+
* All rights reserved.
|
|
4
|
+
*
|
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
/* zstd_decompress_internal:
|
|
13
|
+
* objects and definitions shared within lib/decompress modules */
|
|
14
|
+
|
|
15
|
+
#ifndef ZSTD_DECOMPRESS_INTERNAL_H
|
|
16
|
+
#define ZSTD_DECOMPRESS_INTERNAL_H
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
/*-*******************************************************
|
|
20
|
+
* Dependencies
|
|
21
|
+
*********************************************************/
|
|
22
|
+
#include "../common/mem.h" /* BYTE, U16, U32 */
|
|
23
|
+
#include "../common/zstd_internal.h" /* constants : MaxLL, MaxML, MaxOff, LLFSELog, etc. */
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
/*-*******************************************************
|
|
28
|
+
* Constants
|
|
29
|
+
*********************************************************/
|
|
30
|
+
static UNUSED_ATTR const U32 LL_base[MaxLL+1] = {
|
|
31
|
+
0, 1, 2, 3, 4, 5, 6, 7,
|
|
32
|
+
8, 9, 10, 11, 12, 13, 14, 15,
|
|
33
|
+
16, 18, 20, 22, 24, 28, 32, 40,
|
|
34
|
+
48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000,
|
|
35
|
+
0x2000, 0x4000, 0x8000, 0x10000 };
|
|
36
|
+
|
|
37
|
+
static UNUSED_ATTR const U32 OF_base[MaxOff+1] = {
|
|
38
|
+
0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D,
|
|
39
|
+
0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD,
|
|
40
|
+
0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD,
|
|
41
|
+
0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD };
|
|
42
|
+
|
|
43
|
+
static UNUSED_ATTR const U8 OF_bits[MaxOff+1] = {
|
|
44
|
+
0, 1, 2, 3, 4, 5, 6, 7,
|
|
45
|
+
8, 9, 10, 11, 12, 13, 14, 15,
|
|
46
|
+
16, 17, 18, 19, 20, 21, 22, 23,
|
|
47
|
+
24, 25, 26, 27, 28, 29, 30, 31 };
|
|
48
|
+
|
|
49
|
+
static UNUSED_ATTR const U32 ML_base[MaxML+1] = {
|
|
50
|
+
3, 4, 5, 6, 7, 8, 9, 10,
|
|
51
|
+
11, 12, 13, 14, 15, 16, 17, 18,
|
|
52
|
+
19, 20, 21, 22, 23, 24, 25, 26,
|
|
53
|
+
27, 28, 29, 30, 31, 32, 33, 34,
|
|
54
|
+
35, 37, 39, 41, 43, 47, 51, 59,
|
|
55
|
+
67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803,
|
|
56
|
+
0x1003, 0x2003, 0x4003, 0x8003, 0x10003 };
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
/*-*******************************************************
|
|
60
|
+
* Decompression types
|
|
61
|
+
*********************************************************/
|
|
62
|
+
typedef struct {
|
|
63
|
+
U32 fastMode;
|
|
64
|
+
U32 tableLog;
|
|
65
|
+
} ZSTD_seqSymbol_header;
|
|
66
|
+
|
|
67
|
+
typedef struct {
|
|
68
|
+
U16 nextState;
|
|
69
|
+
BYTE nbAdditionalBits;
|
|
70
|
+
BYTE nbBits;
|
|
71
|
+
U32 baseValue;
|
|
72
|
+
} ZSTD_seqSymbol;
|
|
73
|
+
|
|
74
|
+
#define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log)))
|
|
75
|
+
|
|
76
|
+
#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE (sizeof(S16) * (MaxSeq + 1) + (1u << MaxFSELog) + sizeof(U64))
|
|
77
|
+
#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 ((ZSTD_BUILD_FSE_TABLE_WKSP_SIZE + sizeof(U32) - 1) / sizeof(U32))
|
|
78
|
+
#define ZSTD_HUFFDTABLE_CAPACITY_LOG 12
|
|
79
|
+
|
|
80
|
+
typedef struct {
|
|
81
|
+
ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */
|
|
82
|
+
ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */
|
|
83
|
+
ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */
|
|
84
|
+
HUF_DTable hufTable[HUF_DTABLE_SIZE(ZSTD_HUFFDTABLE_CAPACITY_LOG)]; /* can accommodate HUF_decompress4X */
|
|
85
|
+
U32 rep[ZSTD_REP_NUM];
|
|
86
|
+
U32 workspace[ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32];
|
|
87
|
+
} ZSTD_entropyDTables_t;
|
|
88
|
+
|
|
89
|
+
typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
|
|
90
|
+
ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock,
|
|
91
|
+
ZSTDds_decompressLastBlock, ZSTDds_checkChecksum,
|
|
92
|
+
ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage;
|
|
93
|
+
|
|
94
|
+
typedef enum { zdss_init=0, zdss_loadHeader,
|
|
95
|
+
zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
|
|
96
|
+
|
|
97
|
+
typedef enum {
|
|
98
|
+
ZSTD_use_indefinitely = -1, /* Use the dictionary indefinitely */
|
|
99
|
+
ZSTD_dont_use = 0, /* Do not use the dictionary (if one exists free it) */
|
|
100
|
+
ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */
|
|
101
|
+
} ZSTD_dictUses_e;
|
|
102
|
+
|
|
103
|
+
/* Hashset for storing references to multiple ZSTD_DDict within ZSTD_DCtx */
|
|
104
|
+
typedef struct {
|
|
105
|
+
const ZSTD_DDict** ddictPtrTable;
|
|
106
|
+
size_t ddictPtrTableSize;
|
|
107
|
+
size_t ddictPtrCount;
|
|
108
|
+
} ZSTD_DDictHashSet;
|
|
109
|
+
|
|
110
|
+
#ifndef ZSTD_DECODER_INTERNAL_BUFFER
|
|
111
|
+
# define ZSTD_DECODER_INTERNAL_BUFFER (1 << 16)
|
|
112
|
+
#endif
|
|
113
|
+
|
|
114
|
+
#define ZSTD_LBMIN 64
|
|
115
|
+
#define ZSTD_LBMAX (128 << 10)
|
|
116
|
+
|
|
117
|
+
/* extra buffer, compensates when dst is not large enough to store litBuffer */
|
|
118
|
+
#define ZSTD_LITBUFFEREXTRASIZE BOUNDED(ZSTD_LBMIN, ZSTD_DECODER_INTERNAL_BUFFER, ZSTD_LBMAX)
|
|
119
|
+
|
|
120
|
+
typedef enum {
|
|
121
|
+
ZSTD_not_in_dst = 0, /* Stored entirely within litExtraBuffer */
|
|
122
|
+
ZSTD_in_dst = 1, /* Stored entirely within dst (in memory after current output write) */
|
|
123
|
+
ZSTD_split = 2 /* Split between litExtraBuffer and dst */
|
|
124
|
+
} ZSTD_litLocation_e;
|
|
125
|
+
|
|
126
|
+
struct ZSTD_DCtx_s
|
|
127
|
+
{
|
|
128
|
+
const ZSTD_seqSymbol* LLTptr;
|
|
129
|
+
const ZSTD_seqSymbol* MLTptr;
|
|
130
|
+
const ZSTD_seqSymbol* OFTptr;
|
|
131
|
+
const HUF_DTable* HUFptr;
|
|
132
|
+
ZSTD_entropyDTables_t entropy;
|
|
133
|
+
U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */
|
|
134
|
+
const void* previousDstEnd; /* detect continuity */
|
|
135
|
+
const void* prefixStart; /* start of current segment */
|
|
136
|
+
const void* virtualStart; /* virtual start of previous segment if it was just before current one */
|
|
137
|
+
const void* dictEnd; /* end of previous segment */
|
|
138
|
+
size_t expected;
|
|
139
|
+
ZSTD_FrameHeader fParams;
|
|
140
|
+
U64 processedCSize;
|
|
141
|
+
U64 decodedSize;
|
|
142
|
+
blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */
|
|
143
|
+
ZSTD_dStage stage;
|
|
144
|
+
U32 litEntropy;
|
|
145
|
+
U32 fseEntropy;
|
|
146
|
+
XXH64_state_t xxhState;
|
|
147
|
+
size_t headerSize;
|
|
148
|
+
ZSTD_format_e format;
|
|
149
|
+
ZSTD_forceIgnoreChecksum_e forceIgnoreChecksum; /* User specified: if == 1, will ignore checksums in compressed frame. Default == 0 */
|
|
150
|
+
U32 validateChecksum; /* if == 1, will validate checksum. Is == 1 if (fParams.checksumFlag == 1) and (forceIgnoreChecksum == 0). */
|
|
151
|
+
const BYTE* litPtr;
|
|
152
|
+
ZSTD_customMem customMem;
|
|
153
|
+
size_t litSize;
|
|
154
|
+
size_t rleSize;
|
|
155
|
+
size_t staticSize;
|
|
156
|
+
int isFrameDecompression;
|
|
157
|
+
#if DYNAMIC_BMI2
|
|
158
|
+
int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */
|
|
159
|
+
#endif
|
|
160
|
+
|
|
161
|
+
/* dictionary */
|
|
162
|
+
ZSTD_DDict* ddictLocal;
|
|
163
|
+
const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */
|
|
164
|
+
U32 dictID;
|
|
165
|
+
int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */
|
|
166
|
+
ZSTD_dictUses_e dictUses;
|
|
167
|
+
ZSTD_DDictHashSet* ddictSet; /* Hash set for multiple ddicts */
|
|
168
|
+
ZSTD_refMultipleDDicts_e refMultipleDDicts; /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */
|
|
169
|
+
int disableHufAsm;
|
|
170
|
+
int maxBlockSizeParam;
|
|
171
|
+
|
|
172
|
+
/* streaming */
|
|
173
|
+
ZSTD_dStreamStage streamStage;
|
|
174
|
+
char* inBuff;
|
|
175
|
+
size_t inBuffSize;
|
|
176
|
+
size_t inPos;
|
|
177
|
+
size_t maxWindowSize;
|
|
178
|
+
char* outBuff;
|
|
179
|
+
size_t outBuffSize;
|
|
180
|
+
size_t outStart;
|
|
181
|
+
size_t outEnd;
|
|
182
|
+
size_t lhSize;
|
|
183
|
+
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
|
|
184
|
+
void* legacyContext;
|
|
185
|
+
U32 previousLegacyVersion;
|
|
186
|
+
U32 legacyVersion;
|
|
187
|
+
#endif
|
|
188
|
+
U32 hostageByte;
|
|
189
|
+
int noForwardProgress;
|
|
190
|
+
ZSTD_bufferMode_e outBufferMode;
|
|
191
|
+
ZSTD_outBuffer expectedOutBuffer;
|
|
192
|
+
|
|
193
|
+
/* workspace */
|
|
194
|
+
BYTE* litBuffer;
|
|
195
|
+
const BYTE* litBufferEnd;
|
|
196
|
+
ZSTD_litLocation_e litBufferLocation;
|
|
197
|
+
BYTE litExtraBuffer[ZSTD_LITBUFFEREXTRASIZE + WILDCOPY_OVERLENGTH]; /* literal buffer can be split between storage within dst and within this scratch buffer */
|
|
198
|
+
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
|
|
199
|
+
|
|
200
|
+
size_t oversizedDuration;
|
|
201
|
+
|
|
202
|
+
#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
|
|
203
|
+
void const* dictContentBeginForFuzzing;
|
|
204
|
+
void const* dictContentEndForFuzzing;
|
|
205
|
+
#endif
|
|
206
|
+
|
|
207
|
+
/* Tracing */
|
|
208
|
+
#if ZSTD_TRACE
|
|
209
|
+
ZSTD_TraceCtx traceCtx;
|
|
210
|
+
#endif
|
|
211
|
+
}; /* typedef'd to ZSTD_DCtx within "zstd.h" */
|
|
212
|
+
|
|
213
|
+
MEM_STATIC int ZSTD_DCtx_get_bmi2(const struct ZSTD_DCtx_s *dctx) {
|
|
214
|
+
#if DYNAMIC_BMI2
|
|
215
|
+
return dctx->bmi2;
|
|
216
|
+
#else
|
|
217
|
+
(void)dctx;
|
|
218
|
+
return 0;
|
|
219
|
+
#endif
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/*-*******************************************************
|
|
223
|
+
* Shared internal functions
|
|
224
|
+
*********************************************************/
|
|
225
|
+
|
|
226
|
+
/*! ZSTD_loadDEntropy() :
|
|
227
|
+
* dict : must point at beginning of a valid zstd dictionary.
|
|
228
|
+
* @return : size of dictionary header (size of magic number + dict ID + entropy tables) */
|
|
229
|
+
size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
|
|
230
|
+
const void* const dict, size_t const dictSize);
|
|
231
|
+
|
|
232
|
+
/*! ZSTD_checkContinuity() :
|
|
233
|
+
* check if next `dst` follows previous position, where decompression ended.
|
|
234
|
+
* If yes, do nothing (continue on current segment).
|
|
235
|
+
* If not, classify previous segment as "external dictionary", and start a new segment.
|
|
236
|
+
* This function cannot fail. */
|
|
237
|
+
void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize);
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
#endif /* ZSTD_DECOMPRESS_INTERNAL_H */
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
3
|
+
* All rights reserved.
|
|
4
|
+
*
|
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/* ***************************************************************
|
|
12
|
+
* NOTES/WARNINGS
|
|
13
|
+
******************************************************************/
|
|
14
|
+
/* The streaming API defined here is deprecated.
|
|
15
|
+
* Consider migrating towards ZSTD_compressStream() API in `zstd.h`
|
|
16
|
+
* See 'lib/README.md'.
|
|
17
|
+
*****************************************************************/
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
#if defined (__cplusplus)
|
|
21
|
+
extern "C" {
|
|
22
|
+
#endif
|
|
23
|
+
|
|
24
|
+
#ifndef ZSTD_BUFFERED_H_23987
|
|
25
|
+
#define ZSTD_BUFFERED_H_23987
|
|
26
|
+
|
|
27
|
+
/* *************************************
|
|
28
|
+
* Dependencies
|
|
29
|
+
***************************************/
|
|
30
|
+
#include <stddef.h> /* size_t */
|
|
31
|
+
#include "../zstd.h" /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
/* ***************************************************************
|
|
35
|
+
* Compiler specifics
|
|
36
|
+
*****************************************************************/
|
|
37
|
+
/* Deprecation warnings */
|
|
38
|
+
/* Should these warnings be a problem,
|
|
39
|
+
* it is generally possible to disable them,
|
|
40
|
+
* typically with -Wno-deprecated-declarations for gcc
|
|
41
|
+
* or _CRT_SECURE_NO_WARNINGS in Visual.
|
|
42
|
+
* Otherwise, it's also possible to define ZBUFF_DISABLE_DEPRECATE_WARNINGS
|
|
43
|
+
*/
|
|
44
|
+
#ifdef ZBUFF_DISABLE_DEPRECATE_WARNINGS
|
|
45
|
+
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API /* disable deprecation warnings */
|
|
46
|
+
#else
|
|
47
|
+
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
|
|
48
|
+
# define ZBUFF_DEPRECATED(message) [[deprecated(message)]] ZSTDLIB_API
|
|
49
|
+
# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__)
|
|
50
|
+
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated(message)))
|
|
51
|
+
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
|
52
|
+
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated))
|
|
53
|
+
# elif defined(_MSC_VER)
|
|
54
|
+
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __declspec(deprecated(message))
|
|
55
|
+
# else
|
|
56
|
+
# pragma message("WARNING: You need to implement ZBUFF_DEPRECATED for this compiler")
|
|
57
|
+
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API
|
|
58
|
+
# endif
|
|
59
|
+
#endif /* ZBUFF_DISABLE_DEPRECATE_WARNINGS */
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
/* *************************************
|
|
63
|
+
* Streaming functions
|
|
64
|
+
***************************************/
|
|
65
|
+
/* This is the easier "buffered" streaming API,
|
|
66
|
+
* using an internal buffer to lift all restrictions on user-provided buffers
|
|
67
|
+
* which can be any size, any place, for both input and output.
|
|
68
|
+
* ZBUFF and ZSTD are 100% interoperable,
|
|
69
|
+
* frames created by one can be decoded by the other one */
|
|
70
|
+
|
|
71
|
+
typedef ZSTD_CStream ZBUFF_CCtx;
|
|
72
|
+
ZBUFF_DEPRECATED("use ZSTD_createCStream") ZBUFF_CCtx* ZBUFF_createCCtx(void);
|
|
73
|
+
ZBUFF_DEPRECATED("use ZSTD_freeCStream") size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx);
|
|
74
|
+
|
|
75
|
+
ZBUFF_DEPRECATED("use ZSTD_initCStream") size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel);
|
|
76
|
+
ZBUFF_DEPRECATED("use ZSTD_initCStream_usingDict") size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
|
|
77
|
+
|
|
78
|
+
ZBUFF_DEPRECATED("use ZSTD_compressStream") size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr);
|
|
79
|
+
ZBUFF_DEPRECATED("use ZSTD_flushStream") size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
|
|
80
|
+
ZBUFF_DEPRECATED("use ZSTD_endStream") size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
|
|
81
|
+
|
|
82
|
+
/*-*************************************************
|
|
83
|
+
* Streaming compression - howto
|
|
84
|
+
*
|
|
85
|
+
* A ZBUFF_CCtx object is required to track streaming operation.
|
|
86
|
+
* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources.
|
|
87
|
+
* ZBUFF_CCtx objects can be reused multiple times.
|
|
88
|
+
*
|
|
89
|
+
* Start by initializing ZBUF_CCtx.
|
|
90
|
+
* Use ZBUFF_compressInit() to start a new compression operation.
|
|
91
|
+
* Use ZBUFF_compressInitDictionary() for a compression which requires a dictionary.
|
|
92
|
+
*
|
|
93
|
+
* Use ZBUFF_compressContinue() repetitively to consume input stream.
|
|
94
|
+
* *srcSizePtr and *dstCapacityPtr can be any size.
|
|
95
|
+
* The function will report how many bytes were read or written within *srcSizePtr and *dstCapacityPtr.
|
|
96
|
+
* Note that it may not consume the entire input, in which case it's up to the caller to present again remaining data.
|
|
97
|
+
* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each call, so save its content if it matters or change @dst .
|
|
98
|
+
* @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency)
|
|
99
|
+
* or an error code, which can be tested using ZBUFF_isError().
|
|
100
|
+
*
|
|
101
|
+
* At any moment, it's possible to flush whatever data remains within buffer, using ZBUFF_compressFlush().
|
|
102
|
+
* The nb of bytes written into `dst` will be reported into *dstCapacityPtr.
|
|
103
|
+
* Note that the function cannot output more than *dstCapacityPtr,
|
|
104
|
+
* therefore, some content might still be left into internal buffer if *dstCapacityPtr is too small.
|
|
105
|
+
* @return : nb of bytes still present into internal buffer (0 if it's empty)
|
|
106
|
+
* or an error code, which can be tested using ZBUFF_isError().
|
|
107
|
+
*
|
|
108
|
+
* ZBUFF_compressEnd() instructs to finish a frame.
|
|
109
|
+
* It will perform a flush and write frame epilogue.
|
|
110
|
+
* The epilogue is required for decoders to consider a frame completed.
|
|
111
|
+
* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *dstCapacityPtr is too small.
|
|
112
|
+
* In which case, call again ZBUFF_compressFlush() to complete the flush.
|
|
113
|
+
* @return : nb of bytes still present into internal buffer (0 if it's empty)
|
|
114
|
+
* or an error code, which can be tested using ZBUFF_isError().
|
|
115
|
+
*
|
|
116
|
+
* Hint : _recommended buffer_ sizes (not compulsory) : ZBUFF_recommendedCInSize() / ZBUFF_recommendedCOutSize()
|
|
117
|
+
* input : ZBUFF_recommendedCInSize==128 KB block size is the internal unit, use this value to reduce intermediate stages (better latency)
|
|
118
|
+
* output : ZBUFF_recommendedCOutSize==ZSTD_compressBound(128 KB) + 3 + 3 : ensures it's always possible to write/flush/end a full block. Skip some buffering.
|
|
119
|
+
* By using both, it ensures that input will be entirely consumed, and output will always contain the result, reducing intermediate buffering.
|
|
120
|
+
* **************************************************/
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
typedef ZSTD_DStream ZBUFF_DCtx;
|
|
124
|
+
ZBUFF_DEPRECATED("use ZSTD_createDStream") ZBUFF_DCtx* ZBUFF_createDCtx(void);
|
|
125
|
+
ZBUFF_DEPRECATED("use ZSTD_freeDStream") size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx);
|
|
126
|
+
|
|
127
|
+
ZBUFF_DEPRECATED("use ZSTD_initDStream") size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx);
|
|
128
|
+
ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* dctx, const void* dict, size_t dictSize);
|
|
129
|
+
|
|
130
|
+
ZBUFF_DEPRECATED("use ZSTD_decompressStream") size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx,
|
|
131
|
+
void* dst, size_t* dstCapacityPtr,
|
|
132
|
+
const void* src, size_t* srcSizePtr);
|
|
133
|
+
|
|
134
|
+
/*-***************************************************************************
|
|
135
|
+
* Streaming decompression howto
|
|
136
|
+
*
|
|
137
|
+
* A ZBUFF_DCtx object is required to track streaming operations.
|
|
138
|
+
* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources.
|
|
139
|
+
* Use ZBUFF_decompressInit() to start a new decompression operation,
|
|
140
|
+
* or ZBUFF_decompressInitDictionary() if decompression requires a dictionary.
|
|
141
|
+
* Note that ZBUFF_DCtx objects can be re-init multiple times.
|
|
142
|
+
*
|
|
143
|
+
* Use ZBUFF_decompressContinue() repetitively to consume your input.
|
|
144
|
+
* *srcSizePtr and *dstCapacityPtr can be any size.
|
|
145
|
+
* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
|
|
146
|
+
* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
|
|
147
|
+
* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`.
|
|
148
|
+
* @return : 0 when a frame is completely decoded and fully flushed,
|
|
149
|
+
* 1 when there is still some data left within internal buffer to flush,
|
|
150
|
+
* >1 when more data is expected, with value being a suggested next input size (it's just a hint, which helps latency),
|
|
151
|
+
* or an error code, which can be tested using ZBUFF_isError().
|
|
152
|
+
*
|
|
153
|
+
* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize()
|
|
154
|
+
* output : ZBUFF_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
|
|
155
|
+
* input : ZBUFF_recommendedDInSize == 128KB + 3;
|
|
156
|
+
* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
|
|
157
|
+
* *******************************************************************************/
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
/* *************************************
|
|
161
|
+
* Tool functions
|
|
162
|
+
***************************************/
|
|
163
|
+
ZBUFF_DEPRECATED("use ZSTD_isError") unsigned ZBUFF_isError(size_t errorCode);
|
|
164
|
+
ZBUFF_DEPRECATED("use ZSTD_getErrorName") const char* ZBUFF_getErrorName(size_t errorCode);
|
|
165
|
+
|
|
166
|
+
/** Functions below provide recommended buffer sizes for Compression or Decompression operations.
|
|
167
|
+
* These sizes are just hints, they tend to offer better latency */
|
|
168
|
+
ZBUFF_DEPRECATED("use ZSTD_CStreamInSize") size_t ZBUFF_recommendedCInSize(void);
|
|
169
|
+
ZBUFF_DEPRECATED("use ZSTD_CStreamOutSize") size_t ZBUFF_recommendedCOutSize(void);
|
|
170
|
+
ZBUFF_DEPRECATED("use ZSTD_DStreamInSize") size_t ZBUFF_recommendedDInSize(void);
|
|
171
|
+
ZBUFF_DEPRECATED("use ZSTD_DStreamOutSize") size_t ZBUFF_recommendedDOutSize(void);
|
|
172
|
+
|
|
173
|
+
#endif /* ZSTD_BUFFERED_H_23987 */
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
#ifdef ZBUFF_STATIC_LINKING_ONLY
|
|
177
|
+
#ifndef ZBUFF_STATIC_H_30298098432
|
|
178
|
+
#define ZBUFF_STATIC_H_30298098432
|
|
179
|
+
|
|
180
|
+
/* ====================================================================================
|
|
181
|
+
* The definitions in this section are considered experimental.
|
|
182
|
+
* They should never be used in association with a dynamic library, as they may change in the future.
|
|
183
|
+
* They are provided for advanced usages.
|
|
184
|
+
* Use them only in association with static linking.
|
|
185
|
+
* ==================================================================================== */
|
|
186
|
+
|
|
187
|
+
/*--- Dependency ---*/
|
|
188
|
+
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_customMem */
|
|
189
|
+
#include "../zstd.h"
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
/*--- Custom memory allocator ---*/
|
|
193
|
+
/*! ZBUFF_createCCtx_advanced() :
|
|
194
|
+
* Create a ZBUFF compression context using external alloc and free functions */
|
|
195
|
+
ZBUFF_DEPRECATED("use ZSTD_createCStream_advanced") ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem);
|
|
196
|
+
|
|
197
|
+
/*! ZBUFF_createDCtx_advanced() :
|
|
198
|
+
* Create a ZBUFF decompression context using external alloc and free functions */
|
|
199
|
+
ZBUFF_DEPRECATED("use ZSTD_createDStream_advanced") ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem);
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
/*--- Advanced Streaming Initialization ---*/
|
|
203
|
+
ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
|
|
204
|
+
const void* dict, size_t dictSize,
|
|
205
|
+
ZSTD_parameters params, unsigned long long pledgedSrcSize);
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
#endif /* ZBUFF_STATIC_H_30298098432 */
|
|
209
|
+
#endif /* ZBUFF_STATIC_LINKING_ONLY */
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
#if defined (__cplusplus)
|
|
213
|
+
}
|
|
214
|
+
#endif
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
3
|
+
* All rights reserved.
|
|
4
|
+
*
|
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/*-*************************************
|
|
12
|
+
* Dependencies
|
|
13
|
+
***************************************/
|
|
14
|
+
#include "../common/error_private.h"
|
|
15
|
+
#include "zbuff.h"
|
|
16
|
+
|
|
17
|
+
/*-****************************************
|
|
18
|
+
* ZBUFF Error Management (deprecated)
|
|
19
|
+
******************************************/
|
|
20
|
+
|
|
21
|
+
/*! ZBUFF_isError() :
|
|
22
|
+
* tells if a return value is an error code */
|
|
23
|
+
unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
|
|
24
|
+
/*! ZBUFF_getErrorName() :
|
|
25
|
+
* provides error code string from function result (useful for debugging) */
|
|
26
|
+
const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
|