zstd-ruby 1.4.4.0 → 1.4.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/ext/zstdruby/libzstd/Makefile +123 -58
- data/ext/zstdruby/libzstd/README.md +34 -14
- data/ext/zstdruby/libzstd/common/bitstream.h +31 -37
- data/ext/zstdruby/libzstd/common/compiler.h +19 -3
- data/ext/zstdruby/libzstd/common/cpu.h +1 -1
- data/ext/zstdruby/libzstd/common/debug.c +11 -31
- data/ext/zstdruby/libzstd/common/debug.h +11 -31
- data/ext/zstdruby/libzstd/common/entropy_common.c +13 -33
- data/ext/zstdruby/libzstd/common/error_private.c +2 -1
- data/ext/zstdruby/libzstd/common/error_private.h +6 -2
- data/ext/zstdruby/libzstd/common/fse.h +11 -31
- data/ext/zstdruby/libzstd/common/fse_decompress.c +12 -37
- data/ext/zstdruby/libzstd/common/huf.h +15 -33
- data/ext/zstdruby/libzstd/common/mem.h +1 -1
- data/ext/zstdruby/libzstd/common/pool.c +1 -1
- data/ext/zstdruby/libzstd/common/pool.h +2 -2
- data/ext/zstdruby/libzstd/common/threading.c +4 -3
- data/ext/zstdruby/libzstd/common/threading.h +4 -3
- data/ext/zstdruby/libzstd/common/xxhash.c +15 -33
- data/ext/zstdruby/libzstd/common/xxhash.h +11 -31
- data/ext/zstdruby/libzstd/common/zstd_common.c +1 -1
- data/ext/zstdruby/libzstd/common/zstd_errors.h +2 -1
- data/ext/zstdruby/libzstd/common/zstd_internal.h +112 -15
- data/ext/zstdruby/libzstd/compress/fse_compress.c +17 -40
- data/ext/zstdruby/libzstd/compress/hist.c +15 -35
- data/ext/zstdruby/libzstd/compress/hist.h +12 -32
- data/ext/zstdruby/libzstd/compress/huf_compress.c +92 -92
- data/ext/zstdruby/libzstd/compress/zstd_compress.c +450 -275
- data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +136 -14
- data/ext/zstdruby/libzstd/compress/zstd_compress_literals.c +10 -6
- data/ext/zstdruby/libzstd/compress/zstd_compress_literals.h +1 -1
- data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.c +24 -20
- data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.h +10 -3
- data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.c +845 -0
- data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.h +32 -0
- data/ext/zstdruby/libzstd/compress/zstd_cwksp.h +3 -13
- data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +11 -8
- data/ext/zstdruby/libzstd/compress/zstd_double_fast.h +2 -2
- data/ext/zstdruby/libzstd/compress/zstd_fast.c +36 -24
- data/ext/zstdruby/libzstd/compress/zstd_fast.h +2 -2
- data/ext/zstdruby/libzstd/compress/zstd_lazy.c +34 -11
- data/ext/zstdruby/libzstd/compress/zstd_lazy.h +1 -1
- data/ext/zstdruby/libzstd/compress/zstd_ldm.c +27 -5
- data/ext/zstdruby/libzstd/compress/zstd_ldm.h +7 -2
- data/ext/zstdruby/libzstd/compress/zstd_opt.c +38 -84
- data/ext/zstdruby/libzstd/compress/zstd_opt.h +1 -1
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +48 -21
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +2 -2
- data/ext/zstdruby/libzstd/decompress/huf_decompress.c +76 -62
- data/ext/zstdruby/libzstd/decompress/zstd_ddict.c +12 -8
- data/ext/zstdruby/libzstd/decompress/zstd_ddict.h +2 -2
- data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +264 -148
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.c +312 -203
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.h +3 -3
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_internal.h +18 -4
- data/ext/zstdruby/libzstd/deprecated/zbuff.h +3 -3
- data/ext/zstdruby/libzstd/deprecated/zbuff_common.c +2 -2
- data/ext/zstdruby/libzstd/deprecated/zbuff_compress.c +1 -1
- data/ext/zstdruby/libzstd/deprecated/zbuff_decompress.c +1 -1
- data/ext/zstdruby/libzstd/dictBuilder/cover.c +5 -5
- data/ext/zstdruby/libzstd/dictBuilder/cover.h +14 -4
- data/ext/zstdruby/libzstd/dictBuilder/fastcover.c +14 -4
- data/ext/zstdruby/libzstd/dictBuilder/zdict.c +33 -9
- data/ext/zstdruby/libzstd/dictBuilder/zdict.h +51 -28
- data/ext/zstdruby/libzstd/dll/example/Makefile +2 -1
- data/ext/zstdruby/libzstd/legacy/zstd_legacy.h +4 -4
- data/ext/zstdruby/libzstd/legacy/zstd_v01.c +18 -12
- data/ext/zstdruby/libzstd/legacy/zstd_v01.h +1 -1
- data/ext/zstdruby/libzstd/legacy/zstd_v02.c +10 -6
- data/ext/zstdruby/libzstd/legacy/zstd_v02.h +1 -1
- data/ext/zstdruby/libzstd/legacy/zstd_v03.c +10 -6
- data/ext/zstdruby/libzstd/legacy/zstd_v03.h +1 -1
- data/ext/zstdruby/libzstd/legacy/zstd_v04.c +13 -7
- data/ext/zstdruby/libzstd/legacy/zstd_v04.h +1 -1
- data/ext/zstdruby/libzstd/legacy/zstd_v05.c +17 -13
- data/ext/zstdruby/libzstd/legacy/zstd_v05.h +2 -2
- data/ext/zstdruby/libzstd/legacy/zstd_v06.c +17 -13
- data/ext/zstdruby/libzstd/legacy/zstd_v06.h +1 -1
- data/ext/zstdruby/libzstd/legacy/zstd_v07.c +22 -14
- data/ext/zstdruby/libzstd/legacy/zstd_v07.h +1 -1
- data/ext/zstdruby/libzstd/libzstd.pc.in +2 -2
- data/ext/zstdruby/libzstd/zstd.h +62 -21
- data/lib/zstd-ruby/version.rb +1 -1
- metadata +7 -5
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/*
|
|
2
|
-
* Copyright (c) 2016-
|
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
|
@@ -16,8 +16,8 @@
|
|
|
16
16
|
* Dependencies
|
|
17
17
|
*********************************************************/
|
|
18
18
|
#include <stddef.h> /* size_t */
|
|
19
|
-
#include "zstd.h" /* DCtx, and some public functions */
|
|
20
|
-
#include "zstd_internal.h" /* blockProperties_t, and some public functions */
|
|
19
|
+
#include "../zstd.h" /* DCtx, and some public functions */
|
|
20
|
+
#include "../common/zstd_internal.h" /* blockProperties_t, and some public functions */
|
|
21
21
|
#include "zstd_decompress_internal.h" /* ZSTD_seqSymbol */
|
|
22
22
|
|
|
23
23
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/*
|
|
2
|
-
* Copyright (c) 2016-
|
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
|
@@ -19,8 +19,8 @@
|
|
|
19
19
|
/*-*******************************************************
|
|
20
20
|
* Dependencies
|
|
21
21
|
*********************************************************/
|
|
22
|
-
#include "mem.h" /* BYTE, U16, U32 */
|
|
23
|
-
#include "zstd_internal.h" /* ZSTD_seqSymbol */
|
|
22
|
+
#include "../common/mem.h" /* BYTE, U16, U32 */
|
|
23
|
+
#include "../common/zstd_internal.h" /* ZSTD_seqSymbol */
|
|
24
24
|
|
|
25
25
|
|
|
26
26
|
|
|
@@ -95,6 +95,11 @@ typedef enum {
|
|
|
95
95
|
ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */
|
|
96
96
|
} ZSTD_dictUses_e;
|
|
97
97
|
|
|
98
|
+
typedef enum {
|
|
99
|
+
ZSTD_obm_buffered = 0, /* Buffer the output */
|
|
100
|
+
ZSTD_obm_stable = 1 /* ZSTD_outBuffer is stable */
|
|
101
|
+
} ZSTD_outBufferMode_e;
|
|
102
|
+
|
|
98
103
|
struct ZSTD_DCtx_s
|
|
99
104
|
{
|
|
100
105
|
const ZSTD_seqSymbol* LLTptr;
|
|
@@ -147,10 +152,19 @@ struct ZSTD_DCtx_s
|
|
|
147
152
|
U32 legacyVersion;
|
|
148
153
|
U32 hostageByte;
|
|
149
154
|
int noForwardProgress;
|
|
155
|
+
ZSTD_outBufferMode_e outBufferMode;
|
|
156
|
+
ZSTD_outBuffer expectedOutBuffer;
|
|
150
157
|
|
|
151
158
|
/* workspace */
|
|
152
159
|
BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH];
|
|
153
160
|
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
|
|
161
|
+
|
|
162
|
+
size_t oversizedDuration;
|
|
163
|
+
|
|
164
|
+
#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
|
|
165
|
+
void const* dictContentBeginForFuzzing;
|
|
166
|
+
void const* dictContentEndForFuzzing;
|
|
167
|
+
#endif
|
|
154
168
|
}; /* typedef'd to ZSTD_DCtx within "zstd.h" */
|
|
155
169
|
|
|
156
170
|
|
|
@@ -160,7 +174,7 @@ struct ZSTD_DCtx_s
|
|
|
160
174
|
|
|
161
175
|
/*! ZSTD_loadDEntropy() :
|
|
162
176
|
* dict : must point at beginning of a valid zstd dictionary.
|
|
163
|
-
* @return : size of entropy tables
|
|
177
|
+
* @return : size of dictionary header (size of magic number + dict ID + entropy tables) */
|
|
164
178
|
size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
|
|
165
179
|
const void* const dict, size_t const dictSize);
|
|
166
180
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/*
|
|
2
|
-
* Copyright (c) 2016-
|
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
|
@@ -28,7 +28,7 @@ extern "C" {
|
|
|
28
28
|
* Dependencies
|
|
29
29
|
***************************************/
|
|
30
30
|
#include <stddef.h> /* size_t */
|
|
31
|
-
#include "zstd.h" /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */
|
|
31
|
+
#include "../zstd.h" /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */
|
|
32
32
|
|
|
33
33
|
|
|
34
34
|
/* ***************************************************************
|
|
@@ -186,7 +186,7 @@ ZBUFF_DEPRECATED("use ZSTD_DStreamOutSize") size_t ZBUFF_recommendedDOutSize(voi
|
|
|
186
186
|
|
|
187
187
|
/*--- Dependency ---*/
|
|
188
188
|
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_customMem */
|
|
189
|
-
#include "zstd.h"
|
|
189
|
+
#include "../zstd.h"
|
|
190
190
|
|
|
191
191
|
|
|
192
192
|
/*--- Custom memory allocator ---*/
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/*
|
|
2
|
-
* Copyright (c) 2016-
|
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
/*-*************************************
|
|
12
12
|
* Dependencies
|
|
13
13
|
***************************************/
|
|
14
|
-
#include "error_private.h"
|
|
14
|
+
#include "../common/error_private.h"
|
|
15
15
|
#include "zbuff.h"
|
|
16
16
|
|
|
17
17
|
/*-****************************************
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/*
|
|
2
|
-
* Copyright (c) 2016-
|
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
|
@@ -26,11 +26,11 @@
|
|
|
26
26
|
#include <string.h> /* memset */
|
|
27
27
|
#include <time.h> /* clock */
|
|
28
28
|
|
|
29
|
-
#include "mem.h" /* read */
|
|
30
|
-
#include "pool.h"
|
|
31
|
-
#include "threading.h"
|
|
29
|
+
#include "../common/mem.h" /* read */
|
|
30
|
+
#include "../common/pool.h"
|
|
31
|
+
#include "../common/threading.h"
|
|
32
32
|
#include "cover.h"
|
|
33
|
-
#include "zstd_internal.h" /* includes zstd.h */
|
|
33
|
+
#include "../common/zstd_internal.h" /* includes zstd.h */
|
|
34
34
|
#ifndef ZDICT_STATIC_LINKING_ONLY
|
|
35
35
|
#define ZDICT_STATIC_LINKING_ONLY
|
|
36
36
|
#endif
|
|
@@ -1,11 +1,21 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) 2017-2020, Facebook, Inc.
|
|
3
|
+
* All rights reserved.
|
|
4
|
+
*
|
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
|
9
|
+
*/
|
|
10
|
+
|
|
1
11
|
#include <stdio.h> /* fprintf */
|
|
2
12
|
#include <stdlib.h> /* malloc, free, qsort */
|
|
3
13
|
#include <string.h> /* memset */
|
|
4
14
|
#include <time.h> /* clock */
|
|
5
|
-
#include "mem.h" /* read */
|
|
6
|
-
#include "pool.h"
|
|
7
|
-
#include "threading.h"
|
|
8
|
-
#include "zstd_internal.h" /* includes zstd.h */
|
|
15
|
+
#include "../common/mem.h" /* read */
|
|
16
|
+
#include "../common/pool.h"
|
|
17
|
+
#include "../common/threading.h"
|
|
18
|
+
#include "../common/zstd_internal.h" /* includes zstd.h */
|
|
9
19
|
#ifndef ZDICT_STATIC_LINKING_ONLY
|
|
10
20
|
#define ZDICT_STATIC_LINKING_ONLY
|
|
11
21
|
#endif
|
|
@@ -1,3 +1,13 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) 2018-2020, Facebook, Inc.
|
|
3
|
+
* All rights reserved.
|
|
4
|
+
*
|
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
|
9
|
+
*/
|
|
10
|
+
|
|
1
11
|
/*-*************************************
|
|
2
12
|
* Dependencies
|
|
3
13
|
***************************************/
|
|
@@ -6,11 +16,11 @@
|
|
|
6
16
|
#include <string.h> /* memset */
|
|
7
17
|
#include <time.h> /* clock */
|
|
8
18
|
|
|
9
|
-
#include "mem.h" /* read */
|
|
10
|
-
#include "pool.h"
|
|
11
|
-
#include "threading.h"
|
|
19
|
+
#include "../common/mem.h" /* read */
|
|
20
|
+
#include "../common/pool.h"
|
|
21
|
+
#include "../common/threading.h"
|
|
12
22
|
#include "cover.h"
|
|
13
|
-
#include "zstd_internal.h" /* includes zstd.h */
|
|
23
|
+
#include "../common/zstd_internal.h" /* includes zstd.h */
|
|
14
24
|
#ifndef ZDICT_STATIC_LINKING_ONLY
|
|
15
25
|
#define ZDICT_STATIC_LINKING_ONLY
|
|
16
26
|
#endif
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/*
|
|
2
|
-
* Copyright (c) 2016-
|
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
|
@@ -37,17 +37,18 @@
|
|
|
37
37
|
#include <stdio.h> /* fprintf, fopen, ftello64 */
|
|
38
38
|
#include <time.h> /* clock */
|
|
39
39
|
|
|
40
|
-
#include "mem.h" /* read */
|
|
41
|
-
#include "fse.h" /* FSE_normalizeCount, FSE_writeNCount */
|
|
40
|
+
#include "../common/mem.h" /* read */
|
|
41
|
+
#include "../common/fse.h" /* FSE_normalizeCount, FSE_writeNCount */
|
|
42
42
|
#define HUF_STATIC_LINKING_ONLY
|
|
43
|
-
#include "huf.h" /* HUF_buildCTable, HUF_writeCTable */
|
|
44
|
-
#include "zstd_internal.h" /* includes zstd.h */
|
|
45
|
-
#include "xxhash.h" /* XXH64 */
|
|
43
|
+
#include "../common/huf.h" /* HUF_buildCTable, HUF_writeCTable */
|
|
44
|
+
#include "../common/zstd_internal.h" /* includes zstd.h */
|
|
45
|
+
#include "../common/xxhash.h" /* XXH64 */
|
|
46
46
|
#include "divsufsort.h"
|
|
47
47
|
#ifndef ZDICT_STATIC_LINKING_ONLY
|
|
48
48
|
# define ZDICT_STATIC_LINKING_ONLY
|
|
49
49
|
#endif
|
|
50
50
|
#include "zdict.h"
|
|
51
|
+
#include "../compress/zstd_compress_internal.h" /* ZSTD_loadCEntropy() */
|
|
51
52
|
|
|
52
53
|
|
|
53
54
|
/*-*************************************
|
|
@@ -99,6 +100,29 @@ unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize)
|
|
|
99
100
|
return MEM_readLE32((const char*)dictBuffer + 4);
|
|
100
101
|
}
|
|
101
102
|
|
|
103
|
+
size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize)
|
|
104
|
+
{
|
|
105
|
+
size_t headerSize;
|
|
106
|
+
if (dictSize <= 8 || MEM_readLE32(dictBuffer) != ZSTD_MAGIC_DICTIONARY) return ERROR(dictionary_corrupted);
|
|
107
|
+
|
|
108
|
+
{ unsigned offcodeMaxValue = MaxOff;
|
|
109
|
+
ZSTD_compressedBlockState_t* bs = (ZSTD_compressedBlockState_t*)malloc(sizeof(ZSTD_compressedBlockState_t));
|
|
110
|
+
U32* wksp = (U32*)malloc(HUF_WORKSPACE_SIZE);
|
|
111
|
+
short* offcodeNCount = (short*)malloc((MaxOff+1)*sizeof(short));
|
|
112
|
+
if (!bs || !wksp || !offcodeNCount) {
|
|
113
|
+
headerSize = ERROR(memory_allocation);
|
|
114
|
+
} else {
|
|
115
|
+
ZSTD_reset_compressedBlockState(bs);
|
|
116
|
+
headerSize = ZSTD_loadCEntropy(bs, wksp, offcodeNCount, &offcodeMaxValue, dictBuffer, dictSize);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
free(bs);
|
|
120
|
+
free(wksp);
|
|
121
|
+
free(offcodeNCount);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
return headerSize;
|
|
125
|
+
}
|
|
102
126
|
|
|
103
127
|
/*-********************************************************
|
|
104
128
|
* Dictionary training functions
|
|
@@ -588,12 +612,12 @@ typedef struct
|
|
|
588
612
|
|
|
589
613
|
#define MAXREPOFFSET 1024
|
|
590
614
|
|
|
591
|
-
static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params,
|
|
615
|
+
static void ZDICT_countEStats(EStats_ress_t esr, const ZSTD_parameters* params,
|
|
592
616
|
unsigned* countLit, unsigned* offsetcodeCount, unsigned* matchlengthCount, unsigned* litlengthCount, U32* repOffsets,
|
|
593
617
|
const void* src, size_t srcSize,
|
|
594
618
|
U32 notificationLevel)
|
|
595
619
|
{
|
|
596
|
-
size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_MAX, 1 << params
|
|
620
|
+
size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_MAX, 1 << params->cParams.windowLog);
|
|
597
621
|
size_t cSize;
|
|
598
622
|
|
|
599
623
|
if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */
|
|
@@ -731,7 +755,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
|
|
731
755
|
|
|
732
756
|
/* collect stats on all samples */
|
|
733
757
|
for (u=0; u<nbFiles; u++) {
|
|
734
|
-
ZDICT_countEStats(esr, params,
|
|
758
|
+
ZDICT_countEStats(esr, ¶ms,
|
|
735
759
|
countLit, offcodeCount, matchLengthCount, litLengthCount, repOffset,
|
|
736
760
|
(const char*)srcBuffer + pos, fileSizes[u],
|
|
737
761
|
notificationLevel);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/*
|
|
2
|
-
* Copyright (c) 2016-
|
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
|
@@ -61,9 +61,57 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCap
|
|
|
61
61
|
const void* samplesBuffer,
|
|
62
62
|
const size_t* samplesSizes, unsigned nbSamples);
|
|
63
63
|
|
|
64
|
+
typedef struct {
|
|
65
|
+
int compressionLevel; /*< optimize for a specific zstd compression level; 0 means default */
|
|
66
|
+
unsigned notificationLevel; /*< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
|
|
67
|
+
unsigned dictID; /*< force dictID value; 0 means auto mode (32-bits random value) */
|
|
68
|
+
} ZDICT_params_t;
|
|
69
|
+
|
|
70
|
+
/*! ZDICT_finalizeDictionary():
|
|
71
|
+
* Given a custom content as a basis for dictionary, and a set of samples,
|
|
72
|
+
* finalize dictionary by adding headers and statistics according to the zstd
|
|
73
|
+
* dictionary format.
|
|
74
|
+
*
|
|
75
|
+
* Samples must be stored concatenated in a flat buffer `samplesBuffer`,
|
|
76
|
+
* supplied with an array of sizes `samplesSizes`, providing the size of each
|
|
77
|
+
* sample in order. The samples are used to construct the statistics, so they
|
|
78
|
+
* should be representative of what you will compress with this dictionary.
|
|
79
|
+
*
|
|
80
|
+
* The compression level can be set in `parameters`. You should pass the
|
|
81
|
+
* compression level you expect to use in production. The statistics for each
|
|
82
|
+
* compression level differ, so tuning the dictionary for the compression level
|
|
83
|
+
* can help quite a bit.
|
|
84
|
+
*
|
|
85
|
+
* You can set an explicit dictionary ID in `parameters`, or allow us to pick
|
|
86
|
+
* a random dictionary ID for you, but we can't guarantee no collisions.
|
|
87
|
+
*
|
|
88
|
+
* The dstDictBuffer and the dictContent may overlap, and the content will be
|
|
89
|
+
* appended to the end of the header. If the header + the content doesn't fit in
|
|
90
|
+
* maxDictSize the beginning of the content is truncated to make room, since it
|
|
91
|
+
* is presumed that the most profitable content is at the end of the dictionary,
|
|
92
|
+
* since that is the cheapest to reference.
|
|
93
|
+
*
|
|
94
|
+
* `dictContentSize` must be >= ZDICT_CONTENTSIZE_MIN bytes.
|
|
95
|
+
* `maxDictSize` must be >= max(dictContentSize, ZSTD_DICTSIZE_MIN).
|
|
96
|
+
*
|
|
97
|
+
* @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`),
|
|
98
|
+
* or an error code, which can be tested by ZDICT_isError().
|
|
99
|
+
* Note: ZDICT_finalizeDictionary() will push notifications into stderr if
|
|
100
|
+
* instructed to, using notificationLevel>0.
|
|
101
|
+
* NOTE: This function currently may fail in several edge cases including:
|
|
102
|
+
* * Not enough samples
|
|
103
|
+
* * Samples are uncompressible
|
|
104
|
+
* * Samples are all exactly the same
|
|
105
|
+
*/
|
|
106
|
+
ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dstDictBuffer, size_t maxDictSize,
|
|
107
|
+
const void* dictContent, size_t dictContentSize,
|
|
108
|
+
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
|
109
|
+
ZDICT_params_t parameters);
|
|
110
|
+
|
|
64
111
|
|
|
65
112
|
/*====== Helper functions ======*/
|
|
66
113
|
ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize); /**< extracts dictID; @return zero if error (not a valid dictionary) */
|
|
114
|
+
ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize); /* returns dict header size; returns a ZSTD error code on failure */
|
|
67
115
|
ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode);
|
|
68
116
|
ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
|
|
69
117
|
|
|
@@ -78,11 +126,8 @@ ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
|
|
|
78
126
|
* Use them only in association with static linking.
|
|
79
127
|
* ==================================================================================== */
|
|
80
128
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
unsigned notificationLevel; /* Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
|
|
84
|
-
unsigned dictID; /* force dictID value; 0 means auto mode (32-bits random value) */
|
|
85
|
-
} ZDICT_params_t;
|
|
129
|
+
#define ZDICT_CONTENTSIZE_MIN 128
|
|
130
|
+
#define ZDICT_DICTSIZE_MIN 256
|
|
86
131
|
|
|
87
132
|
/*! ZDICT_cover_params_t:
|
|
88
133
|
* k and d are the only required parameters.
|
|
@@ -198,28 +243,6 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
|
|
|
198
243
|
const size_t* samplesSizes, unsigned nbSamples,
|
|
199
244
|
ZDICT_fastCover_params_t* parameters);
|
|
200
245
|
|
|
201
|
-
/*! ZDICT_finalizeDictionary():
|
|
202
|
-
* Given a custom content as a basis for dictionary, and a set of samples,
|
|
203
|
-
* finalize dictionary by adding headers and statistics.
|
|
204
|
-
*
|
|
205
|
-
* Samples must be stored concatenated in a flat buffer `samplesBuffer`,
|
|
206
|
-
* supplied with an array of sizes `samplesSizes`, providing the size of each sample in order.
|
|
207
|
-
*
|
|
208
|
-
* dictContentSize must be >= ZDICT_CONTENTSIZE_MIN bytes.
|
|
209
|
-
* maxDictSize must be >= dictContentSize, and must be >= ZDICT_DICTSIZE_MIN bytes.
|
|
210
|
-
*
|
|
211
|
-
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`),
|
|
212
|
-
* or an error code, which can be tested by ZDICT_isError().
|
|
213
|
-
* Note: ZDICT_finalizeDictionary() will push notifications into stderr if instructed to, using notificationLevel>0.
|
|
214
|
-
* Note 2: dictBuffer and dictContent can overlap
|
|
215
|
-
*/
|
|
216
|
-
#define ZDICT_CONTENTSIZE_MIN 128
|
|
217
|
-
#define ZDICT_DICTSIZE_MIN 256
|
|
218
|
-
ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
|
|
219
|
-
const void* dictContent, size_t dictContentSize,
|
|
220
|
-
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
|
221
|
-
ZDICT_params_t parameters);
|
|
222
|
-
|
|
223
246
|
typedef struct {
|
|
224
247
|
unsigned selectivityLevel; /* 0 means default; larger => select more => larger dictionary */
|
|
225
248
|
ZDICT_params_t zParams;
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
# ################################################################
|
|
2
|
-
# Copyright (c) 2016-
|
|
2
|
+
# Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
|
3
3
|
# All rights reserved.
|
|
4
4
|
#
|
|
5
5
|
# This source code is licensed under both the BSD-style license (found in the
|
|
6
6
|
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
|
7
7
|
# in the COPYING file in the root directory of this source tree).
|
|
8
|
+
# You may select, at your option, one of the above-listed licenses.
|
|
8
9
|
# ################################################################
|
|
9
10
|
|
|
10
11
|
VOID := /dev/null
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/*
|
|
2
|
-
* Copyright (c) 2016-
|
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
|
@@ -18,9 +18,9 @@ extern "C" {
|
|
|
18
18
|
/* *************************************
|
|
19
19
|
* Includes
|
|
20
20
|
***************************************/
|
|
21
|
-
#include "mem.h" /* MEM_STATIC */
|
|
22
|
-
#include "error_private.h" /* ERROR */
|
|
23
|
-
#include "zstd_internal.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTD_frameSizeInfo */
|
|
21
|
+
#include "../common/mem.h" /* MEM_STATIC */
|
|
22
|
+
#include "../common/error_private.h" /* ERROR */
|
|
23
|
+
#include "../common/zstd_internal.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTD_frameSizeInfo */
|
|
24
24
|
|
|
25
25
|
#if !defined (ZSTD_LEGACY_SUPPORT) || (ZSTD_LEGACY_SUPPORT == 0)
|
|
26
26
|
# undef ZSTD_LEGACY_SUPPORT
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/*
|
|
2
|
-
* Copyright (c) 2016-
|
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
******************************************/
|
|
15
15
|
#include <stddef.h> /* size_t, ptrdiff_t */
|
|
16
16
|
#include "zstd_v01.h"
|
|
17
|
-
#include "error_private.h"
|
|
17
|
+
#include "../common/error_private.h"
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
/******************************************
|
|
@@ -257,7 +257,7 @@ static U64 FSE_read64(const void* memPtr)
|
|
|
257
257
|
U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
|
258
258
|
}
|
|
259
259
|
|
|
260
|
-
#endif
|
|
260
|
+
#endif /* FSE_FORCE_MEMORY_ACCESS */
|
|
261
261
|
|
|
262
262
|
static U16 FSE_readLE16(const void* memPtr)
|
|
263
263
|
{
|
|
@@ -1078,7 +1078,7 @@ static size_t HUF_decompress_usingDTable( /* -3% slower when non static */
|
|
|
1078
1078
|
BYTE* const ostart = (BYTE*) dst;
|
|
1079
1079
|
BYTE* op = ostart;
|
|
1080
1080
|
BYTE* const omax = op + maxDstSize;
|
|
1081
|
-
BYTE* const olimit = omax-15;
|
|
1081
|
+
BYTE* const olimit = maxDstSize < 15 ? op : omax-15;
|
|
1082
1082
|
|
|
1083
1083
|
const void* ptr = DTable;
|
|
1084
1084
|
const HUF_DElt* const dt = (const HUF_DElt*)(ptr)+1;
|
|
@@ -1092,7 +1092,7 @@ static size_t HUF_decompress_usingDTable( /* -3% slower when non static */
|
|
|
1092
1092
|
const size_t length1 = FSE_readLE16(jumpTable);
|
|
1093
1093
|
const size_t length2 = FSE_readLE16(jumpTable+1);
|
|
1094
1094
|
const size_t length3 = FSE_readLE16(jumpTable+2);
|
|
1095
|
-
const size_t length4 = cSrcSize - 6 - length1 - length2 - length3;
|
|
1095
|
+
const size_t length4 = cSrcSize - 6 - length1 - length2 - length3; /* check coherency !! */
|
|
1096
1096
|
const char* const start1 = (const char*)(cSrc) + 6;
|
|
1097
1097
|
const char* const start2 = start1 + length1;
|
|
1098
1098
|
const char* const start3 = start2 + length2;
|
|
@@ -1150,11 +1150,11 @@ static size_t HUF_decompress_usingDTable( /* -3% slower when non static */
|
|
|
1150
1150
|
|
|
1151
1151
|
/* tail */
|
|
1152
1152
|
{
|
|
1153
|
-
|
|
1153
|
+
/* bitTail = bitD1; */ /* *much* slower : -20% !??! */
|
|
1154
1154
|
FSE_DStream_t bitTail;
|
|
1155
1155
|
bitTail.ptr = bitD1.ptr;
|
|
1156
1156
|
bitTail.bitsConsumed = bitD1.bitsConsumed;
|
|
1157
|
-
bitTail.bitContainer = bitD1.bitContainer;
|
|
1157
|
+
bitTail.bitContainer = bitD1.bitContainer; /* required in case of FSE_DStream_endOfBuffer */
|
|
1158
1158
|
bitTail.start = start1;
|
|
1159
1159
|
for ( ; (FSE_reloadDStream(&bitTail) < FSE_DStream_completed) && (op<omax) ; op++)
|
|
1160
1160
|
{
|
|
@@ -1483,7 +1483,9 @@ static size_t ZSTDv01_getcBlockSize(const void* src, size_t srcSize, blockProper
|
|
|
1483
1483
|
static size_t ZSTD_copyUncompressedBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
|
1484
1484
|
{
|
|
1485
1485
|
if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
|
|
1486
|
-
|
|
1486
|
+
if (srcSize > 0) {
|
|
1487
|
+
memcpy(dst, src, srcSize);
|
|
1488
|
+
}
|
|
1487
1489
|
return srcSize;
|
|
1488
1490
|
}
|
|
1489
1491
|
|
|
@@ -1502,7 +1504,7 @@ static size_t ZSTD_decompressLiterals(void* ctx,
|
|
|
1502
1504
|
if (srcSize <= 3) return ERROR(corruption_detected);
|
|
1503
1505
|
|
|
1504
1506
|
litSize = ip[1] + (ip[0]<<8);
|
|
1505
|
-
litSize += ((ip[-3] >> 3) & 7) << 16;
|
|
1507
|
+
litSize += ((ip[-3] >> 3) & 7) << 16; /* mmmmh.... */
|
|
1506
1508
|
op = oend - litSize;
|
|
1507
1509
|
|
|
1508
1510
|
(void)ctx;
|
|
@@ -1541,7 +1543,9 @@ static size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
|
|
|
1541
1543
|
size_t rleSize = litbp.origSize;
|
|
1542
1544
|
if (rleSize>maxDstSize) return ERROR(dstSize_tooSmall);
|
|
1543
1545
|
if (!srcSize) return ERROR(srcSize_wrong);
|
|
1544
|
-
|
|
1546
|
+
if (rleSize > 0) {
|
|
1547
|
+
memset(oend - rleSize, *ip, rleSize);
|
|
1548
|
+
}
|
|
1545
1549
|
*litStart = oend - rleSize;
|
|
1546
1550
|
*litSize = rleSize;
|
|
1547
1551
|
ip++;
|
|
@@ -1901,8 +1905,10 @@ static size_t ZSTD_decompressSequences(
|
|
|
1901
1905
|
{
|
|
1902
1906
|
size_t lastLLSize = litEnd - litPtr;
|
|
1903
1907
|
if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
|
|
1904
|
-
if (
|
|
1905
|
-
|
|
1908
|
+
if (lastLLSize > 0) {
|
|
1909
|
+
if (op != litPtr) memmove(op, litPtr, lastLLSize);
|
|
1910
|
+
op += lastLLSize;
|
|
1911
|
+
}
|
|
1906
1912
|
}
|
|
1907
1913
|
}
|
|
1908
1914
|
|