zstd-ruby 1.4.4.0 → 1.4.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/ext/zstdruby/libzstd/Makefile +123 -58
- data/ext/zstdruby/libzstd/README.md +34 -14
- data/ext/zstdruby/libzstd/common/bitstream.h +31 -37
- data/ext/zstdruby/libzstd/common/compiler.h +19 -3
- data/ext/zstdruby/libzstd/common/cpu.h +1 -1
- data/ext/zstdruby/libzstd/common/debug.c +11 -31
- data/ext/zstdruby/libzstd/common/debug.h +11 -31
- data/ext/zstdruby/libzstd/common/entropy_common.c +13 -33
- data/ext/zstdruby/libzstd/common/error_private.c +2 -1
- data/ext/zstdruby/libzstd/common/error_private.h +6 -2
- data/ext/zstdruby/libzstd/common/fse.h +11 -31
- data/ext/zstdruby/libzstd/common/fse_decompress.c +12 -37
- data/ext/zstdruby/libzstd/common/huf.h +15 -33
- data/ext/zstdruby/libzstd/common/mem.h +1 -1
- data/ext/zstdruby/libzstd/common/pool.c +1 -1
- data/ext/zstdruby/libzstd/common/pool.h +2 -2
- data/ext/zstdruby/libzstd/common/threading.c +4 -3
- data/ext/zstdruby/libzstd/common/threading.h +4 -3
- data/ext/zstdruby/libzstd/common/xxhash.c +15 -33
- data/ext/zstdruby/libzstd/common/xxhash.h +11 -31
- data/ext/zstdruby/libzstd/common/zstd_common.c +1 -1
- data/ext/zstdruby/libzstd/common/zstd_errors.h +2 -1
- data/ext/zstdruby/libzstd/common/zstd_internal.h +112 -15
- data/ext/zstdruby/libzstd/compress/fse_compress.c +17 -40
- data/ext/zstdruby/libzstd/compress/hist.c +15 -35
- data/ext/zstdruby/libzstd/compress/hist.h +12 -32
- data/ext/zstdruby/libzstd/compress/huf_compress.c +92 -92
- data/ext/zstdruby/libzstd/compress/zstd_compress.c +450 -275
- data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +136 -14
- data/ext/zstdruby/libzstd/compress/zstd_compress_literals.c +10 -6
- data/ext/zstdruby/libzstd/compress/zstd_compress_literals.h +1 -1
- data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.c +24 -20
- data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.h +10 -3
- data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.c +845 -0
- data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.h +32 -0
- data/ext/zstdruby/libzstd/compress/zstd_cwksp.h +3 -13
- data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +11 -8
- data/ext/zstdruby/libzstd/compress/zstd_double_fast.h +2 -2
- data/ext/zstdruby/libzstd/compress/zstd_fast.c +36 -24
- data/ext/zstdruby/libzstd/compress/zstd_fast.h +2 -2
- data/ext/zstdruby/libzstd/compress/zstd_lazy.c +34 -11
- data/ext/zstdruby/libzstd/compress/zstd_lazy.h +1 -1
- data/ext/zstdruby/libzstd/compress/zstd_ldm.c +27 -5
- data/ext/zstdruby/libzstd/compress/zstd_ldm.h +7 -2
- data/ext/zstdruby/libzstd/compress/zstd_opt.c +38 -84
- data/ext/zstdruby/libzstd/compress/zstd_opt.h +1 -1
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +48 -21
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +2 -2
- data/ext/zstdruby/libzstd/decompress/huf_decompress.c +76 -62
- data/ext/zstdruby/libzstd/decompress/zstd_ddict.c +12 -8
- data/ext/zstdruby/libzstd/decompress/zstd_ddict.h +2 -2
- data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +264 -148
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.c +312 -203
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.h +3 -3
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_internal.h +18 -4
- data/ext/zstdruby/libzstd/deprecated/zbuff.h +3 -3
- data/ext/zstdruby/libzstd/deprecated/zbuff_common.c +2 -2
- data/ext/zstdruby/libzstd/deprecated/zbuff_compress.c +1 -1
- data/ext/zstdruby/libzstd/deprecated/zbuff_decompress.c +1 -1
- data/ext/zstdruby/libzstd/dictBuilder/cover.c +5 -5
- data/ext/zstdruby/libzstd/dictBuilder/cover.h +14 -4
- data/ext/zstdruby/libzstd/dictBuilder/fastcover.c +14 -4
- data/ext/zstdruby/libzstd/dictBuilder/zdict.c +33 -9
- data/ext/zstdruby/libzstd/dictBuilder/zdict.h +51 -28
- data/ext/zstdruby/libzstd/dll/example/Makefile +2 -1
- data/ext/zstdruby/libzstd/legacy/zstd_legacy.h +4 -4
- data/ext/zstdruby/libzstd/legacy/zstd_v01.c +18 -12
- data/ext/zstdruby/libzstd/legacy/zstd_v01.h +1 -1
- data/ext/zstdruby/libzstd/legacy/zstd_v02.c +10 -6
- data/ext/zstdruby/libzstd/legacy/zstd_v02.h +1 -1
- data/ext/zstdruby/libzstd/legacy/zstd_v03.c +10 -6
- data/ext/zstdruby/libzstd/legacy/zstd_v03.h +1 -1
- data/ext/zstdruby/libzstd/legacy/zstd_v04.c +13 -7
- data/ext/zstdruby/libzstd/legacy/zstd_v04.h +1 -1
- data/ext/zstdruby/libzstd/legacy/zstd_v05.c +17 -13
- data/ext/zstdruby/libzstd/legacy/zstd_v05.h +2 -2
- data/ext/zstdruby/libzstd/legacy/zstd_v06.c +17 -13
- data/ext/zstdruby/libzstd/legacy/zstd_v06.h +1 -1
- data/ext/zstdruby/libzstd/legacy/zstd_v07.c +22 -14
- data/ext/zstdruby/libzstd/legacy/zstd_v07.h +1 -1
- data/ext/zstdruby/libzstd/libzstd.pc.in +2 -2
- data/ext/zstdruby/libzstd/zstd.h +62 -21
- data/lib/zstd-ruby/version.rb +1 -1
- metadata +7 -5
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c) 2016-
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -16,8 +16,8 @@
|
|
16
16
|
* Dependencies
|
17
17
|
*********************************************************/
|
18
18
|
#include <stddef.h> /* size_t */
|
19
|
-
#include "zstd.h" /* DCtx, and some public functions */
|
20
|
-
#include "zstd_internal.h" /* blockProperties_t, and some public functions */
|
19
|
+
#include "../zstd.h" /* DCtx, and some public functions */
|
20
|
+
#include "../common/zstd_internal.h" /* blockProperties_t, and some public functions */
|
21
21
|
#include "zstd_decompress_internal.h" /* ZSTD_seqSymbol */
|
22
22
|
|
23
23
|
|
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c) 2016-
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -19,8 +19,8 @@
|
|
19
19
|
/*-*******************************************************
|
20
20
|
* Dependencies
|
21
21
|
*********************************************************/
|
22
|
-
#include "mem.h" /* BYTE, U16, U32 */
|
23
|
-
#include "zstd_internal.h" /* ZSTD_seqSymbol */
|
22
|
+
#include "../common/mem.h" /* BYTE, U16, U32 */
|
23
|
+
#include "../common/zstd_internal.h" /* ZSTD_seqSymbol */
|
24
24
|
|
25
25
|
|
26
26
|
|
@@ -95,6 +95,11 @@ typedef enum {
|
|
95
95
|
ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */
|
96
96
|
} ZSTD_dictUses_e;
|
97
97
|
|
98
|
+
typedef enum {
|
99
|
+
ZSTD_obm_buffered = 0, /* Buffer the output */
|
100
|
+
ZSTD_obm_stable = 1 /* ZSTD_outBuffer is stable */
|
101
|
+
} ZSTD_outBufferMode_e;
|
102
|
+
|
98
103
|
struct ZSTD_DCtx_s
|
99
104
|
{
|
100
105
|
const ZSTD_seqSymbol* LLTptr;
|
@@ -147,10 +152,19 @@ struct ZSTD_DCtx_s
|
|
147
152
|
U32 legacyVersion;
|
148
153
|
U32 hostageByte;
|
149
154
|
int noForwardProgress;
|
155
|
+
ZSTD_outBufferMode_e outBufferMode;
|
156
|
+
ZSTD_outBuffer expectedOutBuffer;
|
150
157
|
|
151
158
|
/* workspace */
|
152
159
|
BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH];
|
153
160
|
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
|
161
|
+
|
162
|
+
size_t oversizedDuration;
|
163
|
+
|
164
|
+
#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
|
165
|
+
void const* dictContentBeginForFuzzing;
|
166
|
+
void const* dictContentEndForFuzzing;
|
167
|
+
#endif
|
154
168
|
}; /* typedef'd to ZSTD_DCtx within "zstd.h" */
|
155
169
|
|
156
170
|
|
@@ -160,7 +174,7 @@ struct ZSTD_DCtx_s
|
|
160
174
|
|
161
175
|
/*! ZSTD_loadDEntropy() :
|
162
176
|
* dict : must point at beginning of a valid zstd dictionary.
|
163
|
-
* @return : size of entropy tables
|
177
|
+
* @return : size of dictionary header (size of magic number + dict ID + entropy tables) */
|
164
178
|
size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
|
165
179
|
const void* const dict, size_t const dictSize);
|
166
180
|
|
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c) 2016-
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -28,7 +28,7 @@ extern "C" {
|
|
28
28
|
* Dependencies
|
29
29
|
***************************************/
|
30
30
|
#include <stddef.h> /* size_t */
|
31
|
-
#include "zstd.h" /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */
|
31
|
+
#include "../zstd.h" /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */
|
32
32
|
|
33
33
|
|
34
34
|
/* ***************************************************************
|
@@ -186,7 +186,7 @@ ZBUFF_DEPRECATED("use ZSTD_DStreamOutSize") size_t ZBUFF_recommendedDOutSize(voi
|
|
186
186
|
|
187
187
|
/*--- Dependency ---*/
|
188
188
|
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_customMem */
|
189
|
-
#include "zstd.h"
|
189
|
+
#include "../zstd.h"
|
190
190
|
|
191
191
|
|
192
192
|
/*--- Custom memory allocator ---*/
|
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c) 2016-
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -11,7 +11,7 @@
|
|
11
11
|
/*-*************************************
|
12
12
|
* Dependencies
|
13
13
|
***************************************/
|
14
|
-
#include "error_private.h"
|
14
|
+
#include "../common/error_private.h"
|
15
15
|
#include "zbuff.h"
|
16
16
|
|
17
17
|
/*-****************************************
|
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c) 2016-
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -26,11 +26,11 @@
|
|
26
26
|
#include <string.h> /* memset */
|
27
27
|
#include <time.h> /* clock */
|
28
28
|
|
29
|
-
#include "mem.h" /* read */
|
30
|
-
#include "pool.h"
|
31
|
-
#include "threading.h"
|
29
|
+
#include "../common/mem.h" /* read */
|
30
|
+
#include "../common/pool.h"
|
31
|
+
#include "../common/threading.h"
|
32
32
|
#include "cover.h"
|
33
|
-
#include "zstd_internal.h" /* includes zstd.h */
|
33
|
+
#include "../common/zstd_internal.h" /* includes zstd.h */
|
34
34
|
#ifndef ZDICT_STATIC_LINKING_ONLY
|
35
35
|
#define ZDICT_STATIC_LINKING_ONLY
|
36
36
|
#endif
|
@@ -1,11 +1,21 @@
|
|
1
|
+
/*
|
2
|
+
* Copyright (c) 2017-2020, Facebook, Inc.
|
3
|
+
* All rights reserved.
|
4
|
+
*
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
9
|
+
*/
|
10
|
+
|
1
11
|
#include <stdio.h> /* fprintf */
|
2
12
|
#include <stdlib.h> /* malloc, free, qsort */
|
3
13
|
#include <string.h> /* memset */
|
4
14
|
#include <time.h> /* clock */
|
5
|
-
#include "mem.h" /* read */
|
6
|
-
#include "pool.h"
|
7
|
-
#include "threading.h"
|
8
|
-
#include "zstd_internal.h" /* includes zstd.h */
|
15
|
+
#include "../common/mem.h" /* read */
|
16
|
+
#include "../common/pool.h"
|
17
|
+
#include "../common/threading.h"
|
18
|
+
#include "../common/zstd_internal.h" /* includes zstd.h */
|
9
19
|
#ifndef ZDICT_STATIC_LINKING_ONLY
|
10
20
|
#define ZDICT_STATIC_LINKING_ONLY
|
11
21
|
#endif
|
@@ -1,3 +1,13 @@
|
|
1
|
+
/*
|
2
|
+
* Copyright (c) 2018-2020, Facebook, Inc.
|
3
|
+
* All rights reserved.
|
4
|
+
*
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
9
|
+
*/
|
10
|
+
|
1
11
|
/*-*************************************
|
2
12
|
* Dependencies
|
3
13
|
***************************************/
|
@@ -6,11 +16,11 @@
|
|
6
16
|
#include <string.h> /* memset */
|
7
17
|
#include <time.h> /* clock */
|
8
18
|
|
9
|
-
#include "mem.h" /* read */
|
10
|
-
#include "pool.h"
|
11
|
-
#include "threading.h"
|
19
|
+
#include "../common/mem.h" /* read */
|
20
|
+
#include "../common/pool.h"
|
21
|
+
#include "../common/threading.h"
|
12
22
|
#include "cover.h"
|
13
|
-
#include "zstd_internal.h" /* includes zstd.h */
|
23
|
+
#include "../common/zstd_internal.h" /* includes zstd.h */
|
14
24
|
#ifndef ZDICT_STATIC_LINKING_ONLY
|
15
25
|
#define ZDICT_STATIC_LINKING_ONLY
|
16
26
|
#endif
|
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c) 2016-
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -37,17 +37,18 @@
|
|
37
37
|
#include <stdio.h> /* fprintf, fopen, ftello64 */
|
38
38
|
#include <time.h> /* clock */
|
39
39
|
|
40
|
-
#include "mem.h" /* read */
|
41
|
-
#include "fse.h" /* FSE_normalizeCount, FSE_writeNCount */
|
40
|
+
#include "../common/mem.h" /* read */
|
41
|
+
#include "../common/fse.h" /* FSE_normalizeCount, FSE_writeNCount */
|
42
42
|
#define HUF_STATIC_LINKING_ONLY
|
43
|
-
#include "huf.h" /* HUF_buildCTable, HUF_writeCTable */
|
44
|
-
#include "zstd_internal.h" /* includes zstd.h */
|
45
|
-
#include "xxhash.h" /* XXH64 */
|
43
|
+
#include "../common/huf.h" /* HUF_buildCTable, HUF_writeCTable */
|
44
|
+
#include "../common/zstd_internal.h" /* includes zstd.h */
|
45
|
+
#include "../common/xxhash.h" /* XXH64 */
|
46
46
|
#include "divsufsort.h"
|
47
47
|
#ifndef ZDICT_STATIC_LINKING_ONLY
|
48
48
|
# define ZDICT_STATIC_LINKING_ONLY
|
49
49
|
#endif
|
50
50
|
#include "zdict.h"
|
51
|
+
#include "../compress/zstd_compress_internal.h" /* ZSTD_loadCEntropy() */
|
51
52
|
|
52
53
|
|
53
54
|
/*-*************************************
|
@@ -99,6 +100,29 @@ unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize)
|
|
99
100
|
return MEM_readLE32((const char*)dictBuffer + 4);
|
100
101
|
}
|
101
102
|
|
103
|
+
size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize)
|
104
|
+
{
|
105
|
+
size_t headerSize;
|
106
|
+
if (dictSize <= 8 || MEM_readLE32(dictBuffer) != ZSTD_MAGIC_DICTIONARY) return ERROR(dictionary_corrupted);
|
107
|
+
|
108
|
+
{ unsigned offcodeMaxValue = MaxOff;
|
109
|
+
ZSTD_compressedBlockState_t* bs = (ZSTD_compressedBlockState_t*)malloc(sizeof(ZSTD_compressedBlockState_t));
|
110
|
+
U32* wksp = (U32*)malloc(HUF_WORKSPACE_SIZE);
|
111
|
+
short* offcodeNCount = (short*)malloc((MaxOff+1)*sizeof(short));
|
112
|
+
if (!bs || !wksp || !offcodeNCount) {
|
113
|
+
headerSize = ERROR(memory_allocation);
|
114
|
+
} else {
|
115
|
+
ZSTD_reset_compressedBlockState(bs);
|
116
|
+
headerSize = ZSTD_loadCEntropy(bs, wksp, offcodeNCount, &offcodeMaxValue, dictBuffer, dictSize);
|
117
|
+
}
|
118
|
+
|
119
|
+
free(bs);
|
120
|
+
free(wksp);
|
121
|
+
free(offcodeNCount);
|
122
|
+
}
|
123
|
+
|
124
|
+
return headerSize;
|
125
|
+
}
|
102
126
|
|
103
127
|
/*-********************************************************
|
104
128
|
* Dictionary training functions
|
@@ -588,12 +612,12 @@ typedef struct
|
|
588
612
|
|
589
613
|
#define MAXREPOFFSET 1024
|
590
614
|
|
591
|
-
static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params,
|
615
|
+
static void ZDICT_countEStats(EStats_ress_t esr, const ZSTD_parameters* params,
|
592
616
|
unsigned* countLit, unsigned* offsetcodeCount, unsigned* matchlengthCount, unsigned* litlengthCount, U32* repOffsets,
|
593
617
|
const void* src, size_t srcSize,
|
594
618
|
U32 notificationLevel)
|
595
619
|
{
|
596
|
-
size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_MAX, 1 << params
|
620
|
+
size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_MAX, 1 << params->cParams.windowLog);
|
597
621
|
size_t cSize;
|
598
622
|
|
599
623
|
if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */
|
@@ -731,7 +755,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
|
731
755
|
|
732
756
|
/* collect stats on all samples */
|
733
757
|
for (u=0; u<nbFiles; u++) {
|
734
|
-
ZDICT_countEStats(esr, params,
|
758
|
+
ZDICT_countEStats(esr, ¶ms,
|
735
759
|
countLit, offcodeCount, matchLengthCount, litLengthCount, repOffset,
|
736
760
|
(const char*)srcBuffer + pos, fileSizes[u],
|
737
761
|
notificationLevel);
|
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c) 2016-
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -61,9 +61,57 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCap
|
|
61
61
|
const void* samplesBuffer,
|
62
62
|
const size_t* samplesSizes, unsigned nbSamples);
|
63
63
|
|
64
|
+
typedef struct {
|
65
|
+
int compressionLevel; /*< optimize for a specific zstd compression level; 0 means default */
|
66
|
+
unsigned notificationLevel; /*< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
|
67
|
+
unsigned dictID; /*< force dictID value; 0 means auto mode (32-bits random value) */
|
68
|
+
} ZDICT_params_t;
|
69
|
+
|
70
|
+
/*! ZDICT_finalizeDictionary():
|
71
|
+
* Given a custom content as a basis for dictionary, and a set of samples,
|
72
|
+
* finalize dictionary by adding headers and statistics according to the zstd
|
73
|
+
* dictionary format.
|
74
|
+
*
|
75
|
+
* Samples must be stored concatenated in a flat buffer `samplesBuffer`,
|
76
|
+
* supplied with an array of sizes `samplesSizes`, providing the size of each
|
77
|
+
* sample in order. The samples are used to construct the statistics, so they
|
78
|
+
* should be representative of what you will compress with this dictionary.
|
79
|
+
*
|
80
|
+
* The compression level can be set in `parameters`. You should pass the
|
81
|
+
* compression level you expect to use in production. The statistics for each
|
82
|
+
* compression level differ, so tuning the dictionary for the compression level
|
83
|
+
* can help quite a bit.
|
84
|
+
*
|
85
|
+
* You can set an explicit dictionary ID in `parameters`, or allow us to pick
|
86
|
+
* a random dictionary ID for you, but we can't guarantee no collisions.
|
87
|
+
*
|
88
|
+
* The dstDictBuffer and the dictContent may overlap, and the content will be
|
89
|
+
* appended to the end of the header. If the header + the content doesn't fit in
|
90
|
+
* maxDictSize the beginning of the content is truncated to make room, since it
|
91
|
+
* is presumed that the most profitable content is at the end of the dictionary,
|
92
|
+
* since that is the cheapest to reference.
|
93
|
+
*
|
94
|
+
* `dictContentSize` must be >= ZDICT_CONTENTSIZE_MIN bytes.
|
95
|
+
* `maxDictSize` must be >= max(dictContentSize, ZSTD_DICTSIZE_MIN).
|
96
|
+
*
|
97
|
+
* @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`),
|
98
|
+
* or an error code, which can be tested by ZDICT_isError().
|
99
|
+
* Note: ZDICT_finalizeDictionary() will push notifications into stderr if
|
100
|
+
* instructed to, using notificationLevel>0.
|
101
|
+
* NOTE: This function currently may fail in several edge cases including:
|
102
|
+
* * Not enough samples
|
103
|
+
* * Samples are uncompressible
|
104
|
+
* * Samples are all exactly the same
|
105
|
+
*/
|
106
|
+
ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dstDictBuffer, size_t maxDictSize,
|
107
|
+
const void* dictContent, size_t dictContentSize,
|
108
|
+
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
109
|
+
ZDICT_params_t parameters);
|
110
|
+
|
64
111
|
|
65
112
|
/*====== Helper functions ======*/
|
66
113
|
ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize); /**< extracts dictID; @return zero if error (not a valid dictionary) */
|
114
|
+
ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize); /* returns dict header size; returns a ZSTD error code on failure */
|
67
115
|
ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode);
|
68
116
|
ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
|
69
117
|
|
@@ -78,11 +126,8 @@ ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
|
|
78
126
|
* Use them only in association with static linking.
|
79
127
|
* ==================================================================================== */
|
80
128
|
|
81
|
-
|
82
|
-
|
83
|
-
unsigned notificationLevel; /* Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
|
84
|
-
unsigned dictID; /* force dictID value; 0 means auto mode (32-bits random value) */
|
85
|
-
} ZDICT_params_t;
|
129
|
+
#define ZDICT_CONTENTSIZE_MIN 128
|
130
|
+
#define ZDICT_DICTSIZE_MIN 256
|
86
131
|
|
87
132
|
/*! ZDICT_cover_params_t:
|
88
133
|
* k and d are the only required parameters.
|
@@ -198,28 +243,6 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
|
|
198
243
|
const size_t* samplesSizes, unsigned nbSamples,
|
199
244
|
ZDICT_fastCover_params_t* parameters);
|
200
245
|
|
201
|
-
/*! ZDICT_finalizeDictionary():
|
202
|
-
* Given a custom content as a basis for dictionary, and a set of samples,
|
203
|
-
* finalize dictionary by adding headers and statistics.
|
204
|
-
*
|
205
|
-
* Samples must be stored concatenated in a flat buffer `samplesBuffer`,
|
206
|
-
* supplied with an array of sizes `samplesSizes`, providing the size of each sample in order.
|
207
|
-
*
|
208
|
-
* dictContentSize must be >= ZDICT_CONTENTSIZE_MIN bytes.
|
209
|
-
* maxDictSize must be >= dictContentSize, and must be >= ZDICT_DICTSIZE_MIN bytes.
|
210
|
-
*
|
211
|
-
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`),
|
212
|
-
* or an error code, which can be tested by ZDICT_isError().
|
213
|
-
* Note: ZDICT_finalizeDictionary() will push notifications into stderr if instructed to, using notificationLevel>0.
|
214
|
-
* Note 2: dictBuffer and dictContent can overlap
|
215
|
-
*/
|
216
|
-
#define ZDICT_CONTENTSIZE_MIN 128
|
217
|
-
#define ZDICT_DICTSIZE_MIN 256
|
218
|
-
ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
|
219
|
-
const void* dictContent, size_t dictContentSize,
|
220
|
-
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
221
|
-
ZDICT_params_t parameters);
|
222
|
-
|
223
246
|
typedef struct {
|
224
247
|
unsigned selectivityLevel; /* 0 means default; larger => select more => larger dictionary */
|
225
248
|
ZDICT_params_t zParams;
|
@@ -1,10 +1,11 @@
|
|
1
1
|
# ################################################################
|
2
|
-
# Copyright (c) 2016-
|
2
|
+
# Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
# All rights reserved.
|
4
4
|
#
|
5
5
|
# This source code is licensed under both the BSD-style license (found in the
|
6
6
|
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
7
7
|
# in the COPYING file in the root directory of this source tree).
|
8
|
+
# You may select, at your option, one of the above-listed licenses.
|
8
9
|
# ################################################################
|
9
10
|
|
10
11
|
VOID := /dev/null
|
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c) 2016-
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -18,9 +18,9 @@ extern "C" {
|
|
18
18
|
/* *************************************
|
19
19
|
* Includes
|
20
20
|
***************************************/
|
21
|
-
#include "mem.h" /* MEM_STATIC */
|
22
|
-
#include "error_private.h" /* ERROR */
|
23
|
-
#include "zstd_internal.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTD_frameSizeInfo */
|
21
|
+
#include "../common/mem.h" /* MEM_STATIC */
|
22
|
+
#include "../common/error_private.h" /* ERROR */
|
23
|
+
#include "../common/zstd_internal.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTD_frameSizeInfo */
|
24
24
|
|
25
25
|
#if !defined (ZSTD_LEGACY_SUPPORT) || (ZSTD_LEGACY_SUPPORT == 0)
|
26
26
|
# undef ZSTD_LEGACY_SUPPORT
|
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c) 2016-
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -14,7 +14,7 @@
|
|
14
14
|
******************************************/
|
15
15
|
#include <stddef.h> /* size_t, ptrdiff_t */
|
16
16
|
#include "zstd_v01.h"
|
17
|
-
#include "error_private.h"
|
17
|
+
#include "../common/error_private.h"
|
18
18
|
|
19
19
|
|
20
20
|
/******************************************
|
@@ -257,7 +257,7 @@ static U64 FSE_read64(const void* memPtr)
|
|
257
257
|
U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
258
258
|
}
|
259
259
|
|
260
|
-
#endif
|
260
|
+
#endif /* FSE_FORCE_MEMORY_ACCESS */
|
261
261
|
|
262
262
|
static U16 FSE_readLE16(const void* memPtr)
|
263
263
|
{
|
@@ -1078,7 +1078,7 @@ static size_t HUF_decompress_usingDTable( /* -3% slower when non static */
|
|
1078
1078
|
BYTE* const ostart = (BYTE*) dst;
|
1079
1079
|
BYTE* op = ostart;
|
1080
1080
|
BYTE* const omax = op + maxDstSize;
|
1081
|
-
BYTE* const olimit = omax-15;
|
1081
|
+
BYTE* const olimit = maxDstSize < 15 ? op : omax-15;
|
1082
1082
|
|
1083
1083
|
const void* ptr = DTable;
|
1084
1084
|
const HUF_DElt* const dt = (const HUF_DElt*)(ptr)+1;
|
@@ -1092,7 +1092,7 @@ static size_t HUF_decompress_usingDTable( /* -3% slower when non static */
|
|
1092
1092
|
const size_t length1 = FSE_readLE16(jumpTable);
|
1093
1093
|
const size_t length2 = FSE_readLE16(jumpTable+1);
|
1094
1094
|
const size_t length3 = FSE_readLE16(jumpTable+2);
|
1095
|
-
const size_t length4 = cSrcSize - 6 - length1 - length2 - length3;
|
1095
|
+
const size_t length4 = cSrcSize - 6 - length1 - length2 - length3; /* check coherency !! */
|
1096
1096
|
const char* const start1 = (const char*)(cSrc) + 6;
|
1097
1097
|
const char* const start2 = start1 + length1;
|
1098
1098
|
const char* const start3 = start2 + length2;
|
@@ -1150,11 +1150,11 @@ static size_t HUF_decompress_usingDTable( /* -3% slower when non static */
|
|
1150
1150
|
|
1151
1151
|
/* tail */
|
1152
1152
|
{
|
1153
|
-
|
1153
|
+
/* bitTail = bitD1; */ /* *much* slower : -20% !??! */
|
1154
1154
|
FSE_DStream_t bitTail;
|
1155
1155
|
bitTail.ptr = bitD1.ptr;
|
1156
1156
|
bitTail.bitsConsumed = bitD1.bitsConsumed;
|
1157
|
-
bitTail.bitContainer = bitD1.bitContainer;
|
1157
|
+
bitTail.bitContainer = bitD1.bitContainer; /* required in case of FSE_DStream_endOfBuffer */
|
1158
1158
|
bitTail.start = start1;
|
1159
1159
|
for ( ; (FSE_reloadDStream(&bitTail) < FSE_DStream_completed) && (op<omax) ; op++)
|
1160
1160
|
{
|
@@ -1483,7 +1483,9 @@ static size_t ZSTDv01_getcBlockSize(const void* src, size_t srcSize, blockProper
|
|
1483
1483
|
static size_t ZSTD_copyUncompressedBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
1484
1484
|
{
|
1485
1485
|
if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
|
1486
|
-
|
1486
|
+
if (srcSize > 0) {
|
1487
|
+
memcpy(dst, src, srcSize);
|
1488
|
+
}
|
1487
1489
|
return srcSize;
|
1488
1490
|
}
|
1489
1491
|
|
@@ -1502,7 +1504,7 @@ static size_t ZSTD_decompressLiterals(void* ctx,
|
|
1502
1504
|
if (srcSize <= 3) return ERROR(corruption_detected);
|
1503
1505
|
|
1504
1506
|
litSize = ip[1] + (ip[0]<<8);
|
1505
|
-
litSize += ((ip[-3] >> 3) & 7) << 16;
|
1507
|
+
litSize += ((ip[-3] >> 3) & 7) << 16; /* mmmmh.... */
|
1506
1508
|
op = oend - litSize;
|
1507
1509
|
|
1508
1510
|
(void)ctx;
|
@@ -1541,7 +1543,9 @@ static size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
|
|
1541
1543
|
size_t rleSize = litbp.origSize;
|
1542
1544
|
if (rleSize>maxDstSize) return ERROR(dstSize_tooSmall);
|
1543
1545
|
if (!srcSize) return ERROR(srcSize_wrong);
|
1544
|
-
|
1546
|
+
if (rleSize > 0) {
|
1547
|
+
memset(oend - rleSize, *ip, rleSize);
|
1548
|
+
}
|
1545
1549
|
*litStart = oend - rleSize;
|
1546
1550
|
*litSize = rleSize;
|
1547
1551
|
ip++;
|
@@ -1901,8 +1905,10 @@ static size_t ZSTD_decompressSequences(
|
|
1901
1905
|
{
|
1902
1906
|
size_t lastLLSize = litEnd - litPtr;
|
1903
1907
|
if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
|
1904
|
-
if (
|
1905
|
-
|
1908
|
+
if (lastLLSize > 0) {
|
1909
|
+
if (op != litPtr) memmove(op, litPtr, lastLLSize);
|
1910
|
+
op += lastLLSize;
|
1911
|
+
}
|
1906
1912
|
}
|
1907
1913
|
}
|
1908
1914
|
|