zstd-ruby 1.4.4.0 → 1.5.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/dependabot.yml +8 -0
- data/.github/workflows/ruby.yml +35 -0
- data/README.md +2 -2
- data/ext/zstdruby/extconf.rb +1 -0
- data/ext/zstdruby/libzstd/BUCK +5 -7
- data/ext/zstdruby/libzstd/Makefile +241 -173
- data/ext/zstdruby/libzstd/README.md +76 -18
- data/ext/zstdruby/libzstd/common/bitstream.h +75 -57
- data/ext/zstdruby/libzstd/common/compiler.h +196 -20
- data/ext/zstdruby/libzstd/common/cpu.h +1 -3
- data/ext/zstdruby/libzstd/common/debug.c +11 -31
- data/ext/zstdruby/libzstd/common/debug.h +22 -49
- data/ext/zstdruby/libzstd/common/entropy_common.c +208 -76
- data/ext/zstdruby/libzstd/common/error_private.c +3 -1
- data/ext/zstdruby/libzstd/common/error_private.h +87 -4
- data/ext/zstdruby/libzstd/common/fse.h +51 -42
- data/ext/zstdruby/libzstd/common/fse_decompress.c +149 -57
- data/ext/zstdruby/libzstd/common/huf.h +60 -54
- data/ext/zstdruby/libzstd/common/mem.h +87 -98
- data/ext/zstdruby/libzstd/common/pool.c +23 -17
- data/ext/zstdruby/libzstd/common/pool.h +3 -3
- data/ext/zstdruby/libzstd/common/portability_macros.h +131 -0
- data/ext/zstdruby/libzstd/common/threading.c +10 -8
- data/ext/zstdruby/libzstd/common/threading.h +4 -3
- data/ext/zstdruby/libzstd/common/xxhash.c +15 -873
- data/ext/zstdruby/libzstd/common/xxhash.h +5572 -191
- data/ext/zstdruby/libzstd/common/zstd_common.c +10 -10
- data/ext/zstdruby/libzstd/common/zstd_deps.h +111 -0
- data/ext/zstdruby/libzstd/common/zstd_internal.h +252 -108
- data/ext/zstdruby/libzstd/common/zstd_trace.h +163 -0
- data/ext/zstdruby/libzstd/compress/clevels.h +134 -0
- data/ext/zstdruby/libzstd/compress/fse_compress.c +105 -85
- data/ext/zstdruby/libzstd/compress/hist.c +41 -63
- data/ext/zstdruby/libzstd/compress/hist.h +13 -33
- data/ext/zstdruby/libzstd/compress/huf_compress.c +831 -259
- data/ext/zstdruby/libzstd/compress/zstd_compress.c +3213 -1007
- data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +493 -71
- data/ext/zstdruby/libzstd/compress/zstd_compress_literals.c +21 -16
- data/ext/zstdruby/libzstd/compress/zstd_compress_literals.h +4 -2
- data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.c +51 -24
- data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.h +10 -3
- data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.c +573 -0
- data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.h +32 -0
- data/ext/zstdruby/libzstd/compress/zstd_cwksp.h +208 -81
- data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +315 -137
- data/ext/zstdruby/libzstd/compress/zstd_double_fast.h +2 -2
- data/ext/zstdruby/libzstd/compress/zstd_fast.c +319 -128
- data/ext/zstdruby/libzstd/compress/zstd_fast.h +2 -2
- data/ext/zstdruby/libzstd/compress/zstd_lazy.c +1156 -171
- data/ext/zstdruby/libzstd/compress/zstd_lazy.h +59 -1
- data/ext/zstdruby/libzstd/compress/zstd_ldm.c +331 -206
- data/ext/zstdruby/libzstd/compress/zstd_ldm.h +15 -3
- data/ext/zstdruby/libzstd/compress/zstd_ldm_geartab.h +106 -0
- data/ext/zstdruby/libzstd/compress/zstd_opt.c +403 -226
- data/ext/zstdruby/libzstd/compress/zstd_opt.h +1 -1
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +188 -453
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +32 -114
- data/ext/zstdruby/libzstd/decompress/huf_decompress.c +1065 -410
- data/ext/zstdruby/libzstd/decompress/huf_decompress_amd64.S +571 -0
- data/ext/zstdruby/libzstd/decompress/zstd_ddict.c +20 -16
- data/ext/zstdruby/libzstd/decompress/zstd_ddict.h +3 -3
- data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +691 -230
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.c +1072 -323
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.h +16 -7
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_internal.h +71 -10
- data/ext/zstdruby/libzstd/deprecated/zbuff.h +3 -3
- data/ext/zstdruby/libzstd/deprecated/zbuff_common.c +2 -2
- data/ext/zstdruby/libzstd/deprecated/zbuff_compress.c +24 -4
- data/ext/zstdruby/libzstd/deprecated/zbuff_decompress.c +1 -1
- data/ext/zstdruby/libzstd/dictBuilder/cover.c +57 -40
- data/ext/zstdruby/libzstd/dictBuilder/cover.h +20 -9
- data/ext/zstdruby/libzstd/dictBuilder/divsufsort.c +1 -1
- data/ext/zstdruby/libzstd/dictBuilder/fastcover.c +54 -35
- data/ext/zstdruby/libzstd/dictBuilder/zdict.c +151 -57
- data/ext/zstdruby/libzstd/dll/example/Makefile +2 -1
- data/ext/zstdruby/libzstd/dll/example/README.md +16 -22
- data/ext/zstdruby/libzstd/legacy/zstd_legacy.h +4 -4
- data/ext/zstdruby/libzstd/legacy/zstd_v01.c +25 -19
- data/ext/zstdruby/libzstd/legacy/zstd_v01.h +1 -1
- data/ext/zstdruby/libzstd/legacy/zstd_v02.c +18 -14
- data/ext/zstdruby/libzstd/legacy/zstd_v02.h +1 -1
- data/ext/zstdruby/libzstd/legacy/zstd_v03.c +18 -14
- data/ext/zstdruby/libzstd/legacy/zstd_v03.h +1 -1
- data/ext/zstdruby/libzstd/legacy/zstd_v04.c +22 -16
- data/ext/zstdruby/libzstd/legacy/zstd_v04.h +1 -1
- data/ext/zstdruby/libzstd/legacy/zstd_v05.c +29 -25
- data/ext/zstdruby/libzstd/legacy/zstd_v05.h +2 -2
- data/ext/zstdruby/libzstd/legacy/zstd_v06.c +29 -25
- data/ext/zstdruby/libzstd/legacy/zstd_v06.h +1 -1
- data/ext/zstdruby/libzstd/legacy/zstd_v07.c +34 -26
- data/ext/zstdruby/libzstd/legacy/zstd_v07.h +1 -1
- data/ext/zstdruby/libzstd/libzstd.mk +185 -0
- data/ext/zstdruby/libzstd/libzstd.pc.in +4 -3
- data/ext/zstdruby/libzstd/modulemap/module.modulemap +4 -0
- data/ext/zstdruby/libzstd/{dictBuilder/zdict.h → zdict.h} +201 -31
- data/ext/zstdruby/libzstd/zstd.h +760 -234
- data/ext/zstdruby/libzstd/{common/zstd_errors.h → zstd_errors.h} +3 -1
- data/ext/zstdruby/zstdruby.c +2 -2
- data/lib/zstd-ruby/version.rb +1 -1
- metadata +20 -9
- data/.travis.yml +0 -14
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c)
|
2
|
+
* Copyright (c) Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -20,11 +20,10 @@
|
|
20
20
|
|
21
21
|
|
22
22
|
/* ====== Dependencies ====== */
|
23
|
-
#include
|
24
|
-
#include
|
25
|
-
#include "
|
26
|
-
#include "
|
27
|
-
#include "threading.h" /* mutex */
|
23
|
+
#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset, INT_MAX, UINT_MAX */
|
24
|
+
#include "../common/mem.h" /* MEM_STATIC */
|
25
|
+
#include "../common/pool.h" /* threadpool */
|
26
|
+
#include "../common/threading.h" /* mutex */
|
28
27
|
#include "zstd_compress_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
|
29
28
|
#include "zstd_ldm.h"
|
30
29
|
#include "zstdmt_compress.h"
|
@@ -106,11 +105,11 @@ typedef struct ZSTDMT_bufferPool_s {
|
|
106
105
|
static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned nbWorkers, ZSTD_customMem cMem)
|
107
106
|
{
|
108
107
|
unsigned const maxNbBuffers = 2*nbWorkers + 3;
|
109
|
-
ZSTDMT_bufferPool* const bufPool = (ZSTDMT_bufferPool*)
|
108
|
+
ZSTDMT_bufferPool* const bufPool = (ZSTDMT_bufferPool*)ZSTD_customCalloc(
|
110
109
|
sizeof(ZSTDMT_bufferPool) + (maxNbBuffers-1) * sizeof(buffer_t), cMem);
|
111
110
|
if (bufPool==NULL) return NULL;
|
112
111
|
if (ZSTD_pthread_mutex_init(&bufPool->poolMutex, NULL)) {
|
113
|
-
|
112
|
+
ZSTD_customFree(bufPool, cMem);
|
114
113
|
return NULL;
|
115
114
|
}
|
116
115
|
bufPool->bufferSize = 64 KB;
|
@@ -127,10 +126,10 @@ static void ZSTDMT_freeBufferPool(ZSTDMT_bufferPool* bufPool)
|
|
127
126
|
if (!bufPool) return; /* compatibility with free on NULL */
|
128
127
|
for (u=0; u<bufPool->totalBuffers; u++) {
|
129
128
|
DEBUGLOG(4, "free buffer %2u (address:%08X)", u, (U32)(size_t)bufPool->bTable[u].start);
|
130
|
-
|
129
|
+
ZSTD_customFree(bufPool->bTable[u].start, bufPool->cMem);
|
131
130
|
}
|
132
131
|
ZSTD_pthread_mutex_destroy(&bufPool->poolMutex);
|
133
|
-
|
132
|
+
ZSTD_customFree(bufPool, bufPool->cMem);
|
134
133
|
}
|
135
134
|
|
136
135
|
/* only works at initialization, not during compression */
|
@@ -201,13 +200,13 @@ static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool)
|
|
201
200
|
}
|
202
201
|
/* size conditions not respected : scratch this buffer, create new one */
|
203
202
|
DEBUGLOG(5, "ZSTDMT_getBuffer: existing buffer does not meet size conditions => freeing");
|
204
|
-
|
203
|
+
ZSTD_customFree(buf.start, bufPool->cMem);
|
205
204
|
}
|
206
205
|
ZSTD_pthread_mutex_unlock(&bufPool->poolMutex);
|
207
206
|
/* create new buffer */
|
208
207
|
DEBUGLOG(5, "ZSTDMT_getBuffer: create a new buffer");
|
209
208
|
{ buffer_t buffer;
|
210
|
-
void* const start =
|
209
|
+
void* const start = ZSTD_customMalloc(bSize, bufPool->cMem);
|
211
210
|
buffer.start = start; /* note : start can be NULL if malloc fails ! */
|
212
211
|
buffer.capacity = (start==NULL) ? 0 : bSize;
|
213
212
|
if (start==NULL) {
|
@@ -229,13 +228,13 @@ static buffer_t ZSTDMT_resizeBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buffer)
|
|
229
228
|
{
|
230
229
|
size_t const bSize = bufPool->bufferSize;
|
231
230
|
if (buffer.capacity < bSize) {
|
232
|
-
void* const start =
|
231
|
+
void* const start = ZSTD_customMalloc(bSize, bufPool->cMem);
|
233
232
|
buffer_t newBuffer;
|
234
233
|
newBuffer.start = start;
|
235
234
|
newBuffer.capacity = start == NULL ? 0 : bSize;
|
236
235
|
if (start != NULL) {
|
237
236
|
assert(newBuffer.capacity >= buffer.capacity);
|
238
|
-
|
237
|
+
ZSTD_memcpy(newBuffer.start, buffer.start, buffer.capacity);
|
239
238
|
DEBUGLOG(5, "ZSTDMT_resizeBuffer: created buffer of size %u", (U32)bSize);
|
240
239
|
return newBuffer;
|
241
240
|
}
|
@@ -261,14 +260,12 @@ static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf)
|
|
261
260
|
ZSTD_pthread_mutex_unlock(&bufPool->poolMutex);
|
262
261
|
/* Reached bufferPool capacity (should not happen) */
|
263
262
|
DEBUGLOG(5, "ZSTDMT_releaseBuffer: pool capacity reached => freeing ");
|
264
|
-
|
263
|
+
ZSTD_customFree(buf.start, bufPool->cMem);
|
265
264
|
}
|
266
265
|
|
267
266
|
|
268
267
|
/* ===== Seq Pool Wrapper ====== */
|
269
268
|
|
270
|
-
static rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0};
|
271
|
-
|
272
269
|
typedef ZSTDMT_bufferPool ZSTDMT_seqPool;
|
273
270
|
|
274
271
|
static size_t ZSTDMT_sizeof_seqPool(ZSTDMT_seqPool* seqPool)
|
@@ -278,7 +275,7 @@ static size_t ZSTDMT_sizeof_seqPool(ZSTDMT_seqPool* seqPool)
|
|
278
275
|
|
279
276
|
static rawSeqStore_t bufferToSeq(buffer_t buffer)
|
280
277
|
{
|
281
|
-
rawSeqStore_t seq =
|
278
|
+
rawSeqStore_t seq = kNullRawSeqStore;
|
282
279
|
seq.seq = (rawSeq*)buffer.start;
|
283
280
|
seq.capacity = buffer.capacity / sizeof(rawSeq);
|
284
281
|
return seq;
|
@@ -354,7 +351,7 @@ static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool)
|
|
354
351
|
for (cid=0; cid<pool->totalCCtx; cid++)
|
355
352
|
ZSTD_freeCCtx(pool->cctx[cid]); /* note : compatible with free on NULL */
|
356
353
|
ZSTD_pthread_mutex_destroy(&pool->poolMutex);
|
357
|
-
|
354
|
+
ZSTD_customFree(pool, pool->cMem);
|
358
355
|
}
|
359
356
|
|
360
357
|
/* ZSTDMT_createCCtxPool() :
|
@@ -362,12 +359,12 @@ static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool)
|
|
362
359
|
static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(int nbWorkers,
|
363
360
|
ZSTD_customMem cMem)
|
364
361
|
{
|
365
|
-
ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*)
|
362
|
+
ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) ZSTD_customCalloc(
|
366
363
|
sizeof(ZSTDMT_CCtxPool) + (nbWorkers-1)*sizeof(ZSTD_CCtx*), cMem);
|
367
364
|
assert(nbWorkers > 0);
|
368
365
|
if (!cctxPool) return NULL;
|
369
366
|
if (ZSTD_pthread_mutex_init(&cctxPool->poolMutex, NULL)) {
|
370
|
-
|
367
|
+
ZSTD_customFree(cctxPool, cMem);
|
371
368
|
return NULL;
|
372
369
|
}
|
373
370
|
cctxPool->cMem = cMem;
|
@@ -461,52 +458,72 @@ typedef struct {
|
|
461
458
|
ZSTD_window_t ldmWindow; /* A thread-safe copy of ldmState.window */
|
462
459
|
} serialState_t;
|
463
460
|
|
464
|
-
static int
|
461
|
+
static int
|
462
|
+
ZSTDMT_serialState_reset(serialState_t* serialState,
|
463
|
+
ZSTDMT_seqPool* seqPool,
|
464
|
+
ZSTD_CCtx_params params,
|
465
|
+
size_t jobSize,
|
466
|
+
const void* dict, size_t const dictSize,
|
467
|
+
ZSTD_dictContentType_e dictContentType)
|
465
468
|
{
|
466
469
|
/* Adjust parameters */
|
467
|
-
if (params.ldmParams.enableLdm) {
|
470
|
+
if (params.ldmParams.enableLdm == ZSTD_ps_enable) {
|
468
471
|
DEBUGLOG(4, "LDM window size = %u KB", (1U << params.cParams.windowLog) >> 10);
|
469
472
|
ZSTD_ldm_adjustParameters(¶ms.ldmParams, ¶ms.cParams);
|
470
473
|
assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog);
|
471
474
|
assert(params.ldmParams.hashRateLog < 32);
|
472
|
-
serialState->ldmState.hashPower =
|
473
|
-
ZSTD_rollingHash_primePower(params.ldmParams.minMatchLength);
|
474
475
|
} else {
|
475
|
-
|
476
|
+
ZSTD_memset(¶ms.ldmParams, 0, sizeof(params.ldmParams));
|
476
477
|
}
|
477
478
|
serialState->nextJobID = 0;
|
478
479
|
if (params.fParams.checksumFlag)
|
479
480
|
XXH64_reset(&serialState->xxhState, 0);
|
480
|
-
if (params.ldmParams.enableLdm) {
|
481
|
+
if (params.ldmParams.enableLdm == ZSTD_ps_enable) {
|
481
482
|
ZSTD_customMem cMem = params.customMem;
|
482
483
|
unsigned const hashLog = params.ldmParams.hashLog;
|
483
484
|
size_t const hashSize = ((size_t)1 << hashLog) * sizeof(ldmEntry_t);
|
484
485
|
unsigned const bucketLog =
|
485
486
|
params.ldmParams.hashLog - params.ldmParams.bucketSizeLog;
|
486
|
-
size_t const bucketSize = (size_t)1 << bucketLog;
|
487
487
|
unsigned const prevBucketLog =
|
488
488
|
serialState->params.ldmParams.hashLog -
|
489
489
|
serialState->params.ldmParams.bucketSizeLog;
|
490
|
+
size_t const numBuckets = (size_t)1 << bucketLog;
|
490
491
|
/* Size the seq pool tables */
|
491
492
|
ZSTDMT_setNbSeq(seqPool, ZSTD_ldm_getMaxNbSeq(params.ldmParams, jobSize));
|
492
493
|
/* Reset the window */
|
493
|
-
|
494
|
-
serialState->ldmWindow = serialState->ldmState.window;
|
494
|
+
ZSTD_window_init(&serialState->ldmState.window);
|
495
495
|
/* Resize tables and output space if necessary. */
|
496
496
|
if (serialState->ldmState.hashTable == NULL || serialState->params.ldmParams.hashLog < hashLog) {
|
497
|
-
|
498
|
-
serialState->ldmState.hashTable = (ldmEntry_t*)
|
497
|
+
ZSTD_customFree(serialState->ldmState.hashTable, cMem);
|
498
|
+
serialState->ldmState.hashTable = (ldmEntry_t*)ZSTD_customMalloc(hashSize, cMem);
|
499
499
|
}
|
500
500
|
if (serialState->ldmState.bucketOffsets == NULL || prevBucketLog < bucketLog) {
|
501
|
-
|
502
|
-
serialState->ldmState.bucketOffsets = (BYTE*)
|
501
|
+
ZSTD_customFree(serialState->ldmState.bucketOffsets, cMem);
|
502
|
+
serialState->ldmState.bucketOffsets = (BYTE*)ZSTD_customMalloc(numBuckets, cMem);
|
503
503
|
}
|
504
504
|
if (!serialState->ldmState.hashTable || !serialState->ldmState.bucketOffsets)
|
505
505
|
return 1;
|
506
506
|
/* Zero the tables */
|
507
|
-
|
508
|
-
|
507
|
+
ZSTD_memset(serialState->ldmState.hashTable, 0, hashSize);
|
508
|
+
ZSTD_memset(serialState->ldmState.bucketOffsets, 0, numBuckets);
|
509
|
+
|
510
|
+
/* Update window state and fill hash table with dict */
|
511
|
+
serialState->ldmState.loadedDictEnd = 0;
|
512
|
+
if (dictSize > 0) {
|
513
|
+
if (dictContentType == ZSTD_dct_rawContent) {
|
514
|
+
BYTE const* const dictEnd = (const BYTE*)dict + dictSize;
|
515
|
+
ZSTD_window_update(&serialState->ldmState.window, dict, dictSize, /* forceNonContiguous */ 0);
|
516
|
+
ZSTD_ldm_fillHashTable(&serialState->ldmState, (const BYTE*)dict, dictEnd, ¶ms.ldmParams);
|
517
|
+
serialState->ldmState.loadedDictEnd = params.forceWindow ? 0 : (U32)(dictEnd - serialState->ldmState.window.base);
|
518
|
+
} else {
|
519
|
+
/* don't even load anything */
|
520
|
+
}
|
521
|
+
}
|
522
|
+
|
523
|
+
/* Initialize serialState's copy of ldmWindow. */
|
524
|
+
serialState->ldmWindow = serialState->ldmState.window;
|
509
525
|
}
|
526
|
+
|
510
527
|
serialState->params = params;
|
511
528
|
serialState->params.jobSize = (U32)jobSize;
|
512
529
|
return 0;
|
@@ -515,7 +532,7 @@ static int ZSTDMT_serialState_reset(serialState_t* serialState, ZSTDMT_seqPool*
|
|
515
532
|
static int ZSTDMT_serialState_init(serialState_t* serialState)
|
516
533
|
{
|
517
534
|
int initError = 0;
|
518
|
-
|
535
|
+
ZSTD_memset(serialState, 0, sizeof(*serialState));
|
519
536
|
initError |= ZSTD_pthread_mutex_init(&serialState->mutex, NULL);
|
520
537
|
initError |= ZSTD_pthread_cond_init(&serialState->cond, NULL);
|
521
538
|
initError |= ZSTD_pthread_mutex_init(&serialState->ldmWindowMutex, NULL);
|
@@ -530,8 +547,8 @@ static void ZSTDMT_serialState_free(serialState_t* serialState)
|
|
530
547
|
ZSTD_pthread_cond_destroy(&serialState->cond);
|
531
548
|
ZSTD_pthread_mutex_destroy(&serialState->ldmWindowMutex);
|
532
549
|
ZSTD_pthread_cond_destroy(&serialState->ldmWindowCond);
|
533
|
-
|
534
|
-
|
550
|
+
ZSTD_customFree(serialState->ldmState.hashTable, cMem);
|
551
|
+
ZSTD_customFree(serialState->ldmState.bucketOffsets, cMem);
|
535
552
|
}
|
536
553
|
|
537
554
|
static void ZSTDMT_serialState_update(serialState_t* serialState,
|
@@ -547,12 +564,12 @@ static void ZSTDMT_serialState_update(serialState_t* serialState,
|
|
547
564
|
/* A future job may error and skip our job */
|
548
565
|
if (serialState->nextJobID == jobID) {
|
549
566
|
/* It is now our turn, do any processing necessary */
|
550
|
-
if (serialState->params.ldmParams.enableLdm) {
|
567
|
+
if (serialState->params.ldmParams.enableLdm == ZSTD_ps_enable) {
|
551
568
|
size_t error;
|
552
569
|
assert(seqStore.seq != NULL && seqStore.pos == 0 &&
|
553
570
|
seqStore.size == 0 && seqStore.capacity > 0);
|
554
571
|
assert(src.size <= serialState->params.jobSize);
|
555
|
-
ZSTD_window_update(&serialState->ldmState.window, src.start, src.size);
|
572
|
+
ZSTD_window_update(&serialState->ldmState.window, src.start, src.size, /* forceNonContiguous */ 0);
|
556
573
|
error = ZSTD_ldm_generateSequences(
|
557
574
|
&serialState->ldmState, &seqStore,
|
558
575
|
&serialState->params.ldmParams, src.start, src.size);
|
@@ -577,7 +594,7 @@ static void ZSTDMT_serialState_update(serialState_t* serialState,
|
|
577
594
|
if (seqStore.size > 0) {
|
578
595
|
size_t const err = ZSTD_referenceExternalSequences(
|
579
596
|
jobCCtx, seqStore.seq, seqStore.size);
|
580
|
-
assert(serialState->params.ldmParams.enableLdm);
|
597
|
+
assert(serialState->params.ldmParams.enableLdm == ZSTD_ps_enable);
|
581
598
|
assert(!ZSTD_isError(err));
|
582
599
|
(void)err;
|
583
600
|
}
|
@@ -655,7 +672,7 @@ static void ZSTDMT_compressionJob(void* jobDescription)
|
|
655
672
|
if (dstBuff.start==NULL) JOB_ERROR(ERROR(memory_allocation));
|
656
673
|
job->dstBuff = dstBuff; /* this value can be read in ZSTDMT_flush, when it copies the whole job */
|
657
674
|
}
|
658
|
-
if (jobParams.ldmParams.enableLdm && rawSeqStore.seq == NULL)
|
675
|
+
if (jobParams.ldmParams.enableLdm == ZSTD_ps_enable && rawSeqStore.seq == NULL)
|
659
676
|
JOB_ERROR(ERROR(memory_allocation));
|
660
677
|
|
661
678
|
/* Don't compute the checksum for chunks, since we compute it externally,
|
@@ -663,7 +680,9 @@ static void ZSTDMT_compressionJob(void* jobDescription)
|
|
663
680
|
*/
|
664
681
|
if (job->jobID != 0) jobParams.fParams.checksumFlag = 0;
|
665
682
|
/* Don't run LDM for the chunks, since we handle it externally */
|
666
|
-
jobParams.ldmParams.enableLdm =
|
683
|
+
jobParams.ldmParams.enableLdm = ZSTD_ps_disable;
|
684
|
+
/* Correct nbWorkers to 0. */
|
685
|
+
jobParams.nbWorkers = 0;
|
667
686
|
|
668
687
|
|
669
688
|
/* init */
|
@@ -676,6 +695,10 @@ static void ZSTDMT_compressionJob(void* jobDescription)
|
|
676
695
|
{ size_t const forceWindowError = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_forceMaxWindow, !job->firstJob);
|
677
696
|
if (ZSTD_isError(forceWindowError)) JOB_ERROR(forceWindowError);
|
678
697
|
}
|
698
|
+
if (!job->firstJob) {
|
699
|
+
size_t const err = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_deterministicRefPrefix, 0);
|
700
|
+
if (ZSTD_isError(err)) JOB_ERROR(err);
|
701
|
+
}
|
679
702
|
{ size_t const initError = ZSTD_compressBegin_advanced_internal(cctx,
|
680
703
|
job->prefix.start, job->prefix.size, ZSTD_dct_rawContent, /* load dictionary in "content-only" mode (no header analysis) */
|
681
704
|
ZSTD_dtlm_fast,
|
@@ -731,6 +754,13 @@ static void ZSTDMT_compressionJob(void* jobDescription)
|
|
731
754
|
if (ZSTD_isError(cSize)) JOB_ERROR(cSize);
|
732
755
|
lastCBlockSize = cSize;
|
733
756
|
} }
|
757
|
+
if (!job->firstJob) {
|
758
|
+
/* Double check that we don't have an ext-dict, because then our
|
759
|
+
* repcode invalidation doesn't work.
|
760
|
+
*/
|
761
|
+
assert(!ZSTD_window_hasExtDict(cctx->blockState.matchState.window));
|
762
|
+
}
|
763
|
+
ZSTD_CCtx_trace(cctx, 0);
|
734
764
|
|
735
765
|
_endJob:
|
736
766
|
ZSTDMT_serialState_ensureFinished(job->serial, job->jobID, job->cSize);
|
@@ -777,6 +807,15 @@ typedef struct {
|
|
777
807
|
static const roundBuff_t kNullRoundBuff = {NULL, 0, 0};
|
778
808
|
|
779
809
|
#define RSYNC_LENGTH 32
|
810
|
+
/* Don't create chunks smaller than the zstd block size.
|
811
|
+
* This stops us from regressing compression ratio too much,
|
812
|
+
* and ensures our output fits in ZSTD_compressBound().
|
813
|
+
*
|
814
|
+
* If this is shrunk < ZSTD_BLOCKSIZELOG_MIN then
|
815
|
+
* ZSTD_COMPRESSBOUND() will need to be updated.
|
816
|
+
*/
|
817
|
+
#define RSYNC_MIN_BLOCK_LOG ZSTD_BLOCKSIZELOG_MAX
|
818
|
+
#define RSYNC_MIN_BLOCK_SIZE (1<<RSYNC_MIN_BLOCK_LOG)
|
780
819
|
|
781
820
|
typedef struct {
|
782
821
|
U64 hash;
|
@@ -798,7 +837,6 @@ struct ZSTDMT_CCtx_s {
|
|
798
837
|
roundBuff_t roundBuff;
|
799
838
|
serialState_t serial;
|
800
839
|
rsyncState_t rsync;
|
801
|
-
unsigned singleBlockingThread;
|
802
840
|
unsigned jobIDMask;
|
803
841
|
unsigned doneJobID;
|
804
842
|
unsigned nextJobID;
|
@@ -810,6 +848,7 @@ struct ZSTDMT_CCtx_s {
|
|
810
848
|
ZSTD_customMem cMem;
|
811
849
|
ZSTD_CDict* cdictLocal;
|
812
850
|
const ZSTD_CDict* cdict;
|
851
|
+
unsigned providedFactory: 1;
|
813
852
|
};
|
814
853
|
|
815
854
|
static void ZSTDMT_freeJobsTable(ZSTDMT_jobDescription* jobTable, U32 nbJobs, ZSTD_customMem cMem)
|
@@ -820,7 +859,7 @@ static void ZSTDMT_freeJobsTable(ZSTDMT_jobDescription* jobTable, U32 nbJobs, ZS
|
|
820
859
|
ZSTD_pthread_mutex_destroy(&jobTable[jobNb].job_mutex);
|
821
860
|
ZSTD_pthread_cond_destroy(&jobTable[jobNb].job_cond);
|
822
861
|
}
|
823
|
-
|
862
|
+
ZSTD_customFree(jobTable, cMem);
|
824
863
|
}
|
825
864
|
|
826
865
|
/* ZSTDMT_allocJobsTable()
|
@@ -832,7 +871,7 @@ static ZSTDMT_jobDescription* ZSTDMT_createJobsTable(U32* nbJobsPtr, ZSTD_custom
|
|
832
871
|
U32 const nbJobs = 1 << nbJobsLog2;
|
833
872
|
U32 jobNb;
|
834
873
|
ZSTDMT_jobDescription* const jobTable = (ZSTDMT_jobDescription*)
|
835
|
-
|
874
|
+
ZSTD_customCalloc(nbJobs * sizeof(ZSTDMT_jobDescription), cMem);
|
836
875
|
int initError = 0;
|
837
876
|
if (jobTable==NULL) return NULL;
|
838
877
|
*nbJobsPtr = nbJobs;
|
@@ -863,12 +902,12 @@ static size_t ZSTDMT_expandJobsTable (ZSTDMT_CCtx* mtctx, U32 nbWorkers) {
|
|
863
902
|
|
864
903
|
/* ZSTDMT_CCtxParam_setNbWorkers():
|
865
904
|
* Internal use only */
|
866
|
-
size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorkers)
|
905
|
+
static size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorkers)
|
867
906
|
{
|
868
907
|
return ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, (int)nbWorkers);
|
869
908
|
}
|
870
909
|
|
871
|
-
MEM_STATIC ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced_internal(unsigned nbWorkers, ZSTD_customMem cMem)
|
910
|
+
MEM_STATIC ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced_internal(unsigned nbWorkers, ZSTD_customMem cMem, ZSTD_threadPool* pool)
|
872
911
|
{
|
873
912
|
ZSTDMT_CCtx* mtctx;
|
874
913
|
U32 nbJobs = nbWorkers + 2;
|
@@ -881,12 +920,19 @@ MEM_STATIC ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced_internal(unsigned nbWorkers,
|
|
881
920
|
/* invalid custom allocator */
|
882
921
|
return NULL;
|
883
922
|
|
884
|
-
mtctx = (ZSTDMT_CCtx*)
|
923
|
+
mtctx = (ZSTDMT_CCtx*) ZSTD_customCalloc(sizeof(ZSTDMT_CCtx), cMem);
|
885
924
|
if (!mtctx) return NULL;
|
886
925
|
ZSTDMT_CCtxParam_setNbWorkers(&mtctx->params, nbWorkers);
|
887
926
|
mtctx->cMem = cMem;
|
888
927
|
mtctx->allJobsCompleted = 1;
|
889
|
-
|
928
|
+
if (pool != NULL) {
|
929
|
+
mtctx->factory = pool;
|
930
|
+
mtctx->providedFactory = 1;
|
931
|
+
}
|
932
|
+
else {
|
933
|
+
mtctx->factory = POOL_create_advanced(nbWorkers, 0, cMem);
|
934
|
+
mtctx->providedFactory = 0;
|
935
|
+
}
|
890
936
|
mtctx->jobs = ZSTDMT_createJobsTable(&nbJobs, cMem);
|
891
937
|
assert(nbJobs > 0); assert((nbJobs & (nbJobs - 1)) == 0); /* ensure nbJobs is a power of 2 */
|
892
938
|
mtctx->jobIDMask = nbJobs - 1;
|
@@ -903,22 +949,18 @@ MEM_STATIC ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced_internal(unsigned nbWorkers,
|
|
903
949
|
return mtctx;
|
904
950
|
}
|
905
951
|
|
906
|
-
ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, ZSTD_customMem cMem)
|
952
|
+
ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, ZSTD_customMem cMem, ZSTD_threadPool* pool)
|
907
953
|
{
|
908
954
|
#ifdef ZSTD_MULTITHREAD
|
909
|
-
return ZSTDMT_createCCtx_advanced_internal(nbWorkers, cMem);
|
955
|
+
return ZSTDMT_createCCtx_advanced_internal(nbWorkers, cMem, pool);
|
910
956
|
#else
|
911
957
|
(void)nbWorkers;
|
912
958
|
(void)cMem;
|
959
|
+
(void)pool;
|
913
960
|
return NULL;
|
914
961
|
#endif
|
915
962
|
}
|
916
963
|
|
917
|
-
ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbWorkers)
|
918
|
-
{
|
919
|
-
return ZSTDMT_createCCtx_advanced(nbWorkers, ZSTD_defaultCMem);
|
920
|
-
}
|
921
|
-
|
922
964
|
|
923
965
|
/* ZSTDMT_releaseAllJobResources() :
|
924
966
|
* note : ensure all workers are killed first ! */
|
@@ -935,7 +977,7 @@ static void ZSTDMT_releaseAllJobResources(ZSTDMT_CCtx* mtctx)
|
|
935
977
|
ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[jobID].dstBuff);
|
936
978
|
|
937
979
|
/* Clear the job description, but keep the mutex/cond */
|
938
|
-
|
980
|
+
ZSTD_memset(&mtctx->jobs[jobID], 0, sizeof(mtctx->jobs[jobID]));
|
939
981
|
mtctx->jobs[jobID].job_mutex = mutex;
|
940
982
|
mtctx->jobs[jobID].job_cond = cond;
|
941
983
|
}
|
@@ -962,7 +1004,8 @@ static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* mtctx)
|
|
962
1004
|
size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx)
|
963
1005
|
{
|
964
1006
|
if (mtctx==NULL) return 0; /* compatible with free on NULL */
|
965
|
-
|
1007
|
+
if (!mtctx->providedFactory)
|
1008
|
+
POOL_free(mtctx->factory); /* stop and free worker threads */
|
966
1009
|
ZSTDMT_releaseAllJobResources(mtctx); /* release job resources into pools first */
|
967
1010
|
ZSTDMT_freeJobsTable(mtctx->jobs, mtctx->jobIDMask+1, mtctx->cMem);
|
968
1011
|
ZSTDMT_freeBufferPool(mtctx->bufPool);
|
@@ -971,8 +1014,8 @@ size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx)
|
|
971
1014
|
ZSTDMT_serialState_free(&mtctx->serial);
|
972
1015
|
ZSTD_freeCDict(mtctx->cdictLocal);
|
973
1016
|
if (mtctx->roundBuff.buffer)
|
974
|
-
|
975
|
-
|
1017
|
+
ZSTD_customFree(mtctx->roundBuff.buffer, mtctx->cMem);
|
1018
|
+
ZSTD_customFree(mtctx, mtctx->cMem);
|
976
1019
|
return 0;
|
977
1020
|
}
|
978
1021
|
|
@@ -989,72 +1032,13 @@ size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx)
|
|
989
1032
|
+ mtctx->roundBuff.capacity;
|
990
1033
|
}
|
991
1034
|
|
992
|
-
/* Internal only */
|
993
|
-
size_t
|
994
|
-
ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params,
|
995
|
-
ZSTDMT_parameter parameter,
|
996
|
-
int value)
|
997
|
-
{
|
998
|
-
DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter");
|
999
|
-
switch(parameter)
|
1000
|
-
{
|
1001
|
-
case ZSTDMT_p_jobSize :
|
1002
|
-
DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter : set jobSize to %i", value);
|
1003
|
-
return ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, value);
|
1004
|
-
case ZSTDMT_p_overlapLog :
|
1005
|
-
DEBUGLOG(4, "ZSTDMT_p_overlapLog : %i", value);
|
1006
|
-
return ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, value);
|
1007
|
-
case ZSTDMT_p_rsyncable :
|
1008
|
-
DEBUGLOG(4, "ZSTD_p_rsyncable : %i", value);
|
1009
|
-
return ZSTD_CCtxParams_setParameter(params, ZSTD_c_rsyncable, value);
|
1010
|
-
default :
|
1011
|
-
return ERROR(parameter_unsupported);
|
1012
|
-
}
|
1013
|
-
}
|
1014
|
-
|
1015
|
-
size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int value)
|
1016
|
-
{
|
1017
|
-
DEBUGLOG(4, "ZSTDMT_setMTCtxParameter");
|
1018
|
-
return ZSTDMT_CCtxParam_setMTCtxParameter(&mtctx->params, parameter, value);
|
1019
|
-
}
|
1020
|
-
|
1021
|
-
size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int* value)
|
1022
|
-
{
|
1023
|
-
switch (parameter) {
|
1024
|
-
case ZSTDMT_p_jobSize:
|
1025
|
-
return ZSTD_CCtxParams_getParameter(&mtctx->params, ZSTD_c_jobSize, value);
|
1026
|
-
case ZSTDMT_p_overlapLog:
|
1027
|
-
return ZSTD_CCtxParams_getParameter(&mtctx->params, ZSTD_c_overlapLog, value);
|
1028
|
-
case ZSTDMT_p_rsyncable:
|
1029
|
-
return ZSTD_CCtxParams_getParameter(&mtctx->params, ZSTD_c_rsyncable, value);
|
1030
|
-
default:
|
1031
|
-
return ERROR(parameter_unsupported);
|
1032
|
-
}
|
1033
|
-
}
|
1034
|
-
|
1035
|
-
/* Sets parameters relevant to the compression job,
|
1036
|
-
* initializing others to default values. */
|
1037
|
-
static ZSTD_CCtx_params ZSTDMT_initJobCCtxParams(const ZSTD_CCtx_params* params)
|
1038
|
-
{
|
1039
|
-
ZSTD_CCtx_params jobParams = *params;
|
1040
|
-
/* Clear parameters related to multithreading */
|
1041
|
-
jobParams.forceWindow = 0;
|
1042
|
-
jobParams.nbWorkers = 0;
|
1043
|
-
jobParams.jobSize = 0;
|
1044
|
-
jobParams.overlapLog = 0;
|
1045
|
-
jobParams.rsyncable = 0;
|
1046
|
-
memset(&jobParams.ldmParams, 0, sizeof(ldmParams_t));
|
1047
|
-
memset(&jobParams.customMem, 0, sizeof(ZSTD_customMem));
|
1048
|
-
return jobParams;
|
1049
|
-
}
|
1050
|
-
|
1051
1035
|
|
1052
1036
|
/* ZSTDMT_resize() :
|
1053
1037
|
* @return : error code if fails, 0 on success */
|
1054
1038
|
static size_t ZSTDMT_resize(ZSTDMT_CCtx* mtctx, unsigned nbWorkers)
|
1055
1039
|
{
|
1056
1040
|
if (POOL_resize(mtctx->factory, nbWorkers)) return ERROR(memory_allocation);
|
1057
|
-
FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbWorkers) );
|
1041
|
+
FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbWorkers) , "");
|
1058
1042
|
mtctx->bufPool = ZSTDMT_expandBufferPool(mtctx->bufPool, nbWorkers);
|
1059
1043
|
if (mtctx->bufPool == NULL) return ERROR(memory_allocation);
|
1060
1044
|
mtctx->cctxPool = ZSTDMT_expandCCtxPool(mtctx->cctxPool, nbWorkers);
|
@@ -1076,7 +1060,7 @@ void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_p
|
|
1076
1060
|
DEBUGLOG(5, "ZSTDMT_updateCParams_whileCompressing (level:%i)",
|
1077
1061
|
compressionLevel);
|
1078
1062
|
mtctx->params.compressionLevel = compressionLevel;
|
1079
|
-
{ ZSTD_compressionParameters cParams = ZSTD_getCParamsFromCCtxParams(cctxParams, 0,
|
1063
|
+
{ ZSTD_compressionParameters cParams = ZSTD_getCParamsFromCCtxParams(cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
|
1080
1064
|
cParams.windowLog = saved_wlog;
|
1081
1065
|
mtctx->params.cParams = cParams;
|
1082
1066
|
}
|
@@ -1160,11 +1144,11 @@ size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx)
|
|
1160
1144
|
static unsigned ZSTDMT_computeTargetJobLog(const ZSTD_CCtx_params* params)
|
1161
1145
|
{
|
1162
1146
|
unsigned jobLog;
|
1163
|
-
if (params->ldmParams.enableLdm) {
|
1147
|
+
if (params->ldmParams.enableLdm == ZSTD_ps_enable) {
|
1164
1148
|
/* In Long Range Mode, the windowLog is typically oversized.
|
1165
1149
|
* In which case, it's preferable to determine the jobSize
|
1166
|
-
* based on
|
1167
|
-
jobLog = MAX(21, params->cParams.chainLog +
|
1150
|
+
* based on cycleLog instead. */
|
1151
|
+
jobLog = MAX(21, ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy) + 3);
|
1168
1152
|
} else {
|
1169
1153
|
jobLog = MAX(20, params->cParams.windowLog + 2);
|
1170
1154
|
}
|
@@ -1204,7 +1188,7 @@ static size_t ZSTDMT_computeOverlapSize(const ZSTD_CCtx_params* params)
|
|
1204
1188
|
int const overlapRLog = 9 - ZSTDMT_overlapLog(params->overlapLog, params->cParams.strategy);
|
1205
1189
|
int ovLog = (overlapRLog >= 8) ? 0 : (params->cParams.windowLog - overlapRLog);
|
1206
1190
|
assert(0 <= overlapRLog && overlapRLog <= 8);
|
1207
|
-
if (params->ldmParams.enableLdm) {
|
1191
|
+
if (params->ldmParams.enableLdm == ZSTD_ps_enable) {
|
1208
1192
|
/* In Long Range Mode, the windowLog is typically oversized.
|
1209
1193
|
* In which case, it's preferable to determine the jobSize
|
1210
1194
|
* based on chainLog instead.
|
@@ -1218,172 +1202,6 @@ static size_t ZSTDMT_computeOverlapSize(const ZSTD_CCtx_params* params)
|
|
1218
1202
|
return (ovLog==0) ? 0 : (size_t)1 << ovLog;
|
1219
1203
|
}
|
1220
1204
|
|
1221
|
-
static unsigned
|
1222
|
-
ZSTDMT_computeNbJobs(const ZSTD_CCtx_params* params, size_t srcSize, unsigned nbWorkers)
|
1223
|
-
{
|
1224
|
-
assert(nbWorkers>0);
|
1225
|
-
{ size_t const jobSizeTarget = (size_t)1 << ZSTDMT_computeTargetJobLog(params);
|
1226
|
-
size_t const jobMaxSize = jobSizeTarget << 2;
|
1227
|
-
size_t const passSizeMax = jobMaxSize * nbWorkers;
|
1228
|
-
unsigned const multiplier = (unsigned)(srcSize / passSizeMax) + 1;
|
1229
|
-
unsigned const nbJobsLarge = multiplier * nbWorkers;
|
1230
|
-
unsigned const nbJobsMax = (unsigned)(srcSize / jobSizeTarget) + 1;
|
1231
|
-
unsigned const nbJobsSmall = MIN(nbJobsMax, nbWorkers);
|
1232
|
-
return (multiplier>1) ? nbJobsLarge : nbJobsSmall;
|
1233
|
-
} }
|
1234
|
-
|
1235
|
-
/* ZSTDMT_compress_advanced_internal() :
|
1236
|
-
* This is a blocking function : it will only give back control to caller after finishing its compression job.
|
1237
|
-
*/
|
1238
|
-
static size_t ZSTDMT_compress_advanced_internal(
|
1239
|
-
ZSTDMT_CCtx* mtctx,
|
1240
|
-
void* dst, size_t dstCapacity,
|
1241
|
-
const void* src, size_t srcSize,
|
1242
|
-
const ZSTD_CDict* cdict,
|
1243
|
-
ZSTD_CCtx_params params)
|
1244
|
-
{
|
1245
|
-
ZSTD_CCtx_params const jobParams = ZSTDMT_initJobCCtxParams(¶ms);
|
1246
|
-
size_t const overlapSize = ZSTDMT_computeOverlapSize(¶ms);
|
1247
|
-
unsigned const nbJobs = ZSTDMT_computeNbJobs(¶ms, srcSize, params.nbWorkers);
|
1248
|
-
size_t const proposedJobSize = (srcSize + (nbJobs-1)) / nbJobs;
|
1249
|
-
size_t const avgJobSize = (((proposedJobSize-1) & 0x1FFFF) < 0x7FFF) ? proposedJobSize + 0xFFFF : proposedJobSize; /* avoid too small last block */
|
1250
|
-
const char* const srcStart = (const char*)src;
|
1251
|
-
size_t remainingSrcSize = srcSize;
|
1252
|
-
unsigned const compressWithinDst = (dstCapacity >= ZSTD_compressBound(srcSize)) ? nbJobs : (unsigned)(dstCapacity / ZSTD_compressBound(avgJobSize)); /* presumes avgJobSize >= 256 KB, which should be the case */
|
1253
|
-
size_t frameStartPos = 0, dstBufferPos = 0;
|
1254
|
-
assert(jobParams.nbWorkers == 0);
|
1255
|
-
assert(mtctx->cctxPool->totalCCtx == params.nbWorkers);
|
1256
|
-
|
1257
|
-
params.jobSize = (U32)avgJobSize;
|
1258
|
-
DEBUGLOG(4, "ZSTDMT_compress_advanced_internal: nbJobs=%2u (rawSize=%u bytes; fixedSize=%u) ",
|
1259
|
-
nbJobs, (U32)proposedJobSize, (U32)avgJobSize);
|
1260
|
-
|
1261
|
-
if ((nbJobs==1) | (params.nbWorkers<=1)) { /* fallback to single-thread mode : this is a blocking invocation anyway */
|
1262
|
-
ZSTD_CCtx* const cctx = mtctx->cctxPool->cctx[0];
|
1263
|
-
DEBUGLOG(4, "ZSTDMT_compress_advanced_internal: fallback to single-thread mode");
|
1264
|
-
if (cdict) return ZSTD_compress_usingCDict_advanced(cctx, dst, dstCapacity, src, srcSize, cdict, jobParams.fParams);
|
1265
|
-
return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, NULL, 0, &jobParams);
|
1266
|
-
}
|
1267
|
-
|
1268
|
-
assert(avgJobSize >= 256 KB); /* condition for ZSTD_compressBound(A) + ZSTD_compressBound(B) <= ZSTD_compressBound(A+B), required to compress directly into Dst (no additional buffer) */
|
1269
|
-
ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(avgJobSize) );
|
1270
|
-
if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, avgJobSize))
|
1271
|
-
return ERROR(memory_allocation);
|
1272
|
-
|
1273
|
-
FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbJobs) ); /* only expands if necessary */
|
1274
|
-
|
1275
|
-
{ unsigned u;
|
1276
|
-
for (u=0; u<nbJobs; u++) {
|
1277
|
-
size_t const jobSize = MIN(remainingSrcSize, avgJobSize);
|
1278
|
-
size_t const dstBufferCapacity = ZSTD_compressBound(jobSize);
|
1279
|
-
buffer_t const dstAsBuffer = { (char*)dst + dstBufferPos, dstBufferCapacity };
|
1280
|
-
buffer_t const dstBuffer = u < compressWithinDst ? dstAsBuffer : g_nullBuffer;
|
1281
|
-
size_t dictSize = u ? overlapSize : 0;
|
1282
|
-
|
1283
|
-
mtctx->jobs[u].prefix.start = srcStart + frameStartPos - dictSize;
|
1284
|
-
mtctx->jobs[u].prefix.size = dictSize;
|
1285
|
-
mtctx->jobs[u].src.start = srcStart + frameStartPos;
|
1286
|
-
mtctx->jobs[u].src.size = jobSize; assert(jobSize > 0); /* avoid job.src.size == 0 */
|
1287
|
-
mtctx->jobs[u].consumed = 0;
|
1288
|
-
mtctx->jobs[u].cSize = 0;
|
1289
|
-
mtctx->jobs[u].cdict = (u==0) ? cdict : NULL;
|
1290
|
-
mtctx->jobs[u].fullFrameSize = srcSize;
|
1291
|
-
mtctx->jobs[u].params = jobParams;
|
1292
|
-
/* do not calculate checksum within sections, but write it in header for first section */
|
1293
|
-
mtctx->jobs[u].dstBuff = dstBuffer;
|
1294
|
-
mtctx->jobs[u].cctxPool = mtctx->cctxPool;
|
1295
|
-
mtctx->jobs[u].bufPool = mtctx->bufPool;
|
1296
|
-
mtctx->jobs[u].seqPool = mtctx->seqPool;
|
1297
|
-
mtctx->jobs[u].serial = &mtctx->serial;
|
1298
|
-
mtctx->jobs[u].jobID = u;
|
1299
|
-
mtctx->jobs[u].firstJob = (u==0);
|
1300
|
-
mtctx->jobs[u].lastJob = (u==nbJobs-1);
|
1301
|
-
|
1302
|
-
DEBUGLOG(5, "ZSTDMT_compress_advanced_internal: posting job %u (%u bytes)", u, (U32)jobSize);
|
1303
|
-
DEBUG_PRINTHEX(6, mtctx->jobs[u].prefix.start, 12);
|
1304
|
-
POOL_add(mtctx->factory, ZSTDMT_compressionJob, &mtctx->jobs[u]);
|
1305
|
-
|
1306
|
-
frameStartPos += jobSize;
|
1307
|
-
dstBufferPos += dstBufferCapacity;
|
1308
|
-
remainingSrcSize -= jobSize;
|
1309
|
-
} }
|
1310
|
-
|
1311
|
-
/* collect result */
|
1312
|
-
{ size_t error = 0, dstPos = 0;
|
1313
|
-
unsigned jobID;
|
1314
|
-
for (jobID=0; jobID<nbJobs; jobID++) {
|
1315
|
-
DEBUGLOG(5, "waiting for job %u ", jobID);
|
1316
|
-
ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobs[jobID].job_mutex);
|
1317
|
-
while (mtctx->jobs[jobID].consumed < mtctx->jobs[jobID].src.size) {
|
1318
|
-
DEBUGLOG(5, "waiting for jobCompleted signal from job %u", jobID);
|
1319
|
-
ZSTD_pthread_cond_wait(&mtctx->jobs[jobID].job_cond, &mtctx->jobs[jobID].job_mutex);
|
1320
|
-
}
|
1321
|
-
ZSTD_pthread_mutex_unlock(&mtctx->jobs[jobID].job_mutex);
|
1322
|
-
DEBUGLOG(5, "ready to write job %u ", jobID);
|
1323
|
-
|
1324
|
-
{ size_t const cSize = mtctx->jobs[jobID].cSize;
|
1325
|
-
if (ZSTD_isError(cSize)) error = cSize;
|
1326
|
-
if ((!error) && (dstPos + cSize > dstCapacity)) error = ERROR(dstSize_tooSmall);
|
1327
|
-
if (jobID) { /* note : job 0 is written directly at dst, which is correct position */
|
1328
|
-
if (!error)
|
1329
|
-
memmove((char*)dst + dstPos, mtctx->jobs[jobID].dstBuff.start, cSize); /* may overlap when job compressed within dst */
|
1330
|
-
if (jobID >= compressWithinDst) { /* job compressed into its own buffer, which must be released */
|
1331
|
-
DEBUGLOG(5, "releasing buffer %u>=%u", jobID, compressWithinDst);
|
1332
|
-
ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[jobID].dstBuff);
|
1333
|
-
} }
|
1334
|
-
mtctx->jobs[jobID].dstBuff = g_nullBuffer;
|
1335
|
-
mtctx->jobs[jobID].cSize = 0;
|
1336
|
-
dstPos += cSize ;
|
1337
|
-
}
|
1338
|
-
} /* for (jobID=0; jobID<nbJobs; jobID++) */
|
1339
|
-
|
1340
|
-
DEBUGLOG(4, "checksumFlag : %u ", params.fParams.checksumFlag);
|
1341
|
-
if (params.fParams.checksumFlag) {
|
1342
|
-
U32 const checksum = (U32)XXH64_digest(&mtctx->serial.xxhState);
|
1343
|
-
if (dstPos + 4 > dstCapacity) {
|
1344
|
-
error = ERROR(dstSize_tooSmall);
|
1345
|
-
} else {
|
1346
|
-
DEBUGLOG(4, "writing checksum : %08X \n", checksum);
|
1347
|
-
MEM_writeLE32((char*)dst + dstPos, checksum);
|
1348
|
-
dstPos += 4;
|
1349
|
-
} }
|
1350
|
-
|
1351
|
-
if (!error) DEBUGLOG(4, "compressed size : %u ", (U32)dstPos);
|
1352
|
-
return error ? error : dstPos;
|
1353
|
-
}
|
1354
|
-
}
|
1355
|
-
|
1356
|
-
size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
|
1357
|
-
void* dst, size_t dstCapacity,
|
1358
|
-
const void* src, size_t srcSize,
|
1359
|
-
const ZSTD_CDict* cdict,
|
1360
|
-
ZSTD_parameters params,
|
1361
|
-
int overlapLog)
|
1362
|
-
{
|
1363
|
-
ZSTD_CCtx_params cctxParams = mtctx->params;
|
1364
|
-
cctxParams.cParams = params.cParams;
|
1365
|
-
cctxParams.fParams = params.fParams;
|
1366
|
-
assert(ZSTD_OVERLAPLOG_MIN <= overlapLog && overlapLog <= ZSTD_OVERLAPLOG_MAX);
|
1367
|
-
cctxParams.overlapLog = overlapLog;
|
1368
|
-
return ZSTDMT_compress_advanced_internal(mtctx,
|
1369
|
-
dst, dstCapacity,
|
1370
|
-
src, srcSize,
|
1371
|
-
cdict, cctxParams);
|
1372
|
-
}
|
1373
|
-
|
1374
|
-
|
1375
|
-
size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
|
1376
|
-
void* dst, size_t dstCapacity,
|
1377
|
-
const void* src, size_t srcSize,
|
1378
|
-
int compressionLevel)
|
1379
|
-
{
|
1380
|
-
ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, 0);
|
1381
|
-
int const overlapLog = ZSTDMT_overlapLog_default(params.cParams.strategy);
|
1382
|
-
params.fParams.contentSizeFlag = 1;
|
1383
|
-
return ZSTDMT_compress_advanced(mtctx, dst, dstCapacity, src, srcSize, NULL, params, overlapLog);
|
1384
|
-
}
|
1385
|
-
|
1386
|
-
|
1387
1205
|
/* ====================================== */
|
1388
1206
|
/* ======= Streaming API ======= */
|
1389
1207
|
/* ====================================== */
|
@@ -1403,21 +1221,11 @@ size_t ZSTDMT_initCStream_internal(
|
|
1403
1221
|
|
1404
1222
|
/* init */
|
1405
1223
|
if (params.nbWorkers != mtctx->params.nbWorkers)
|
1406
|
-
FORWARD_IF_ERROR( ZSTDMT_resize(mtctx, params.nbWorkers) );
|
1224
|
+
FORWARD_IF_ERROR( ZSTDMT_resize(mtctx, params.nbWorkers) , "");
|
1407
1225
|
|
1408
1226
|
if (params.jobSize != 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN;
|
1409
1227
|
if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = (size_t)ZSTDMT_JOBSIZE_MAX;
|
1410
1228
|
|
1411
|
-
mtctx->singleBlockingThread = (pledgedSrcSize <= ZSTDMT_JOBSIZE_MIN); /* do not trigger multi-threading when srcSize is too small */
|
1412
|
-
if (mtctx->singleBlockingThread) {
|
1413
|
-
ZSTD_CCtx_params const singleThreadParams = ZSTDMT_initJobCCtxParams(¶ms);
|
1414
|
-
DEBUGLOG(5, "ZSTDMT_initCStream_internal: switch to single blocking thread mode");
|
1415
|
-
assert(singleThreadParams.nbWorkers == 0);
|
1416
|
-
return ZSTD_initCStream_internal(mtctx->cctxPool->cctx[0],
|
1417
|
-
dict, dictSize, cdict,
|
1418
|
-
&singleThreadParams, pledgedSrcSize);
|
1419
|
-
}
|
1420
|
-
|
1421
1229
|
DEBUGLOG(4, "ZSTDMT_initCStream_internal: %u workers", params.nbWorkers);
|
1422
1230
|
|
1423
1231
|
if (mtctx->allJobsCompleted == 0) { /* previous compression not correctly finished */
|
@@ -1451,9 +1259,11 @@ size_t ZSTDMT_initCStream_internal(
|
|
1451
1259
|
|
1452
1260
|
if (params.rsyncable) {
|
1453
1261
|
/* Aim for the targetsectionSize as the average job size. */
|
1454
|
-
U32 const
|
1455
|
-
U32 const rsyncBits = ZSTD_highbit32(
|
1456
|
-
|
1262
|
+
U32 const jobSizeKB = (U32)(mtctx->targetSectionSize >> 10);
|
1263
|
+
U32 const rsyncBits = (assert(jobSizeKB >= 1), ZSTD_highbit32(jobSizeKB) + 10);
|
1264
|
+
/* We refuse to create jobs < RSYNC_MIN_BLOCK_SIZE bytes, so make sure our
|
1265
|
+
* expected job size is at least 4x larger. */
|
1266
|
+
assert(rsyncBits >= RSYNC_MIN_BLOCK_LOG + 2);
|
1457
1267
|
DEBUGLOG(4, "rsyncLog = %u", rsyncBits);
|
1458
1268
|
mtctx->rsync.hash = 0;
|
1459
1269
|
mtctx->rsync.hitMask = (1ULL << rsyncBits) - 1;
|
@@ -1465,7 +1275,7 @@ size_t ZSTDMT_initCStream_internal(
|
|
1465
1275
|
ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(mtctx->targetSectionSize));
|
1466
1276
|
{
|
1467
1277
|
/* If ldm is enabled we need windowSize space. */
|
1468
|
-
size_t const windowSize = mtctx->params.ldmParams.enableLdm ? (1U << mtctx->params.cParams.windowLog) : 0;
|
1278
|
+
size_t const windowSize = mtctx->params.ldmParams.enableLdm == ZSTD_ps_enable ? (1U << mtctx->params.cParams.windowLog) : 0;
|
1469
1279
|
/* Two buffers of slack, plus extra space for the overlap
|
1470
1280
|
* This is the minimum slack that LDM works with. One extra because
|
1471
1281
|
* flush might waste up to targetSectionSize-1 bytes. Another extra
|
@@ -1480,8 +1290,8 @@ size_t ZSTDMT_initCStream_internal(
|
|
1480
1290
|
size_t const capacity = MAX(windowSize, sectionsSize) + slackSize;
|
1481
1291
|
if (mtctx->roundBuff.capacity < capacity) {
|
1482
1292
|
if (mtctx->roundBuff.buffer)
|
1483
|
-
|
1484
|
-
mtctx->roundBuff.buffer = (BYTE*)
|
1293
|
+
ZSTD_customFree(mtctx->roundBuff.buffer, mtctx->cMem);
|
1294
|
+
mtctx->roundBuff.buffer = (BYTE*)ZSTD_customMalloc(capacity, mtctx->cMem);
|
1485
1295
|
if (mtctx->roundBuff.buffer == NULL) {
|
1486
1296
|
mtctx->roundBuff.capacity = 0;
|
1487
1297
|
return ERROR(memory_allocation);
|
@@ -1500,58 +1310,12 @@ size_t ZSTDMT_initCStream_internal(
|
|
1500
1310
|
mtctx->allJobsCompleted = 0;
|
1501
1311
|
mtctx->consumed = 0;
|
1502
1312
|
mtctx->produced = 0;
|
1503
|
-
if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, mtctx->targetSectionSize
|
1313
|
+
if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, mtctx->targetSectionSize,
|
1314
|
+
dict, dictSize, dictContentType))
|
1504
1315
|
return ERROR(memory_allocation);
|
1505
1316
|
return 0;
|
1506
1317
|
}
|
1507
1318
|
|
1508
|
-
size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
|
1509
|
-
const void* dict, size_t dictSize,
|
1510
|
-
ZSTD_parameters params,
|
1511
|
-
unsigned long long pledgedSrcSize)
|
1512
|
-
{
|
1513
|
-
ZSTD_CCtx_params cctxParams = mtctx->params; /* retrieve sticky params */
|
1514
|
-
DEBUGLOG(4, "ZSTDMT_initCStream_advanced (pledgedSrcSize=%u)", (U32)pledgedSrcSize);
|
1515
|
-
cctxParams.cParams = params.cParams;
|
1516
|
-
cctxParams.fParams = params.fParams;
|
1517
|
-
return ZSTDMT_initCStream_internal(mtctx, dict, dictSize, ZSTD_dct_auto, NULL,
|
1518
|
-
cctxParams, pledgedSrcSize);
|
1519
|
-
}
|
1520
|
-
|
1521
|
-
size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
|
1522
|
-
const ZSTD_CDict* cdict,
|
1523
|
-
ZSTD_frameParameters fParams,
|
1524
|
-
unsigned long long pledgedSrcSize)
|
1525
|
-
{
|
1526
|
-
ZSTD_CCtx_params cctxParams = mtctx->params;
|
1527
|
-
if (cdict==NULL) return ERROR(dictionary_wrong); /* method incompatible with NULL cdict */
|
1528
|
-
cctxParams.cParams = ZSTD_getCParamsFromCDict(cdict);
|
1529
|
-
cctxParams.fParams = fParams;
|
1530
|
-
return ZSTDMT_initCStream_internal(mtctx, NULL, 0 /*dictSize*/, ZSTD_dct_auto, cdict,
|
1531
|
-
cctxParams, pledgedSrcSize);
|
1532
|
-
}
|
1533
|
-
|
1534
|
-
|
1535
|
-
/* ZSTDMT_resetCStream() :
|
1536
|
-
* pledgedSrcSize can be zero == unknown (for the time being)
|
1537
|
-
* prefer using ZSTD_CONTENTSIZE_UNKNOWN,
|
1538
|
-
* as `0` might mean "empty" in the future */
|
1539
|
-
size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize)
|
1540
|
-
{
|
1541
|
-
if (!pledgedSrcSize) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
|
1542
|
-
return ZSTDMT_initCStream_internal(mtctx, NULL, 0, ZSTD_dct_auto, 0, mtctx->params,
|
1543
|
-
pledgedSrcSize);
|
1544
|
-
}
|
1545
|
-
|
1546
|
-
size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel) {
|
1547
|
-
ZSTD_parameters const params = ZSTD_getParams(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0);
|
1548
|
-
ZSTD_CCtx_params cctxParams = mtctx->params; /* retrieve sticky params */
|
1549
|
-
DEBUGLOG(4, "ZSTDMT_initCStream (cLevel=%i)", compressionLevel);
|
1550
|
-
cctxParams.cParams = params.cParams;
|
1551
|
-
cctxParams.fParams = params.fParams;
|
1552
|
-
return ZSTDMT_initCStream_internal(mtctx, NULL, 0, ZSTD_dct_auto, NULL, cctxParams, ZSTD_CONTENTSIZE_UNKNOWN);
|
1553
|
-
}
|
1554
|
-
|
1555
1319
|
|
1556
1320
|
/* ZSTDMT_writeLastEmptyBlock()
|
1557
1321
|
* Write a single empty block with an end-of-frame to finish a frame.
|
@@ -1714,9 +1478,11 @@ static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, u
|
|
1714
1478
|
assert(mtctx->doneJobID < mtctx->nextJobID);
|
1715
1479
|
assert(cSize >= mtctx->jobs[wJobID].dstFlushed);
|
1716
1480
|
assert(mtctx->jobs[wJobID].dstBuff.start != NULL);
|
1717
|
-
|
1718
|
-
|
1719
|
-
|
1481
|
+
if (toFlush > 0) {
|
1482
|
+
ZSTD_memcpy((char*)output->dst + output->pos,
|
1483
|
+
(const char*)mtctx->jobs[wJobID].dstBuff.start + mtctx->jobs[wJobID].dstFlushed,
|
1484
|
+
toFlush);
|
1485
|
+
}
|
1720
1486
|
output->pos += toFlush;
|
1721
1487
|
mtctx->jobs[wJobID].dstFlushed += toFlush; /* can write : this value is only used by mtctx */
|
1722
1488
|
|
@@ -1784,17 +1550,21 @@ static range_t ZSTDMT_getInputDataInUse(ZSTDMT_CCtx* mtctx)
|
|
1784
1550
|
static int ZSTDMT_isOverlapped(buffer_t buffer, range_t range)
|
1785
1551
|
{
|
1786
1552
|
BYTE const* const bufferStart = (BYTE const*)buffer.start;
|
1787
|
-
BYTE const* const bufferEnd = bufferStart + buffer.capacity;
|
1788
1553
|
BYTE const* const rangeStart = (BYTE const*)range.start;
|
1789
|
-
BYTE const* const rangeEnd = rangeStart + range.size;
|
1790
1554
|
|
1791
1555
|
if (rangeStart == NULL || bufferStart == NULL)
|
1792
1556
|
return 0;
|
1793
|
-
/* Empty ranges cannot overlap */
|
1794
|
-
if (bufferStart == bufferEnd || rangeStart == rangeEnd)
|
1795
|
-
return 0;
|
1796
1557
|
|
1797
|
-
|
1558
|
+
{
|
1559
|
+
BYTE const* const bufferEnd = bufferStart + buffer.capacity;
|
1560
|
+
BYTE const* const rangeEnd = rangeStart + range.size;
|
1561
|
+
|
1562
|
+
/* Empty ranges cannot overlap */
|
1563
|
+
if (bufferStart == bufferEnd || rangeStart == rangeEnd)
|
1564
|
+
return 0;
|
1565
|
+
|
1566
|
+
return bufferStart < rangeEnd && rangeStart < bufferEnd;
|
1567
|
+
}
|
1798
1568
|
}
|
1799
1569
|
|
1800
1570
|
static int ZSTDMT_doesOverlapWindow(buffer_t buffer, ZSTD_window_t window)
|
@@ -1821,7 +1591,7 @@ static int ZSTDMT_doesOverlapWindow(buffer_t buffer, ZSTD_window_t window)
|
|
1821
1591
|
|
1822
1592
|
static void ZSTDMT_waitForLdmComplete(ZSTDMT_CCtx* mtctx, buffer_t buffer)
|
1823
1593
|
{
|
1824
|
-
if (mtctx->params.ldmParams.enableLdm) {
|
1594
|
+
if (mtctx->params.ldmParams.enableLdm == ZSTD_ps_enable) {
|
1825
1595
|
ZSTD_pthread_mutex_t* mutex = &mtctx->serial.ldmWindowMutex;
|
1826
1596
|
DEBUGLOG(5, "ZSTDMT_waitForLdmComplete");
|
1827
1597
|
DEBUGLOG(5, "source [0x%zx, 0x%zx)",
|
@@ -1867,7 +1637,7 @@ static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx)
|
|
1867
1637
|
return 0;
|
1868
1638
|
}
|
1869
1639
|
ZSTDMT_waitForLdmComplete(mtctx, buffer);
|
1870
|
-
|
1640
|
+
ZSTD_memmove(start, mtctx->inBuff.prefix.start, prefixSize);
|
1871
1641
|
mtctx->inBuff.prefix.start = start;
|
1872
1642
|
mtctx->roundBuff.pos = prefixSize;
|
1873
1643
|
}
|
@@ -1924,6 +1694,11 @@ findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuffer const input)
|
|
1924
1694
|
if (!mtctx->params.rsyncable)
|
1925
1695
|
/* Rsync is disabled. */
|
1926
1696
|
return syncPoint;
|
1697
|
+
if (mtctx->inBuff.filled + input.size - input.pos < RSYNC_MIN_BLOCK_SIZE)
|
1698
|
+
/* We don't emit synchronization points if it would produce too small blocks.
|
1699
|
+
* We don't have enough input to find a synchronization point, so don't look.
|
1700
|
+
*/
|
1701
|
+
return syncPoint;
|
1927
1702
|
if (mtctx->inBuff.filled + syncPoint.toLoad < RSYNC_LENGTH)
|
1928
1703
|
/* Not enough to compute the hash.
|
1929
1704
|
* We will miss any synchronization points in this RSYNC_LENGTH byte
|
@@ -1934,23 +1709,41 @@ findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuffer const input)
|
|
1934
1709
|
*/
|
1935
1710
|
return syncPoint;
|
1936
1711
|
/* Initialize the loop variables. */
|
1937
|
-
if (mtctx->inBuff.filled
|
1938
|
-
/* We
|
1712
|
+
if (mtctx->inBuff.filled < RSYNC_MIN_BLOCK_SIZE) {
|
1713
|
+
/* We don't need to scan the first RSYNC_MIN_BLOCK_SIZE positions
|
1714
|
+
* because they can't possibly be a sync point. So we can start
|
1715
|
+
* part way through the input buffer.
|
1716
|
+
*/
|
1717
|
+
pos = RSYNC_MIN_BLOCK_SIZE - mtctx->inBuff.filled;
|
1718
|
+
if (pos >= RSYNC_LENGTH) {
|
1719
|
+
prev = istart + pos - RSYNC_LENGTH;
|
1720
|
+
hash = ZSTD_rollingHash_compute(prev, RSYNC_LENGTH);
|
1721
|
+
} else {
|
1722
|
+
assert(mtctx->inBuff.filled >= RSYNC_LENGTH);
|
1723
|
+
prev = (BYTE const*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled - RSYNC_LENGTH;
|
1724
|
+
hash = ZSTD_rollingHash_compute(prev + pos, (RSYNC_LENGTH - pos));
|
1725
|
+
hash = ZSTD_rollingHash_append(hash, istart, pos);
|
1726
|
+
}
|
1727
|
+
} else {
|
1728
|
+
/* We have enough bytes buffered to initialize the hash,
|
1729
|
+
* and are have processed enough bytes to find a sync point.
|
1939
1730
|
* Start scanning at the beginning of the input.
|
1940
1731
|
*/
|
1732
|
+
assert(mtctx->inBuff.filled >= RSYNC_MIN_BLOCK_SIZE);
|
1733
|
+
assert(RSYNC_MIN_BLOCK_SIZE >= RSYNC_LENGTH);
|
1941
1734
|
pos = 0;
|
1942
1735
|
prev = (BYTE const*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled - RSYNC_LENGTH;
|
1943
1736
|
hash = ZSTD_rollingHash_compute(prev, RSYNC_LENGTH);
|
1944
|
-
|
1945
|
-
|
1946
|
-
|
1947
|
-
|
1948
|
-
|
1949
|
-
|
1950
|
-
|
1951
|
-
|
1952
|
-
|
1953
|
-
|
1737
|
+
if ((hash & hitMask) == hitMask) {
|
1738
|
+
/* We're already at a sync point so don't load any more until
|
1739
|
+
* we're able to flush this sync point.
|
1740
|
+
* This likely happened because the job table was full so we
|
1741
|
+
* couldn't add our job.
|
1742
|
+
*/
|
1743
|
+
syncPoint.toLoad = 0;
|
1744
|
+
syncPoint.flush = 1;
|
1745
|
+
return syncPoint;
|
1746
|
+
}
|
1954
1747
|
}
|
1955
1748
|
/* Starting with the hash of the previous RSYNC_LENGTH bytes, roll
|
1956
1749
|
* through the input. If we hit a synchronization point, then cut the
|
@@ -1962,8 +1755,9 @@ findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuffer const input)
|
|
1962
1755
|
*/
|
1963
1756
|
for (; pos < syncPoint.toLoad; ++pos) {
|
1964
1757
|
BYTE const toRemove = pos < RSYNC_LENGTH ? prev[pos] : istart[pos - RSYNC_LENGTH];
|
1965
|
-
|
1758
|
+
assert(pos < RSYNC_LENGTH || ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash);
|
1966
1759
|
hash = ZSTD_rollingHash_rotate(hash, toRemove, istart[pos], primePower);
|
1760
|
+
assert(mtctx->inBuff.filled + pos >= RSYNC_MIN_BLOCK_SIZE);
|
1967
1761
|
if ((hash & hitMask) == hitMask) {
|
1968
1762
|
syncPoint.toLoad = pos + 1;
|
1969
1763
|
syncPoint.flush = 1;
|
@@ -1995,34 +1789,11 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
|
|
1995
1789
|
assert(output->pos <= output->size);
|
1996
1790
|
assert(input->pos <= input->size);
|
1997
1791
|
|
1998
|
-
if (mtctx->singleBlockingThread) { /* delegate to single-thread (synchronous) */
|
1999
|
-
return ZSTD_compressStream2(mtctx->cctxPool->cctx[0], output, input, endOp);
|
2000
|
-
}
|
2001
|
-
|
2002
1792
|
if ((mtctx->frameEnded) && (endOp==ZSTD_e_continue)) {
|
2003
1793
|
/* current frame being ended. Only flush/end are allowed */
|
2004
1794
|
return ERROR(stage_wrong);
|
2005
1795
|
}
|
2006
1796
|
|
2007
|
-
/* single-pass shortcut (note : synchronous-mode) */
|
2008
|
-
if ( (!mtctx->params.rsyncable) /* rsyncable mode is disabled */
|
2009
|
-
&& (mtctx->nextJobID == 0) /* just started */
|
2010
|
-
&& (mtctx->inBuff.filled == 0) /* nothing buffered */
|
2011
|
-
&& (!mtctx->jobReady) /* no job already created */
|
2012
|
-
&& (endOp == ZSTD_e_end) /* end order */
|
2013
|
-
&& (output->size - output->pos >= ZSTD_compressBound(input->size - input->pos)) ) { /* enough space in dst */
|
2014
|
-
size_t const cSize = ZSTDMT_compress_advanced_internal(mtctx,
|
2015
|
-
(char*)output->dst + output->pos, output->size - output->pos,
|
2016
|
-
(const char*)input->src + input->pos, input->size - input->pos,
|
2017
|
-
mtctx->cdict, mtctx->params);
|
2018
|
-
if (ZSTD_isError(cSize)) return cSize;
|
2019
|
-
input->pos = input->size;
|
2020
|
-
output->pos += cSize;
|
2021
|
-
mtctx->allJobsCompleted = 1;
|
2022
|
-
mtctx->frameEnded = 1;
|
2023
|
-
return 0;
|
2024
|
-
}
|
2025
|
-
|
2026
1797
|
/* fill input buffer */
|
2027
1798
|
if ( (!mtctx->jobReady)
|
2028
1799
|
&& (input->size > input->pos) ) { /* support NULL input */
|
@@ -2045,13 +1816,21 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
|
|
2045
1816
|
assert(mtctx->inBuff.buffer.capacity >= mtctx->targetSectionSize);
|
2046
1817
|
DEBUGLOG(5, "ZSTDMT_compressStream_generic: adding %u bytes on top of %u to buffer of size %u",
|
2047
1818
|
(U32)syncPoint.toLoad, (U32)mtctx->inBuff.filled, (U32)mtctx->targetSectionSize);
|
2048
|
-
|
1819
|
+
ZSTD_memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, syncPoint.toLoad);
|
2049
1820
|
input->pos += syncPoint.toLoad;
|
2050
1821
|
mtctx->inBuff.filled += syncPoint.toLoad;
|
2051
1822
|
forwardInputProgress = syncPoint.toLoad>0;
|
2052
1823
|
}
|
2053
|
-
|
2054
|
-
|
1824
|
+
}
|
1825
|
+
if ((input->pos < input->size) && (endOp == ZSTD_e_end)) {
|
1826
|
+
/* Can't end yet because the input is not fully consumed.
|
1827
|
+
* We are in one of these cases:
|
1828
|
+
* - mtctx->inBuff is NULL & empty: we couldn't get an input buffer so don't create a new job.
|
1829
|
+
* - We filled the input buffer: flush this job but don't end the frame.
|
1830
|
+
* - We hit a synchronization point: flush this job but don't end the frame.
|
1831
|
+
*/
|
1832
|
+
assert(mtctx->inBuff.filled == 0 || mtctx->inBuff.filled == mtctx->targetSectionSize || mtctx->params.rsyncable);
|
1833
|
+
endOp = ZSTD_e_flush;
|
2055
1834
|
}
|
2056
1835
|
|
2057
1836
|
if ( (mtctx->jobReady)
|
@@ -2060,7 +1839,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
|
|
2060
1839
|
|| ((endOp == ZSTD_e_end) && (!mtctx->frameEnded)) ) { /* must finish the frame with a zero-size block */
|
2061
1840
|
size_t const jobSize = mtctx->inBuff.filled;
|
2062
1841
|
assert(mtctx->inBuff.filled <= mtctx->targetSectionSize);
|
2063
|
-
FORWARD_IF_ERROR( ZSTDMT_createCompressionJob(mtctx, jobSize, endOp) );
|
1842
|
+
FORWARD_IF_ERROR( ZSTDMT_createCompressionJob(mtctx, jobSize, endOp) , "");
|
2064
1843
|
}
|
2065
1844
|
|
2066
1845
|
/* check for potential compressed data ready to be flushed */
|
@@ -2070,47 +1849,3 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
|
|
2070
1849
|
return remainingToFlush;
|
2071
1850
|
}
|
2072
1851
|
}
|
2073
|
-
|
2074
|
-
|
2075
|
-
size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
|
2076
|
-
{
|
2077
|
-
FORWARD_IF_ERROR( ZSTDMT_compressStream_generic(mtctx, output, input, ZSTD_e_continue) );
|
2078
|
-
|
2079
|
-
/* recommended next input size : fill current input buffer */
|
2080
|
-
return mtctx->targetSectionSize - mtctx->inBuff.filled; /* note : could be zero when input buffer is fully filled and no more availability to create new job */
|
2081
|
-
}
|
2082
|
-
|
2083
|
-
|
2084
|
-
static size_t ZSTDMT_flushStream_internal(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_EndDirective endFrame)
|
2085
|
-
{
|
2086
|
-
size_t const srcSize = mtctx->inBuff.filled;
|
2087
|
-
DEBUGLOG(5, "ZSTDMT_flushStream_internal");
|
2088
|
-
|
2089
|
-
if ( mtctx->jobReady /* one job ready for a worker to pick up */
|
2090
|
-
|| (srcSize > 0) /* still some data within input buffer */
|
2091
|
-
|| ((endFrame==ZSTD_e_end) && !mtctx->frameEnded)) { /* need a last 0-size block to end frame */
|
2092
|
-
DEBUGLOG(5, "ZSTDMT_flushStream_internal : create a new job (%u bytes, end:%u)",
|
2093
|
-
(U32)srcSize, (U32)endFrame);
|
2094
|
-
FORWARD_IF_ERROR( ZSTDMT_createCompressionJob(mtctx, srcSize, endFrame) );
|
2095
|
-
}
|
2096
|
-
|
2097
|
-
/* check if there is any data available to flush */
|
2098
|
-
return ZSTDMT_flushProduced(mtctx, output, 1 /* blockToFlush */, endFrame);
|
2099
|
-
}
|
2100
|
-
|
2101
|
-
|
2102
|
-
size_t ZSTDMT_flushStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output)
|
2103
|
-
{
|
2104
|
-
DEBUGLOG(5, "ZSTDMT_flushStream");
|
2105
|
-
if (mtctx->singleBlockingThread)
|
2106
|
-
return ZSTD_flushStream(mtctx->cctxPool->cctx[0], output);
|
2107
|
-
return ZSTDMT_flushStream_internal(mtctx, output, ZSTD_e_flush);
|
2108
|
-
}
|
2109
|
-
|
2110
|
-
size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output)
|
2111
|
-
{
|
2112
|
-
DEBUGLOG(4, "ZSTDMT_endStream");
|
2113
|
-
if (mtctx->singleBlockingThread)
|
2114
|
-
return ZSTD_endStream(mtctx->cctxPool->cctx[0], output);
|
2115
|
-
return ZSTDMT_flushStream_internal(mtctx, output, ZSTD_e_end);
|
2116
|
-
}
|