zstdlib 0.6.0 → 0.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGES.md +5 -0
- data/ext/zstdlib/extconf.rb +1 -1
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/bitstream.h +31 -37
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/compiler.h +19 -3
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/cpu.h +1 -1
- data/ext/zstdlib/zstd-1.4.5/lib/common/debug.c +24 -0
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/debug.h +11 -31
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/entropy_common.c +13 -33
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/error_private.c +2 -1
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/error_private.h +6 -2
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/fse.h +11 -31
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/fse_decompress.c +12 -37
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/huf.h +15 -33
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/mem.h +1 -1
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/pool.c +1 -1
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/pool.h +2 -2
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/threading.c +4 -3
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/threading.h +4 -3
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/xxhash.c +15 -33
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/xxhash.h +11 -31
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/zstd_common.c +1 -1
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/zstd_errors.h +2 -1
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/common/zstd_internal.h +112 -15
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/fse_compress.c +17 -40
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/hist.c +15 -35
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/hist.h +12 -32
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/huf_compress.c +92 -92
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_compress.c +450 -275
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_compress_internal.h +136 -14
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_compress_literals.c +10 -6
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_compress_literals.h +1 -1
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_compress_sequences.c +24 -20
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_compress_sequences.h +10 -3
- data/ext/zstdlib/zstd-1.4.5/lib/compress/zstd_compress_superblock.c +845 -0
- data/ext/zstdlib/zstd-1.4.5/lib/compress/zstd_compress_superblock.h +32 -0
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_cwksp.h +3 -13
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_double_fast.c +11 -8
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_double_fast.h +2 -2
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_fast.c +36 -24
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_fast.h +2 -2
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_lazy.c +34 -11
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_lazy.h +1 -1
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_ldm.c +27 -5
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_ldm.h +7 -2
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_opt.c +38 -84
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstd_opt.h +1 -1
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstdmt_compress.c +48 -21
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/compress/zstdmt_compress.h +2 -2
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/huf_decompress.c +76 -62
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/zstd_ddict.c +12 -8
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/zstd_ddict.h +2 -2
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/zstd_decompress.c +264 -148
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/zstd_decompress_block.c +312 -203
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/zstd_decompress_block.h +3 -3
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/decompress/zstd_decompress_internal.h +18 -4
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/lib/zstd.h +62 -21
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/gzclose.c +0 -0
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/gzcompatibility.h +1 -1
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/gzguts.h +0 -0
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/gzlib.c +0 -0
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/gzread.c +0 -0
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/gzwrite.c +0 -0
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/zstd_zlibwrapper.c +1 -1
- data/ext/zstdlib/{zstd-1.4.4 → zstd-1.4.5}/zlibWrapper/zstd_zlibwrapper.h +1 -1
- metadata +65 -63
- data/ext/zstdlib/zstd-1.4.4/lib/common/debug.c +0 -44
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c) 2016-
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -18,7 +18,7 @@
|
|
18
18
|
/*-*************************************
|
19
19
|
* Dependencies
|
20
20
|
***************************************/
|
21
|
-
#include "zstd_internal.h"
|
21
|
+
#include "../common/zstd_internal.h"
|
22
22
|
#include "zstd_cwksp.h"
|
23
23
|
#ifdef ZSTD_MULTITHREAD
|
24
24
|
# include "zstdmt_compress.h"
|
@@ -166,6 +166,7 @@ typedef struct {
|
|
166
166
|
typedef struct {
|
167
167
|
ZSTD_window_t window; /* State for the window round buffer management */
|
168
168
|
ldmEntry_t* hashTable;
|
169
|
+
U32 loadedDictEnd;
|
169
170
|
BYTE* bucketOffsets; /* Next position in bucket to insert entry */
|
170
171
|
U64 hashPower; /* Used to compute the rolling hash.
|
171
172
|
* Depends on ldmParams.minMatchLength */
|
@@ -249,6 +250,7 @@ struct ZSTD_CCtx_s {
|
|
249
250
|
size_t staticSize;
|
250
251
|
SeqCollector seqCollector;
|
251
252
|
int isFirstBlock;
|
253
|
+
int initialized;
|
252
254
|
|
253
255
|
seqStore_t seqStore; /* sequences storage ptrs */
|
254
256
|
ldmState_t ldmState; /* long distance matching state */
|
@@ -324,6 +326,31 @@ MEM_STATIC U32 ZSTD_MLcode(U32 mlBase)
|
|
324
326
|
return (mlBase > 127) ? ZSTD_highbit32(mlBase) + ML_deltaCode : ML_Code[mlBase];
|
325
327
|
}
|
326
328
|
|
329
|
+
typedef struct repcodes_s {
|
330
|
+
U32 rep[3];
|
331
|
+
} repcodes_t;
|
332
|
+
|
333
|
+
MEM_STATIC repcodes_t ZSTD_updateRep(U32 const rep[3], U32 const offset, U32 const ll0)
|
334
|
+
{
|
335
|
+
repcodes_t newReps;
|
336
|
+
if (offset >= ZSTD_REP_NUM) { /* full offset */
|
337
|
+
newReps.rep[2] = rep[1];
|
338
|
+
newReps.rep[1] = rep[0];
|
339
|
+
newReps.rep[0] = offset - ZSTD_REP_MOVE;
|
340
|
+
} else { /* repcode */
|
341
|
+
U32 const repCode = offset + ll0;
|
342
|
+
if (repCode > 0) { /* note : if repCode==0, no change */
|
343
|
+
U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode];
|
344
|
+
newReps.rep[2] = (repCode >= 2) ? rep[1] : rep[2];
|
345
|
+
newReps.rep[1] = rep[0];
|
346
|
+
newReps.rep[0] = currentOffset;
|
347
|
+
} else { /* repCode == 0 */
|
348
|
+
memcpy(&newReps, rep, sizeof(newReps));
|
349
|
+
}
|
350
|
+
}
|
351
|
+
return newReps;
|
352
|
+
}
|
353
|
+
|
327
354
|
/* ZSTD_cParam_withinBounds:
|
328
355
|
* @return 1 if value is within cParam bounds,
|
329
356
|
* 0 otherwise */
|
@@ -336,6 +363,30 @@ MEM_STATIC int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value)
|
|
336
363
|
return 1;
|
337
364
|
}
|
338
365
|
|
366
|
+
/* ZSTD_noCompressBlock() :
|
367
|
+
* Writes uncompressed block to dst buffer from given src.
|
368
|
+
* Returns the size of the block */
|
369
|
+
MEM_STATIC size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock)
|
370
|
+
{
|
371
|
+
U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(srcSize << 3);
|
372
|
+
RETURN_ERROR_IF(srcSize + ZSTD_blockHeaderSize > dstCapacity,
|
373
|
+
dstSize_tooSmall, "dst buf too small for uncompressed block");
|
374
|
+
MEM_writeLE24(dst, cBlockHeader24);
|
375
|
+
memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize);
|
376
|
+
return ZSTD_blockHeaderSize + srcSize;
|
377
|
+
}
|
378
|
+
|
379
|
+
MEM_STATIC size_t ZSTD_rleCompressBlock (void* dst, size_t dstCapacity, BYTE src, size_t srcSize, U32 lastBlock)
|
380
|
+
{
|
381
|
+
BYTE* const op = (BYTE*)dst;
|
382
|
+
U32 const cBlockHeader = lastBlock + (((U32)bt_rle)<<1) + (U32)(srcSize << 3);
|
383
|
+
RETURN_ERROR_IF(dstCapacity < 4, dstSize_tooSmall, "");
|
384
|
+
MEM_writeLE24(op, cBlockHeader);
|
385
|
+
op[3] = src;
|
386
|
+
return 4;
|
387
|
+
}
|
388
|
+
|
389
|
+
|
339
390
|
/* ZSTD_minGain() :
|
340
391
|
* minimum compression required
|
341
392
|
* to generate a compress block or a compressed literals section.
|
@@ -348,6 +399,21 @@ MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat)
|
|
348
399
|
return (srcSize >> minlog) + 2;
|
349
400
|
}
|
350
401
|
|
402
|
+
MEM_STATIC int ZSTD_disableLiteralsCompression(const ZSTD_CCtx_params* cctxParams)
|
403
|
+
{
|
404
|
+
switch (cctxParams->literalCompressionMode) {
|
405
|
+
case ZSTD_lcm_huffman:
|
406
|
+
return 0;
|
407
|
+
case ZSTD_lcm_uncompressed:
|
408
|
+
return 1;
|
409
|
+
default:
|
410
|
+
assert(0 /* impossible: pre-validated */);
|
411
|
+
/* fall-through */
|
412
|
+
case ZSTD_lcm_auto:
|
413
|
+
return (cctxParams->cParams.strategy == ZSTD_fast) && (cctxParams->cParams.targetLength > 0);
|
414
|
+
}
|
415
|
+
}
|
416
|
+
|
351
417
|
/*! ZSTD_safecopyLiterals() :
|
352
418
|
* memcpy() function that won't read beyond more than WILDCOPY_OVERLENGTH bytes past ilimit_w.
|
353
419
|
* Only called when the sequence ends past ilimit_w, so it only needs to be optimized for single
|
@@ -433,8 +499,7 @@ static unsigned ZSTD_NbCommonBytes (size_t val)
|
|
433
499
|
if (MEM_64bits()) {
|
434
500
|
# if defined(_MSC_VER) && defined(_WIN64)
|
435
501
|
unsigned long r = 0;
|
436
|
-
_BitScanForward64( &r, (U64)val );
|
437
|
-
return (unsigned)(r>>3);
|
502
|
+
return _BitScanForward64( &r, (U64)val ) ? (unsigned)(r >> 3) : 0;
|
438
503
|
# elif defined(__GNUC__) && (__GNUC__ >= 4)
|
439
504
|
return (__builtin_ctzll((U64)val) >> 3);
|
440
505
|
# else
|
@@ -451,8 +516,7 @@ static unsigned ZSTD_NbCommonBytes (size_t val)
|
|
451
516
|
} else { /* 32 bits */
|
452
517
|
# if defined(_MSC_VER)
|
453
518
|
unsigned long r=0;
|
454
|
-
_BitScanForward( &r, (U32)val );
|
455
|
-
return (unsigned)(r>>3);
|
519
|
+
return _BitScanForward( &r, (U32)val ) ? (unsigned)(r >> 3) : 0;
|
456
520
|
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
457
521
|
return (__builtin_ctz((U32)val) >> 3);
|
458
522
|
# else
|
@@ -467,8 +531,7 @@ static unsigned ZSTD_NbCommonBytes (size_t val)
|
|
467
531
|
if (MEM_64bits()) {
|
468
532
|
# if defined(_MSC_VER) && defined(_WIN64)
|
469
533
|
unsigned long r = 0;
|
470
|
-
_BitScanReverse64( &r, val );
|
471
|
-
return (unsigned)(r>>3);
|
534
|
+
return _BitScanReverse64( &r, val ) ? (unsigned)(r >> 3) : 0;
|
472
535
|
# elif defined(__GNUC__) && (__GNUC__ >= 4)
|
473
536
|
return (__builtin_clzll(val) >> 3);
|
474
537
|
# else
|
@@ -482,8 +545,7 @@ static unsigned ZSTD_NbCommonBytes (size_t val)
|
|
482
545
|
} else { /* 32 bits */
|
483
546
|
# if defined(_MSC_VER)
|
484
547
|
unsigned long r = 0;
|
485
|
-
_BitScanReverse( &r, (unsigned long)val );
|
486
|
-
return (unsigned)(r>>3);
|
548
|
+
return _BitScanReverse( &r, (unsigned long)val ) ? (unsigned)(r >> 3) : 0;
|
487
549
|
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
488
550
|
return (__builtin_clz((U32)val) >> 3);
|
489
551
|
# else
|
@@ -730,7 +792,10 @@ MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog,
|
|
730
792
|
*/
|
731
793
|
U32 const cycleMask = (1U << cycleLog) - 1;
|
732
794
|
U32 const current = (U32)((BYTE const*)src - window->base);
|
733
|
-
U32 const
|
795
|
+
U32 const currentCycle0 = current & cycleMask;
|
796
|
+
/* Exclude zero so that newCurrent - maxDist >= 1. */
|
797
|
+
U32 const currentCycle1 = currentCycle0 == 0 ? (1U << cycleLog) : currentCycle0;
|
798
|
+
U32 const newCurrent = currentCycle1 + maxDist;
|
734
799
|
U32 const correction = current - newCurrent;
|
735
800
|
assert((maxDist & cycleMask) == 0);
|
736
801
|
assert(current > newCurrent);
|
@@ -739,8 +804,17 @@ MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog,
|
|
739
804
|
|
740
805
|
window->base += correction;
|
741
806
|
window->dictBase += correction;
|
742
|
-
window->lowLimit
|
743
|
-
window->
|
807
|
+
if (window->lowLimit <= correction) window->lowLimit = 1;
|
808
|
+
else window->lowLimit -= correction;
|
809
|
+
if (window->dictLimit <= correction) window->dictLimit = 1;
|
810
|
+
else window->dictLimit -= correction;
|
811
|
+
|
812
|
+
/* Ensure we can still reference the full window. */
|
813
|
+
assert(newCurrent >= maxDist);
|
814
|
+
assert(newCurrent - maxDist >= 1);
|
815
|
+
/* Ensure that lowLimit and dictLimit didn't underflow. */
|
816
|
+
assert(window->lowLimit <= newCurrent);
|
817
|
+
assert(window->dictLimit <= newCurrent);
|
744
818
|
|
745
819
|
DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction,
|
746
820
|
window->lowLimit);
|
@@ -844,6 +918,15 @@ ZSTD_checkDictValidity(const ZSTD_window_t* window,
|
|
844
918
|
} } }
|
845
919
|
}
|
846
920
|
|
921
|
+
MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) {
|
922
|
+
memset(window, 0, sizeof(*window));
|
923
|
+
window->base = (BYTE const*)"";
|
924
|
+
window->dictBase = (BYTE const*)"";
|
925
|
+
window->dictLimit = 1; /* start from 1, so that 1st position is valid */
|
926
|
+
window->lowLimit = 1; /* it ensures first and later CCtx usages compress the same */
|
927
|
+
window->nextSrc = window->base + 1; /* see issue #1241 */
|
928
|
+
}
|
929
|
+
|
847
930
|
/**
|
848
931
|
* ZSTD_window_update():
|
849
932
|
* Updates the window by appending [src, src + srcSize) to the window.
|
@@ -857,6 +940,10 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
|
|
857
940
|
BYTE const* const ip = (BYTE const*)src;
|
858
941
|
U32 contiguous = 1;
|
859
942
|
DEBUGLOG(5, "ZSTD_window_update");
|
943
|
+
if (srcSize == 0)
|
944
|
+
return contiguous;
|
945
|
+
assert(window->base != NULL);
|
946
|
+
assert(window->dictBase != NULL);
|
860
947
|
/* Check if blocks follow each other */
|
861
948
|
if (src != window->nextSrc) {
|
862
949
|
/* not contiguous */
|
@@ -867,7 +954,7 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
|
|
867
954
|
window->dictLimit = (U32)distanceFromBase;
|
868
955
|
window->dictBase = window->base;
|
869
956
|
window->base = ip - distanceFromBase;
|
870
|
-
|
957
|
+
/* ms->nextToUpdate = window->dictLimit; */
|
871
958
|
if (window->dictLimit - window->lowLimit < HASH_READ_SIZE) window->lowLimit = window->dictLimit; /* too small extDict */
|
872
959
|
contiguous = 0;
|
873
960
|
}
|
@@ -883,6 +970,9 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
|
|
883
970
|
return contiguous;
|
884
971
|
}
|
885
972
|
|
973
|
+
/**
|
974
|
+
* Returns the lowest allowed match index. It may either be in the ext-dict or the prefix.
|
975
|
+
*/
|
886
976
|
MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 current, unsigned windowLog)
|
887
977
|
{
|
888
978
|
U32 const maxDistance = 1U << windowLog;
|
@@ -893,6 +983,19 @@ MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 current
|
|
893
983
|
return matchLowest;
|
894
984
|
}
|
895
985
|
|
986
|
+
/**
|
987
|
+
* Returns the lowest allowed match index in the prefix.
|
988
|
+
*/
|
989
|
+
MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_matchState_t* ms, U32 current, unsigned windowLog)
|
990
|
+
{
|
991
|
+
U32 const maxDistance = 1U << windowLog;
|
992
|
+
U32 const lowestValid = ms->window.dictLimit;
|
993
|
+
U32 const withinWindow = (current - lowestValid > maxDistance) ? current - maxDistance : lowestValid;
|
994
|
+
U32 const isDictionary = (ms->loadedDictEnd != 0);
|
995
|
+
U32 const matchLowest = isDictionary ? lowestValid : withinWindow;
|
996
|
+
return matchLowest;
|
997
|
+
}
|
998
|
+
|
896
999
|
|
897
1000
|
|
898
1001
|
/* debug functions */
|
@@ -931,6 +1034,21 @@ MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max)
|
|
931
1034
|
}
|
932
1035
|
#endif
|
933
1036
|
|
1037
|
+
/* ===============================================================
|
1038
|
+
* Shared internal declarations
|
1039
|
+
* These prototypes may be called from sources not in lib/compress
|
1040
|
+
* =============================================================== */
|
1041
|
+
|
1042
|
+
/* ZSTD_loadCEntropy() :
|
1043
|
+
* dict : must point at beginning of a valid zstd dictionary.
|
1044
|
+
* return : size of dictionary header (size of magic number + dict ID + entropy tables)
|
1045
|
+
* assumptions : magic number supposed already checked
|
1046
|
+
* and dictSize >= 8 */
|
1047
|
+
size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace,
|
1048
|
+
short* offcodeNCount, unsigned* offcodeMaxValue,
|
1049
|
+
const void* const dict, size_t dictSize);
|
1050
|
+
|
1051
|
+
void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs);
|
934
1052
|
|
935
1053
|
/* ==============================================================
|
936
1054
|
* Private declarations
|
@@ -940,6 +1058,7 @@ MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max)
|
|
940
1058
|
/* ZSTD_getCParamsFromCCtxParams() :
|
941
1059
|
* cParams are built depending on compressionLevel, src size hints,
|
942
1060
|
* LDM and manually set compression parameters.
|
1061
|
+
* Note: srcSizeHint == 0 means 0!
|
943
1062
|
*/
|
944
1063
|
ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
|
945
1064
|
const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize);
|
@@ -999,5 +1118,8 @@ size_t ZSTD_writeLastEmptyBlock(void* dst, size_t dstCapacity);
|
|
999
1118
|
*/
|
1000
1119
|
size_t ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSeq);
|
1001
1120
|
|
1121
|
+
/** ZSTD_cycleLog() :
|
1122
|
+
* condition for correct operation : hashLog > 1 */
|
1123
|
+
U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat);
|
1002
1124
|
|
1003
1125
|
#endif /* ZSTD_COMPRESS_H */
|
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c) 2016-
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -18,7 +18,7 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src,
|
|
18
18
|
BYTE* const ostart = (BYTE* const)dst;
|
19
19
|
U32 const flSize = 1 + (srcSize>31) + (srcSize>4095);
|
20
20
|
|
21
|
-
RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall);
|
21
|
+
RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall, "");
|
22
22
|
|
23
23
|
switch(flSize)
|
24
24
|
{
|
@@ -36,6 +36,7 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src,
|
|
36
36
|
}
|
37
37
|
|
38
38
|
memcpy(ostart + flSize, src, srcSize);
|
39
|
+
DEBUGLOG(5, "Raw literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize));
|
39
40
|
return srcSize + flSize;
|
40
41
|
}
|
41
42
|
|
@@ -62,6 +63,7 @@ size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void*
|
|
62
63
|
}
|
63
64
|
|
64
65
|
ostart[flSize] = *(const BYTE*)src;
|
66
|
+
DEBUGLOG(5, "RLE literals: %u -> %u", (U32)srcSize, (U32)flSize + 1);
|
65
67
|
return flSize+1;
|
66
68
|
}
|
67
69
|
|
@@ -80,8 +82,8 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
|
|
80
82
|
symbolEncodingType_e hType = set_compressed;
|
81
83
|
size_t cLitSize;
|
82
84
|
|
83
|
-
DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i)",
|
84
|
-
disableLiteralCompression);
|
85
|
+
DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i srcSize=%u)",
|
86
|
+
disableLiteralCompression, (U32)srcSize);
|
85
87
|
|
86
88
|
/* Prepare nextEntropy assuming reusing the existing table */
|
87
89
|
memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
|
@@ -102,14 +104,15 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
|
|
102
104
|
cLitSize = singleStream ?
|
103
105
|
HUF_compress1X_repeat(
|
104
106
|
ostart+lhSize, dstCapacity-lhSize, src, srcSize,
|
105
|
-
|
107
|
+
HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize,
|
106
108
|
(HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2) :
|
107
109
|
HUF_compress4X_repeat(
|
108
110
|
ostart+lhSize, dstCapacity-lhSize, src, srcSize,
|
109
|
-
|
111
|
+
HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize,
|
110
112
|
(HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2);
|
111
113
|
if (repeat != HUF_repeat_none) {
|
112
114
|
/* reused the existing table */
|
115
|
+
DEBUGLOG(5, "Reusing previous huffman table");
|
113
116
|
hType = set_repeat;
|
114
117
|
}
|
115
118
|
}
|
@@ -150,5 +153,6 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
|
|
150
153
|
default: /* not possible : lhSize is {3,4,5} */
|
151
154
|
assert(0);
|
152
155
|
}
|
156
|
+
DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)srcSize, (U32)(lhSize+cLitSize));
|
153
157
|
return lhSize+cLitSize;
|
154
158
|
}
|
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c) 2016-
|
2
|
+
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -60,7 +60,7 @@ static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max,
|
|
60
60
|
BYTE wksp[FSE_NCOUNTBOUND];
|
61
61
|
S16 norm[MaxSeq + 1];
|
62
62
|
const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max);
|
63
|
-
FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max));
|
63
|
+
FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max), "");
|
64
64
|
return FSE_writeNCount(wksp, sizeof(wksp), norm, max, tableLog);
|
65
65
|
}
|
66
66
|
|
@@ -86,7 +86,7 @@ static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t
|
|
86
86
|
* Returns the cost in bits of encoding the distribution in count using ctable.
|
87
87
|
* Returns an error if ctable cannot represent all the symbols in count.
|
88
88
|
*/
|
89
|
-
|
89
|
+
size_t ZSTD_fseBitCost(
|
90
90
|
FSE_CTable const* ctable,
|
91
91
|
unsigned const* count,
|
92
92
|
unsigned const max)
|
@@ -96,18 +96,22 @@ static size_t ZSTD_fseBitCost(
|
|
96
96
|
unsigned s;
|
97
97
|
FSE_CState_t cstate;
|
98
98
|
FSE_initCState(&cstate, ctable);
|
99
|
-
|
100
|
-
|
99
|
+
if (ZSTD_getFSEMaxSymbolValue(ctable) < max) {
|
100
|
+
DEBUGLOG(5, "Repeat FSE_CTable has maxSymbolValue %u < %u",
|
101
101
|
ZSTD_getFSEMaxSymbolValue(ctable), max);
|
102
|
+
return ERROR(GENERIC);
|
103
|
+
}
|
102
104
|
for (s = 0; s <= max; ++s) {
|
103
105
|
unsigned const tableLog = cstate.stateLog;
|
104
106
|
unsigned const badCost = (tableLog + 1) << kAccuracyLog;
|
105
107
|
unsigned const bitCost = FSE_bitCost(cstate.symbolTT, tableLog, s, kAccuracyLog);
|
106
108
|
if (count[s] == 0)
|
107
109
|
continue;
|
108
|
-
|
109
|
-
|
110
|
-
|
110
|
+
if (bitCost >= badCost) {
|
111
|
+
DEBUGLOG(5, "Repeat FSE_CTable has Prob[%u] == 0", s);
|
112
|
+
return ERROR(GENERIC);
|
113
|
+
}
|
114
|
+
cost += (size_t)count[s] * bitCost;
|
111
115
|
}
|
112
116
|
return cost >> kAccuracyLog;
|
113
117
|
}
|
@@ -117,15 +121,15 @@ static size_t ZSTD_fseBitCost(
|
|
117
121
|
* table described by norm. The max symbol support by norm is assumed >= max.
|
118
122
|
* norm must be valid for every symbol with non-zero probability in count.
|
119
123
|
*/
|
120
|
-
|
121
|
-
|
124
|
+
size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog,
|
125
|
+
unsigned const* count, unsigned const max)
|
122
126
|
{
|
123
127
|
unsigned const shift = 8 - accuracyLog;
|
124
128
|
size_t cost = 0;
|
125
129
|
unsigned s;
|
126
130
|
assert(accuracyLog <= 8);
|
127
131
|
for (s = 0; s <= max; ++s) {
|
128
|
-
unsigned const normAcc = norm[s] != -1 ? norm[s] : 1;
|
132
|
+
unsigned const normAcc = (norm[s] != -1) ? (unsigned)norm[s] : 1;
|
129
133
|
unsigned const norm256 = normAcc << shift;
|
130
134
|
assert(norm256 > 0);
|
131
135
|
assert(norm256 < 256);
|
@@ -230,15 +234,15 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
|
|
230
234
|
|
231
235
|
switch (type) {
|
232
236
|
case set_rle:
|
233
|
-
FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max));
|
234
|
-
RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall);
|
237
|
+
FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max), "");
|
238
|
+
RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall, "not enough space");
|
235
239
|
*op = codeTable[0];
|
236
240
|
return 1;
|
237
241
|
case set_repeat:
|
238
242
|
memcpy(nextCTable, prevCTable, prevCTableSize);
|
239
243
|
return 0;
|
240
244
|
case set_basic:
|
241
|
-
FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize)); /* note : could be pre-calculated */
|
245
|
+
FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize), ""); /* note : could be pre-calculated */
|
242
246
|
return 0;
|
243
247
|
case set_compressed: {
|
244
248
|
S16 norm[MaxSeq + 1];
|
@@ -249,14 +253,14 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
|
|
249
253
|
nbSeq_1--;
|
250
254
|
}
|
251
255
|
assert(nbSeq_1 > 1);
|
252
|
-
FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max));
|
256
|
+
FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max), "");
|
253
257
|
{ size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */
|
254
|
-
FORWARD_IF_ERROR(NCountSize);
|
255
|
-
FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, entropyWorkspace, entropyWorkspaceSize));
|
258
|
+
FORWARD_IF_ERROR(NCountSize, "FSE_writeNCount failed");
|
259
|
+
FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, entropyWorkspace, entropyWorkspaceSize), "");
|
256
260
|
return NCountSize;
|
257
261
|
}
|
258
262
|
}
|
259
|
-
default: assert(0); RETURN_ERROR(GENERIC);
|
263
|
+
default: assert(0); RETURN_ERROR(GENERIC, "impossible to reach");
|
260
264
|
}
|
261
265
|
}
|
262
266
|
|
@@ -290,7 +294,7 @@ ZSTD_encodeSequences_body(
|
|
290
294
|
if (MEM_32bits()) BIT_flushBits(&blockStream);
|
291
295
|
if (longOffsets) {
|
292
296
|
U32 const ofBits = ofCodeTable[nbSeq-1];
|
293
|
-
|
297
|
+
unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
|
294
298
|
if (extraBits) {
|
295
299
|
BIT_addBits(&blockStream, sequences[nbSeq-1].offset, extraBits);
|
296
300
|
BIT_flushBits(&blockStream);
|
@@ -327,7 +331,7 @@ ZSTD_encodeSequences_body(
|
|
327
331
|
BIT_addBits(&blockStream, sequences[n].matchLength, mlBits);
|
328
332
|
if (MEM_32bits() || (ofBits+mlBits+llBits > 56)) BIT_flushBits(&blockStream);
|
329
333
|
if (longOffsets) {
|
330
|
-
|
334
|
+
unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
|
331
335
|
if (extraBits) {
|
332
336
|
BIT_addBits(&blockStream, sequences[n].offset, extraBits);
|
333
337
|
BIT_flushBits(&blockStream); /* (7)*/
|