extzstd 0.1.1 → 0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/HISTORY.ja.md +18 -0
- data/README.md +15 -50
- data/contrib/zstd/CONTRIBUTING.md +1 -1
- data/contrib/zstd/COPYING +339 -0
- data/contrib/zstd/Makefile +82 -51
- data/contrib/zstd/NEWS +92 -5
- data/contrib/zstd/README.md +50 -41
- data/contrib/zstd/appveyor.yml +164 -102
- data/contrib/zstd/circle.yml +10 -22
- data/contrib/zstd/lib/BUCK +31 -10
- data/contrib/zstd/lib/Makefile +57 -31
- data/contrib/zstd/lib/README.md +68 -37
- data/contrib/zstd/lib/common/bitstream.h +130 -76
- data/contrib/zstd/lib/common/compiler.h +86 -0
- data/contrib/zstd/lib/common/error_private.c +15 -11
- data/contrib/zstd/lib/common/error_private.h +8 -8
- data/contrib/zstd/lib/common/fse.h +19 -9
- data/contrib/zstd/lib/common/fse_decompress.c +3 -22
- data/contrib/zstd/lib/common/huf.h +68 -26
- data/contrib/zstd/lib/common/mem.h +23 -35
- data/contrib/zstd/lib/common/pool.c +123 -63
- data/contrib/zstd/lib/common/pool.h +19 -10
- data/contrib/zstd/lib/common/threading.c +11 -16
- data/contrib/zstd/lib/common/threading.h +52 -33
- data/contrib/zstd/lib/common/xxhash.c +28 -22
- data/contrib/zstd/lib/common/zstd_common.c +40 -27
- data/contrib/zstd/lib/common/zstd_errors.h +43 -34
- data/contrib/zstd/lib/common/zstd_internal.h +131 -123
- data/contrib/zstd/lib/compress/fse_compress.c +17 -33
- data/contrib/zstd/lib/compress/huf_compress.c +15 -9
- data/contrib/zstd/lib/compress/zstd_compress.c +2096 -2363
- data/contrib/zstd/lib/compress/zstd_compress_internal.h +462 -0
- data/contrib/zstd/lib/compress/zstd_double_fast.c +309 -0
- data/contrib/zstd/lib/compress/zstd_double_fast.h +29 -0
- data/contrib/zstd/lib/compress/zstd_fast.c +243 -0
- data/contrib/zstd/lib/compress/zstd_fast.h +31 -0
- data/contrib/zstd/lib/compress/zstd_lazy.c +765 -0
- data/contrib/zstd/lib/compress/zstd_lazy.h +39 -0
- data/contrib/zstd/lib/compress/zstd_ldm.c +707 -0
- data/contrib/zstd/lib/compress/zstd_ldm.h +68 -0
- data/contrib/zstd/lib/compress/zstd_opt.c +785 -0
- data/contrib/zstd/lib/compress/zstd_opt.h +19 -908
- data/contrib/zstd/lib/compress/zstdmt_compress.c +737 -327
- data/contrib/zstd/lib/compress/zstdmt_compress.h +88 -26
- data/contrib/zstd/lib/decompress/huf_decompress.c +158 -50
- data/contrib/zstd/lib/decompress/zstd_decompress.c +884 -699
- data/contrib/zstd/lib/deprecated/zbuff.h +5 -4
- data/contrib/zstd/lib/deprecated/zbuff_common.c +5 -5
- data/contrib/zstd/lib/deprecated/zbuff_compress.c +6 -4
- data/contrib/zstd/lib/deprecated/zbuff_decompress.c +5 -4
- data/contrib/zstd/lib/dictBuilder/cover.c +93 -77
- data/contrib/zstd/lib/dictBuilder/zdict.c +107 -92
- data/contrib/zstd/lib/dictBuilder/zdict.h +112 -102
- data/contrib/zstd/lib/legacy/zstd_legacy.h +9 -4
- data/contrib/zstd/lib/legacy/zstd_v01.c +7 -6
- data/contrib/zstd/lib/legacy/zstd_v01.h +5 -4
- data/contrib/zstd/lib/legacy/zstd_v02.c +27 -99
- data/contrib/zstd/lib/legacy/zstd_v02.h +5 -4
- data/contrib/zstd/lib/legacy/zstd_v03.c +26 -98
- data/contrib/zstd/lib/legacy/zstd_v03.h +5 -4
- data/contrib/zstd/lib/legacy/zstd_v04.c +22 -91
- data/contrib/zstd/lib/legacy/zstd_v04.h +5 -4
- data/contrib/zstd/lib/legacy/zstd_v05.c +23 -99
- data/contrib/zstd/lib/legacy/zstd_v05.h +5 -4
- data/contrib/zstd/lib/legacy/zstd_v06.c +22 -96
- data/contrib/zstd/lib/legacy/zstd_v06.h +5 -4
- data/contrib/zstd/lib/legacy/zstd_v07.c +19 -95
- data/contrib/zstd/lib/legacy/zstd_v07.h +5 -4
- data/contrib/zstd/lib/zstd.h +895 -271
- data/ext/extconf.rb +11 -2
- data/ext/extzstd.c +45 -128
- data/ext/extzstd.h +74 -31
- data/ext/extzstd_stream.c +401 -142
- data/ext/zstd_common.c +5 -0
- data/ext/zstd_compress.c +8 -0
- data/ext/zstd_decompress.c +1 -0
- data/ext/zstd_dictbuilder.c +2 -0
- data/lib/extzstd/version.rb +1 -1
- data/lib/extzstd.rb +48 -1
- data/test/test_basic.rb +9 -1
- metadata +17 -7
- data/HISTORY.ja +0 -10
- data/contrib/zstd/LICENSE-examples +0 -11
- data/contrib/zstd/PATENTS +0 -33
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
|
|
1
|
+
/*
|
|
2
2
|
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
|
-
* This source code is licensed under the BSD-style license found in the
|
|
6
|
-
* LICENSE file in the root directory of this source tree
|
|
7
|
-
*
|
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
|
8
9
|
*/
|
|
9
10
|
|
|
10
11
|
|
|
@@ -326,13 +327,6 @@ size_t ZSTDv05_decompress_usingPreparedDCtx(
|
|
|
326
327
|
* Streaming functions (direct mode)
|
|
327
328
|
****************************************/
|
|
328
329
|
size_t ZSTDv05_decompressBegin(ZSTDv05_DCtx* dctx);
|
|
329
|
-
size_t ZSTDv05_decompressBegin_usingDict(ZSTDv05_DCtx* dctx, const void* dict, size_t dictSize);
|
|
330
|
-
void ZSTDv05_copyDCtx(ZSTDv05_DCtx* dctx, const ZSTDv05_DCtx* preparedDCtx);
|
|
331
|
-
|
|
332
|
-
size_t ZSTDv05_getFrameParams(ZSTDv05_parameters* params, const void* src, size_t srcSize);
|
|
333
|
-
|
|
334
|
-
size_t ZSTDv05_nextSrcSizeToDecompress(ZSTDv05_DCtx* dctx);
|
|
335
|
-
size_t ZSTDv05_decompressContinue(ZSTDv05_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
|
336
330
|
|
|
337
331
|
/*
|
|
338
332
|
Streaming decompression, direct mode (bufferless)
|
|
@@ -742,18 +736,6 @@ MEM_STATIC BITv05_DStream_status BITv05_reloadDStream(BITv05_DStream_t* bitD);
|
|
|
742
736
|
MEM_STATIC unsigned BITv05_endOfDStream(const BITv05_DStream_t* bitD);
|
|
743
737
|
|
|
744
738
|
|
|
745
|
-
/*!
|
|
746
|
-
* Start by invoking BITv05_initDStream().
|
|
747
|
-
* A chunk of the bitStream is then stored into a local register.
|
|
748
|
-
* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
|
|
749
|
-
* You can then retrieve bitFields stored into the local register, **in reverse order**.
|
|
750
|
-
* Local register is explicitly reloaded from memory by the BITv05_reloadDStream() method.
|
|
751
|
-
* A reload guarantee a minimum of ((8*sizeof(size_t))-7) bits when its result is BITv05_DStream_unfinished.
|
|
752
|
-
* Otherwise, it can be less than that, so proceed accordingly.
|
|
753
|
-
* Checking if DStream has reached its end can be performed with BITv05_endOfDStream()
|
|
754
|
-
*/
|
|
755
|
-
|
|
756
|
-
|
|
757
739
|
/*-****************************************
|
|
758
740
|
* unsafe API
|
|
759
741
|
******************************************/
|
|
@@ -765,7 +747,7 @@ MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, unsigned nbBits);
|
|
|
765
747
|
/*-**************************************************************
|
|
766
748
|
* Helper functions
|
|
767
749
|
****************************************************************/
|
|
768
|
-
MEM_STATIC unsigned BITv05_highbit32 (
|
|
750
|
+
MEM_STATIC unsigned BITv05_highbit32 (U32 val)
|
|
769
751
|
{
|
|
770
752
|
# if defined(_MSC_VER) /* Visual */
|
|
771
753
|
unsigned long r=0;
|
|
@@ -818,13 +800,13 @@ MEM_STATIC size_t BITv05_initDStream(BITv05_DStream_t* bitD, const void* srcBuff
|
|
|
818
800
|
bitD->bitContainer = *(const BYTE*)(bitD->start);
|
|
819
801
|
switch(srcSize)
|
|
820
802
|
{
|
|
821
|
-
case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16)
|
|
822
|
-
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24)
|
|
823
|
-
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32)
|
|
824
|
-
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
|
|
825
|
-
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
|
|
826
|
-
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
|
|
827
|
-
default
|
|
803
|
+
case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);/* fall-through */
|
|
804
|
+
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);/* fall-through */
|
|
805
|
+
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);/* fall-through */
|
|
806
|
+
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24; /* fall-through */
|
|
807
|
+
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16; /* fall-through */
|
|
808
|
+
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8; /* fall-through */
|
|
809
|
+
default: break;
|
|
828
810
|
}
|
|
829
811
|
contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
|
|
830
812
|
if (contain32 == 0) return ERROR(GENERIC); /* endMark not present */
|
|
@@ -835,13 +817,6 @@ MEM_STATIC size_t BITv05_initDStream(BITv05_DStream_t* bitD, const void* srcBuff
|
|
|
835
817
|
return srcSize;
|
|
836
818
|
}
|
|
837
819
|
|
|
838
|
-
/*!BITv05_lookBits
|
|
839
|
-
* Provides next n bits from local register
|
|
840
|
-
* local register is not modified (bits are still present for next read/look)
|
|
841
|
-
* On 32-bits, maxNbBits==25
|
|
842
|
-
* On 64-bits, maxNbBits==57
|
|
843
|
-
* @return : value extracted
|
|
844
|
-
*/
|
|
845
820
|
MEM_STATIC size_t BITv05_lookBits(BITv05_DStream_t* bitD, U32 nbBits)
|
|
846
821
|
{
|
|
847
822
|
const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
|
@@ -861,11 +836,6 @@ MEM_STATIC void BITv05_skipBits(BITv05_DStream_t* bitD, U32 nbBits)
|
|
|
861
836
|
bitD->bitsConsumed += nbBits;
|
|
862
837
|
}
|
|
863
838
|
|
|
864
|
-
/*!BITv05_readBits
|
|
865
|
-
* Read next n bits from local register.
|
|
866
|
-
* pay attention to not read more than nbBits contained into local register.
|
|
867
|
-
* @return : extracted value.
|
|
868
|
-
*/
|
|
869
839
|
MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, U32 nbBits)
|
|
870
840
|
{
|
|
871
841
|
size_t value = BITv05_lookBits(bitD, nbBits);
|
|
@@ -884,8 +854,8 @@ MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, U32 nbBits)
|
|
|
884
854
|
|
|
885
855
|
MEM_STATIC BITv05_DStream_status BITv05_reloadDStream(BITv05_DStream_t* bitD)
|
|
886
856
|
{
|
|
887
|
-
|
|
888
|
-
|
|
857
|
+
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
|
|
858
|
+
return BITv05_DStream_overflow;
|
|
889
859
|
|
|
890
860
|
if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) {
|
|
891
861
|
bitD->ptr -= bitD->bitsConsumed >> 3;
|
|
@@ -1001,54 +971,6 @@ static unsigned char FSEv05_decodeSymbol(FSEv05_DState_t* DStatePtr, BITv05_DStr
|
|
|
1001
971
|
|
|
1002
972
|
static unsigned FSEv05_endOfDState(const FSEv05_DState_t* DStatePtr);
|
|
1003
973
|
|
|
1004
|
-
/*!
|
|
1005
|
-
Let's now decompose FSEv05_decompress_usingDTable() into its unitary components.
|
|
1006
|
-
You will decode FSEv05-encoded symbols from the bitStream,
|
|
1007
|
-
and also any other bitFields you put in, **in reverse order**.
|
|
1008
|
-
|
|
1009
|
-
You will need a few variables to track your bitStream. They are :
|
|
1010
|
-
|
|
1011
|
-
BITv05_DStream_t DStream; // Stream context
|
|
1012
|
-
FSEv05_DState_t DState; // State context. Multiple ones are possible
|
|
1013
|
-
FSEv05_DTable* DTablePtr; // Decoding table, provided by FSEv05_buildDTable()
|
|
1014
|
-
|
|
1015
|
-
The first thing to do is to init the bitStream.
|
|
1016
|
-
errorCode = BITv05_initDStream(&DStream, srcBuffer, srcSize);
|
|
1017
|
-
|
|
1018
|
-
You should then retrieve your initial state(s)
|
|
1019
|
-
(in reverse flushing order if you have several ones) :
|
|
1020
|
-
errorCode = FSEv05_initDState(&DState, &DStream, DTablePtr);
|
|
1021
|
-
|
|
1022
|
-
You can then decode your data, symbol after symbol.
|
|
1023
|
-
For information the maximum number of bits read by FSEv05_decodeSymbol() is 'tableLog'.
|
|
1024
|
-
Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
|
|
1025
|
-
unsigned char symbol = FSEv05_decodeSymbol(&DState, &DStream);
|
|
1026
|
-
|
|
1027
|
-
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
|
|
1028
|
-
Note : maximum allowed nbBits is 25, for 32-bits compatibility
|
|
1029
|
-
size_t bitField = BITv05_readBits(&DStream, nbBits);
|
|
1030
|
-
|
|
1031
|
-
All above operations only read from local register (which size depends on size_t).
|
|
1032
|
-
Refueling the register from memory is manually performed by the reload method.
|
|
1033
|
-
endSignal = FSEv05_reloadDStream(&DStream);
|
|
1034
|
-
|
|
1035
|
-
BITv05_reloadDStream() result tells if there is still some more data to read from DStream.
|
|
1036
|
-
BITv05_DStream_unfinished : there is still some data left into the DStream.
|
|
1037
|
-
BITv05_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
|
|
1038
|
-
BITv05_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
|
|
1039
|
-
BITv05_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
|
|
1040
|
-
|
|
1041
|
-
When reaching end of buffer (BITv05_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
|
|
1042
|
-
to properly detect the exact end of stream.
|
|
1043
|
-
After each decoded symbol, check if DStream is fully consumed using this simple test :
|
|
1044
|
-
BITv05_reloadDStream(&DStream) >= BITv05_DStream_completed
|
|
1045
|
-
|
|
1046
|
-
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
|
|
1047
|
-
Checking if DStream has reached its end is performed by :
|
|
1048
|
-
BITv05_endOfDStream(&DStream);
|
|
1049
|
-
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
|
|
1050
|
-
FSEv05_endOfDState(&DState);
|
|
1051
|
-
*/
|
|
1052
974
|
|
|
1053
975
|
|
|
1054
976
|
/* *****************************************
|
|
@@ -2895,7 +2817,7 @@ static size_t ZSTDv05_decodeFrameHeader_Part2(ZSTDv05_DCtx* zc, const void* src,
|
|
|
2895
2817
|
if (srcSize != zc->headerSize)
|
|
2896
2818
|
return ERROR(srcSize_wrong);
|
|
2897
2819
|
result = ZSTDv05_getFrameParams(&(zc->params), src, srcSize);
|
|
2898
|
-
if ((MEM_32bits()) && (zc->params.windowLog > 25)) return ERROR(
|
|
2820
|
+
if ((MEM_32bits()) && (zc->params.windowLog > 25)) return ERROR(frameParameter_unsupported);
|
|
2899
2821
|
return result;
|
|
2900
2822
|
}
|
|
2901
2823
|
|
|
@@ -3955,7 +3877,7 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst
|
|
|
3955
3877
|
zbc->stage = ZBUFFv05ds_decodeHeader;
|
|
3956
3878
|
break;
|
|
3957
3879
|
}
|
|
3958
|
-
|
|
3880
|
+
/* fall-through */
|
|
3959
3881
|
case ZBUFFv05ds_loadHeader:
|
|
3960
3882
|
/* complete header from src */
|
|
3961
3883
|
{
|
|
@@ -3973,7 +3895,7 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst
|
|
|
3973
3895
|
}
|
|
3974
3896
|
// zbc->stage = ZBUFFv05ds_decodeHeader; break; /* useless : stage follows */
|
|
3975
3897
|
}
|
|
3976
|
-
|
|
3898
|
+
/* fall-through */
|
|
3977
3899
|
case ZBUFFv05ds_decodeHeader:
|
|
3978
3900
|
/* apply header to create / resize buffers */
|
|
3979
3901
|
{
|
|
@@ -4000,7 +3922,7 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst
|
|
|
4000
3922
|
break;
|
|
4001
3923
|
}
|
|
4002
3924
|
zbc->stage = ZBUFFv05ds_read;
|
|
4003
|
-
|
|
3925
|
+
/* fall-through */
|
|
4004
3926
|
case ZBUFFv05ds_read:
|
|
4005
3927
|
{
|
|
4006
3928
|
size_t neededInSize = ZSTDv05_nextSrcSizeToDecompress(zbc->zc);
|
|
@@ -4024,7 +3946,7 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst
|
|
|
4024
3946
|
if (ip==iend) { notDone = 0; break; } /* no more input */
|
|
4025
3947
|
zbc->stage = ZBUFFv05ds_load;
|
|
4026
3948
|
}
|
|
4027
|
-
|
|
3949
|
+
/* fall-through */
|
|
4028
3950
|
case ZBUFFv05ds_load:
|
|
4029
3951
|
{
|
|
4030
3952
|
size_t neededInSize = ZSTDv05_nextSrcSizeToDecompress(zbc->zc);
|
|
@@ -4045,7 +3967,9 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst
|
|
|
4045
3967
|
zbc->outEnd = zbc->outStart + decodedSize;
|
|
4046
3968
|
zbc->stage = ZBUFFv05ds_flush;
|
|
4047
3969
|
// break; /* ZBUFFv05ds_flush follows */
|
|
4048
|
-
|
|
3970
|
+
}
|
|
3971
|
+
}
|
|
3972
|
+
/* fall-through */
|
|
4049
3973
|
case ZBUFFv05ds_flush:
|
|
4050
3974
|
{
|
|
4051
3975
|
size_t toFlushSize = zbc->outEnd - zbc->outStart;
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
|
|
1
|
+
/*
|
|
2
2
|
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
|
-
* This source code is licensed under the BSD-style license found in the
|
|
6
|
-
* LICENSE file in the root directory of this source tree
|
|
7
|
-
*
|
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
|
8
9
|
*/
|
|
9
10
|
|
|
10
11
|
#ifndef ZSTDv05_H
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
|
|
1
|
+
/*
|
|
2
2
|
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
|
-
* This source code is licensed under the BSD-style license found in the
|
|
6
|
-
* LICENSE file in the root directory of this source tree
|
|
7
|
-
*
|
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
|
8
9
|
*/
|
|
9
10
|
|
|
10
11
|
|
|
@@ -838,16 +839,6 @@ MEM_STATIC BITv06_DStream_status BITv06_reloadDStream(BITv06_DStream_t* bitD);
|
|
|
838
839
|
MEM_STATIC unsigned BITv06_endOfDStream(const BITv06_DStream_t* bitD);
|
|
839
840
|
|
|
840
841
|
|
|
841
|
-
/* Start by invoking BITv06_initDStream().
|
|
842
|
-
* A chunk of the bitStream is then stored into a local register.
|
|
843
|
-
* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
|
|
844
|
-
* You can then retrieve bitFields stored into the local register, **in reverse order**.
|
|
845
|
-
* Local register is explicitly reloaded from memory by the BITv06_reloadDStream() method.
|
|
846
|
-
* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BITv06_DStream_unfinished.
|
|
847
|
-
* Otherwise, it can be less than that, so proceed accordingly.
|
|
848
|
-
* Checking if DStream has reached its end can be performed with BITv06_endOfDStream().
|
|
849
|
-
*/
|
|
850
|
-
|
|
851
842
|
|
|
852
843
|
/*-****************************************
|
|
853
844
|
* unsafe API
|
|
@@ -860,7 +851,7 @@ MEM_STATIC size_t BITv06_readBitsFast(BITv06_DStream_t* bitD, unsigned nbBits);
|
|
|
860
851
|
/*-**************************************************************
|
|
861
852
|
* Internal functions
|
|
862
853
|
****************************************************************/
|
|
863
|
-
MEM_STATIC unsigned BITv06_highbit32 (
|
|
854
|
+
MEM_STATIC unsigned BITv06_highbit32 ( U32 val)
|
|
864
855
|
{
|
|
865
856
|
# if defined(_MSC_VER) /* Visual */
|
|
866
857
|
unsigned long r=0;
|
|
@@ -910,13 +901,13 @@ MEM_STATIC size_t BITv06_initDStream(BITv06_DStream_t* bitD, const void* srcBuff
|
|
|
910
901
|
bitD->bitContainer = *(const BYTE*)(bitD->start);
|
|
911
902
|
switch(srcSize)
|
|
912
903
|
{
|
|
913
|
-
case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16)
|
|
914
|
-
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24)
|
|
915
|
-
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32)
|
|
916
|
-
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
|
|
917
|
-
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
|
|
918
|
-
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8;
|
|
919
|
-
default
|
|
904
|
+
case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);/* fall-through */
|
|
905
|
+
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);/* fall-through */
|
|
906
|
+
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);/* fall-through */
|
|
907
|
+
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; /* fall-through */
|
|
908
|
+
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; /* fall-through */
|
|
909
|
+
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; /* fall-through */
|
|
910
|
+
default: break;
|
|
920
911
|
}
|
|
921
912
|
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
|
|
922
913
|
if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */
|
|
@@ -928,13 +919,6 @@ MEM_STATIC size_t BITv06_initDStream(BITv06_DStream_t* bitD, const void* srcBuff
|
|
|
928
919
|
}
|
|
929
920
|
|
|
930
921
|
|
|
931
|
-
/*! BITv06_lookBits() :
|
|
932
|
-
* Provides next n bits from local register.
|
|
933
|
-
* local register is not modified.
|
|
934
|
-
* On 32-bits, maxNbBits==24.
|
|
935
|
-
* On 64-bits, maxNbBits==56.
|
|
936
|
-
* @return : value extracted
|
|
937
|
-
*/
|
|
938
922
|
MEM_STATIC size_t BITv06_lookBits(const BITv06_DStream_t* bitD, U32 nbBits)
|
|
939
923
|
{
|
|
940
924
|
U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
|
@@ -954,11 +938,6 @@ MEM_STATIC void BITv06_skipBits(BITv06_DStream_t* bitD, U32 nbBits)
|
|
|
954
938
|
bitD->bitsConsumed += nbBits;
|
|
955
939
|
}
|
|
956
940
|
|
|
957
|
-
/*! BITv06_readBits() :
|
|
958
|
-
* Read (consume) next n bits from local register and update.
|
|
959
|
-
* Pay attention to not read more than nbBits contained into local register.
|
|
960
|
-
* @return : extracted value.
|
|
961
|
-
*/
|
|
962
941
|
MEM_STATIC size_t BITv06_readBits(BITv06_DStream_t* bitD, U32 nbBits)
|
|
963
942
|
{
|
|
964
943
|
size_t const value = BITv06_lookBits(bitD, nbBits);
|
|
@@ -975,15 +954,10 @@ MEM_STATIC size_t BITv06_readBitsFast(BITv06_DStream_t* bitD, U32 nbBits)
|
|
|
975
954
|
return value;
|
|
976
955
|
}
|
|
977
956
|
|
|
978
|
-
/*! BITv06_reloadDStream() :
|
|
979
|
-
* Refill `BITv06_DStream_t` from src buffer previously defined (see BITv06_initDStream() ).
|
|
980
|
-
* This function is safe, it guarantees it will not read beyond src buffer.
|
|
981
|
-
* @return : status of `BITv06_DStream_t` internal register.
|
|
982
|
-
if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
|
|
983
957
|
MEM_STATIC BITv06_DStream_status BITv06_reloadDStream(BITv06_DStream_t* bitD)
|
|
984
958
|
{
|
|
985
|
-
|
|
986
|
-
|
|
959
|
+
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
|
|
960
|
+
return BITv06_DStream_overflow;
|
|
987
961
|
|
|
988
962
|
if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) {
|
|
989
963
|
bitD->ptr -= bitD->bitsConsumed >> 3;
|
|
@@ -1102,55 +1076,6 @@ static void FSEv06_initDState(FSEv06_DState_t* DStatePtr, BITv06_DStream_t*
|
|
|
1102
1076
|
|
|
1103
1077
|
static unsigned char FSEv06_decodeSymbol(FSEv06_DState_t* DStatePtr, BITv06_DStream_t* bitD);
|
|
1104
1078
|
|
|
1105
|
-
/*!
|
|
1106
|
-
Let's now decompose FSEv06_decompress_usingDTable() into its unitary components.
|
|
1107
|
-
You will decode FSE-encoded symbols from the bitStream,
|
|
1108
|
-
and also any other bitFields you put in, **in reverse order**.
|
|
1109
|
-
|
|
1110
|
-
You will need a few variables to track your bitStream. They are :
|
|
1111
|
-
|
|
1112
|
-
BITv06_DStream_t DStream; // Stream context
|
|
1113
|
-
FSEv06_DState_t DState; // State context. Multiple ones are possible
|
|
1114
|
-
FSEv06_DTable* DTablePtr; // Decoding table, provided by FSEv06_buildDTable()
|
|
1115
|
-
|
|
1116
|
-
The first thing to do is to init the bitStream.
|
|
1117
|
-
errorCode = BITv06_initDStream(&DStream, srcBuffer, srcSize);
|
|
1118
|
-
|
|
1119
|
-
You should then retrieve your initial state(s)
|
|
1120
|
-
(in reverse flushing order if you have several ones) :
|
|
1121
|
-
errorCode = FSEv06_initDState(&DState, &DStream, DTablePtr);
|
|
1122
|
-
|
|
1123
|
-
You can then decode your data, symbol after symbol.
|
|
1124
|
-
For information the maximum number of bits read by FSEv06_decodeSymbol() is 'tableLog'.
|
|
1125
|
-
Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
|
|
1126
|
-
unsigned char symbol = FSEv06_decodeSymbol(&DState, &DStream);
|
|
1127
|
-
|
|
1128
|
-
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
|
|
1129
|
-
Note : maximum allowed nbBits is 25, for 32-bits compatibility
|
|
1130
|
-
size_t bitField = BITv06_readBits(&DStream, nbBits);
|
|
1131
|
-
|
|
1132
|
-
All above operations only read from local register (which size depends on size_t).
|
|
1133
|
-
Refueling the register from memory is manually performed by the reload method.
|
|
1134
|
-
endSignal = FSEv06_reloadDStream(&DStream);
|
|
1135
|
-
|
|
1136
|
-
BITv06_reloadDStream() result tells if there is still some more data to read from DStream.
|
|
1137
|
-
BITv06_DStream_unfinished : there is still some data left into the DStream.
|
|
1138
|
-
BITv06_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
|
|
1139
|
-
BITv06_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
|
|
1140
|
-
BITv06_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
|
|
1141
|
-
|
|
1142
|
-
When reaching end of buffer (BITv06_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
|
|
1143
|
-
to properly detect the exact end of stream.
|
|
1144
|
-
After each decoded symbol, check if DStream is fully consumed using this simple test :
|
|
1145
|
-
BITv06_reloadDStream(&DStream) >= BITv06_DStream_completed
|
|
1146
|
-
|
|
1147
|
-
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
|
|
1148
|
-
Checking if DStream has reached its end is performed by :
|
|
1149
|
-
BITv06_endOfDStream(&DStream);
|
|
1150
|
-
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
|
|
1151
|
-
FSEv06_endOfDState(&DState);
|
|
1152
|
-
*/
|
|
1153
|
-
|
|
1154
1079
|
|
|
1155
1080
|
/* *****************************************
|
|
1156
1081
|
* FSE unsafe API
|
|
@@ -3084,7 +3009,7 @@ size_t ZSTDv06_getFrameParams(ZSTDv06_frameParams* fparamsPtr, const void* src,
|
|
|
3084
3009
|
static size_t ZSTDv06_decodeFrameHeader(ZSTDv06_DCtx* zc, const void* src, size_t srcSize)
|
|
3085
3010
|
{
|
|
3086
3011
|
size_t const result = ZSTDv06_getFrameParams(&(zc->fParams), src, srcSize);
|
|
3087
|
-
if ((MEM_32bits()) && (zc->fParams.windowLog > 25)) return ERROR(
|
|
3012
|
+
if ((MEM_32bits()) && (zc->fParams.windowLog > 25)) return ERROR(frameParameter_unsupported);
|
|
3088
3013
|
return result;
|
|
3089
3014
|
}
|
|
3090
3015
|
|
|
@@ -3789,7 +3714,7 @@ size_t ZSTDv06_decompressContinue(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapac
|
|
|
3789
3714
|
return 0;
|
|
3790
3715
|
}
|
|
3791
3716
|
dctx->expected = 0; /* not necessary to copy more */
|
|
3792
|
-
|
|
3717
|
+
/* fall-through */
|
|
3793
3718
|
case ZSTDds_decodeFrameHeader:
|
|
3794
3719
|
{ size_t result;
|
|
3795
3720
|
memcpy(dctx->headerBuffer + ZSTDv06_frameHeaderSize_min, src, dctx->expected);
|
|
@@ -4116,7 +4041,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
|
|
|
4116
4041
|
if (zbd->outBuff == NULL) return ERROR(memory_allocation);
|
|
4117
4042
|
} } }
|
|
4118
4043
|
zbd->stage = ZBUFFds_read;
|
|
4119
|
-
|
|
4044
|
+
/* fall-through */
|
|
4120
4045
|
case ZBUFFds_read:
|
|
4121
4046
|
{ size_t const neededInSize = ZSTDv06_nextSrcSizeToDecompress(zbd->zd);
|
|
4122
4047
|
if (neededInSize==0) { /* end of frame */
|
|
@@ -4138,7 +4063,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
|
|
|
4138
4063
|
if (ip==iend) { notDone = 0; break; } /* no more input */
|
|
4139
4064
|
zbd->stage = ZBUFFds_load;
|
|
4140
4065
|
}
|
|
4141
|
-
|
|
4066
|
+
/* fall-through */
|
|
4142
4067
|
case ZBUFFds_load:
|
|
4143
4068
|
{ size_t const neededInSize = ZSTDv06_nextSrcSizeToDecompress(zbd->zd);
|
|
4144
4069
|
size_t const toLoad = neededInSize - zbd->inPos; /* should always be <= remaining space within inBuff */
|
|
@@ -4159,8 +4084,9 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
|
|
|
4159
4084
|
zbd->outEnd = zbd->outStart + decodedSize;
|
|
4160
4085
|
zbd->stage = ZBUFFds_flush;
|
|
4161
4086
|
// break; /* ZBUFFds_flush follows */
|
|
4162
|
-
|
|
4163
|
-
|
|
4087
|
+
}
|
|
4088
|
+
}
|
|
4089
|
+
/* fall-through */
|
|
4164
4090
|
case ZBUFFds_flush:
|
|
4165
4091
|
{ size_t const toFlushSize = zbd->outEnd - zbd->outStart;
|
|
4166
4092
|
size_t const flushedSize = ZBUFFv06_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize);
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
|
|
1
|
+
/*
|
|
2
2
|
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
|
-
* This source code is licensed under the BSD-style license found in the
|
|
6
|
-
* LICENSE file in the root directory of this source tree
|
|
7
|
-
*
|
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
|
8
9
|
*/
|
|
9
10
|
|
|
10
11
|
#ifndef ZSTDv06_H
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
|
|
1
|
+
/*
|
|
2
2
|
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
|
3
3
|
* All rights reserved.
|
|
4
4
|
*
|
|
5
|
-
* This source code is licensed under the BSD-style license found in the
|
|
6
|
-
* LICENSE file in the root directory of this source tree
|
|
7
|
-
*
|
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
|
8
9
|
*/
|
|
9
10
|
|
|
10
11
|
|
|
@@ -510,16 +511,6 @@ MEM_STATIC BITv07_DStream_status BITv07_reloadDStream(BITv07_DStream_t* bitD);
|
|
|
510
511
|
MEM_STATIC unsigned BITv07_endOfDStream(const BITv07_DStream_t* bitD);
|
|
511
512
|
|
|
512
513
|
|
|
513
|
-
/* Start by invoking BITv07_initDStream().
|
|
514
|
-
* A chunk of the bitStream is then stored into a local register.
|
|
515
|
-
* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
|
|
516
|
-
* You can then retrieve bitFields stored into the local register, **in reverse order**.
|
|
517
|
-
* Local register is explicitly reloaded from memory by the BITv07_reloadDStream() method.
|
|
518
|
-
* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BITv07_DStream_unfinished.
|
|
519
|
-
* Otherwise, it can be less than that, so proceed accordingly.
|
|
520
|
-
* Checking if DStream has reached its end can be performed with BITv07_endOfDStream().
|
|
521
|
-
*/
|
|
522
|
-
|
|
523
514
|
|
|
524
515
|
/*-****************************************
|
|
525
516
|
* unsafe API
|
|
@@ -532,7 +523,7 @@ MEM_STATIC size_t BITv07_readBitsFast(BITv07_DStream_t* bitD, unsigned nbBits);
|
|
|
532
523
|
/*-**************************************************************
|
|
533
524
|
* Internal functions
|
|
534
525
|
****************************************************************/
|
|
535
|
-
MEM_STATIC unsigned BITv07_highbit32 (
|
|
526
|
+
MEM_STATIC unsigned BITv07_highbit32 (U32 val)
|
|
536
527
|
{
|
|
537
528
|
# if defined(_MSC_VER) /* Visual */
|
|
538
529
|
unsigned long r=0;
|
|
@@ -580,13 +571,13 @@ MEM_STATIC size_t BITv07_initDStream(BITv07_DStream_t* bitD, const void* srcBuff
|
|
|
580
571
|
bitD->bitContainer = *(const BYTE*)(bitD->start);
|
|
581
572
|
switch(srcSize)
|
|
582
573
|
{
|
|
583
|
-
case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16)
|
|
584
|
-
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24)
|
|
585
|
-
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32)
|
|
586
|
-
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
|
|
587
|
-
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
|
|
588
|
-
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8;
|
|
589
|
-
default
|
|
574
|
+
case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);/* fall-through */
|
|
575
|
+
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);/* fall-through */
|
|
576
|
+
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);/* fall-through */
|
|
577
|
+
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; /* fall-through */
|
|
578
|
+
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; /* fall-through */
|
|
579
|
+
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; /* fall-through */
|
|
580
|
+
default: break;
|
|
590
581
|
}
|
|
591
582
|
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
|
|
592
583
|
bitD->bitsConsumed = lastByte ? 8 - BITv07_highbit32(lastByte) : 0;
|
|
@@ -598,13 +589,6 @@ MEM_STATIC size_t BITv07_initDStream(BITv07_DStream_t* bitD, const void* srcBuff
|
|
|
598
589
|
}
|
|
599
590
|
|
|
600
591
|
|
|
601
|
-
/*! BITv07_lookBits() :
|
|
602
|
-
* Provides next n bits from local register.
|
|
603
|
-
* local register is not modified.
|
|
604
|
-
* On 32-bits, maxNbBits==24.
|
|
605
|
-
* On 64-bits, maxNbBits==56.
|
|
606
|
-
* @return : value extracted
|
|
607
|
-
*/
|
|
608
592
|
MEM_STATIC size_t BITv07_lookBits(const BITv07_DStream_t* bitD, U32 nbBits)
|
|
609
593
|
{
|
|
610
594
|
U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
|
@@ -624,11 +608,6 @@ MEM_STATIC void BITv07_skipBits(BITv07_DStream_t* bitD, U32 nbBits)
|
|
|
624
608
|
bitD->bitsConsumed += nbBits;
|
|
625
609
|
}
|
|
626
610
|
|
|
627
|
-
/*! BITv07_readBits() :
|
|
628
|
-
* Read (consume) next n bits from local register and update.
|
|
629
|
-
* Pay attention to not read more than nbBits contained into local register.
|
|
630
|
-
* @return : extracted value.
|
|
631
|
-
*/
|
|
632
611
|
MEM_STATIC size_t BITv07_readBits(BITv07_DStream_t* bitD, U32 nbBits)
|
|
633
612
|
{
|
|
634
613
|
size_t const value = BITv07_lookBits(bitD, nbBits);
|
|
@@ -645,11 +624,6 @@ MEM_STATIC size_t BITv07_readBitsFast(BITv07_DStream_t* bitD, U32 nbBits)
|
|
|
645
624
|
return value;
|
|
646
625
|
}
|
|
647
626
|
|
|
648
|
-
/*! BITv07_reloadDStream() :
|
|
649
|
-
* Refill `BITv07_DStream_t` from src buffer previously defined (see BITv07_initDStream() ).
|
|
650
|
-
* This function is safe, it guarantees it will not read beyond src buffer.
|
|
651
|
-
* @return : status of `BITv07_DStream_t` internal register.
|
|
652
|
-
if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
|
|
653
627
|
MEM_STATIC BITv07_DStream_status BITv07_reloadDStream(BITv07_DStream_t* bitD)
|
|
654
628
|
{
|
|
655
629
|
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */
|
|
@@ -873,55 +847,6 @@ static void FSEv07_initDState(FSEv07_DState_t* DStatePtr, BITv07_DStream_t*
|
|
|
873
847
|
static unsigned char FSEv07_decodeSymbol(FSEv07_DState_t* DStatePtr, BITv07_DStream_t* bitD);
|
|
874
848
|
|
|
875
849
|
|
|
876
|
-
/**<
|
|
877
|
-
Let's now decompose FSEv07_decompress_usingDTable() into its unitary components.
|
|
878
|
-
You will decode FSE-encoded symbols from the bitStream,
|
|
879
|
-
and also any other bitFields you put in, **in reverse order**.
|
|
880
|
-
|
|
881
|
-
You will need a few variables to track your bitStream. They are :
|
|
882
|
-
|
|
883
|
-
BITv07_DStream_t DStream; // Stream context
|
|
884
|
-
FSEv07_DState_t DState; // State context. Multiple ones are possible
|
|
885
|
-
FSEv07_DTable* DTablePtr; // Decoding table, provided by FSEv07_buildDTable()
|
|
886
|
-
|
|
887
|
-
The first thing to do is to init the bitStream.
|
|
888
|
-
errorCode = BITv07_initDStream(&DStream, srcBuffer, srcSize);
|
|
889
|
-
|
|
890
|
-
You should then retrieve your initial state(s)
|
|
891
|
-
(in reverse flushing order if you have several ones) :
|
|
892
|
-
errorCode = FSEv07_initDState(&DState, &DStream, DTablePtr);
|
|
893
|
-
|
|
894
|
-
You can then decode your data, symbol after symbol.
|
|
895
|
-
For information the maximum number of bits read by FSEv07_decodeSymbol() is 'tableLog'.
|
|
896
|
-
Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
|
|
897
|
-
unsigned char symbol = FSEv07_decodeSymbol(&DState, &DStream);
|
|
898
|
-
|
|
899
|
-
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
|
|
900
|
-
Note : maximum allowed nbBits is 25, for 32-bits compatibility
|
|
901
|
-
size_t bitField = BITv07_readBits(&DStream, nbBits);
|
|
902
|
-
|
|
903
|
-
All above operations only read from local register (which size depends on size_t).
|
|
904
|
-
Refueling the register from memory is manually performed by the reload method.
|
|
905
|
-
endSignal = FSEv07_reloadDStream(&DStream);
|
|
906
|
-
|
|
907
|
-
BITv07_reloadDStream() result tells if there is still some more data to read from DStream.
|
|
908
|
-
BITv07_DStream_unfinished : there is still some data left into the DStream.
|
|
909
|
-
BITv07_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
|
|
910
|
-
BITv07_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
|
|
911
|
-
BITv07_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
|
|
912
|
-
|
|
913
|
-
When reaching end of buffer (BITv07_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
|
|
914
|
-
to properly detect the exact end of stream.
|
|
915
|
-
After each decoded symbol, check if DStream is fully consumed using this simple test :
|
|
916
|
-
BITv07_reloadDStream(&DStream) >= BITv07_DStream_completed
|
|
917
|
-
|
|
918
|
-
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
|
|
919
|
-
Checking if DStream has reached its end is performed by :
|
|
920
|
-
BITv07_endOfDStream(&DStream);
|
|
921
|
-
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
|
|
922
|
-
FSEv07_endOfDState(&DState);
|
|
923
|
-
*/
|
|
924
|
-
|
|
925
850
|
|
|
926
851
|
/* *****************************************
|
|
927
852
|
* FSE unsafe API
|
|
@@ -2920,8 +2845,6 @@ typedef struct {
|
|
|
2920
2845
|
void ZSTDv07_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq);
|
|
2921
2846
|
|
|
2922
2847
|
/* custom memory allocation functions */
|
|
2923
|
-
void* ZSTDv07_defaultAllocFunction(void* opaque, size_t size);
|
|
2924
|
-
void ZSTDv07_defaultFreeFunction(void* opaque, void* address);
|
|
2925
2848
|
static const ZSTDv07_customMem defaultCustomMem = { ZSTDv07_defaultAllocFunction, ZSTDv07_defaultFreeFunction, NULL };
|
|
2926
2849
|
|
|
2927
2850
|
#endif /* ZSTDv07_CCOMMON_H_MODULE */
|
|
@@ -4047,7 +3970,7 @@ size_t ZSTDv07_decompressContinue(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapac
|
|
|
4047
3970
|
return 0;
|
|
4048
3971
|
}
|
|
4049
3972
|
dctx->expected = 0; /* not necessary to copy more */
|
|
4050
|
-
|
|
3973
|
+
/* fall-through */
|
|
4051
3974
|
case ZSTDds_decodeFrameHeader:
|
|
4052
3975
|
{ size_t result;
|
|
4053
3976
|
memcpy(dctx->headerBuffer + ZSTDv07_frameHeaderSize_min, src, dctx->expected);
|
|
@@ -4494,7 +4417,7 @@ size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd,
|
|
|
4494
4417
|
} } }
|
|
4495
4418
|
zbd->stage = ZBUFFds_read;
|
|
4496
4419
|
/* pass-through */
|
|
4497
|
-
|
|
4420
|
+
/* fall-through */
|
|
4498
4421
|
case ZBUFFds_read:
|
|
4499
4422
|
{ size_t const neededInSize = ZSTDv07_nextSrcSizeToDecompress(zbd->zd);
|
|
4500
4423
|
if (neededInSize==0) { /* end of frame */
|
|
@@ -4517,7 +4440,7 @@ size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd,
|
|
|
4517
4440
|
if (ip==iend) { notDone = 0; break; } /* no more input */
|
|
4518
4441
|
zbd->stage = ZBUFFds_load;
|
|
4519
4442
|
}
|
|
4520
|
-
|
|
4443
|
+
/* fall-through */
|
|
4521
4444
|
case ZBUFFds_load:
|
|
4522
4445
|
{ size_t const neededInSize = ZSTDv07_nextSrcSizeToDecompress(zbd->zd);
|
|
4523
4446
|
size_t const toLoad = neededInSize - zbd->inPos; /* should always be <= remaining space within inBuff */
|
|
@@ -4540,8 +4463,9 @@ size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd,
|
|
|
4540
4463
|
zbd->stage = ZBUFFds_flush;
|
|
4541
4464
|
/* break; */
|
|
4542
4465
|
/* pass-through */
|
|
4543
|
-
|
|
4544
|
-
|
|
4466
|
+
}
|
|
4467
|
+
}
|
|
4468
|
+
/* fall-through */
|
|
4545
4469
|
case ZBUFFds_flush:
|
|
4546
4470
|
{ size_t const toFlushSize = zbd->outEnd - zbd->outStart;
|
|
4547
4471
|
size_t const flushedSize = ZBUFFv07_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize);
|