zstd-ruby 1.2.0.0 → 1.3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/ext/zstdruby/libzstd/Makefile +7 -5
- data/ext/zstdruby/libzstd/common/bitstream.h +23 -9
- data/ext/zstdruby/libzstd/common/error_private.c +4 -1
- data/ext/zstdruby/libzstd/common/huf.h +20 -0
- data/ext/zstdruby/libzstd/common/mem.h +0 -14
- data/ext/zstdruby/libzstd/common/pool.c +12 -0
- data/ext/zstdruby/libzstd/common/pool.h +5 -0
- data/ext/zstdruby/libzstd/common/threading.c +0 -1
- data/ext/zstdruby/libzstd/common/zstd_common.c +25 -18
- data/ext/zstdruby/libzstd/common/zstd_errors.h +15 -7
- data/ext/zstdruby/libzstd/common/zstd_internal.h +59 -9
- data/ext/zstdruby/libzstd/compress/huf_compress.c +7 -3
- data/ext/zstdruby/libzstd/compress/zstd_compress.c +1082 -487
- data/ext/zstdruby/libzstd/compress/zstd_opt.h +30 -15
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +362 -158
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +49 -13
- data/ext/zstdruby/libzstd/decompress/huf_decompress.c +150 -26
- data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +380 -258
- data/ext/zstdruby/libzstd/dictBuilder/cover.c +23 -37
- data/ext/zstdruby/libzstd/dictBuilder/zdict.c +30 -40
- data/ext/zstdruby/libzstd/dictBuilder/zdict.h +104 -95
- data/ext/zstdruby/libzstd/legacy/zstd_v04.c +11 -10
- data/ext/zstdruby/libzstd/legacy/zstd_v05.c +14 -19
- data/ext/zstdruby/libzstd/legacy/zstd_v06.c +13 -12
- data/ext/zstdruby/libzstd/legacy/zstd_v07.c +13 -14
- data/ext/zstdruby/libzstd/zstd.h +507 -166
- data/lib/zstd-ruby/version.rb +1 -1
- metadata +2 -2
@@ -15,25 +15,34 @@
|
|
15
15
|
#endif
|
16
16
|
|
17
17
|
|
18
|
-
/* Note : All prototypes defined in this file
|
19
|
-
*
|
18
|
+
/* Note : All prototypes defined in this file are labelled experimental.
|
19
|
+
* No guarantee of API continuity is provided on any of them.
|
20
|
+
* In fact, the expectation is that these prototypes will be replaced
|
21
|
+
* by ZSTD_compress_generic() API in the near future */
|
20
22
|
|
21
23
|
/* === Dependencies === */
|
22
|
-
#include <stddef.h>
|
24
|
+
#include <stddef.h> /* size_t */
|
23
25
|
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */
|
24
|
-
#include "zstd.h"
|
26
|
+
#include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
|
25
27
|
|
26
28
|
|
27
|
-
/* ===
|
28
|
-
|
29
|
+
/* === Memory management === */
|
29
30
|
typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx;
|
30
31
|
ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbThreads);
|
31
|
-
ZSTDLIB_API
|
32
|
+
ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads,
|
33
|
+
ZSTD_customMem cMem);
|
34
|
+
ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx);
|
35
|
+
|
36
|
+
ZSTDLIB_API size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx);
|
37
|
+
|
38
|
+
|
39
|
+
/* === Simple buffer-to-butter one-pass function === */
|
40
|
+
|
41
|
+
ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
|
42
|
+
void* dst, size_t dstCapacity,
|
43
|
+
const void* src, size_t srcSize,
|
44
|
+
int compressionLevel);
|
32
45
|
|
33
|
-
ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* cctx,
|
34
|
-
void* dst, size_t dstCapacity,
|
35
|
-
const void* src, size_t srcSize,
|
36
|
-
int compressionLevel);
|
37
46
|
|
38
47
|
|
39
48
|
/* === Streaming functions === */
|
@@ -53,8 +62,22 @@ ZSTDLIB_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output);
|
|
53
62
|
# define ZSTDMT_SECTION_SIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */
|
54
63
|
#endif
|
55
64
|
|
56
|
-
ZSTDLIB_API size_t
|
57
|
-
|
65
|
+
ZSTDLIB_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
|
66
|
+
void* dst, size_t dstCapacity,
|
67
|
+
const void* src, size_t srcSize,
|
68
|
+
const ZSTD_CDict* cdict,
|
69
|
+
ZSTD_parameters const params,
|
70
|
+
unsigned overlapRLog);
|
71
|
+
|
72
|
+
ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
|
73
|
+
const void* dict, size_t dictSize, /* dict can be released after init, a local copy is preserved within zcs */
|
74
|
+
ZSTD_parameters params,
|
75
|
+
unsigned long long pledgedSrcSize); /* pledgedSrcSize is optional and can be zero == unknown */
|
76
|
+
|
77
|
+
ZSTDLIB_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
|
78
|
+
const ZSTD_CDict* cdict,
|
79
|
+
ZSTD_frameParameters fparams,
|
80
|
+
unsigned long long pledgedSrcSize); /* note : zero means empty */
|
58
81
|
|
59
82
|
/* ZSDTMT_parameter :
|
60
83
|
* List of parameters that can be set using ZSTDMT_setMTCtxParameter() */
|
@@ -71,6 +94,19 @@ typedef enum {
|
|
71
94
|
ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter, unsigned value);
|
72
95
|
|
73
96
|
|
97
|
+
/*! ZSTDMT_compressStream_generic() :
|
98
|
+
* Combines ZSTDMT_compressStream() with ZSTDMT_flushStream() or ZSTDMT_endStream()
|
99
|
+
* depending on flush directive.
|
100
|
+
* @return : minimum amount of data still to be flushed
|
101
|
+
* 0 if fully flushed
|
102
|
+
* or an error code */
|
103
|
+
ZSTDLIB_API size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
|
104
|
+
ZSTD_outBuffer* output,
|
105
|
+
ZSTD_inBuffer* input,
|
106
|
+
ZSTD_EndDirective endOp);
|
107
|
+
|
108
|
+
|
109
|
+
|
74
110
|
#if defined (__cplusplus)
|
75
111
|
}
|
76
112
|
#endif
|
@@ -67,6 +67,12 @@
|
|
67
67
|
#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
|
68
68
|
|
69
69
|
|
70
|
+
/* **************************************************************
|
71
|
+
* Byte alignment for workSpace management
|
72
|
+
****************************************************************/
|
73
|
+
#define HUF_ALIGN(x, a) HUF_ALIGN_MASK((x), (a) - 1)
|
74
|
+
#define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask))
|
75
|
+
|
70
76
|
/*-***************************/
|
71
77
|
/* generic DTableDesc */
|
72
78
|
/*-***************************/
|
@@ -87,16 +93,28 @@ static DTableDesc HUF_getDTableDesc(const HUF_DTable* table)
|
|
87
93
|
|
88
94
|
typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX2; /* single-symbol decoding */
|
89
95
|
|
90
|
-
size_t
|
96
|
+
size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize)
|
91
97
|
{
|
92
|
-
BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
|
93
|
-
U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
|
94
98
|
U32 tableLog = 0;
|
95
99
|
U32 nbSymbols = 0;
|
96
100
|
size_t iSize;
|
97
101
|
void* const dtPtr = DTable + 1;
|
98
102
|
HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr;
|
99
103
|
|
104
|
+
U32* rankVal;
|
105
|
+
BYTE* huffWeight;
|
106
|
+
size_t spaceUsed32 = 0;
|
107
|
+
|
108
|
+
rankVal = (U32 *)workSpace + spaceUsed32;
|
109
|
+
spaceUsed32 += HUF_TABLELOG_ABSOLUTEMAX + 1;
|
110
|
+
huffWeight = (BYTE *)((U32 *)workSpace + spaceUsed32);
|
111
|
+
spaceUsed32 += HUF_ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2;
|
112
|
+
|
113
|
+
if ((spaceUsed32 << 2) > wkspSize)
|
114
|
+
return ERROR(tableLog_tooLarge);
|
115
|
+
workSpace = (U32 *)workSpace + spaceUsed32;
|
116
|
+
wkspSize -= (spaceUsed32 << 2);
|
117
|
+
|
100
118
|
HUF_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable));
|
101
119
|
/* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
|
102
120
|
|
@@ -135,6 +153,13 @@ size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize)
|
|
135
153
|
return iSize;
|
136
154
|
}
|
137
155
|
|
156
|
+
size_t HUF_readDTableX2(HUF_DTable* DTable, const void* src, size_t srcSize)
|
157
|
+
{
|
158
|
+
U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
|
159
|
+
return HUF_readDTableX2_wksp(DTable, src, srcSize,
|
160
|
+
workSpace, sizeof(workSpace));
|
161
|
+
}
|
162
|
+
|
138
163
|
|
139
164
|
static BYTE HUF_decodeSymbolX2(BIT_DStream_t* Dstream, const HUF_DEltX2* dt, const U32 dtLog)
|
140
165
|
{
|
@@ -212,11 +237,13 @@ size_t HUF_decompress1X2_usingDTable(
|
|
212
237
|
return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
|
213
238
|
}
|
214
239
|
|
215
|
-
size_t
|
240
|
+
size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize,
|
241
|
+
const void* cSrc, size_t cSrcSize,
|
242
|
+
void* workSpace, size_t wkspSize)
|
216
243
|
{
|
217
244
|
const BYTE* ip = (const BYTE*) cSrc;
|
218
245
|
|
219
|
-
size_t const hSize =
|
246
|
+
size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, workSpace, wkspSize);
|
220
247
|
if (HUF_isError(hSize)) return hSize;
|
221
248
|
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
|
222
249
|
ip += hSize; cSrcSize -= hSize;
|
@@ -224,6 +251,15 @@ size_t HUF_decompress1X2_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, cons
|
|
224
251
|
return HUF_decompress1X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx);
|
225
252
|
}
|
226
253
|
|
254
|
+
|
255
|
+
size_t HUF_decompress1X2_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize,
|
256
|
+
const void* cSrc, size_t cSrcSize)
|
257
|
+
{
|
258
|
+
U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
|
259
|
+
return HUF_decompress1X2_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize,
|
260
|
+
workSpace, sizeof(workSpace));
|
261
|
+
}
|
262
|
+
|
227
263
|
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
228
264
|
{
|
229
265
|
HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX);
|
@@ -335,11 +371,14 @@ size_t HUF_decompress4X2_usingDTable(
|
|
335
371
|
}
|
336
372
|
|
337
373
|
|
338
|
-
size_t
|
374
|
+
size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
|
375
|
+
const void* cSrc, size_t cSrcSize,
|
376
|
+
void* workSpace, size_t wkspSize)
|
339
377
|
{
|
340
378
|
const BYTE* ip = (const BYTE*) cSrc;
|
341
379
|
|
342
|
-
size_t const hSize =
|
380
|
+
size_t const hSize = HUF_readDTableX2_wksp (dctx, cSrc, cSrcSize,
|
381
|
+
workSpace, wkspSize);
|
343
382
|
if (HUF_isError(hSize)) return hSize;
|
344
383
|
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
|
345
384
|
ip += hSize; cSrcSize -= hSize;
|
@@ -347,6 +386,13 @@ size_t HUF_decompress4X2_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, cons
|
|
347
386
|
return HUF_decompress4X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, dctx);
|
348
387
|
}
|
349
388
|
|
389
|
+
|
390
|
+
size_t HUF_decompress4X2_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
391
|
+
{
|
392
|
+
U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
|
393
|
+
return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
|
394
|
+
workSpace, sizeof(workSpace));
|
395
|
+
}
|
350
396
|
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
351
397
|
{
|
352
398
|
HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX);
|
@@ -403,7 +449,8 @@ static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 co
|
|
403
449
|
} }
|
404
450
|
}
|
405
451
|
|
406
|
-
typedef U32
|
452
|
+
typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1];
|
453
|
+
typedef rankValCol_t rankVal_t[HUF_TABLELOG_MAX];
|
407
454
|
|
408
455
|
static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog,
|
409
456
|
const sortedSymbol_t* sortedList, const U32 sortedListSize,
|
@@ -447,20 +494,43 @@ static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog,
|
|
447
494
|
}
|
448
495
|
}
|
449
496
|
|
450
|
-
size_t
|
497
|
+
size_t HUF_readDTableX4_wksp(HUF_DTable* DTable, const void* src,
|
498
|
+
size_t srcSize, void* workSpace,
|
499
|
+
size_t wkspSize)
|
451
500
|
{
|
452
|
-
BYTE weightList[HUF_SYMBOLVALUE_MAX + 1];
|
453
|
-
sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1];
|
454
|
-
U32 rankStats[HUF_TABLELOG_MAX + 1] = { 0 };
|
455
|
-
U32 rankStart0[HUF_TABLELOG_MAX + 2] = { 0 };
|
456
|
-
U32* const rankStart = rankStart0+1;
|
457
|
-
rankVal_t rankVal;
|
458
501
|
U32 tableLog, maxW, sizeOfSort, nbSymbols;
|
459
502
|
DTableDesc dtd = HUF_getDTableDesc(DTable);
|
460
503
|
U32 const maxTableLog = dtd.maxTableLog;
|
461
504
|
size_t iSize;
|
462
505
|
void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */
|
463
506
|
HUF_DEltX4* const dt = (HUF_DEltX4*)dtPtr;
|
507
|
+
U32 *rankStart;
|
508
|
+
|
509
|
+
rankValCol_t* rankVal;
|
510
|
+
U32* rankStats;
|
511
|
+
U32* rankStart0;
|
512
|
+
sortedSymbol_t* sortedSymbol;
|
513
|
+
BYTE* weightList;
|
514
|
+
size_t spaceUsed32 = 0;
|
515
|
+
|
516
|
+
rankVal = (rankValCol_t *)((U32 *)workSpace + spaceUsed32);
|
517
|
+
spaceUsed32 += (sizeof(rankValCol_t) * HUF_TABLELOG_MAX) >> 2;
|
518
|
+
rankStats = (U32 *)workSpace + spaceUsed32;
|
519
|
+
spaceUsed32 += HUF_TABLELOG_MAX + 1;
|
520
|
+
rankStart0 = (U32 *)workSpace + spaceUsed32;
|
521
|
+
spaceUsed32 += HUF_TABLELOG_MAX + 2;
|
522
|
+
sortedSymbol = (sortedSymbol_t *)workSpace + (spaceUsed32 * sizeof(U32)) / sizeof(sortedSymbol_t);
|
523
|
+
spaceUsed32 += HUF_ALIGN(sizeof(sortedSymbol_t) * (HUF_SYMBOLVALUE_MAX + 1), sizeof(U32)) >> 2;
|
524
|
+
weightList = (BYTE *)((U32 *)workSpace + spaceUsed32);
|
525
|
+
spaceUsed32 += HUF_ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2;
|
526
|
+
|
527
|
+
if ((spaceUsed32 << 2) > wkspSize)
|
528
|
+
return ERROR(tableLog_tooLarge);
|
529
|
+
workSpace = (U32 *)workSpace + spaceUsed32;
|
530
|
+
wkspSize -= (spaceUsed32 << 2);
|
531
|
+
|
532
|
+
rankStart = rankStart0 + 1;
|
533
|
+
memset(rankStats, 0, sizeof(U32) * (2 * HUF_TABLELOG_MAX + 2 + 1));
|
464
534
|
|
465
535
|
HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */
|
466
536
|
if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
|
@@ -527,6 +597,12 @@ size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize)
|
|
527
597
|
return iSize;
|
528
598
|
}
|
529
599
|
|
600
|
+
size_t HUF_readDTableX4(HUF_DTable* DTable, const void* src, size_t srcSize)
|
601
|
+
{
|
602
|
+
U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
|
603
|
+
return HUF_readDTableX4_wksp(DTable, src, srcSize,
|
604
|
+
workSpace, sizeof(workSpace));
|
605
|
+
}
|
530
606
|
|
531
607
|
static U32 HUF_decodeSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog)
|
532
608
|
{
|
@@ -545,7 +621,8 @@ static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DE
|
|
545
621
|
if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) {
|
546
622
|
BIT_skipBits(DStream, dt[val].nbBits);
|
547
623
|
if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8))
|
548
|
-
|
624
|
+
/* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */
|
625
|
+
DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8);
|
549
626
|
} }
|
550
627
|
return 1;
|
551
628
|
}
|
@@ -626,11 +703,14 @@ size_t HUF_decompress1X4_usingDTable(
|
|
626
703
|
return HUF_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
|
627
704
|
}
|
628
705
|
|
629
|
-
size_t
|
706
|
+
size_t HUF_decompress1X4_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize,
|
707
|
+
const void* cSrc, size_t cSrcSize,
|
708
|
+
void* workSpace, size_t wkspSize)
|
630
709
|
{
|
631
710
|
const BYTE* ip = (const BYTE*) cSrc;
|
632
711
|
|
633
|
-
size_t const hSize =
|
712
|
+
size_t const hSize = HUF_readDTableX4_wksp(DCtx, cSrc, cSrcSize,
|
713
|
+
workSpace, wkspSize);
|
634
714
|
if (HUF_isError(hSize)) return hSize;
|
635
715
|
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
|
636
716
|
ip += hSize; cSrcSize -= hSize;
|
@@ -638,6 +718,15 @@ size_t HUF_decompress1X4_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, cons
|
|
638
718
|
return HUF_decompress1X4_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx);
|
639
719
|
}
|
640
720
|
|
721
|
+
|
722
|
+
size_t HUF_decompress1X4_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize,
|
723
|
+
const void* cSrc, size_t cSrcSize)
|
724
|
+
{
|
725
|
+
U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
|
726
|
+
return HUF_decompress1X4_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize,
|
727
|
+
workSpace, sizeof(workSpace));
|
728
|
+
}
|
729
|
+
|
641
730
|
size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
642
731
|
{
|
643
732
|
HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_TABLELOG_MAX);
|
@@ -748,11 +837,14 @@ size_t HUF_decompress4X4_usingDTable(
|
|
748
837
|
}
|
749
838
|
|
750
839
|
|
751
|
-
size_t
|
840
|
+
size_t HUF_decompress4X4_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
|
841
|
+
const void* cSrc, size_t cSrcSize,
|
842
|
+
void* workSpace, size_t wkspSize)
|
752
843
|
{
|
753
844
|
const BYTE* ip = (const BYTE*) cSrc;
|
754
845
|
|
755
|
-
size_t hSize =
|
846
|
+
size_t hSize = HUF_readDTableX4_wksp(dctx, cSrc, cSrcSize,
|
847
|
+
workSpace, wkspSize);
|
756
848
|
if (HUF_isError(hSize)) return hSize;
|
757
849
|
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
|
758
850
|
ip += hSize; cSrcSize -= hSize;
|
@@ -760,6 +852,15 @@ size_t HUF_decompress4X4_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, cons
|
|
760
852
|
return HUF_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx);
|
761
853
|
}
|
762
854
|
|
855
|
+
|
856
|
+
size_t HUF_decompress4X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize,
|
857
|
+
const void* cSrc, size_t cSrcSize)
|
858
|
+
{
|
859
|
+
U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
|
860
|
+
return HUF_decompress4X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
|
861
|
+
workSpace, sizeof(workSpace));
|
862
|
+
}
|
863
|
+
|
763
864
|
size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
764
865
|
{
|
765
866
|
HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_TABLELOG_MAX);
|
@@ -861,19 +962,32 @@ size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const
|
|
861
962
|
}
|
862
963
|
}
|
863
964
|
|
864
|
-
size_t HUF_decompress4X_hufOnly
|
965
|
+
size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
966
|
+
{
|
967
|
+
U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
|
968
|
+
return HUF_decompress4X_hufOnly_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
|
969
|
+
workSpace, sizeof(workSpace));
|
970
|
+
}
|
971
|
+
|
972
|
+
|
973
|
+
size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst,
|
974
|
+
size_t dstSize, const void* cSrc,
|
975
|
+
size_t cSrcSize, void* workSpace,
|
976
|
+
size_t wkspSize)
|
865
977
|
{
|
866
978
|
/* validation checks */
|
867
979
|
if (dstSize == 0) return ERROR(dstSize_tooSmall);
|
868
980
|
if ((cSrcSize >= dstSize) || (cSrcSize <= 1)) return ERROR(corruption_detected); /* invalid */
|
869
981
|
|
870
982
|
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
|
871
|
-
return algoNb ?
|
872
|
-
|
983
|
+
return algoNb ? HUF_decompress4X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize):
|
984
|
+
HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize);
|
873
985
|
}
|
874
986
|
}
|
875
987
|
|
876
|
-
size_t
|
988
|
+
size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
|
989
|
+
const void* cSrc, size_t cSrcSize,
|
990
|
+
void* workSpace, size_t wkspSize)
|
877
991
|
{
|
878
992
|
/* validation checks */
|
879
993
|
if (dstSize == 0) return ERROR(dstSize_tooSmall);
|
@@ -882,7 +996,17 @@ size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const
|
|
882
996
|
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
|
883
997
|
|
884
998
|
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
|
885
|
-
return algoNb ?
|
886
|
-
|
999
|
+
return algoNb ? HUF_decompress1X4_DCtx_wksp(dctx, dst, dstSize, cSrc,
|
1000
|
+
cSrcSize, workSpace, wkspSize):
|
1001
|
+
HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
|
1002
|
+
cSrcSize, workSpace, wkspSize);
|
887
1003
|
}
|
888
1004
|
}
|
1005
|
+
|
1006
|
+
size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize,
|
1007
|
+
const void* cSrc, size_t cSrcSize)
|
1008
|
+
{
|
1009
|
+
U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
|
1010
|
+
return HUF_decompress1X_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
|
1011
|
+
workSpace, sizeof(workSpace));
|
1012
|
+
}
|
@@ -53,8 +53,7 @@
|
|
53
53
|
# include "zstd_legacy.h"
|
54
54
|
#endif
|
55
55
|
|
56
|
-
|
57
|
-
#if defined(_MSC_VER)
|
56
|
+
#if defined(_MSC_VER) && !defined(_M_IA64) /* _mm_prefetch() is not defined for ia64 */
|
58
57
|
# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
|
59
58
|
# define ZSTD_PREFETCH(ptr) _mm_prefetch((const char*)ptr, _MM_HINT_T0)
|
60
59
|
#elif defined(__GNUC__)
|
@@ -63,8 +62,9 @@
|
|
63
62
|
# define ZSTD_PREFETCH(ptr) /* disabled */
|
64
63
|
#endif
|
65
64
|
|
65
|
+
|
66
66
|
/*-*************************************
|
67
|
-
*
|
67
|
+
* Errors
|
68
68
|
***************************************/
|
69
69
|
#define ZSTD_isError ERR_isError /* for inlining */
|
70
70
|
#define FSE_isError ERR_isError
|
@@ -85,11 +85,15 @@ typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
|
|
85
85
|
ZSTDds_decompressLastBlock, ZSTDds_checkChecksum,
|
86
86
|
ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage;
|
87
87
|
|
88
|
+
typedef enum { zdss_init=0, zdss_loadHeader,
|
89
|
+
zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
|
90
|
+
|
88
91
|
typedef struct {
|
89
92
|
FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
|
90
93
|
FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
|
91
94
|
FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
|
92
95
|
HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */
|
96
|
+
U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
|
93
97
|
U32 rep[ZSTD_REP_NUM];
|
94
98
|
} ZSTD_entropyTables_t;
|
95
99
|
|
@@ -105,7 +109,7 @@ struct ZSTD_DCtx_s
|
|
105
109
|
const void* vBase; /* virtual start of previous segment if it was just before current one */
|
106
110
|
const void* dictEnd; /* end of previous segment */
|
107
111
|
size_t expected;
|
108
|
-
|
112
|
+
ZSTD_frameHeader fParams;
|
109
113
|
blockType_e bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
|
110
114
|
ZSTD_dStage stage;
|
111
115
|
U32 litEntropy;
|
@@ -117,11 +121,39 @@ struct ZSTD_DCtx_s
|
|
117
121
|
ZSTD_customMem customMem;
|
118
122
|
size_t litSize;
|
119
123
|
size_t rleSize;
|
120
|
-
|
124
|
+
size_t staticSize;
|
125
|
+
|
126
|
+
/* streaming */
|
127
|
+
ZSTD_DDict* ddictLocal;
|
128
|
+
const ZSTD_DDict* ddict;
|
129
|
+
ZSTD_dStreamStage streamStage;
|
130
|
+
char* inBuff;
|
131
|
+
size_t inBuffSize;
|
132
|
+
size_t inPos;
|
133
|
+
size_t maxWindowSize;
|
134
|
+
char* outBuff;
|
135
|
+
size_t outBuffSize;
|
136
|
+
size_t outStart;
|
137
|
+
size_t outEnd;
|
138
|
+
size_t blockSize;
|
139
|
+
size_t lhSize;
|
140
|
+
void* legacyContext;
|
141
|
+
U32 previousLegacyVersion;
|
142
|
+
U32 legacyVersion;
|
143
|
+
U32 hostageByte;
|
144
|
+
|
145
|
+
/* workspace */
|
146
|
+
BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH];
|
121
147
|
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
|
122
148
|
}; /* typedef'd to ZSTD_DCtx within "zstd.h" */
|
123
149
|
|
124
|
-
size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx)
|
150
|
+
size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx)
|
151
|
+
{
|
152
|
+
if (dctx==NULL) return 0; /* support sizeof NULL */
|
153
|
+
return sizeof(*dctx)
|
154
|
+
+ ZSTD_sizeof_DDict(dctx->ddictLocal)
|
155
|
+
+ dctx->inBuffSize + dctx->outBuffSize;
|
156
|
+
}
|
125
157
|
|
126
158
|
size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); }
|
127
159
|
|
@@ -145,40 +177,76 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
|
|
145
177
|
return 0;
|
146
178
|
}
|
147
179
|
|
180
|
+
static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
|
181
|
+
{
|
182
|
+
ZSTD_decompressBegin(dctx); /* cannot fail */
|
183
|
+
dctx->staticSize = 0;
|
184
|
+
dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
|
185
|
+
dctx->ddict = NULL;
|
186
|
+
dctx->ddictLocal = NULL;
|
187
|
+
dctx->inBuff = NULL;
|
188
|
+
dctx->inBuffSize = 0;
|
189
|
+
dctx->outBuffSize= 0;
|
190
|
+
dctx->streamStage = zdss_init;
|
191
|
+
}
|
192
|
+
|
148
193
|
ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
|
149
194
|
{
|
150
|
-
|
195
|
+
if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
|
151
196
|
|
152
|
-
|
153
|
-
|
197
|
+
{ ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(*dctx), customMem);
|
198
|
+
if (!dctx) return NULL;
|
199
|
+
dctx->customMem = customMem;
|
200
|
+
dctx->legacyContext = NULL;
|
201
|
+
dctx->previousLegacyVersion = 0;
|
202
|
+
ZSTD_initDCtx_internal(dctx);
|
203
|
+
return dctx;
|
204
|
+
}
|
205
|
+
}
|
154
206
|
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
207
|
+
ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize)
|
208
|
+
{
|
209
|
+
ZSTD_DCtx* dctx = (ZSTD_DCtx*) workspace;
|
210
|
+
|
211
|
+
if ((size_t)workspace & 7) return NULL; /* 8-aligned */
|
212
|
+
if (workspaceSize < sizeof(ZSTD_DCtx)) return NULL; /* minimum size */
|
213
|
+
|
214
|
+
ZSTD_initDCtx_internal(dctx);
|
215
|
+
dctx->staticSize = workspaceSize;
|
216
|
+
dctx->inBuff = (char*)(dctx+1);
|
159
217
|
return dctx;
|
160
218
|
}
|
161
219
|
|
162
220
|
ZSTD_DCtx* ZSTD_createDCtx(void)
|
163
221
|
{
|
164
|
-
return ZSTD_createDCtx_advanced(
|
222
|
+
return ZSTD_createDCtx_advanced(ZSTD_defaultCMem);
|
165
223
|
}
|
166
224
|
|
167
225
|
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
|
168
226
|
{
|
169
227
|
if (dctx==NULL) return 0; /* support free on NULL */
|
170
|
-
|
171
|
-
|
228
|
+
if (dctx->staticSize) return ERROR(memory_allocation); /* not compatible with static DCtx */
|
229
|
+
{ ZSTD_customMem const cMem = dctx->customMem;
|
230
|
+
ZSTD_freeDDict(dctx->ddictLocal);
|
231
|
+
dctx->ddictLocal = NULL;
|
232
|
+
ZSTD_free(dctx->inBuff, cMem);
|
233
|
+
dctx->inBuff = NULL;
|
234
|
+
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
|
235
|
+
if (dctx->legacyContext)
|
236
|
+
ZSTD_freeLegacyStreamContext(dctx->legacyContext, dctx->previousLegacyVersion);
|
237
|
+
#endif
|
238
|
+
ZSTD_free(dctx, cMem);
|
239
|
+
return 0;
|
240
|
+
}
|
172
241
|
}
|
173
242
|
|
243
|
+
/* no longer useful */
|
174
244
|
void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
|
175
245
|
{
|
176
|
-
size_t const
|
177
|
-
memcpy(dstDCtx, srcDCtx,
|
246
|
+
size_t const toCopy = (size_t)((char*)(&dstDCtx->inBuff) - (char*)dstDCtx);
|
247
|
+
memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */
|
178
248
|
}
|
179
249
|
|
180
|
-
static void ZSTD_refDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict);
|
181
|
-
|
182
250
|
|
183
251
|
/*-*************************************************************
|
184
252
|
* Decompression section
|
@@ -206,7 +274,7 @@ unsigned ZSTD_isFrame(const void* buffer, size_t size)
|
|
206
274
|
/** ZSTD_frameHeaderSize() :
|
207
275
|
* srcSize must be >= ZSTD_frameHeaderSize_prefix.
|
208
276
|
* @return : size of the Frame Header */
|
209
|
-
|
277
|
+
size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
|
210
278
|
{
|
211
279
|
if (srcSize < ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong);
|
212
280
|
{ BYTE const fhd = ((const BYTE*)src)[4];
|
@@ -219,22 +287,24 @@ static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
|
|
219
287
|
}
|
220
288
|
|
221
289
|
|
222
|
-
/**
|
290
|
+
/** ZSTD_getFrameHeader() :
|
223
291
|
* decode Frame Header, or require larger `srcSize`.
|
224
|
-
* @return : 0, `
|
292
|
+
* @return : 0, `zfhPtr` is correctly filled,
|
225
293
|
* >0, `srcSize` is too small, result is expected `srcSize`,
|
226
294
|
* or an error code, which can be tested using ZSTD_isError() */
|
227
|
-
size_t
|
295
|
+
size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize)
|
228
296
|
{
|
229
297
|
const BYTE* ip = (const BYTE*)src;
|
230
|
-
|
231
298
|
if (srcSize < ZSTD_frameHeaderSize_prefix) return ZSTD_frameHeaderSize_prefix;
|
299
|
+
|
232
300
|
if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) {
|
233
301
|
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
302
|
+
/* skippable frame */
|
303
|
+
if (srcSize < ZSTD_skippableHeaderSize)
|
304
|
+
return ZSTD_skippableHeaderSize; /* magic number + frame length */
|
305
|
+
memset(zfhPtr, 0, sizeof(*zfhPtr));
|
306
|
+
zfhPtr->frameContentSize = MEM_readLE32((const char *)src + 4);
|
307
|
+
zfhPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */
|
238
308
|
return 0;
|
239
309
|
}
|
240
310
|
return ERROR(prefix_unknown);
|
@@ -254,11 +324,13 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
|
|
254
324
|
U32 windowSize = 0;
|
255
325
|
U32 dictID = 0;
|
256
326
|
U64 frameContentSize = 0;
|
257
|
-
if ((fhdByte & 0x08) != 0)
|
327
|
+
if ((fhdByte & 0x08) != 0)
|
328
|
+
return ERROR(frameParameter_unsupported); /* reserved bits, must be zero */
|
258
329
|
if (!singleSegment) {
|
259
330
|
BYTE const wlByte = ip[pos++];
|
260
331
|
U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
|
261
|
-
if (windowLog > ZSTD_WINDOWLOG_MAX)
|
332
|
+
if (windowLog > ZSTD_WINDOWLOG_MAX)
|
333
|
+
return ERROR(frameParameter_windowTooLarge);
|
262
334
|
windowSize = (1U << windowLog);
|
263
335
|
windowSize += (windowSize >> 3) * (wlByte&7);
|
264
336
|
}
|
@@ -281,10 +353,10 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
|
|
281
353
|
}
|
282
354
|
if (!windowSize) windowSize = (U32)frameContentSize;
|
283
355
|
if (windowSize > windowSizeMax) return ERROR(frameParameter_windowTooLarge);
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
356
|
+
zfhPtr->frameContentSize = frameContentSize;
|
357
|
+
zfhPtr->windowSize = windowSize;
|
358
|
+
zfhPtr->dictID = dictID;
|
359
|
+
zfhPtr->checksumFlag = checksumFlag;
|
288
360
|
}
|
289
361
|
return 0;
|
290
362
|
}
|
@@ -302,9 +374,8 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
|
|
302
374
|
return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret;
|
303
375
|
}
|
304
376
|
#endif
|
305
|
-
{
|
306
|
-
|
307
|
-
if (ZSTD_getFrameParams(&fParams, src, srcSize) != 0) return ZSTD_CONTENTSIZE_ERROR;
|
377
|
+
{ ZSTD_frameHeader fParams;
|
378
|
+
if (ZSTD_getFrameHeader(&fParams, src, srcSize) != 0) return ZSTD_CONTENTSIZE_ERROR;
|
308
379
|
if (fParams.windowSize == 0) {
|
309
380
|
/* Either skippable or empty frame, size == 0 either way */
|
310
381
|
return 0;
|
@@ -323,51 +394,48 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
|
|
323
394
|
* @return : decompressed size of the frames contained */
|
324
395
|
unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
|
325
396
|
{
|
326
|
-
|
327
|
-
unsigned long long totalDstSize = 0;
|
328
|
-
while (srcSize >= ZSTD_frameHeaderSize_prefix) {
|
329
|
-
const U32 magicNumber = MEM_readLE32(src);
|
397
|
+
unsigned long long totalDstSize = 0;
|
330
398
|
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
srcSize -= skippableSize;
|
343
|
-
continue;
|
399
|
+
while (srcSize >= ZSTD_frameHeaderSize_prefix) {
|
400
|
+
const U32 magicNumber = MEM_readLE32(src);
|
401
|
+
|
402
|
+
if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
|
403
|
+
size_t skippableSize;
|
404
|
+
if (srcSize < ZSTD_skippableHeaderSize)
|
405
|
+
return ERROR(srcSize_wrong);
|
406
|
+
skippableSize = MEM_readLE32((const BYTE *)src + 4) +
|
407
|
+
ZSTD_skippableHeaderSize;
|
408
|
+
if (srcSize < skippableSize) {
|
409
|
+
return ZSTD_CONTENTSIZE_ERROR;
|
344
410
|
}
|
345
411
|
|
346
|
-
|
347
|
-
|
348
|
-
|
412
|
+
src = (const BYTE *)src + skippableSize;
|
413
|
+
srcSize -= skippableSize;
|
414
|
+
continue;
|
415
|
+
}
|
349
416
|
|
350
|
-
|
351
|
-
|
352
|
-
totalDstSize += ret;
|
353
|
-
}
|
354
|
-
{
|
355
|
-
size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize);
|
356
|
-
if (ZSTD_isError(frameSrcSize)) {
|
357
|
-
return ZSTD_CONTENTSIZE_ERROR;
|
358
|
-
}
|
417
|
+
{ unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
|
418
|
+
if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret;
|
359
419
|
|
360
|
-
|
361
|
-
|
362
|
-
|
420
|
+
/* check for overflow */
|
421
|
+
if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR;
|
422
|
+
totalDstSize += ret;
|
363
423
|
}
|
424
|
+
{ size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize);
|
425
|
+
if (ZSTD_isError(frameSrcSize)) {
|
426
|
+
return ZSTD_CONTENTSIZE_ERROR;
|
427
|
+
}
|
364
428
|
|
365
|
-
|
366
|
-
|
429
|
+
src = (const BYTE *)src + frameSrcSize;
|
430
|
+
srcSize -= frameSrcSize;
|
367
431
|
}
|
432
|
+
}
|
368
433
|
|
369
|
-
|
434
|
+
if (srcSize) {
|
435
|
+
return ZSTD_CONTENTSIZE_ERROR;
|
370
436
|
}
|
437
|
+
|
438
|
+
return totalDstSize;
|
371
439
|
}
|
372
440
|
|
373
441
|
/** ZSTD_getDecompressedSize() :
|
@@ -389,7 +457,7 @@ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize)
|
|
389
457
|
* @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */
|
390
458
|
static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize)
|
391
459
|
{
|
392
|
-
size_t const result =
|
460
|
+
size_t const result = ZSTD_getFrameHeader(&(dctx->fParams), src, headerSize);
|
393
461
|
if (ZSTD_isError(result)) return result; /* invalid header */
|
394
462
|
if (result>0) return ERROR(srcSize_wrong); /* headerSize too small */
|
395
463
|
if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong);
|
@@ -485,7 +553,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
|
|
485
553
|
litCSize = (lhc >> 22) + (istart[4] << 10);
|
486
554
|
break;
|
487
555
|
}
|
488
|
-
if (litSize >
|
556
|
+
if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected);
|
489
557
|
if (litCSize + lhSize > srcSize) return ERROR(corruption_detected);
|
490
558
|
|
491
559
|
if (HUF_isError((litEncType==set_repeat) ?
|
@@ -493,8 +561,10 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
|
|
493
561
|
HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) :
|
494
562
|
HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) ) :
|
495
563
|
( singleStream ?
|
496
|
-
|
497
|
-
|
564
|
+
HUF_decompress1X2_DCtx_wksp(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize,
|
565
|
+
dctx->entropy.workspace, sizeof(dctx->entropy.workspace)) :
|
566
|
+
HUF_decompress4X_hufOnly_wksp(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize,
|
567
|
+
dctx->entropy.workspace, sizeof(dctx->entropy.workspace)))))
|
498
568
|
return ERROR(corruption_detected);
|
499
569
|
|
500
570
|
dctx->litPtr = dctx->litBuffer;
|
@@ -557,7 +627,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
|
|
557
627
|
if (srcSize<4) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */
|
558
628
|
break;
|
559
629
|
}
|
560
|
-
if (litSize >
|
630
|
+
if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected);
|
561
631
|
memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH);
|
562
632
|
dctx->litPtr = dctx->litBuffer;
|
563
633
|
dctx->litSize = litSize;
|
@@ -684,6 +754,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
|
|
684
754
|
const BYTE* const istart = (const BYTE* const)src;
|
685
755
|
const BYTE* const iend = istart + srcSize;
|
686
756
|
const BYTE* ip = istart;
|
757
|
+
DEBUGLOG(5, "ZSTD_decodeSeqHeaders");
|
687
758
|
|
688
759
|
/* check */
|
689
760
|
if (srcSize < MIN_SEQUENCES_SIZE) return ERROR(srcSize_wrong);
|
@@ -864,12 +935,18 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
|
|
864
935
|
seq.offset = offset;
|
865
936
|
}
|
866
937
|
|
867
|
-
seq.matchLength = ML_base[mlCode]
|
938
|
+
seq.matchLength = ML_base[mlCode]
|
939
|
+
+ ((mlCode>31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */
|
868
940
|
if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&seqState->DStream);
|
869
941
|
|
870
|
-
seq.litLength = LL_base[llCode]
|
871
|
-
|
872
|
-
|
942
|
+
seq.litLength = LL_base[llCode]
|
943
|
+
+ ((llCode>15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */
|
944
|
+
if ( MEM_32bits()
|
945
|
+
|| (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) )
|
946
|
+
BIT_reloadDStream(&seqState->DStream);
|
947
|
+
|
948
|
+
DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u",
|
949
|
+
(U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset);
|
873
950
|
|
874
951
|
/* ANS state update */
|
875
952
|
FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */
|
@@ -909,7 +986,8 @@ size_t ZSTD_execSequence(BYTE* op,
|
|
909
986
|
/* copy Match */
|
910
987
|
if (sequence.offset > (size_t)(oLitEnd - base)) {
|
911
988
|
/* offset beyond prefix -> go into extDict */
|
912
|
-
if (sequence.offset > (size_t)(oLitEnd - vBase))
|
989
|
+
if (sequence.offset > (size_t)(oLitEnd - vBase))
|
990
|
+
return ERROR(corruption_detected);
|
913
991
|
match = dictEnd + (match - base);
|
914
992
|
if (match + sequence.matchLength <= dictEnd) {
|
915
993
|
memmove(oLitEnd, match, sequence.matchLength);
|
@@ -977,9 +1055,12 @@ static size_t ZSTD_decompressSequences(
|
|
977
1055
|
const BYTE* const vBase = (const BYTE*) (dctx->vBase);
|
978
1056
|
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
|
979
1057
|
int nbSeq;
|
1058
|
+
DEBUGLOG(5, "ZSTD_decompressSequences");
|
980
1059
|
|
981
1060
|
/* Build Decoding Tables */
|
982
1061
|
{ size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize);
|
1062
|
+
DEBUGLOG(5, "ZSTD_decodeSeqHeaders: size=%u, nbSeq=%i",
|
1063
|
+
(U32)seqHSize, nbSeq);
|
983
1064
|
if (ZSTD_isError(seqHSize)) return seqHSize;
|
984
1065
|
ip += seqHSize;
|
985
1066
|
}
|
@@ -998,11 +1079,13 @@ static size_t ZSTD_decompressSequences(
|
|
998
1079
|
nbSeq--;
|
999
1080
|
{ seq_t const sequence = ZSTD_decodeSequence(&seqState);
|
1000
1081
|
size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
|
1082
|
+
DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
|
1001
1083
|
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
|
1002
1084
|
op += oneSeqSize;
|
1003
1085
|
} }
|
1004
1086
|
|
1005
1087
|
/* check if reached exact end */
|
1088
|
+
DEBUGLOG(5, "after decode loop, remaining nbSeq : %i", nbSeq);
|
1006
1089
|
if (nbSeq) return ERROR(corruption_detected);
|
1007
1090
|
/* save reps for next block */
|
1008
1091
|
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
|
@@ -1216,7 +1299,7 @@ static size_t ZSTD_decompressSequencesLong(
|
|
1216
1299
|
const BYTE* const base = (const BYTE*) (dctx->base);
|
1217
1300
|
const BYTE* const vBase = (const BYTE*) (dctx->vBase);
|
1218
1301
|
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
|
1219
|
-
unsigned const
|
1302
|
+
unsigned const windowSize32 = (unsigned)dctx->fParams.windowSize;
|
1220
1303
|
int nbSeq;
|
1221
1304
|
|
1222
1305
|
/* Build Decoding Tables */
|
@@ -1246,13 +1329,13 @@ static size_t ZSTD_decompressSequencesLong(
|
|
1246
1329
|
|
1247
1330
|
/* prepare in advance */
|
1248
1331
|
for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && seqNb<seqAdvance; seqNb++) {
|
1249
|
-
sequences[seqNb] = ZSTD_decodeSequenceLong(&seqState,
|
1332
|
+
sequences[seqNb] = ZSTD_decodeSequenceLong(&seqState, windowSize32);
|
1250
1333
|
}
|
1251
1334
|
if (seqNb<seqAdvance) return ERROR(corruption_detected);
|
1252
1335
|
|
1253
1336
|
/* decode and decompress */
|
1254
1337
|
for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && seqNb<nbSeq ; seqNb++) {
|
1255
|
-
seq_t const sequence = ZSTD_decodeSequenceLong(&seqState,
|
1338
|
+
seq_t const sequence = ZSTD_decodeSequenceLong(&seqState, windowSize32);
|
1256
1339
|
size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[(seqNb-ADVANCED_SEQS) & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd);
|
1257
1340
|
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
|
1258
1341
|
ZSTD_PREFETCH(sequence.match);
|
@@ -1289,11 +1372,13 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
|
|
1289
1372
|
const void* src, size_t srcSize)
|
1290
1373
|
{ /* blockType == blockCompressed */
|
1291
1374
|
const BYTE* ip = (const BYTE*)src;
|
1375
|
+
DEBUGLOG(5, "ZSTD_decompressBlock_internal");
|
1292
1376
|
|
1293
|
-
if (srcSize >=
|
1377
|
+
if (srcSize >= ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong);
|
1294
1378
|
|
1295
1379
|
/* Decode literals section */
|
1296
1380
|
{ size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize);
|
1381
|
+
DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : %u", (U32)litCSize);
|
1297
1382
|
if (ZSTD_isError(litCSize)) return litCSize;
|
1298
1383
|
ip += litCSize;
|
1299
1384
|
srcSize -= litCSize;
|
@@ -1364,13 +1449,13 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
|
|
1364
1449
|
const BYTE* ip = (const BYTE*)src;
|
1365
1450
|
const BYTE* const ipstart = ip;
|
1366
1451
|
size_t remainingSize = srcSize;
|
1367
|
-
|
1452
|
+
ZSTD_frameHeader fParams;
|
1368
1453
|
|
1369
1454
|
size_t const headerSize = ZSTD_frameHeaderSize(ip, remainingSize);
|
1370
1455
|
if (ZSTD_isError(headerSize)) return headerSize;
|
1371
1456
|
|
1372
1457
|
/* Frame Header */
|
1373
|
-
{ size_t const ret =
|
1458
|
+
{ size_t const ret = ZSTD_getFrameHeader(&fParams, ip, remainingSize);
|
1374
1459
|
if (ZSTD_isError(ret)) return ret;
|
1375
1460
|
if (ret > 0) return ERROR(srcSize_wrong);
|
1376
1461
|
}
|
@@ -1505,6 +1590,8 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
|
|
1505
1590
|
size_t decodedSize;
|
1506
1591
|
size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
|
1507
1592
|
if (ZSTD_isError(frameSize)) return frameSize;
|
1593
|
+
/* legacy support is incompatible with static dctx */
|
1594
|
+
if (dctx->staticSize) return ERROR(memory_allocation);
|
1508
1595
|
|
1509
1596
|
decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize);
|
1510
1597
|
|
@@ -1540,7 +1627,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
|
|
1540
1627
|
|
1541
1628
|
if (ddict) {
|
1542
1629
|
/* we were called from ZSTD_decompress_usingDDict */
|
1543
|
-
|
1630
|
+
CHECK_F(ZSTD_decompressBegin_usingDDict(dctx, ddict));
|
1544
1631
|
} else {
|
1545
1632
|
/* this will initialize correctly with no dict if dict == NULL, so
|
1546
1633
|
* use this in all cases but ddict */
|
@@ -1580,7 +1667,7 @@ size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const
|
|
1580
1667
|
|
1581
1668
|
size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
1582
1669
|
{
|
1583
|
-
#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE
|
1670
|
+
#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1)
|
1584
1671
|
size_t regenSize;
|
1585
1672
|
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
1586
1673
|
if (dctx==NULL) return ERROR(memory_allocation);
|
@@ -1604,6 +1691,7 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) {
|
|
1604
1691
|
switch(dctx->stage)
|
1605
1692
|
{
|
1606
1693
|
default: /* should not happen */
|
1694
|
+
assert(0);
|
1607
1695
|
case ZSTDds_getFrameHeaderSize:
|
1608
1696
|
case ZSTDds_decodeFrameHeader:
|
1609
1697
|
return ZSTDnit_frameHeader;
|
@@ -1621,21 +1709,24 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) {
|
|
1621
1709
|
}
|
1622
1710
|
}
|
1623
1711
|
|
1624
|
-
int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; }
|
1712
|
+
static int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; }
|
1625
1713
|
|
1626
1714
|
/** ZSTD_decompressContinue() :
|
1627
|
-
*
|
1628
|
-
*
|
1715
|
+
* srcSize : must be the exact nb of bytes expected (see ZSTD_nextSrcSizeToDecompress())
|
1716
|
+
* @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity)
|
1717
|
+
* or an error code, which can be tested using ZSTD_isError() */
|
1629
1718
|
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
1630
1719
|
{
|
1720
|
+
DEBUGLOG(5, "ZSTD_decompressContinue");
|
1631
1721
|
/* Sanity check */
|
1632
|
-
if (srcSize != dctx->expected) return ERROR(srcSize_wrong);
|
1722
|
+
if (srcSize != dctx->expected) return ERROR(srcSize_wrong); /* unauthorized */
|
1633
1723
|
if (dstCapacity) ZSTD_checkContinuity(dctx, dst);
|
1634
1724
|
|
1635
1725
|
switch (dctx->stage)
|
1636
1726
|
{
|
1637
1727
|
case ZSTDds_getFrameHeaderSize :
|
1638
|
-
if (srcSize != ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong); /*
|
1728
|
+
if (srcSize != ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong); /* unauthorized */
|
1729
|
+
assert(src != NULL);
|
1639
1730
|
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
|
1640
1731
|
memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix);
|
1641
1732
|
dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_prefix; /* magic number + skippable frame length */
|
@@ -1653,6 +1744,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
|
|
1653
1744
|
dctx->expected = 0; /* not necessary to copy more */
|
1654
1745
|
|
1655
1746
|
case ZSTDds_decodeFrameHeader:
|
1747
|
+
assert(src != NULL);
|
1656
1748
|
memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected);
|
1657
1749
|
CHECK_F(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize));
|
1658
1750
|
dctx->expected = ZSTD_blockHeaderSize;
|
@@ -1680,17 +1772,19 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
|
|
1680
1772
|
dctx->stage = ZSTDds_getFrameHeaderSize;
|
1681
1773
|
}
|
1682
1774
|
} else {
|
1683
|
-
dctx->expected =
|
1775
|
+
dctx->expected = ZSTD_blockHeaderSize; /* jump to next header */
|
1684
1776
|
dctx->stage = ZSTDds_decodeBlockHeader;
|
1685
1777
|
}
|
1686
1778
|
return 0;
|
1687
1779
|
}
|
1688
1780
|
case ZSTDds_decompressLastBlock:
|
1689
1781
|
case ZSTDds_decompressBlock:
|
1782
|
+
DEBUGLOG(5, "case ZSTDds_decompressBlock");
|
1690
1783
|
{ size_t rSize;
|
1691
1784
|
switch(dctx->bType)
|
1692
1785
|
{
|
1693
1786
|
case bt_compressed:
|
1787
|
+
DEBUGLOG(5, "case bt_compressed");
|
1694
1788
|
rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize);
|
1695
1789
|
break;
|
1696
1790
|
case bt_raw :
|
@@ -1730,7 +1824,8 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
|
|
1730
1824
|
return 0;
|
1731
1825
|
}
|
1732
1826
|
case ZSTDds_decodeSkippableHeader:
|
1733
|
-
{
|
1827
|
+
{ assert(src != NULL);
|
1828
|
+
memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected);
|
1734
1829
|
dctx->expected = MEM_readLE32(dctx->headerBuffer + 4);
|
1735
1830
|
dctx->stage = ZSTDds_skipFrame;
|
1736
1831
|
return 0;
|
@@ -1767,7 +1862,9 @@ static size_t ZSTD_loadEntropy(ZSTD_entropyTables_t* entropy, const void* const
|
|
1767
1862
|
dictPtr += 8; /* skip header = magic + dictID */
|
1768
1863
|
|
1769
1864
|
|
1770
|
-
{ size_t const hSize =
|
1865
|
+
{ size_t const hSize = HUF_readDTableX4_wksp(
|
1866
|
+
entropy->hufTable, dictPtr, dictEnd - dictPtr,
|
1867
|
+
entropy->workspace, sizeof(entropy->workspace));
|
1771
1868
|
if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
|
1772
1869
|
dictPtr += hSize;
|
1773
1870
|
}
|
@@ -1815,7 +1912,7 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
|
|
1815
1912
|
{
|
1816
1913
|
if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize);
|
1817
1914
|
{ U32 const magic = MEM_readLE32(dict);
|
1818
|
-
if (magic !=
|
1915
|
+
if (magic != ZSTD_MAGIC_DICTIONARY) {
|
1819
1916
|
return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */
|
1820
1917
|
} }
|
1821
1918
|
dctx->dictID = MEM_readLE32((const char*)dict + 4);
|
@@ -1862,10 +1959,10 @@ static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict)
|
|
1862
1959
|
return ddict->dictSize;
|
1863
1960
|
}
|
1864
1961
|
|
1865
|
-
|
1962
|
+
size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict)
|
1866
1963
|
{
|
1867
|
-
ZSTD_decompressBegin(dstDCtx);
|
1868
|
-
if (ddict) { /* support
|
1964
|
+
CHECK_F(ZSTD_decompressBegin(dstDCtx));
|
1965
|
+
if (ddict) { /* support begin on NULL */
|
1869
1966
|
dstDCtx->dictID = ddict->dictID;
|
1870
1967
|
dstDCtx->base = ddict->dictContent;
|
1871
1968
|
dstDCtx->vBase = ddict->dictContent;
|
@@ -1886,6 +1983,7 @@ static void ZSTD_refDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict)
|
|
1886
1983
|
dstDCtx->fseEntropy = 0;
|
1887
1984
|
}
|
1888
1985
|
}
|
1986
|
+
return 0;
|
1889
1987
|
}
|
1890
1988
|
|
1891
1989
|
static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict)
|
@@ -1894,7 +1992,7 @@ static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict)
|
|
1894
1992
|
ddict->entropyPresent = 0;
|
1895
1993
|
if (ddict->dictSize < 8) return 0;
|
1896
1994
|
{ U32 const magic = MEM_readLE32(ddict->dictContent);
|
1897
|
-
if (magic !=
|
1995
|
+
if (magic != ZSTD_MAGIC_DICTIONARY) return 0; /* pure content mode */
|
1898
1996
|
}
|
1899
1997
|
ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + 4);
|
1900
1998
|
|
@@ -1905,33 +2003,39 @@ static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict)
|
|
1905
2003
|
}
|
1906
2004
|
|
1907
2005
|
|
2006
|
+
static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, const void* dict, size_t dictSize, unsigned byReference)
|
2007
|
+
{
|
2008
|
+
if ((byReference) || (!dict) || (!dictSize)) {
|
2009
|
+
ddict->dictBuffer = NULL;
|
2010
|
+
ddict->dictContent = dict;
|
2011
|
+
} else {
|
2012
|
+
void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem);
|
2013
|
+
ddict->dictBuffer = internalBuffer;
|
2014
|
+
ddict->dictContent = internalBuffer;
|
2015
|
+
if (!internalBuffer) return ERROR(memory_allocation);
|
2016
|
+
memcpy(internalBuffer, dict, dictSize);
|
2017
|
+
}
|
2018
|
+
ddict->dictSize = dictSize;
|
2019
|
+
ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
|
2020
|
+
|
2021
|
+
/* parse dictionary content */
|
2022
|
+
CHECK_F( ZSTD_loadEntropy_inDDict(ddict) );
|
2023
|
+
|
2024
|
+
return 0;
|
2025
|
+
}
|
2026
|
+
|
1908
2027
|
ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, unsigned byReference, ZSTD_customMem customMem)
|
1909
2028
|
{
|
1910
|
-
if (!customMem.customAlloc
|
1911
|
-
if (!customMem.customAlloc || !customMem.customFree) return NULL;
|
2029
|
+
if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
|
1912
2030
|
|
1913
2031
|
{ ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem);
|
1914
2032
|
if (!ddict) return NULL;
|
1915
2033
|
ddict->cMem = customMem;
|
1916
2034
|
|
1917
|
-
if ((
|
1918
|
-
ddict
|
1919
|
-
|
1920
|
-
} else {
|
1921
|
-
void* const internalBuffer = ZSTD_malloc(dictSize, customMem);
|
1922
|
-
if (!internalBuffer) { ZSTD_freeDDict(ddict); return NULL; }
|
1923
|
-
memcpy(internalBuffer, dict, dictSize);
|
1924
|
-
ddict->dictBuffer = internalBuffer;
|
1925
|
-
ddict->dictContent = internalBuffer;
|
2035
|
+
if (ZSTD_isError( ZSTD_initDDict_internal(ddict, dict, dictSize, byReference) )) {
|
2036
|
+
ZSTD_freeDDict(ddict);
|
2037
|
+
return NULL;
|
1926
2038
|
}
|
1927
|
-
ddict->dictSize = dictSize;
|
1928
|
-
ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
|
1929
|
-
/* parse dictionary content */
|
1930
|
-
{ size_t const errorCode = ZSTD_loadEntropy_inDDict(ddict);
|
1931
|
-
if (ZSTD_isError(errorCode)) {
|
1932
|
-
ZSTD_freeDDict(ddict);
|
1933
|
-
return NULL;
|
1934
|
-
} }
|
1935
2039
|
|
1936
2040
|
return ddict;
|
1937
2041
|
}
|
@@ -1947,7 +2051,6 @@ ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
|
|
1947
2051
|
return ZSTD_createDDict_advanced(dict, dictSize, 0, allocator);
|
1948
2052
|
}
|
1949
2053
|
|
1950
|
-
|
1951
2054
|
/*! ZSTD_createDDict_byReference() :
|
1952
2055
|
* Create a digested dictionary, to start decompression without startup delay.
|
1953
2056
|
* Dictionary content is simply referenced, it will be accessed during decompression.
|
@@ -1959,6 +2062,26 @@ ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize
|
|
1959
2062
|
}
|
1960
2063
|
|
1961
2064
|
|
2065
|
+
ZSTD_DDict* ZSTD_initStaticDDict(void* workspace, size_t workspaceSize,
|
2066
|
+
const void* dict, size_t dictSize,
|
2067
|
+
unsigned byReference)
|
2068
|
+
{
|
2069
|
+
size_t const neededSpace = sizeof(ZSTD_DDict) + (byReference ? 0 : dictSize);
|
2070
|
+
ZSTD_DDict* const ddict = (ZSTD_DDict*)workspace;
|
2071
|
+
assert(workspace != NULL);
|
2072
|
+
assert(dict != NULL);
|
2073
|
+
if ((size_t)workspace & 7) return NULL; /* 8-aligned */
|
2074
|
+
if (workspaceSize < neededSpace) return NULL;
|
2075
|
+
if (!byReference) {
|
2076
|
+
memcpy(ddict+1, dict, dictSize); /* local copy */
|
2077
|
+
dict = ddict+1;
|
2078
|
+
}
|
2079
|
+
if (ZSTD_isError( ZSTD_initDDict_internal(ddict, dict, dictSize, 1 /* byRef */) ))
|
2080
|
+
return NULL;
|
2081
|
+
return ddict;
|
2082
|
+
}
|
2083
|
+
|
2084
|
+
|
1962
2085
|
size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
|
1963
2086
|
{
|
1964
2087
|
if (ddict==NULL) return 0; /* support free on NULL */
|
@@ -1969,6 +2092,14 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
|
|
1969
2092
|
}
|
1970
2093
|
}
|
1971
2094
|
|
2095
|
+
/*! ZSTD_estimateDDictSize() :
|
2096
|
+
* Estimate amount of memory that will be needed to create a dictionary for decompression.
|
2097
|
+
* Note : dictionary created "byReference" are smaller */
|
2098
|
+
size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference)
|
2099
|
+
{
|
2100
|
+
return sizeof(ZSTD_DDict) + (byReference ? 0 : dictSize);
|
2101
|
+
}
|
2102
|
+
|
1972
2103
|
size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
|
1973
2104
|
{
|
1974
2105
|
if (ddict==NULL) return 0; /* support sizeof on NULL */
|
@@ -1982,7 +2113,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
|
|
1982
2113
|
unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
|
1983
2114
|
{
|
1984
2115
|
if (dictSize < 8) return 0;
|
1985
|
-
if (MEM_readLE32(dict) !=
|
2116
|
+
if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0;
|
1986
2117
|
return MEM_readLE32((const char*)dict + 4);
|
1987
2118
|
}
|
1988
2119
|
|
@@ -2008,11 +2139,11 @@ unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
|
|
2008
2139
|
* Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`.
|
2009
2140
|
* - This is not a Zstandard frame.
|
2010
2141
|
* When identifying the exact failure cause, it's possible to use
|
2011
|
-
*
|
2142
|
+
* ZSTD_getFrameHeader(), which will provide a more precise error code. */
|
2012
2143
|
unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize)
|
2013
2144
|
{
|
2014
|
-
|
2015
|
-
size_t const hError =
|
2145
|
+
ZSTD_frameHeader zfp = { 0 , 0 , 0 , 0 };
|
2146
|
+
size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize);
|
2016
2147
|
if (ZSTD_isError(hError)) return 0;
|
2017
2148
|
return zfp.dictID;
|
2018
2149
|
}
|
@@ -2037,88 +2168,35 @@ size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
|
|
2037
2168
|
* Streaming decompression
|
2038
2169
|
*====================================*/
|
2039
2170
|
|
2040
|
-
typedef enum { zdss_init, zdss_loadHeader,
|
2041
|
-
zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
|
2042
|
-
|
2043
|
-
/* *** Resource management *** */
|
2044
|
-
struct ZSTD_DStream_s {
|
2045
|
-
ZSTD_DCtx* dctx;
|
2046
|
-
ZSTD_DDict* ddictLocal;
|
2047
|
-
const ZSTD_DDict* ddict;
|
2048
|
-
ZSTD_frameParams fParams;
|
2049
|
-
ZSTD_dStreamStage stage;
|
2050
|
-
char* inBuff;
|
2051
|
-
size_t inBuffSize;
|
2052
|
-
size_t inPos;
|
2053
|
-
size_t maxWindowSize;
|
2054
|
-
char* outBuff;
|
2055
|
-
size_t outBuffSize;
|
2056
|
-
size_t outStart;
|
2057
|
-
size_t outEnd;
|
2058
|
-
size_t blockSize;
|
2059
|
-
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; /* tmp buffer to store frame header */
|
2060
|
-
size_t lhSize;
|
2061
|
-
ZSTD_customMem customMem;
|
2062
|
-
void* legacyContext;
|
2063
|
-
U32 previousLegacyVersion;
|
2064
|
-
U32 legacyVersion;
|
2065
|
-
U32 hostageByte;
|
2066
|
-
}; /* typedef'd to ZSTD_DStream within "zstd.h" */
|
2067
|
-
|
2068
|
-
|
2069
2171
|
ZSTD_DStream* ZSTD_createDStream(void)
|
2070
2172
|
{
|
2071
|
-
return ZSTD_createDStream_advanced(
|
2173
|
+
return ZSTD_createDStream_advanced(ZSTD_defaultCMem);
|
2072
2174
|
}
|
2073
2175
|
|
2074
|
-
ZSTD_DStream*
|
2176
|
+
ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize)
|
2075
2177
|
{
|
2076
|
-
|
2077
|
-
|
2078
|
-
if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
|
2079
|
-
if (!customMem.customAlloc || !customMem.customFree) return NULL;
|
2178
|
+
return ZSTD_initStaticDCtx(workspace, workspaceSize);
|
2179
|
+
}
|
2080
2180
|
|
2081
|
-
|
2082
|
-
|
2083
|
-
|
2084
|
-
memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem));
|
2085
|
-
zds->dctx = ZSTD_createDCtx_advanced(customMem);
|
2086
|
-
if (zds->dctx == NULL) { ZSTD_freeDStream(zds); return NULL; }
|
2087
|
-
zds->stage = zdss_init;
|
2088
|
-
zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
|
2089
|
-
return zds;
|
2181
|
+
ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem)
|
2182
|
+
{
|
2183
|
+
return ZSTD_createDCtx_advanced(customMem);
|
2090
2184
|
}
|
2091
2185
|
|
2092
2186
|
size_t ZSTD_freeDStream(ZSTD_DStream* zds)
|
2093
2187
|
{
|
2094
|
-
|
2095
|
-
{ ZSTD_customMem const cMem = zds->customMem;
|
2096
|
-
ZSTD_freeDCtx(zds->dctx);
|
2097
|
-
zds->dctx = NULL;
|
2098
|
-
ZSTD_freeDDict(zds->ddictLocal);
|
2099
|
-
zds->ddictLocal = NULL;
|
2100
|
-
ZSTD_free(zds->inBuff, cMem);
|
2101
|
-
zds->inBuff = NULL;
|
2102
|
-
ZSTD_free(zds->outBuff, cMem);
|
2103
|
-
zds->outBuff = NULL;
|
2104
|
-
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
|
2105
|
-
if (zds->legacyContext)
|
2106
|
-
ZSTD_freeLegacyStreamContext(zds->legacyContext, zds->previousLegacyVersion);
|
2107
|
-
#endif
|
2108
|
-
ZSTD_free(zds, cMem);
|
2109
|
-
return 0;
|
2110
|
-
}
|
2188
|
+
return ZSTD_freeDCtx(zds);
|
2111
2189
|
}
|
2112
2190
|
|
2113
2191
|
|
2114
2192
|
/* *** Initialization *** */
|
2115
2193
|
|
2116
|
-
size_t ZSTD_DStreamInSize(void) { return
|
2117
|
-
size_t ZSTD_DStreamOutSize(void) { return
|
2194
|
+
size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; }
|
2195
|
+
size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; }
|
2118
2196
|
|
2119
2197
|
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize)
|
2120
2198
|
{
|
2121
|
-
zds->
|
2199
|
+
zds->streamStage = zdss_loadHeader;
|
2122
2200
|
zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
|
2123
2201
|
ZSTD_freeDDict(zds->ddictLocal);
|
2124
2202
|
if (dict && dictSize >= 8) {
|
@@ -2147,7 +2225,7 @@ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict)
|
|
2147
2225
|
|
2148
2226
|
size_t ZSTD_resetDStream(ZSTD_DStream* zds)
|
2149
2227
|
{
|
2150
|
-
zds->
|
2228
|
+
zds->streamStage = zdss_loadHeader;
|
2151
2229
|
zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
|
2152
2230
|
zds->legacyVersion = 0;
|
2153
2231
|
zds->hostageByte = 0;
|
@@ -2168,11 +2246,24 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
|
|
2168
2246
|
|
2169
2247
|
size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds)
|
2170
2248
|
{
|
2171
|
-
|
2172
|
-
|
2173
|
-
|
2174
|
-
|
2175
|
-
|
2249
|
+
return ZSTD_sizeof_DCtx(zds);
|
2250
|
+
}
|
2251
|
+
|
2252
|
+
size_t ZSTD_estimateDStreamSize(size_t windowSize)
|
2253
|
+
{
|
2254
|
+
size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX);
|
2255
|
+
size_t const inBuffSize = blockSize; /* no block can be larger */
|
2256
|
+
size_t const outBuffSize = windowSize + blockSize + (WILDCOPY_OVERLENGTH * 2);
|
2257
|
+
return sizeof(ZSTD_DStream) + ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize;
|
2258
|
+
}
|
2259
|
+
|
2260
|
+
ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize)
|
2261
|
+
{
|
2262
|
+
ZSTD_frameHeader fh;
|
2263
|
+
size_t const err = ZSTD_getFrameHeader(&fh, src, srcSize);
|
2264
|
+
if (ZSTD_isError(err)) return err;
|
2265
|
+
if (err>0) return ERROR(srcSize_wrong);
|
2266
|
+
return ZSTD_estimateDStreamSize(fh.windowSize);
|
2176
2267
|
}
|
2177
2268
|
|
2178
2269
|
|
@@ -2196,44 +2287,55 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|
2196
2287
|
char* op = ostart;
|
2197
2288
|
U32 someMoreWork = 1;
|
2198
2289
|
|
2290
|
+
DEBUGLOG(5, "ZSTD_decompressStream");
|
2291
|
+
DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos));
|
2199
2292
|
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
|
2200
|
-
if (zds->legacyVersion)
|
2293
|
+
if (zds->legacyVersion) {
|
2294
|
+
/* legacy support is incompatible with static dctx */
|
2295
|
+
if (zds->staticSize) return ERROR(memory_allocation);
|
2201
2296
|
return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input);
|
2297
|
+
}
|
2202
2298
|
#endif
|
2203
2299
|
|
2204
2300
|
while (someMoreWork) {
|
2205
|
-
switch(zds->
|
2301
|
+
switch(zds->streamStage)
|
2206
2302
|
{
|
2207
2303
|
case zdss_init :
|
2208
2304
|
ZSTD_resetDStream(zds); /* transparent reset on starting decoding a new frame */
|
2209
2305
|
/* fall-through */
|
2210
2306
|
|
2211
2307
|
case zdss_loadHeader :
|
2212
|
-
{ size_t const hSize =
|
2213
|
-
if (ZSTD_isError(hSize))
|
2308
|
+
{ size_t const hSize = ZSTD_getFrameHeader(&zds->fParams, zds->headerBuffer, zds->lhSize);
|
2309
|
+
if (ZSTD_isError(hSize)) {
|
2214
2310
|
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
|
2215
|
-
|
2311
|
+
U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
|
2216
2312
|
if (legacyVersion) {
|
2217
2313
|
const void* const dict = zds->ddict ? zds->ddict->dictContent : NULL;
|
2218
2314
|
size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0;
|
2315
|
+
/* legacy support is incompatible with static dctx */
|
2316
|
+
if (zds->staticSize) return ERROR(memory_allocation);
|
2219
2317
|
CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion,
|
2220
2318
|
dict, dictSize));
|
2221
2319
|
zds->legacyVersion = zds->previousLegacyVersion = legacyVersion;
|
2222
2320
|
return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input);
|
2223
2321
|
} else {
|
2224
2322
|
return hSize; /* error */
|
2225
|
-
|
2323
|
+
}
|
2226
2324
|
#else
|
2227
|
-
|
2325
|
+
return hSize;
|
2228
2326
|
#endif
|
2327
|
+
}
|
2229
2328
|
if (hSize != 0) { /* need more input */
|
2230
2329
|
size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */
|
2231
2330
|
if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
|
2232
|
-
|
2233
|
-
|
2331
|
+
if (iend-ip > 0) {
|
2332
|
+
memcpy(zds->headerBuffer + zds->lhSize, ip, iend-ip);
|
2333
|
+
zds->lhSize += iend-ip;
|
2334
|
+
}
|
2234
2335
|
input->pos = input->size;
|
2235
2336
|
return (MAX(ZSTD_frameHeaderSize_min, hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
|
2236
2337
|
}
|
2338
|
+
assert(ip != NULL);
|
2237
2339
|
memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad;
|
2238
2340
|
break;
|
2239
2341
|
} }
|
@@ -2243,74 +2345,90 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|
2243
2345
|
&& (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) {
|
2244
2346
|
size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart);
|
2245
2347
|
if (cSize <= (size_t)(iend-istart)) {
|
2246
|
-
size_t const decompressedSize = ZSTD_decompress_usingDDict(zds
|
2348
|
+
size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, oend-op, istart, cSize, zds->ddict);
|
2247
2349
|
if (ZSTD_isError(decompressedSize)) return decompressedSize;
|
2248
2350
|
ip = istart + cSize;
|
2249
2351
|
op += decompressedSize;
|
2250
|
-
zds->
|
2251
|
-
zds->
|
2352
|
+
zds->expected = 0;
|
2353
|
+
zds->streamStage = zdss_init;
|
2252
2354
|
someMoreWork = 0;
|
2253
2355
|
break;
|
2254
2356
|
} }
|
2255
2357
|
|
2256
|
-
/* Consume header */
|
2257
|
-
|
2258
|
-
|
2259
|
-
|
2260
|
-
|
2261
|
-
|
2262
|
-
|
2358
|
+
/* Consume header (see ZSTDds_decodeFrameHeader) */
|
2359
|
+
DEBUGLOG(4, "Consume header");
|
2360
|
+
CHECK_F(ZSTD_decompressBegin_usingDDict(zds, zds->ddict));
|
2361
|
+
|
2362
|
+
if ((MEM_readLE32(zds->headerBuffer) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
|
2363
|
+
zds->expected = MEM_readLE32(zds->headerBuffer + 4);
|
2364
|
+
zds->stage = ZSTDds_skipFrame;
|
2365
|
+
} else {
|
2366
|
+
CHECK_F(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize));
|
2367
|
+
zds->expected = ZSTD_blockHeaderSize;
|
2368
|
+
zds->stage = ZSTDds_decodeBlockHeader;
|
2369
|
+
}
|
2263
2370
|
|
2371
|
+
/* control buffer memory usage */
|
2372
|
+
DEBUGLOG(4, "Control max buffer memory usage");
|
2264
2373
|
zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
|
2265
2374
|
if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge);
|
2266
2375
|
|
2267
2376
|
/* Adapt buffer sizes to frame header instructions */
|
2268
|
-
{ size_t const blockSize = MIN(zds->fParams.windowSize,
|
2377
|
+
{ size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_MAX);
|
2269
2378
|
size_t const neededOutSize = zds->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
|
2270
2379
|
zds->blockSize = blockSize;
|
2271
|
-
if (zds->inBuffSize < blockSize) {
|
2272
|
-
|
2273
|
-
|
2274
|
-
|
2275
|
-
|
2380
|
+
if ((zds->inBuffSize < blockSize) || (zds->outBuffSize < neededOutSize)) {
|
2381
|
+
size_t const bufferSize = blockSize + neededOutSize;
|
2382
|
+
DEBUGLOG(4, "inBuff : from %u to %u",
|
2383
|
+
(U32)zds->inBuffSize, (U32)blockSize);
|
2384
|
+
DEBUGLOG(4, "outBuff : from %u to %u",
|
2385
|
+
(U32)zds->outBuffSize, (U32)neededOutSize);
|
2386
|
+
if (zds->staticSize) { /* static DCtx */
|
2387
|
+
DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize);
|
2388
|
+
assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */
|
2389
|
+
if (bufferSize > zds->staticSize - sizeof(ZSTD_DCtx))
|
2390
|
+
return ERROR(memory_allocation);
|
2391
|
+
} else {
|
2392
|
+
ZSTD_free(zds->inBuff, zds->customMem);
|
2393
|
+
zds->inBuffSize = 0;
|
2394
|
+
zds->outBuffSize = 0;
|
2395
|
+
zds->inBuff = (char*)ZSTD_malloc(bufferSize, zds->customMem);
|
2396
|
+
if (zds->inBuff == NULL) return ERROR(memory_allocation);
|
2397
|
+
}
|
2276
2398
|
zds->inBuffSize = blockSize;
|
2277
|
-
|
2278
|
-
if (zds->outBuffSize < neededOutSize) {
|
2279
|
-
ZSTD_free(zds->outBuff, zds->customMem);
|
2280
|
-
zds->outBuffSize = 0;
|
2281
|
-
zds->outBuff = (char*)ZSTD_malloc(neededOutSize, zds->customMem);
|
2282
|
-
if (zds->outBuff == NULL) return ERROR(memory_allocation);
|
2399
|
+
zds->outBuff = zds->inBuff + zds->inBuffSize;
|
2283
2400
|
zds->outBuffSize = neededOutSize;
|
2284
2401
|
} }
|
2285
|
-
zds->
|
2402
|
+
zds->streamStage = zdss_read;
|
2286
2403
|
/* pass-through */
|
2287
2404
|
|
2288
2405
|
case zdss_read:
|
2289
|
-
|
2406
|
+
DEBUGLOG(5, "stage zdss_read");
|
2407
|
+
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds);
|
2408
|
+
DEBUGLOG(5, "neededInSize = %u", (U32)neededInSize);
|
2290
2409
|
if (neededInSize==0) { /* end of frame */
|
2291
|
-
zds->
|
2410
|
+
zds->streamStage = zdss_init;
|
2292
2411
|
someMoreWork = 0;
|
2293
2412
|
break;
|
2294
2413
|
}
|
2295
2414
|
if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
|
2296
|
-
const
|
2297
|
-
size_t const decodedSize = ZSTD_decompressContinue(zds
|
2415
|
+
int const isSkipFrame = ZSTD_isSkipFrame(zds);
|
2416
|
+
size_t const decodedSize = ZSTD_decompressContinue(zds,
|
2298
2417
|
zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart),
|
2299
2418
|
ip, neededInSize);
|
2300
2419
|
if (ZSTD_isError(decodedSize)) return decodedSize;
|
2301
2420
|
ip += neededInSize;
|
2302
2421
|
if (!decodedSize && !isSkipFrame) break; /* this was just a header */
|
2303
2422
|
zds->outEnd = zds->outStart + decodedSize;
|
2304
|
-
zds->
|
2423
|
+
zds->streamStage = zdss_flush;
|
2305
2424
|
break;
|
2306
|
-
|
2307
|
-
|
2308
|
-
|
2309
|
-
|
2310
|
-
}
|
2425
|
+
} }
|
2426
|
+
if (ip==iend) { someMoreWork = 0; break; } /* no more input */
|
2427
|
+
zds->streamStage = zdss_load;
|
2428
|
+
/* pass-through */
|
2311
2429
|
|
2312
2430
|
case zdss_load:
|
2313
|
-
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds
|
2431
|
+
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds);
|
2314
2432
|
size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */
|
2315
2433
|
size_t loadedSize;
|
2316
2434
|
if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected); /* should never happen */
|
@@ -2320,17 +2438,17 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|
2320
2438
|
if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */
|
2321
2439
|
|
2322
2440
|
/* decode loaded input */
|
2323
|
-
{ const int isSkipFrame = ZSTD_isSkipFrame(zds
|
2324
|
-
size_t const decodedSize = ZSTD_decompressContinue(zds
|
2441
|
+
{ const int isSkipFrame = ZSTD_isSkipFrame(zds);
|
2442
|
+
size_t const decodedSize = ZSTD_decompressContinue(zds,
|
2325
2443
|
zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart,
|
2326
2444
|
zds->inBuff, neededInSize);
|
2327
2445
|
if (ZSTD_isError(decodedSize)) return decodedSize;
|
2328
2446
|
zds->inPos = 0; /* input is consumed */
|
2329
|
-
if (!decodedSize && !isSkipFrame) { zds->
|
2447
|
+
if (!decodedSize && !isSkipFrame) { zds->streamStage = zdss_read; break; } /* this was just a header */
|
2330
2448
|
zds->outEnd = zds->outStart + decodedSize;
|
2331
|
-
zds->stage = zdss_flush;
|
2332
|
-
/* pass-through */
|
2333
2449
|
} }
|
2450
|
+
zds->streamStage = zdss_flush;
|
2451
|
+
/* pass-through */
|
2334
2452
|
|
2335
2453
|
case zdss_flush:
|
2336
2454
|
{ size_t const toFlushSize = zds->outEnd - zds->outStart;
|
@@ -2338,37 +2456,41 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|
2338
2456
|
op += flushedSize;
|
2339
2457
|
zds->outStart += flushedSize;
|
2340
2458
|
if (flushedSize == toFlushSize) { /* flush completed */
|
2341
|
-
zds->
|
2459
|
+
zds->streamStage = zdss_read;
|
2342
2460
|
if (zds->outStart + zds->blockSize > zds->outBuffSize)
|
2343
2461
|
zds->outStart = zds->outEnd = 0;
|
2344
2462
|
break;
|
2345
|
-
|
2346
|
-
|
2347
|
-
|
2348
|
-
|
2349
|
-
|
2463
|
+
} }
|
2464
|
+
/* cannot complete flush */
|
2465
|
+
someMoreWork = 0;
|
2466
|
+
break;
|
2467
|
+
|
2350
2468
|
default: return ERROR(GENERIC); /* impossible */
|
2351
2469
|
} }
|
2352
2470
|
|
2353
2471
|
/* result */
|
2354
2472
|
input->pos += (size_t)(ip-istart);
|
2355
2473
|
output->pos += (size_t)(op-ostart);
|
2356
|
-
{ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds
|
2474
|
+
{ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds);
|
2357
2475
|
if (!nextSrcSizeHint) { /* frame fully decoded */
|
2358
2476
|
if (zds->outEnd == zds->outStart) { /* output fully flushed */
|
2359
2477
|
if (zds->hostageByte) {
|
2360
|
-
if (input->pos >= input->size) {
|
2478
|
+
if (input->pos >= input->size) {
|
2479
|
+
/* can't release hostage (not present) */
|
2480
|
+
zds->streamStage = zdss_read;
|
2481
|
+
return 1;
|
2482
|
+
}
|
2361
2483
|
input->pos++; /* release hostage */
|
2362
|
-
}
|
2484
|
+
} /* zds->hostageByte */
|
2363
2485
|
return 0;
|
2364
|
-
}
|
2486
|
+
} /* zds->outEnd == zds->outStart */
|
2365
2487
|
if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */
|
2366
2488
|
input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */
|
2367
2489
|
zds->hostageByte=1;
|
2368
2490
|
}
|
2369
2491
|
return 1;
|
2370
|
-
}
|
2371
|
-
nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds
|
2492
|
+
} /* nextSrcSizeHint==0 */
|
2493
|
+
nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds) == ZSTDnit_block); /* preload header of next block */
|
2372
2494
|
if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */
|
2373
2495
|
nextSrcSizeHint -= zds->inPos; /* already loaded*/
|
2374
2496
|
return nextSrcSizeHint;
|