extlz4 0.2.5 → 0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/HISTORY.ja.md +9 -1
- data/README.md +44 -41
- data/contrib/lz4/NEWS +33 -0
- data/contrib/lz4/README.md +30 -24
- data/contrib/lz4/lib/README.md +59 -10
- data/contrib/lz4/lib/lz4.c +1303 -583
- data/contrib/lz4/lib/lz4.h +376 -176
- data/contrib/lz4/lib/lz4frame.c +447 -286
- data/contrib/lz4/lib/lz4frame.h +289 -74
- data/contrib/lz4/lib/lz4frame_static.h +4 -111
- data/contrib/lz4/lib/lz4hc.c +789 -207
- data/contrib/lz4/lib/lz4hc.h +256 -93
- data/contrib/lz4/lib/xxhash.c +376 -240
- data/contrib/lz4/lib/xxhash.h +128 -93
- data/ext/blockapi.c +2 -2
- data/ext/lz4_amalgam.c +0 -23
- data/gemstub.rb +4 -4
- data/lib/extlz4.rb +46 -0
- data/lib/extlz4/version.rb +1 -1
- metadata +33 -10
- data/contrib/lz4/circle.yml +0 -38
- data/contrib/lz4/lib/lz4opt.h +0 -356
data/contrib/lz4/lib/lz4frame.c
CHANGED
@@ -1,41 +1,44 @@
|
|
1
1
|
/*
|
2
|
-
LZ4 auto-framing library
|
3
|
-
Copyright (C) 2011-2016, Yann Collet.
|
4
|
-
|
5
|
-
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
6
|
-
|
7
|
-
Redistribution and use in source and binary forms, with or without
|
8
|
-
modification, are permitted provided that the following conditions are
|
9
|
-
met:
|
10
|
-
|
11
|
-
* Redistributions of source code must retain the above copyright
|
12
|
-
notice, this list of conditions and the following disclaimer.
|
13
|
-
* Redistributions in binary form must reproduce the above
|
14
|
-
copyright notice, this list of conditions and the following disclaimer
|
15
|
-
in the documentation and/or other materials provided with the
|
16
|
-
distribution.
|
17
|
-
|
18
|
-
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
19
|
-
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
20
|
-
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
21
|
-
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
22
|
-
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
23
|
-
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
24
|
-
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
25
|
-
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
26
|
-
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
27
|
-
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
28
|
-
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
29
|
-
|
30
|
-
You can contact the author at :
|
31
|
-
- LZ4 homepage : http://www.lz4.org
|
32
|
-
- LZ4 source repository : https://github.com/lz4/lz4
|
33
|
-
*/
|
2
|
+
* LZ4 auto-framing library
|
3
|
+
* Copyright (C) 2011-2016, Yann Collet.
|
4
|
+
*
|
5
|
+
* BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
6
|
+
*
|
7
|
+
* Redistribution and use in source and binary forms, with or without
|
8
|
+
* modification, are permitted provided that the following conditions are
|
9
|
+
* met:
|
10
|
+
*
|
11
|
+
* - Redistributions of source code must retain the above copyright
|
12
|
+
* notice, this list of conditions and the following disclaimer.
|
13
|
+
* - Redistributions in binary form must reproduce the above
|
14
|
+
* copyright notice, this list of conditions and the following disclaimer
|
15
|
+
* in the documentation and/or other materials provided with the
|
16
|
+
* distribution.
|
17
|
+
*
|
18
|
+
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
19
|
+
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
20
|
+
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
21
|
+
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
22
|
+
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
23
|
+
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
24
|
+
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
25
|
+
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
26
|
+
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
27
|
+
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
28
|
+
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
29
|
+
*
|
30
|
+
* You can contact the author at :
|
31
|
+
* - LZ4 homepage : http://www.lz4.org
|
32
|
+
* - LZ4 source repository : https://github.com/lz4/lz4
|
33
|
+
*/
|
34
34
|
|
35
35
|
/* LZ4F is a stand-alone API to create LZ4-compressed Frames
|
36
|
-
*
|
37
|
-
*
|
38
|
-
*
|
36
|
+
* in full conformance with specification v1.6.1 .
|
37
|
+
* This library rely upon memory management capabilities (malloc, free)
|
38
|
+
* provided either by <stdlib.h>,
|
39
|
+
* or redirected towards another library of user's choice
|
40
|
+
* (see Memory Routines below).
|
41
|
+
*/
|
39
42
|
|
40
43
|
|
41
44
|
/*-************************************
|
@@ -46,20 +49,47 @@ You can contact the author at :
|
|
46
49
|
#endif
|
47
50
|
|
48
51
|
|
52
|
+
/*-************************************
|
53
|
+
* Tuning parameters
|
54
|
+
**************************************/
|
55
|
+
/*
|
56
|
+
* LZ4F_HEAPMODE :
|
57
|
+
* Select how default compression functions will allocate memory for their hash table,
|
58
|
+
* in memory stack (0:default, fastest), or in memory heap (1:requires malloc()).
|
59
|
+
*/
|
60
|
+
#ifndef LZ4F_HEAPMODE
|
61
|
+
# define LZ4F_HEAPMODE 0
|
62
|
+
#endif
|
63
|
+
|
64
|
+
|
49
65
|
/*-************************************
|
50
66
|
* Memory routines
|
51
67
|
**************************************/
|
68
|
+
/*
|
69
|
+
* User may redirect invocations of
|
70
|
+
* malloc(), calloc() and free()
|
71
|
+
* towards another library or solution of their choice
|
72
|
+
* by modifying below section.
|
73
|
+
*/
|
52
74
|
#include <stdlib.h> /* malloc, calloc, free */
|
53
|
-
#
|
54
|
-
#define
|
75
|
+
#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */
|
76
|
+
# define ALLOC(s) malloc(s)
|
77
|
+
# define ALLOC_AND_ZERO(s) calloc(1,(s))
|
78
|
+
# define FREEMEM(p) free(p)
|
79
|
+
#endif
|
80
|
+
|
55
81
|
#include <string.h> /* memset, memcpy, memmove */
|
56
|
-
#
|
82
|
+
#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */
|
83
|
+
# define MEM_INIT(p,v,s) memset((p),(v),(s))
|
84
|
+
#endif
|
57
85
|
|
58
86
|
|
59
87
|
/*-************************************
|
60
|
-
*
|
88
|
+
* Library declarations
|
61
89
|
**************************************/
|
62
|
-
#
|
90
|
+
#define LZ4F_STATIC_LINKING_ONLY
|
91
|
+
#include "lz4frame.h"
|
92
|
+
#define LZ4_STATIC_LINKING_ONLY
|
63
93
|
#include "lz4.h"
|
64
94
|
#define LZ4_HC_STATIC_LINKING_ONLY
|
65
95
|
#include "lz4hc.h"
|
@@ -70,8 +100,29 @@ You can contact the author at :
|
|
70
100
|
/*-************************************
|
71
101
|
* Debug
|
72
102
|
**************************************/
|
103
|
+
#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=1)
|
104
|
+
# include <assert.h>
|
105
|
+
#else
|
106
|
+
# ifndef assert
|
107
|
+
# define assert(condition) ((void)0)
|
108
|
+
# endif
|
109
|
+
#endif
|
110
|
+
|
73
111
|
#define LZ4F_STATIC_ASSERT(c) { enum { LZ4F_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
|
74
112
|
|
113
|
+
#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) && !defined(DEBUGLOG)
|
114
|
+
# include <stdio.h>
|
115
|
+
static int g_debuglog_enable = 1;
|
116
|
+
# define DEBUGLOG(l, ...) { \
|
117
|
+
if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \
|
118
|
+
fprintf(stderr, __FILE__ ": "); \
|
119
|
+
fprintf(stderr, __VA_ARGS__); \
|
120
|
+
fprintf(stderr, " \n"); \
|
121
|
+
} }
|
122
|
+
#else
|
123
|
+
# define DEBUGLOG(l, ...) {} /* disabled */
|
124
|
+
#endif
|
125
|
+
|
75
126
|
|
76
127
|
/*-************************************
|
77
128
|
* Basic Types
|
@@ -97,8 +148,8 @@ static U32 LZ4F_readLE32 (const void* src)
|
|
97
148
|
{
|
98
149
|
const BYTE* const srcPtr = (const BYTE*)src;
|
99
150
|
U32 value32 = srcPtr[0];
|
100
|
-
value32 += (srcPtr[1]<<8
|
101
|
-
value32 += (srcPtr[2]<<16
|
151
|
+
value32 += ((U32)srcPtr[1])<< 8;
|
152
|
+
value32 += ((U32)srcPtr[2])<<16;
|
102
153
|
value32 += ((U32)srcPtr[3])<<24;
|
103
154
|
return value32;
|
104
155
|
}
|
@@ -143,9 +194,11 @@ static void LZ4F_writeLE64 (void* dst, U64 value64)
|
|
143
194
|
/*-************************************
|
144
195
|
* Constants
|
145
196
|
**************************************/
|
146
|
-
#
|
147
|
-
#define
|
148
|
-
#define
|
197
|
+
#ifndef LZ4_SRC_INCLUDED /* avoid double definition */
|
198
|
+
# define KB *(1<<10)
|
199
|
+
# define MB *(1<<20)
|
200
|
+
# define GB *(1<<30)
|
201
|
+
#endif
|
149
202
|
|
150
203
|
#define _1BIT 0x01
|
151
204
|
#define _2BITS 0x03
|
@@ -158,9 +211,10 @@ static void LZ4F_writeLE64 (void* dst, U64 value64)
|
|
158
211
|
#define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U
|
159
212
|
#define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB
|
160
213
|
|
161
|
-
static const size_t minFHSize = 7
|
214
|
+
static const size_t minFHSize = LZ4F_HEADER_SIZE_MIN; /* 7 */
|
162
215
|
static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 19 */
|
163
|
-
static const size_t BHSize = 4;
|
216
|
+
static const size_t BHSize = 4; /* block header : size, and compress flag */
|
217
|
+
static const size_t BFSize = 4; /* block footer : checksum (optional) */
|
164
218
|
|
165
219
|
|
166
220
|
/*-************************************
|
@@ -180,7 +234,8 @@ typedef struct LZ4F_cctx_s
|
|
180
234
|
U64 totalInSize;
|
181
235
|
XXH32_state_t xxh;
|
182
236
|
void* lz4CtxPtr;
|
183
|
-
|
237
|
+
U16 lz4CtxAlloc; /* sized for: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */
|
238
|
+
U16 lz4CtxState; /* in use as: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */
|
184
239
|
} LZ4F_cctx_t;
|
185
240
|
|
186
241
|
|
@@ -220,22 +275,22 @@ unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; }
|
|
220
275
|
|
221
276
|
int LZ4F_compressionLevel_max(void) { return LZ4HC_CLEVEL_MAX; }
|
222
277
|
|
223
|
-
|
224
|
-
/*-************************************
|
225
|
-
* Private functions
|
226
|
-
**************************************/
|
227
|
-
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
|
228
|
-
|
229
|
-
static size_t LZ4F_getBlockSize(unsigned blockSizeID)
|
278
|
+
size_t LZ4F_getBlockSize(unsigned blockSizeID)
|
230
279
|
{
|
231
280
|
static const size_t blockSizes[4] = { 64 KB, 256 KB, 1 MB, 4 MB };
|
232
281
|
|
233
282
|
if (blockSizeID == 0) blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT;
|
234
|
-
blockSizeID
|
235
|
-
|
283
|
+
if (blockSizeID < LZ4F_max64KB || blockSizeID > LZ4F_max4MB)
|
284
|
+
return err0r(LZ4F_ERROR_maxBlockSize_invalid);
|
285
|
+
blockSizeID -= LZ4F_max64KB;
|
236
286
|
return blockSizes[blockSizeID];
|
237
287
|
}
|
238
288
|
|
289
|
+
/*-************************************
|
290
|
+
* Private functions
|
291
|
+
**************************************/
|
292
|
+
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
|
293
|
+
|
239
294
|
static BYTE LZ4F_headerChecksum (const void* header, size_t length)
|
240
295
|
{
|
241
296
|
U32 const xxh = XXH32(header, length, 0);
|
@@ -271,7 +326,7 @@ static size_t LZ4F_compressBound_internal(size_t srcSize,
|
|
271
326
|
size_t alreadyBuffered)
|
272
327
|
{
|
273
328
|
LZ4F_preferences_t prefsNull;
|
274
|
-
|
329
|
+
MEM_INIT(&prefsNull, 0, sizeof(prefsNull));
|
275
330
|
prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */
|
276
331
|
{ const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr;
|
277
332
|
U32 const flush = prefsPtr->autoFlush | (srcSize==0);
|
@@ -281,15 +336,14 @@ static size_t LZ4F_compressBound_internal(size_t srcSize,
|
|
281
336
|
size_t const bufferedSize = MIN(alreadyBuffered, maxBuffered);
|
282
337
|
size_t const maxSrcSize = srcSize + bufferedSize;
|
283
338
|
unsigned const nbFullBlocks = (unsigned)(maxSrcSize / blockSize);
|
284
|
-
size_t const partialBlockSize =
|
339
|
+
size_t const partialBlockSize = maxSrcSize & (blockSize-1);
|
285
340
|
size_t const lastBlockSize = flush ? partialBlockSize : 0;
|
286
341
|
unsigned const nbBlocks = nbFullBlocks + (lastBlockSize>0);
|
287
342
|
|
288
|
-
size_t const
|
289
|
-
size_t const
|
290
|
-
size_t const frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4);
|
343
|
+
size_t const blockCRCSize = BFSize * prefsPtr->frameInfo.blockChecksumFlag;
|
344
|
+
size_t const frameEnd = BHSize + (prefsPtr->frameInfo.contentChecksumFlag*BFSize);
|
291
345
|
|
292
|
-
return ((
|
346
|
+
return ((BHSize + blockCRCSize) * nbBlocks) +
|
293
347
|
(blockSize * nbFullBlocks) + lastBlockSize + frameEnd;
|
294
348
|
}
|
295
349
|
}
|
@@ -300,7 +354,7 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere
|
|
300
354
|
size_t const headerSize = maxFHSize; /* max header size, including optional fields */
|
301
355
|
|
302
356
|
if (preferencesPtr!=NULL) prefs = *preferencesPtr;
|
303
|
-
else
|
357
|
+
else MEM_INIT(&prefs, 0, sizeof(prefs));
|
304
358
|
prefs.autoFlush = 1;
|
305
359
|
|
306
360
|
return headerSize + LZ4F_compressBound_internal(srcSize, &prefs, 0);;
|
@@ -316,27 +370,22 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere
|
|
316
370
|
* @return : number of bytes written into dstBuffer,
|
317
371
|
* or an error code if it fails (can be tested using LZ4F_isError())
|
318
372
|
*/
|
319
|
-
size_t LZ4F_compressFrame_usingCDict(
|
373
|
+
size_t LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx,
|
374
|
+
void* dstBuffer, size_t dstCapacity,
|
320
375
|
const void* srcBuffer, size_t srcSize,
|
321
376
|
const LZ4F_CDict* cdict,
|
322
377
|
const LZ4F_preferences_t* preferencesPtr)
|
323
378
|
{
|
324
|
-
LZ4F_cctx_t cctxI;
|
325
|
-
LZ4_stream_t lz4ctx; /* pretty large on stack */
|
326
379
|
LZ4F_preferences_t prefs;
|
327
380
|
LZ4F_compressOptions_t options;
|
328
381
|
BYTE* const dstStart = (BYTE*) dstBuffer;
|
329
382
|
BYTE* dstPtr = dstStart;
|
330
383
|
BYTE* const dstEnd = dstStart + dstCapacity;
|
331
384
|
|
332
|
-
memset(&cctxI, 0, sizeof(cctxI));
|
333
|
-
cctxI.version = LZ4F_VERSION;
|
334
|
-
cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */
|
335
|
-
|
336
385
|
if (preferencesPtr!=NULL)
|
337
386
|
prefs = *preferencesPtr;
|
338
387
|
else
|
339
|
-
|
388
|
+
MEM_INIT(&prefs, 0, sizeof(prefs));
|
340
389
|
if (prefs.frameInfo.contentSize != 0)
|
341
390
|
prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */
|
342
391
|
|
@@ -345,33 +394,28 @@ size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity,
|
|
345
394
|
if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID))
|
346
395
|
prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* only one block => no need for inter-block link */
|
347
396
|
|
348
|
-
|
349
|
-
cctxI.lz4CtxPtr = &lz4ctx;
|
350
|
-
cctxI.lz4CtxLevel = 1;
|
351
|
-
} /* fast compression context pre-created on stack */
|
352
|
-
|
353
|
-
memset(&options, 0, sizeof(options));
|
397
|
+
MEM_INIT(&options, 0, sizeof(options));
|
354
398
|
options.stableSrc = 1;
|
355
399
|
|
356
400
|
if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs)) /* condition to guarantee success */
|
357
401
|
return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
|
358
402
|
|
359
|
-
{ size_t const headerSize = LZ4F_compressBegin_usingCDict(
|
403
|
+
{ size_t const headerSize = LZ4F_compressBegin_usingCDict(cctx, dstBuffer, dstCapacity, cdict, &prefs); /* write header */
|
360
404
|
if (LZ4F_isError(headerSize)) return headerSize;
|
361
405
|
dstPtr += headerSize; /* header size */ }
|
362
406
|
|
363
|
-
|
407
|
+
assert(dstEnd >= dstPtr);
|
408
|
+
{ size_t const cSize = LZ4F_compressUpdate(cctx, dstPtr, (size_t)(dstEnd-dstPtr), srcBuffer, srcSize, &options);
|
364
409
|
if (LZ4F_isError(cSize)) return cSize;
|
365
410
|
dstPtr += cSize; }
|
366
411
|
|
367
|
-
|
412
|
+
assert(dstEnd >= dstPtr);
|
413
|
+
{ size_t const tailSize = LZ4F_compressEnd(cctx, dstPtr, (size_t)(dstEnd-dstPtr), &options); /* flush last block, and generate suffix */
|
368
414
|
if (LZ4F_isError(tailSize)) return tailSize;
|
369
415
|
dstPtr += tailSize; }
|
370
416
|
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
return (dstPtr - dstStart);
|
417
|
+
assert(dstEnd >= dstStart);
|
418
|
+
return (size_t)(dstPtr - dstStart);
|
375
419
|
}
|
376
420
|
|
377
421
|
|
@@ -386,9 +430,44 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
|
|
386
430
|
const void* srcBuffer, size_t srcSize,
|
387
431
|
const LZ4F_preferences_t* preferencesPtr)
|
388
432
|
{
|
389
|
-
|
390
|
-
|
391
|
-
|
433
|
+
size_t result;
|
434
|
+
#if (LZ4F_HEAPMODE)
|
435
|
+
LZ4F_cctx_t *cctxPtr;
|
436
|
+
result = LZ4F_createCompressionContext(&cctxPtr, LZ4F_VERSION);
|
437
|
+
if (LZ4F_isError(result)) return result;
|
438
|
+
#else
|
439
|
+
LZ4F_cctx_t cctx;
|
440
|
+
LZ4_stream_t lz4ctx;
|
441
|
+
LZ4F_cctx_t *cctxPtr = &cctx;
|
442
|
+
|
443
|
+
DEBUGLOG(4, "LZ4F_compressFrame");
|
444
|
+
MEM_INIT(&cctx, 0, sizeof(cctx));
|
445
|
+
cctx.version = LZ4F_VERSION;
|
446
|
+
cctx.maxBufferSize = 5 MB; /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */
|
447
|
+
if (preferencesPtr == NULL ||
|
448
|
+
preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN)
|
449
|
+
{
|
450
|
+
LZ4_initStream(&lz4ctx, sizeof(lz4ctx));
|
451
|
+
cctxPtr->lz4CtxPtr = &lz4ctx;
|
452
|
+
cctxPtr->lz4CtxAlloc = 1;
|
453
|
+
cctxPtr->lz4CtxState = 1;
|
454
|
+
}
|
455
|
+
#endif
|
456
|
+
|
457
|
+
result = LZ4F_compressFrame_usingCDict(cctxPtr, dstBuffer, dstCapacity,
|
458
|
+
srcBuffer, srcSize,
|
459
|
+
NULL, preferencesPtr);
|
460
|
+
|
461
|
+
#if (LZ4F_HEAPMODE)
|
462
|
+
LZ4F_freeCompressionContext(cctxPtr);
|
463
|
+
#else
|
464
|
+
if (preferencesPtr != NULL &&
|
465
|
+
preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN)
|
466
|
+
{
|
467
|
+
FREEMEM(cctxPtr->lz4CtxPtr);
|
468
|
+
}
|
469
|
+
#endif
|
470
|
+
return result;
|
392
471
|
}
|
393
472
|
|
394
473
|
|
@@ -411,13 +490,14 @@ struct LZ4F_CDict_s {
|
|
411
490
|
LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
|
412
491
|
{
|
413
492
|
const char* dictStart = (const char*)dictBuffer;
|
414
|
-
LZ4F_CDict* cdict = (LZ4F_CDict*)
|
493
|
+
LZ4F_CDict* cdict = (LZ4F_CDict*) ALLOC(sizeof(*cdict));
|
494
|
+
DEBUGLOG(4, "LZ4F_createCDict");
|
415
495
|
if (!cdict) return NULL;
|
416
496
|
if (dictSize > 64 KB) {
|
417
497
|
dictStart += dictSize - 64 KB;
|
418
498
|
dictSize = 64 KB;
|
419
499
|
}
|
420
|
-
cdict->dictContent =
|
500
|
+
cdict->dictContent = ALLOC(dictSize);
|
421
501
|
cdict->fastCtx = LZ4_createStream();
|
422
502
|
cdict->HCCtx = LZ4_createStreamHC();
|
423
503
|
if (!cdict->dictContent || !cdict->fastCtx || !cdict->HCCtx) {
|
@@ -425,9 +505,8 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
|
|
425
505
|
return NULL;
|
426
506
|
}
|
427
507
|
memcpy(cdict->dictContent, dictStart, dictSize);
|
428
|
-
LZ4_resetStream(cdict->fastCtx);
|
429
508
|
LZ4_loadDict (cdict->fastCtx, (const char*)cdict->dictContent, (int)dictSize);
|
430
|
-
|
509
|
+
LZ4_setCompressionLevel(cdict->HCCtx, LZ4HC_CLEVEL_DEFAULT);
|
431
510
|
LZ4_loadDictHC(cdict->HCCtx, (const char*)cdict->dictContent, (int)dictSize);
|
432
511
|
return cdict;
|
433
512
|
}
|
@@ -456,7 +535,7 @@ void LZ4F_freeCDict(LZ4F_CDict* cdict)
|
|
456
535
|
*/
|
457
536
|
LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version)
|
458
537
|
{
|
459
|
-
LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)
|
538
|
+
LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)ALLOC_AND_ZERO(sizeof(LZ4F_cctx_t));
|
460
539
|
if (cctxPtr==NULL) return err0r(LZ4F_ERROR_allocation_failed);
|
461
540
|
|
462
541
|
cctxPtr->version = version;
|
@@ -482,6 +561,36 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
|
|
482
561
|
}
|
483
562
|
|
484
563
|
|
564
|
+
/**
|
565
|
+
* This function prepares the internal LZ4(HC) stream for a new compression,
|
566
|
+
* resetting the context and attaching the dictionary, if there is one.
|
567
|
+
*
|
568
|
+
* It needs to be called at the beginning of each independent compression
|
569
|
+
* stream (i.e., at the beginning of a frame in blockLinked mode, or at the
|
570
|
+
* beginning of each block in blockIndependent mode).
|
571
|
+
*/
|
572
|
+
static void LZ4F_initStream(void* ctx,
|
573
|
+
const LZ4F_CDict* cdict,
|
574
|
+
int level,
|
575
|
+
LZ4F_blockMode_t blockMode) {
|
576
|
+
if (level < LZ4HC_CLEVEL_MIN) {
|
577
|
+
if (cdict != NULL || blockMode == LZ4F_blockLinked) {
|
578
|
+
/* In these cases, we will call LZ4_compress_fast_continue(),
|
579
|
+
* which needs an already reset context. Otherwise, we'll call a
|
580
|
+
* one-shot API. The non-continued APIs internally perform their own
|
581
|
+
* resets at the beginning of their calls, where they know what
|
582
|
+
* tableType they need the context to be in. So in that case this
|
583
|
+
* would be misguided / wasted work. */
|
584
|
+
LZ4_resetStream_fast((LZ4_stream_t*)ctx);
|
585
|
+
}
|
586
|
+
LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);
|
587
|
+
} else {
|
588
|
+
LZ4_resetStreamHC_fast((LZ4_streamHC_t*)ctx, level);
|
589
|
+
LZ4_attach_HC_dictionary((LZ4_streamHC_t *)ctx, cdict ? cdict->HCCtx : NULL);
|
590
|
+
}
|
591
|
+
}
|
592
|
+
|
593
|
+
|
485
594
|
/*! LZ4F_compressBegin_usingCDict() :
|
486
595
|
* init streaming compression and writes frame header into dstBuffer.
|
487
596
|
* dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes.
|
@@ -499,21 +608,35 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
|
|
499
608
|
BYTE* headerStart;
|
500
609
|
|
501
610
|
if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
|
502
|
-
|
611
|
+
MEM_INIT(&prefNull, 0, sizeof(prefNull));
|
503
612
|
if (preferencesPtr == NULL) preferencesPtr = &prefNull;
|
504
613
|
cctxPtr->prefs = *preferencesPtr;
|
505
614
|
|
506
615
|
/* Ctx Management */
|
507
|
-
{
|
508
|
-
if (cctxPtr->
|
616
|
+
{ U16 const ctxTypeID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2;
|
617
|
+
if (cctxPtr->lz4CtxAlloc < ctxTypeID) {
|
509
618
|
FREEMEM(cctxPtr->lz4CtxPtr);
|
510
|
-
if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
|
511
|
-
cctxPtr->lz4CtxPtr =
|
512
|
-
else
|
513
|
-
cctxPtr->lz4CtxPtr =
|
514
|
-
|
515
|
-
cctxPtr->
|
516
|
-
|
619
|
+
if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
|
620
|
+
cctxPtr->lz4CtxPtr = LZ4_createStream();
|
621
|
+
} else {
|
622
|
+
cctxPtr->lz4CtxPtr = LZ4_createStreamHC();
|
623
|
+
}
|
624
|
+
if (cctxPtr->lz4CtxPtr == NULL)
|
625
|
+
return err0r(LZ4F_ERROR_allocation_failed);
|
626
|
+
cctxPtr->lz4CtxAlloc = ctxTypeID;
|
627
|
+
cctxPtr->lz4CtxState = ctxTypeID;
|
628
|
+
} else if (cctxPtr->lz4CtxState != ctxTypeID) {
|
629
|
+
/* otherwise, a sufficient buffer is allocated, but we need to
|
630
|
+
* reset it to the correct context type */
|
631
|
+
if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
|
632
|
+
LZ4_initStream((LZ4_stream_t *) cctxPtr->lz4CtxPtr, sizeof (LZ4_stream_t));
|
633
|
+
} else {
|
634
|
+
LZ4_initStreamHC((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, sizeof(LZ4_streamHC_t));
|
635
|
+
LZ4_setCompressionLevel((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
|
636
|
+
}
|
637
|
+
cctxPtr->lz4CtxState = ctxTypeID;
|
638
|
+
}
|
639
|
+
}
|
517
640
|
|
518
641
|
/* Buffer Management */
|
519
642
|
if (cctxPtr->prefs.frameInfo.blockSizeID == 0)
|
@@ -521,37 +644,28 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
|
|
521
644
|
cctxPtr->maxBlockSize = LZ4F_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID);
|
522
645
|
|
523
646
|
{ size_t const requiredBuffSize = preferencesPtr->autoFlush ?
|
524
|
-
(cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked)
|
525
|
-
cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked)
|
647
|
+
((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) ? 64 KB : 0) : /* only needs past data up to window size */
|
648
|
+
cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) ? 128 KB : 0);
|
526
649
|
|
527
650
|
if (cctxPtr->maxBufferSize < requiredBuffSize) {
|
528
651
|
cctxPtr->maxBufferSize = 0;
|
529
652
|
FREEMEM(cctxPtr->tmpBuff);
|
530
|
-
cctxPtr->tmpBuff = (BYTE*)
|
653
|
+
cctxPtr->tmpBuff = (BYTE*)ALLOC_AND_ZERO(requiredBuffSize);
|
531
654
|
if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed);
|
532
655
|
cctxPtr->maxBufferSize = requiredBuffSize;
|
533
656
|
} }
|
534
657
|
cctxPtr->tmpIn = cctxPtr->tmpBuff;
|
535
658
|
cctxPtr->tmpInSize = 0;
|
536
|
-
XXH32_reset(&(cctxPtr->xxh), 0);
|
659
|
+
(void)XXH32_reset(&(cctxPtr->xxh), 0);
|
537
660
|
|
538
661
|
/* context init */
|
539
662
|
cctxPtr->cdict = cdict;
|
540
663
|
if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) {
|
541
664
|
/* frame init only for blockLinked : blockIndependent will be init at each block */
|
542
|
-
|
543
|
-
|
544
|
-
|
545
|
-
|
546
|
-
memcpy(cctxPtr->lz4CtxPtr, cdict->HCCtx, sizeof(*cdict->HCCtx));
|
547
|
-
LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
|
548
|
-
}
|
549
|
-
} else {
|
550
|
-
if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
|
551
|
-
LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr));
|
552
|
-
else
|
553
|
-
LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel);
|
554
|
-
}
|
665
|
+
LZ4F_initStream(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel, LZ4F_blockLinked);
|
666
|
+
}
|
667
|
+
if (preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN) {
|
668
|
+
LZ4_favorDecompressionSpeed((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, (int)preferencesPtr->favorDecSpeed);
|
555
669
|
}
|
556
670
|
|
557
671
|
/* Magic Number */
|
@@ -563,7 +677,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
|
|
563
677
|
*dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */
|
564
678
|
+ ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5)
|
565
679
|
+ ((cctxPtr->prefs.frameInfo.blockChecksumFlag & _1BIT ) << 4)
|
566
|
-
+ ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3)
|
680
|
+
+ ((unsigned)(cctxPtr->prefs.frameInfo.contentSize > 0) << 3)
|
567
681
|
+ ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2)
|
568
682
|
+ (cctxPtr->prefs.frameInfo.dictID > 0) );
|
569
683
|
/* BD Byte */
|
@@ -580,11 +694,11 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
|
|
580
694
|
dstPtr += 4;
|
581
695
|
}
|
582
696
|
/* Header CRC Byte */
|
583
|
-
*dstPtr = LZ4F_headerChecksum(headerStart, dstPtr - headerStart);
|
697
|
+
*dstPtr = LZ4F_headerChecksum(headerStart, (size_t)(dstPtr - headerStart));
|
584
698
|
dstPtr++;
|
585
699
|
|
586
700
|
cctxPtr->cStage = 1; /* header written, now request input data block */
|
587
|
-
return (dstPtr - dstStart);
|
701
|
+
return (size_t)(dstPtr - dstStart);
|
588
702
|
}
|
589
703
|
|
590
704
|
|
@@ -593,7 +707,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
|
|
593
707
|
* dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes.
|
594
708
|
* preferencesPtr can be NULL, in which case default parameters are selected.
|
595
709
|
* @return : number of bytes written into dstBuffer for the header
|
596
|
-
*
|
710
|
+
* or an error code (can be tested using LZ4F_isError())
|
597
711
|
*/
|
598
712
|
size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr,
|
599
713
|
void* dstBuffer, size_t dstCapacity,
|
@@ -604,10 +718,10 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr,
|
|
604
718
|
}
|
605
719
|
|
606
720
|
|
607
|
-
/*
|
608
|
-
*
|
609
|
-
*
|
610
|
-
*
|
721
|
+
/* LZ4F_compressBound() :
|
722
|
+
* @return minimum capacity of dstBuffer for a given srcSize to handle worst case scenario.
|
723
|
+
* LZ4F_preferences_t structure is optional : if NULL, preferences will be set to cover worst case scenario.
|
724
|
+
* This function cannot fail.
|
611
725
|
*/
|
612
726
|
size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr)
|
613
727
|
{
|
@@ -619,55 +733,59 @@ typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize
|
|
619
733
|
|
620
734
|
|
621
735
|
/*! LZ4F_makeBlock():
|
622
|
-
* compress a single block, add header and checksum
|
623
|
-
* assumption : dst buffer capacity is >= srcSize
|
624
|
-
|
736
|
+
* compress a single block, add header and optional checksum.
|
737
|
+
* assumption : dst buffer capacity is >= BHSize + srcSize + crcSize
|
738
|
+
*/
|
739
|
+
static size_t LZ4F_makeBlock(void* dst,
|
740
|
+
const void* src, size_t srcSize,
|
625
741
|
compressFunc_t compress, void* lz4ctx, int level,
|
626
|
-
|
742
|
+
const LZ4F_CDict* cdict,
|
743
|
+
LZ4F_blockChecksum_t crcFlag)
|
627
744
|
{
|
628
745
|
BYTE* const cSizePtr = (BYTE*)dst;
|
629
|
-
U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+
|
746
|
+
U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize),
|
630
747
|
(int)(srcSize), (int)(srcSize-1),
|
631
748
|
level, cdict);
|
632
|
-
LZ4F_writeLE32(cSizePtr, cSize);
|
633
749
|
if (cSize == 0) { /* compression failed */
|
634
750
|
cSize = (U32)srcSize;
|
635
751
|
LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG);
|
636
|
-
memcpy(cSizePtr+
|
752
|
+
memcpy(cSizePtr+BHSize, src, srcSize);
|
753
|
+
} else {
|
754
|
+
LZ4F_writeLE32(cSizePtr, cSize);
|
637
755
|
}
|
638
756
|
if (crcFlag) {
|
639
|
-
U32 const crc32 = XXH32(cSizePtr+
|
640
|
-
LZ4F_writeLE32(cSizePtr+
|
757
|
+
U32 const crc32 = XXH32(cSizePtr+BHSize, cSize, 0); /* checksum of compressed data */
|
758
|
+
LZ4F_writeLE32(cSizePtr+BHSize+cSize, crc32);
|
641
759
|
}
|
642
|
-
return
|
760
|
+
return BHSize + cSize + ((U32)crcFlag)*BFSize;
|
643
761
|
}
|
644
762
|
|
645
763
|
|
646
764
|
static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
|
647
765
|
{
|
648
|
-
int const acceleration = (level <
|
766
|
+
int const acceleration = (level < 0) ? -level + 1 : 1;
|
767
|
+
LZ4F_initStream(ctx, cdict, level, LZ4F_blockIndependent);
|
649
768
|
if (cdict) {
|
650
|
-
memcpy(ctx, cdict->fastCtx, sizeof(*cdict->fastCtx));
|
651
769
|
return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
|
770
|
+
} else {
|
771
|
+
return LZ4_compress_fast_extState_fastReset(ctx, src, dst, srcSize, dstCapacity, acceleration);
|
652
772
|
}
|
653
|
-
return LZ4_compress_fast_extState(ctx, src, dst, srcSize, dstCapacity, acceleration);
|
654
773
|
}
|
655
774
|
|
656
775
|
static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
|
657
776
|
{
|
658
|
-
int const acceleration = (level <
|
777
|
+
int const acceleration = (level < 0) ? -level + 1 : 1;
|
659
778
|
(void)cdict; /* init once at beginning of frame */
|
660
779
|
return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
|
661
780
|
}
|
662
781
|
|
663
782
|
static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
|
664
783
|
{
|
784
|
+
LZ4F_initStream(ctx, cdict, level, LZ4F_blockIndependent);
|
665
785
|
if (cdict) {
|
666
|
-
memcpy(ctx, cdict->HCCtx, sizeof(*cdict->HCCtx));
|
667
|
-
LZ4_setCompressionLevel((LZ4_streamHC_t*)ctx, level);
|
668
786
|
return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
|
669
787
|
}
|
670
|
-
return
|
788
|
+
return LZ4_compress_HC_extStateHC_fastReset(ctx, src, dst, srcSize, dstCapacity, level);
|
671
789
|
}
|
672
790
|
|
673
791
|
static int LZ4F_compressBlockHC_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
|
@@ -716,10 +834,12 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
|
|
716
834
|
LZ4F_lastBlockStatus lastBlockCompressed = notDone;
|
717
835
|
compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
|
718
836
|
|
837
|
+
DEBUGLOG(4, "LZ4F_compressUpdate (srcSize=%zu)", srcSize);
|
719
838
|
|
720
839
|
if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
|
721
|
-
if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
|
722
|
-
|
840
|
+
if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
|
841
|
+
return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
|
842
|
+
MEM_INIT(&cOptionsNull, 0, sizeof(cOptionsNull));
|
723
843
|
if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull;
|
724
844
|
|
725
845
|
/* complete tmp buffer */
|
@@ -737,9 +857,11 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
|
|
737
857
|
memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy);
|
738
858
|
srcPtr += sizeToCopy;
|
739
859
|
|
740
|
-
dstPtr += LZ4F_makeBlock(dstPtr,
|
860
|
+
dstPtr += LZ4F_makeBlock(dstPtr,
|
861
|
+
cctxPtr->tmpIn, blockSize,
|
741
862
|
compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
|
742
|
-
cctxPtr->cdict,
|
863
|
+
cctxPtr->cdict,
|
864
|
+
cctxPtr->prefs.frameInfo.blockChecksumFlag);
|
743
865
|
|
744
866
|
if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
|
745
867
|
cctxPtr->tmpInSize = 0;
|
@@ -749,18 +871,22 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
|
|
749
871
|
while ((size_t)(srcEnd - srcPtr) >= blockSize) {
|
750
872
|
/* compress full blocks */
|
751
873
|
lastBlockCompressed = fromSrcBuffer;
|
752
|
-
dstPtr += LZ4F_makeBlock(dstPtr,
|
874
|
+
dstPtr += LZ4F_makeBlock(dstPtr,
|
875
|
+
srcPtr, blockSize,
|
753
876
|
compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
|
754
|
-
cctxPtr->cdict,
|
877
|
+
cctxPtr->cdict,
|
878
|
+
cctxPtr->prefs.frameInfo.blockChecksumFlag);
|
755
879
|
srcPtr += blockSize;
|
756
880
|
}
|
757
881
|
|
758
882
|
if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) {
|
759
883
|
/* compress remaining input < blockSize */
|
760
884
|
lastBlockCompressed = fromSrcBuffer;
|
761
|
-
dstPtr += LZ4F_makeBlock(dstPtr,
|
885
|
+
dstPtr += LZ4F_makeBlock(dstPtr,
|
886
|
+
srcPtr, (size_t)(srcEnd - srcPtr),
|
762
887
|
compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
|
763
|
-
cctxPtr->cdict,
|
888
|
+
cctxPtr->cdict,
|
889
|
+
cctxPtr->prefs.frameInfo.blockChecksumFlag);
|
764
890
|
srcPtr = srcEnd;
|
765
891
|
}
|
766
892
|
|
@@ -786,28 +912,30 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
|
|
786
912
|
/* some input data left, necessarily < blockSize */
|
787
913
|
if (srcPtr < srcEnd) {
|
788
914
|
/* fill tmp buffer */
|
789
|
-
size_t const sizeToCopy = srcEnd - srcPtr;
|
915
|
+
size_t const sizeToCopy = (size_t)(srcEnd - srcPtr);
|
790
916
|
memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy);
|
791
917
|
cctxPtr->tmpInSize = sizeToCopy;
|
792
918
|
}
|
793
919
|
|
794
920
|
if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled)
|
795
|
-
XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize);
|
921
|
+
(void)XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize);
|
796
922
|
|
797
923
|
cctxPtr->totalInSize += srcSize;
|
798
|
-
return dstPtr - dstStart;
|
924
|
+
return (size_t)(dstPtr - dstStart);
|
799
925
|
}
|
800
926
|
|
801
927
|
|
802
928
|
/*! LZ4F_flush() :
|
803
|
-
*
|
804
|
-
*
|
805
|
-
* The result of the function is the number of bytes written into dstBuffer
|
806
|
-
*
|
929
|
+
* When compressed data must be sent immediately, without waiting for a block to be filled,
|
930
|
+
* invoke LZ4_flush(), which will immediately compress any remaining data stored within LZ4F_cctx.
|
931
|
+
* The result of the function is the number of bytes written into dstBuffer.
|
932
|
+
* It can be zero, this means there was no data left within LZ4F_cctx.
|
807
933
|
* The function outputs an error code if it fails (can be tested using LZ4F_isError())
|
808
|
-
*
|
934
|
+
* LZ4F_compressOptions_t* is optional. NULL is a valid argument.
|
809
935
|
*/
|
810
|
-
size_t LZ4F_flush(LZ4F_cctx* cctxPtr,
|
936
|
+
size_t LZ4F_flush(LZ4F_cctx* cctxPtr,
|
937
|
+
void* dstBuffer, size_t dstCapacity,
|
938
|
+
const LZ4F_compressOptions_t* compressOptionsPtr)
|
811
939
|
{
|
812
940
|
BYTE* const dstStart = (BYTE*)dstBuffer;
|
813
941
|
BYTE* dstPtr = dstStart;
|
@@ -815,52 +943,65 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const
|
|
815
943
|
|
816
944
|
if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */
|
817
945
|
if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
|
818
|
-
if (dstCapacity < (cctxPtr->tmpInSize +
|
946
|
+
if (dstCapacity < (cctxPtr->tmpInSize + BHSize + BFSize))
|
947
|
+
return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
|
819
948
|
(void)compressOptionsPtr; /* not yet useful */
|
820
949
|
|
821
950
|
/* select compression function */
|
822
951
|
compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
|
823
952
|
|
824
953
|
/* compress tmp buffer */
|
825
|
-
dstPtr += LZ4F_makeBlock(dstPtr,
|
954
|
+
dstPtr += LZ4F_makeBlock(dstPtr,
|
955
|
+
cctxPtr->tmpIn, cctxPtr->tmpInSize,
|
826
956
|
compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
|
827
|
-
cctxPtr->cdict,
|
828
|
-
|
957
|
+
cctxPtr->cdict,
|
958
|
+
cctxPtr->prefs.frameInfo.blockChecksumFlag);
|
959
|
+
assert(((void)"flush overflows dstBuffer!", (size_t)(dstPtr - dstStart) <= dstCapacity));
|
960
|
+
|
961
|
+
if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked)
|
962
|
+
cctxPtr->tmpIn += cctxPtr->tmpInSize;
|
829
963
|
cctxPtr->tmpInSize = 0;
|
830
964
|
|
831
965
|
/* keep tmpIn within limits */
|
832
966
|
if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) { /* necessarily LZ4F_blockLinked */
|
833
|
-
int realDictSize = LZ4F_localSaveDict(cctxPtr);
|
967
|
+
int const realDictSize = LZ4F_localSaveDict(cctxPtr);
|
834
968
|
cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
|
835
969
|
}
|
836
970
|
|
837
|
-
return dstPtr - dstStart;
|
971
|
+
return (size_t)(dstPtr - dstStart);
|
838
972
|
}
|
839
973
|
|
840
974
|
|
841
975
|
/*! LZ4F_compressEnd() :
|
842
|
-
*
|
843
|
-
*
|
844
|
-
*
|
845
|
-
*
|
846
|
-
*
|
847
|
-
*
|
848
|
-
*
|
976
|
+
* When you want to properly finish the compressed frame, just call LZ4F_compressEnd().
|
977
|
+
* It will flush whatever data remained within compressionContext (like LZ4_flush())
|
978
|
+
* but also properly finalize the frame, with an endMark and an (optional) checksum.
|
979
|
+
* LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
|
980
|
+
* @return: the number of bytes written into dstBuffer (necessarily >= 4 (endMark size))
|
981
|
+
* or an error code if it fails (can be tested using LZ4F_isError())
|
982
|
+
* The context can then be used again to compress a new frame, starting with LZ4F_compressBegin().
|
849
983
|
*/
|
850
|
-
size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr,
|
984
|
+
size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr,
|
985
|
+
void* dstBuffer, size_t dstCapacity,
|
986
|
+
const LZ4F_compressOptions_t* compressOptionsPtr)
|
851
987
|
{
|
852
988
|
BYTE* const dstStart = (BYTE*)dstBuffer;
|
853
989
|
BYTE* dstPtr = dstStart;
|
854
990
|
|
855
|
-
size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer,
|
991
|
+
size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr);
|
856
992
|
if (LZ4F_isError(flushSize)) return flushSize;
|
857
993
|
dstPtr += flushSize;
|
858
994
|
|
995
|
+
assert(flushSize <= dstCapacity);
|
996
|
+
dstCapacity -= flushSize;
|
997
|
+
|
998
|
+
if (dstCapacity < 4) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
|
859
999
|
LZ4F_writeLE32(dstPtr, 0);
|
860
|
-
dstPtr+=4; /* endMark */
|
1000
|
+
dstPtr += 4; /* endMark */
|
861
1001
|
|
862
1002
|
if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) {
|
863
1003
|
U32 const xxh = XXH32_digest(&(cctxPtr->xxh));
|
1004
|
+
if (dstCapacity < 8) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
|
864
1005
|
LZ4F_writeLE32(dstPtr, xxh);
|
865
1006
|
dstPtr+=4; /* content Checksum */
|
866
1007
|
}
|
@@ -873,7 +1014,7 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize,
|
|
873
1014
|
return err0r(LZ4F_ERROR_frameSize_wrong);
|
874
1015
|
}
|
875
1016
|
|
876
|
-
return dstPtr - dstStart;
|
1017
|
+
return (size_t)(dstPtr - dstStart);
|
877
1018
|
}
|
878
1019
|
|
879
1020
|
|
@@ -887,8 +1028,7 @@ typedef enum {
|
|
887
1028
|
dstage_getBlockHeader, dstage_storeBlockHeader,
|
888
1029
|
dstage_copyDirect, dstage_getBlockChecksum,
|
889
1030
|
dstage_getCBlock, dstage_storeCBlock,
|
890
|
-
|
891
|
-
dstage_decodeCBlock_intoTmp, dstage_flushOut,
|
1031
|
+
dstage_flushOut,
|
892
1032
|
dstage_getSuffix, dstage_storeSuffix,
|
893
1033
|
dstage_getSFrameSize, dstage_storeSFrameSize,
|
894
1034
|
dstage_skipSkippable
|
@@ -924,7 +1064,7 @@ struct LZ4F_dctx_s {
|
|
924
1064
|
*/
|
925
1065
|
LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber)
|
926
1066
|
{
|
927
|
-
LZ4F_dctx* const dctx = (LZ4F_dctx*)
|
1067
|
+
LZ4F_dctx* const dctx = (LZ4F_dctx*)ALLOC_AND_ZERO(sizeof(LZ4F_dctx));
|
928
1068
|
if (dctx==NULL) return err0r(LZ4F_ERROR_GENERIC);
|
929
1069
|
|
930
1070
|
dctx->version = versionNumber;
|
@@ -955,31 +1095,6 @@ void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx)
|
|
955
1095
|
}
|
956
1096
|
|
957
1097
|
|
958
|
-
/*! LZ4F_headerSize() :
|
959
|
-
* @return : size of frame header
|
960
|
-
* or an error code, which can be tested using LZ4F_isError()
|
961
|
-
*/
|
962
|
-
static size_t LZ4F_headerSize(const void* src, size_t srcSize)
|
963
|
-
{
|
964
|
-
/* minimal srcSize to determine header size */
|
965
|
-
if (srcSize < 5) return err0r(LZ4F_ERROR_frameHeader_incomplete);
|
966
|
-
|
967
|
-
/* special case : skippable frames */
|
968
|
-
if ((LZ4F_readLE32(src) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) return 8;
|
969
|
-
|
970
|
-
/* control magic number */
|
971
|
-
if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER)
|
972
|
-
return err0r(LZ4F_ERROR_frameType_unknown);
|
973
|
-
|
974
|
-
/* Frame Header Size */
|
975
|
-
{ BYTE const FLG = ((const BYTE*)src)[4];
|
976
|
-
U32 const contentSizeFlag = (FLG>>3) & _1BIT;
|
977
|
-
U32 const dictIDFlag = FLG & _1BIT;
|
978
|
-
return minFHSize + (contentSizeFlag*8) + (dictIDFlag*4);
|
979
|
-
}
|
980
|
-
}
|
981
|
-
|
982
|
-
|
983
1098
|
/*! LZ4F_decodeHeader() :
|
984
1099
|
* input : `src` points at the **beginning of the frame**
|
985
1100
|
* output : set internal values of dctx, such as
|
@@ -996,7 +1111,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
|
|
996
1111
|
|
997
1112
|
/* need to decode header to get frameInfo */
|
998
1113
|
if (srcSize < minFHSize) return err0r(LZ4F_ERROR_frameHeader_incomplete); /* minimal frame header size */
|
999
|
-
|
1114
|
+
MEM_INIT(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo));
|
1000
1115
|
|
1001
1116
|
/* special case : skippable frames */
|
1002
1117
|
if ((LZ4F_readLE32(srcPtr) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) {
|
@@ -1031,7 +1146,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
|
|
1031
1146
|
}
|
1032
1147
|
|
1033
1148
|
/* Frame Header Size */
|
1034
|
-
frameHeaderSize = minFHSize + (contentSizeFlag
|
1149
|
+
frameHeaderSize = minFHSize + (contentSizeFlag?8:0) + (dictIDFlag?4:0);
|
1035
1150
|
|
1036
1151
|
if (srcSize < frameHeaderSize) {
|
1037
1152
|
/* not enough input to fully decode frame header */
|
@@ -1052,6 +1167,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
|
|
1052
1167
|
}
|
1053
1168
|
|
1054
1169
|
/* check header */
|
1170
|
+
assert(frameHeaderSize > 5);
|
1055
1171
|
{ BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5);
|
1056
1172
|
if (HC != srcPtr[frameHeaderSize-1])
|
1057
1173
|
return err0r(LZ4F_ERROR_headerChecksum_invalid);
|
@@ -1075,6 +1191,34 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
|
|
1075
1191
|
}
|
1076
1192
|
|
1077
1193
|
|
1194
|
+
/*! LZ4F_headerSize() :
|
1195
|
+
* @return : size of frame header
|
1196
|
+
* or an error code, which can be tested using LZ4F_isError()
|
1197
|
+
*/
|
1198
|
+
size_t LZ4F_headerSize(const void* src, size_t srcSize)
|
1199
|
+
{
|
1200
|
+
if (src == NULL) return err0r(LZ4F_ERROR_srcPtr_wrong);
|
1201
|
+
|
1202
|
+
/* minimal srcSize to determine header size */
|
1203
|
+
if (srcSize < LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH)
|
1204
|
+
return err0r(LZ4F_ERROR_frameHeader_incomplete);
|
1205
|
+
|
1206
|
+
/* special case : skippable frames */
|
1207
|
+
if ((LZ4F_readLE32(src) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START)
|
1208
|
+
return 8;
|
1209
|
+
|
1210
|
+
/* control magic number */
|
1211
|
+
if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER)
|
1212
|
+
return err0r(LZ4F_ERROR_frameType_unknown);
|
1213
|
+
|
1214
|
+
/* Frame Header Size */
|
1215
|
+
{ BYTE const FLG = ((const BYTE*)src)[4];
|
1216
|
+
U32 const contentSizeFlag = (FLG>>3) & _1BIT;
|
1217
|
+
U32 const dictIDFlag = FLG & _1BIT;
|
1218
|
+
return minFHSize + (contentSizeFlag?8:0) + (dictIDFlag?4:0);
|
1219
|
+
}
|
1220
|
+
}
|
1221
|
+
|
1078
1222
|
/*! LZ4F_getFrameInfo() :
|
1079
1223
|
* This function extracts frame parameters (max blockSize, frame checksum, etc.).
|
1080
1224
|
* Usage is optional. Objective is to provide relevant information for allocation purposes.
|
@@ -1090,10 +1234,12 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
|
|
1090
1234
|
* note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped.
|
1091
1235
|
* note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
|
1092
1236
|
*/
|
1093
|
-
LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
|
1094
|
-
|
1237
|
+
LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
|
1238
|
+
LZ4F_frameInfo_t* frameInfoPtr,
|
1239
|
+
const void* srcBuffer, size_t* srcSizePtr)
|
1095
1240
|
{
|
1096
|
-
|
1241
|
+
LZ4F_STATIC_ASSERT(dstage_getFrameHeader < dstage_storeFrameHeader);
|
1242
|
+
if (dctx->dStage > dstage_storeFrameHeader) {
|
1097
1243
|
/* frameInfo already decoded */
|
1098
1244
|
size_t o=0, i=0;
|
1099
1245
|
*srcSizePtr = 0;
|
@@ -1106,7 +1252,6 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoP
|
|
1106
1252
|
*srcSizePtr = 0;
|
1107
1253
|
return err0r(LZ4F_ERROR_frameDecoding_alreadyStarted);
|
1108
1254
|
} else {
|
1109
|
-
size_t decodeResult;
|
1110
1255
|
size_t const hSize = LZ4F_headerSize(srcBuffer, *srcSizePtr);
|
1111
1256
|
if (LZ4F_isError(hSize)) { *srcSizePtr=0; return hSize; }
|
1112
1257
|
if (*srcSizePtr < hSize) {
|
@@ -1114,45 +1259,53 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoP
|
|
1114
1259
|
return err0r(LZ4F_ERROR_frameHeader_incomplete);
|
1115
1260
|
}
|
1116
1261
|
|
1117
|
-
decodeResult = LZ4F_decodeHeader(dctx, srcBuffer, hSize);
|
1118
|
-
|
1119
|
-
|
1120
|
-
|
1121
|
-
|
1122
|
-
|
1123
|
-
|
1124
|
-
|
1125
|
-
|
1126
|
-
} }
|
1262
|
+
{ size_t decodeResult = LZ4F_decodeHeader(dctx, srcBuffer, hSize);
|
1263
|
+
if (LZ4F_isError(decodeResult)) {
|
1264
|
+
*srcSizePtr = 0;
|
1265
|
+
} else {
|
1266
|
+
*srcSizePtr = decodeResult;
|
1267
|
+
decodeResult = BHSize; /* block header size */
|
1268
|
+
}
|
1269
|
+
*frameInfoPtr = dctx->frameInfo;
|
1270
|
+
return decodeResult;
|
1271
|
+
} } }
|
1127
1272
|
}
|
1128
1273
|
|
1129
1274
|
|
1130
1275
|
/* LZ4F_updateDict() :
|
1131
1276
|
* only used for LZ4F_blockLinked mode */
|
1132
|
-
static void LZ4F_updateDict(LZ4F_dctx* dctx,
|
1277
|
+
static void LZ4F_updateDict(LZ4F_dctx* dctx,
|
1278
|
+
const BYTE* dstPtr, size_t dstSize, const BYTE* dstBufferStart,
|
1279
|
+
unsigned withinTmp)
|
1133
1280
|
{
|
1134
1281
|
if (dctx->dictSize==0)
|
1135
1282
|
dctx->dict = (const BYTE*)dstPtr; /* priority to dictionary continuity */
|
1136
1283
|
|
1137
|
-
if (dctx->dict + dctx->dictSize == dstPtr) { /* dictionary continuity */
|
1284
|
+
if (dctx->dict + dctx->dictSize == dstPtr) { /* dictionary continuity, directly within dstBuffer */
|
1138
1285
|
dctx->dictSize += dstSize;
|
1139
1286
|
return;
|
1140
1287
|
}
|
1141
1288
|
|
1142
|
-
|
1143
|
-
|
1144
|
-
dctx->
|
1289
|
+
assert(dstPtr >= dstBufferStart);
|
1290
|
+
if ((size_t)(dstPtr - dstBufferStart) + dstSize >= 64 KB) { /* history in dstBuffer becomes large enough to become dictionary */
|
1291
|
+
dctx->dict = (const BYTE*)dstBufferStart;
|
1292
|
+
dctx->dictSize = (size_t)(dstPtr - dstBufferStart) + dstSize;
|
1145
1293
|
return;
|
1146
1294
|
}
|
1147
1295
|
|
1148
|
-
|
1149
|
-
|
1296
|
+
assert(dstSize < 64 KB); /* if dstSize >= 64 KB, dictionary would be set into dstBuffer directly */
|
1297
|
+
|
1298
|
+
/* dstBuffer does not contain whole useful history (64 KB), so it must be saved within tmpOut */
|
1299
|
+
|
1300
|
+
if ((withinTmp) && (dctx->dict == dctx->tmpOutBuffer)) { /* continue history within tmpOutBuffer */
|
1301
|
+
/* withinTmp expectation : content of [dstPtr,dstSize] is same as [dict+dictSize,dstSize], so we just extend it */
|
1302
|
+
assert(dctx->dict + dctx->dictSize == dctx->tmpOut + dctx->tmpOutStart);
|
1150
1303
|
dctx->dictSize += dstSize;
|
1151
1304
|
return;
|
1152
1305
|
}
|
1153
1306
|
|
1154
1307
|
if (withinTmp) { /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */
|
1155
|
-
size_t const preserveSize = dctx->tmpOut - dctx->tmpOutBuffer;
|
1308
|
+
size_t const preserveSize = (size_t)(dctx->tmpOut - dctx->tmpOutBuffer);
|
1156
1309
|
size_t copySize = 64 KB - dctx->tmpOutSize;
|
1157
1310
|
const BYTE* const oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart;
|
1158
1311
|
if (dctx->tmpOutSize > 64 KB) copySize = 0;
|
@@ -1167,7 +1320,7 @@ static void LZ4F_updateDict(LZ4F_dctx* dctx, const BYTE* dstPtr, size_t dstSize,
|
|
1167
1320
|
|
1168
1321
|
if (dctx->dict == dctx->tmpOutBuffer) { /* copy dst into tmp to complete dict */
|
1169
1322
|
if (dctx->dictSize + dstSize > dctx->maxBufferSize) { /* tmp buffer not large enough */
|
1170
|
-
size_t const preserveSize = 64 KB - dstSize;
|
1323
|
+
size_t const preserveSize = 64 KB - dstSize;
|
1171
1324
|
memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize);
|
1172
1325
|
dctx->dictSize = preserveSize;
|
1173
1326
|
}
|
@@ -1177,7 +1330,7 @@ static void LZ4F_updateDict(LZ4F_dctx* dctx, const BYTE* dstPtr, size_t dstSize,
|
|
1177
1330
|
}
|
1178
1331
|
|
1179
1332
|
/* join dict & dest into tmp */
|
1180
|
-
{ size_t preserveSize = 64 KB - dstSize;
|
1333
|
+
{ size_t preserveSize = 64 KB - dstSize;
|
1181
1334
|
if (preserveSize > dctx->dictSize) preserveSize = dctx->dictSize;
|
1182
1335
|
memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize);
|
1183
1336
|
memcpy(dctx->tmpOutBuffer + preserveSize, dstPtr, dstSize);
|
@@ -1223,7 +1376,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1223
1376
|
size_t nextSrcSizeHint = 1;
|
1224
1377
|
|
1225
1378
|
|
1226
|
-
|
1379
|
+
MEM_INIT(&optionsNull, 0, sizeof(optionsNull));
|
1227
1380
|
if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull;
|
1228
1381
|
*srcSizePtr = 0;
|
1229
1382
|
*dstSizePtr = 0;
|
@@ -1237,14 +1390,14 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1237
1390
|
|
1238
1391
|
case dstage_getFrameHeader:
|
1239
1392
|
if ((size_t)(srcEnd-srcPtr) >= maxFHSize) { /* enough to decode - shortcut */
|
1240
|
-
size_t const hSize = LZ4F_decodeHeader(dctx, srcPtr, srcEnd-srcPtr); /* will update dStage appropriately */
|
1393
|
+
size_t const hSize = LZ4F_decodeHeader(dctx, srcPtr, (size_t)(srcEnd-srcPtr)); /* will update dStage appropriately */
|
1241
1394
|
if (LZ4F_isError(hSize)) return hSize;
|
1242
1395
|
srcPtr += hSize;
|
1243
1396
|
break;
|
1244
1397
|
}
|
1245
1398
|
dctx->tmpInSize = 0;
|
1246
1399
|
if (srcEnd-srcPtr == 0) return minFHSize; /* 0-size input */
|
1247
|
-
dctx->tmpInTarget = minFHSize; /* minimum to
|
1400
|
+
dctx->tmpInTarget = minFHSize; /* minimum size to decode header */
|
1248
1401
|
dctx->dStage = dstage_storeFrameHeader;
|
1249
1402
|
/* fall-through */
|
1250
1403
|
|
@@ -1265,18 +1418,18 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1265
1418
|
break;
|
1266
1419
|
|
1267
1420
|
case dstage_init:
|
1268
|
-
if (dctx->frameInfo.contentChecksumFlag) XXH32_reset(&(dctx->xxh), 0);
|
1421
|
+
if (dctx->frameInfo.contentChecksumFlag) (void)XXH32_reset(&(dctx->xxh), 0);
|
1269
1422
|
/* internal buffers allocation */
|
1270
1423
|
{ size_t const bufferNeeded = dctx->maxBlockSize
|
1271
|
-
+ ((dctx->frameInfo.blockMode==LZ4F_blockLinked)
|
1424
|
+
+ ((dctx->frameInfo.blockMode==LZ4F_blockLinked) ? 128 KB : 0);
|
1272
1425
|
if (bufferNeeded > dctx->maxBufferSize) { /* tmp buffers too small */
|
1273
1426
|
dctx->maxBufferSize = 0; /* ensure allocation will be re-attempted on next entry*/
|
1274
1427
|
FREEMEM(dctx->tmpIn);
|
1275
|
-
dctx->tmpIn = (BYTE*)
|
1428
|
+
dctx->tmpIn = (BYTE*)ALLOC(dctx->maxBlockSize + BFSize /* block checksum */);
|
1276
1429
|
if (dctx->tmpIn == NULL)
|
1277
1430
|
return err0r(LZ4F_ERROR_allocation_failed);
|
1278
1431
|
FREEMEM(dctx->tmpOutBuffer);
|
1279
|
-
dctx->tmpOutBuffer= (BYTE*)
|
1432
|
+
dctx->tmpOutBuffer= (BYTE*)ALLOC(bufferNeeded);
|
1280
1433
|
if (dctx->tmpOutBuffer== NULL)
|
1281
1434
|
return err0r(LZ4F_ERROR_allocation_failed);
|
1282
1435
|
dctx->maxBufferSize = bufferNeeded;
|
@@ -1319,7 +1472,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1319
1472
|
|
1320
1473
|
/* decode block header */
|
1321
1474
|
{ size_t const nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU;
|
1322
|
-
size_t const crcSize = dctx->frameInfo.blockChecksumFlag *
|
1475
|
+
size_t const crcSize = dctx->frameInfo.blockChecksumFlag * BFSize;
|
1323
1476
|
if (nextCBlockSize==0) { /* frameEnd signal, no more block */
|
1324
1477
|
dctx->dStage = dstage_getSuffix;
|
1325
1478
|
break;
|
@@ -1330,7 +1483,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1330
1483
|
/* next block is uncompressed */
|
1331
1484
|
dctx->tmpInTarget = nextCBlockSize;
|
1332
1485
|
if (dctx->frameInfo.blockChecksumFlag) {
|
1333
|
-
XXH32_reset(&dctx->blockChecksum, 0);
|
1486
|
+
(void)XXH32_reset(&dctx->blockChecksum, 0);
|
1334
1487
|
}
|
1335
1488
|
dctx->dStage = dstage_copyDirect;
|
1336
1489
|
break;
|
@@ -1339,7 +1492,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1339
1492
|
dctx->tmpInTarget = nextCBlockSize + crcSize;
|
1340
1493
|
dctx->dStage = dstage_getCBlock;
|
1341
1494
|
if (dstPtr==dstEnd) {
|
1342
|
-
nextSrcSizeHint =
|
1495
|
+
nextSrcSizeHint = BHSize + nextCBlockSize + crcSize;
|
1343
1496
|
doAnotherStage = 0;
|
1344
1497
|
}
|
1345
1498
|
break;
|
@@ -1350,10 +1503,10 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1350
1503
|
size_t const sizeToCopy = MIN(dctx->tmpInTarget, minBuffSize);
|
1351
1504
|
memcpy(dstPtr, srcPtr, sizeToCopy);
|
1352
1505
|
if (dctx->frameInfo.blockChecksumFlag) {
|
1353
|
-
XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy);
|
1506
|
+
(void)XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy);
|
1354
1507
|
}
|
1355
1508
|
if (dctx->frameInfo.contentChecksumFlag)
|
1356
|
-
XXH32_update(&dctx->xxh, srcPtr, sizeToCopy);
|
1509
|
+
(void)XXH32_update(&dctx->xxh, srcPtr, sizeToCopy);
|
1357
1510
|
if (dctx->frameInfo.contentSize)
|
1358
1511
|
dctx->frameRemainingSize -= sizeToCopy;
|
1359
1512
|
|
@@ -1373,7 +1526,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1373
1526
|
}
|
1374
1527
|
dctx->tmpInTarget -= sizeToCopy; /* need to copy more */
|
1375
1528
|
nextSrcSizeHint = dctx->tmpInTarget +
|
1376
|
-
+
|
1529
|
+
+(dctx->frameInfo.blockChecksumFlag ? BFSize : 0)
|
1377
1530
|
+ BHSize /* next header size */;
|
1378
1531
|
doAnotherStage = 0;
|
1379
1532
|
break;
|
@@ -1401,8 +1554,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1401
1554
|
U32 const calcCRC = XXH32_digest(&dctx->blockChecksum);
|
1402
1555
|
if (readCRC != calcCRC)
|
1403
1556
|
return err0r(LZ4F_ERROR_blockChecksum_invalid);
|
1404
|
-
|
1405
|
-
}
|
1557
|
+
} }
|
1406
1558
|
dctx->dStage = dstage_getBlockHeader; /* new block */
|
1407
1559
|
break;
|
1408
1560
|
|
@@ -1415,9 +1567,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1415
1567
|
/* input large enough to read full block directly */
|
1416
1568
|
selectedIn = srcPtr;
|
1417
1569
|
srcPtr += dctx->tmpInTarget;
|
1418
|
-
dctx->dStage = dstage_decodeCBlock;
|
1419
|
-
break;
|
1420
1570
|
|
1571
|
+
if (0) /* jump over next block */
|
1421
1572
|
case dstage_storeCBlock:
|
1422
1573
|
{ size_t const wantedData = dctx->tmpInTarget - dctx->tmpInSize;
|
1423
1574
|
size_t const inputLeft = (size_t)(srcEnd-srcPtr);
|
@@ -1426,51 +1577,54 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1426
1577
|
dctx->tmpInSize += sizeToCopy;
|
1427
1578
|
srcPtr += sizeToCopy;
|
1428
1579
|
if (dctx->tmpInSize < dctx->tmpInTarget) { /* need more input */
|
1429
|
-
nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize)
|
1430
|
-
|
1580
|
+
nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize)
|
1581
|
+
+ (dctx->frameInfo.blockChecksumFlag ? BFSize : 0)
|
1582
|
+
+ BHSize /* next header size */;
|
1583
|
+
doAnotherStage = 0;
|
1431
1584
|
break;
|
1432
1585
|
}
|
1433
1586
|
selectedIn = dctx->tmpIn;
|
1434
|
-
dctx->dStage = dstage_decodeCBlock;
|
1435
1587
|
}
|
1436
|
-
/* fall-through */
|
1437
1588
|
|
1438
|
-
|
1439
|
-
case dstage_decodeCBlock:
|
1589
|
+
/* At this stage, input is large enough to decode a block */
|
1440
1590
|
if (dctx->frameInfo.blockChecksumFlag) {
|
1441
1591
|
dctx->tmpInTarget -= 4;
|
1592
|
+
assert(selectedIn != NULL); /* selectedIn is defined at this stage (either srcPtr, or dctx->tmpIn) */
|
1442
1593
|
{ U32 const readBlockCrc = LZ4F_readLE32(selectedIn + dctx->tmpInTarget);
|
1443
1594
|
U32 const calcBlockCrc = XXH32(selectedIn, dctx->tmpInTarget, 0);
|
1444
1595
|
if (readBlockCrc != calcBlockCrc)
|
1445
1596
|
return err0r(LZ4F_ERROR_blockChecksum_invalid);
|
1446
1597
|
} }
|
1447
|
-
if ((size_t)(dstEnd-dstPtr) < dctx->maxBlockSize) /* not enough place into dst : decode into tmpOut */
|
1448
|
-
dctx->dStage = dstage_decodeCBlock_intoTmp;
|
1449
|
-
else
|
1450
|
-
dctx->dStage = dstage_decodeCBlock_intoDst;
|
1451
|
-
break;
|
1452
1598
|
|
1453
|
-
|
1454
|
-
|
1599
|
+
if ((size_t)(dstEnd-dstPtr) >= dctx->maxBlockSize) {
|
1600
|
+
const char* dict = (const char*)dctx->dict;
|
1601
|
+
size_t dictSize = dctx->dictSize;
|
1602
|
+
int decodedSize;
|
1603
|
+
if (dict && dictSize > 1 GB) {
|
1604
|
+
/* the dictSize param is an int, avoid truncation / sign issues */
|
1605
|
+
dict += dictSize - 64 KB;
|
1606
|
+
dictSize = 64 KB;
|
1607
|
+
}
|
1608
|
+
/* enough capacity in `dst` to decompress directly there */
|
1609
|
+
decodedSize = LZ4_decompress_safe_usingDict(
|
1455
1610
|
(const char*)selectedIn, (char*)dstPtr,
|
1456
1611
|
(int)dctx->tmpInTarget, (int)dctx->maxBlockSize,
|
1457
|
-
|
1612
|
+
dict, (int)dictSize);
|
1458
1613
|
if (decodedSize < 0) return err0r(LZ4F_ERROR_GENERIC); /* decompression failed */
|
1459
1614
|
if (dctx->frameInfo.contentChecksumFlag)
|
1460
|
-
XXH32_update(&(dctx->xxh), dstPtr, decodedSize);
|
1615
|
+
XXH32_update(&(dctx->xxh), dstPtr, (size_t)decodedSize);
|
1461
1616
|
if (dctx->frameInfo.contentSize)
|
1462
|
-
dctx->frameRemainingSize -= decodedSize;
|
1617
|
+
dctx->frameRemainingSize -= (size_t)decodedSize;
|
1463
1618
|
|
1464
1619
|
/* dictionary management */
|
1465
1620
|
if (dctx->frameInfo.blockMode==LZ4F_blockLinked)
|
1466
|
-
LZ4F_updateDict(dctx, dstPtr, decodedSize, dstStart, 0);
|
1621
|
+
LZ4F_updateDict(dctx, dstPtr, (size_t)decodedSize, dstStart, 0);
|
1467
1622
|
|
1468
1623
|
dstPtr += decodedSize;
|
1469
1624
|
dctx->dStage = dstage_getBlockHeader;
|
1470
1625
|
break;
|
1471
1626
|
}
|
1472
1627
|
|
1473
|
-
case dstage_decodeCBlock_intoTmp:
|
1474
1628
|
/* not enough place into dst : decode into tmpOut */
|
1475
1629
|
/* ensure enough place for tmpOut */
|
1476
1630
|
if (dctx->frameInfo.blockMode == LZ4F_blockLinked) {
|
@@ -1483,21 +1637,28 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1483
1637
|
} else { /* dict not within tmp */
|
1484
1638
|
size_t const reservedDictSpace = MIN(dctx->dictSize, 64 KB);
|
1485
1639
|
dctx->tmpOut = dctx->tmpOutBuffer + reservedDictSpace;
|
1486
|
-
|
1487
|
-
}
|
1640
|
+
} }
|
1488
1641
|
|
1489
1642
|
/* Decode block */
|
1490
|
-
{
|
1643
|
+
{ const char* dict = (const char*)dctx->dict;
|
1644
|
+
size_t dictSize = dctx->dictSize;
|
1645
|
+
int decodedSize;
|
1646
|
+
if (dict && dictSize > 1 GB) {
|
1647
|
+
/* the dictSize param is an int, avoid truncation / sign issues */
|
1648
|
+
dict += dictSize - 64 KB;
|
1649
|
+
dictSize = 64 KB;
|
1650
|
+
}
|
1651
|
+
decodedSize = LZ4_decompress_safe_usingDict(
|
1491
1652
|
(const char*)selectedIn, (char*)dctx->tmpOut,
|
1492
1653
|
(int)dctx->tmpInTarget, (int)dctx->maxBlockSize,
|
1493
|
-
|
1654
|
+
dict, (int)dictSize);
|
1494
1655
|
if (decodedSize < 0) /* decompression failed */
|
1495
1656
|
return err0r(LZ4F_ERROR_decompressionFailed);
|
1496
1657
|
if (dctx->frameInfo.contentChecksumFlag)
|
1497
|
-
XXH32_update(&(dctx->xxh), dctx->tmpOut, decodedSize);
|
1658
|
+
XXH32_update(&(dctx->xxh), dctx->tmpOut, (size_t)decodedSize);
|
1498
1659
|
if (dctx->frameInfo.contentSize)
|
1499
|
-
dctx->frameRemainingSize -= decodedSize;
|
1500
|
-
dctx->tmpOutSize = decodedSize;
|
1660
|
+
dctx->frameRemainingSize -= (size_t)decodedSize;
|
1661
|
+
dctx->tmpOutSize = (size_t)decodedSize;
|
1501
1662
|
dctx->tmpOutStart = 0;
|
1502
1663
|
dctx->dStage = dstage_flushOut;
|
1503
1664
|
}
|
@@ -1508,8 +1669,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1508
1669
|
memcpy(dstPtr, dctx->tmpOut + dctx->tmpOutStart, sizeToCopy);
|
1509
1670
|
|
1510
1671
|
/* dictionary management */
|
1511
|
-
if (dctx->frameInfo.blockMode==LZ4F_blockLinked)
|
1512
|
-
LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 1);
|
1672
|
+
if (dctx->frameInfo.blockMode == LZ4F_blockLinked)
|
1673
|
+
LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 1 /*withinTmp*/);
|
1513
1674
|
|
1514
1675
|
dctx->tmpOutStart += sizeToCopy;
|
1515
1676
|
dstPtr += sizeToCopy;
|
@@ -1518,8 +1679,9 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1518
1679
|
dctx->dStage = dstage_getBlockHeader; /* get next block */
|
1519
1680
|
break;
|
1520
1681
|
}
|
1682
|
+
/* could not flush everything : stop there, just request a block header */
|
1683
|
+
doAnotherStage = 0;
|
1521
1684
|
nextSrcSizeHint = BHSize;
|
1522
|
-
doAnotherStage = 0; /* still some data to flush */
|
1523
1685
|
break;
|
1524
1686
|
}
|
1525
1687
|
|
@@ -1556,7 +1718,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1556
1718
|
selectedIn = dctx->tmpIn;
|
1557
1719
|
} /* if (dctx->dStage == dstage_storeSuffix) */
|
1558
1720
|
|
1559
|
-
/* case dstage_checkSuffix: */ /* no direct
|
1721
|
+
/* case dstage_checkSuffix: */ /* no direct entry, avoid initialization risks */
|
1560
1722
|
{ U32 const readCRC = LZ4F_readLE32(selectedIn);
|
1561
1723
|
U32 const resultCRC = XXH32_digest(&(dctx->xxh));
|
1562
1724
|
if (readCRC != resultCRC)
|
@@ -1580,8 +1742,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1580
1742
|
|
1581
1743
|
if (dctx->dStage == dstage_storeSFrameSize)
|
1582
1744
|
case dstage_storeSFrameSize:
|
1583
|
-
{
|
1584
|
-
size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize,
|
1745
|
+
{ size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize,
|
1585
1746
|
(size_t)(srcEnd - srcPtr) );
|
1586
1747
|
memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy);
|
1587
1748
|
srcPtr += sizeToCopy;
|
@@ -1595,7 +1756,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1595
1756
|
selectedIn = dctx->header + 4;
|
1596
1757
|
} /* if (dctx->dStage == dstage_storeSFrameSize) */
|
1597
1758
|
|
1598
|
-
/* case dstage_decodeSFrameSize: */ /* no direct
|
1759
|
+
/* case dstage_decodeSFrameSize: */ /* no direct entry */
|
1599
1760
|
{ size_t const SFrameSize = LZ4F_readLE32(selectedIn);
|
1600
1761
|
dctx->frameInfo.contentSize = SFrameSize;
|
1601
1762
|
dctx->tmpInTarget = SFrameSize;
|
@@ -1614,7 +1775,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1614
1775
|
LZ4F_resetDecompressionContext(dctx);
|
1615
1776
|
break;
|
1616
1777
|
}
|
1617
|
-
}
|
1778
|
+
} /* switch (dctx->dStage) */
|
1618
1779
|
} /* while (doAnotherStage) */
|
1619
1780
|
|
1620
1781
|
/* preserve history within tmp whenever necessary */
|
@@ -1625,7 +1786,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1625
1786
|
&& ((unsigned)(dctx->dStage)-2 < (unsigned)(dstage_getSuffix)-2) ) /* valid stages : [init ... getSuffix[ */
|
1626
1787
|
{
|
1627
1788
|
if (dctx->dStage == dstage_flushOut) {
|
1628
|
-
size_t const preserveSize = dctx->tmpOut - dctx->tmpOutBuffer;
|
1789
|
+
size_t const preserveSize = (size_t)(dctx->tmpOut - dctx->tmpOutBuffer);
|
1629
1790
|
size_t copySize = 64 KB - dctx->tmpOutSize;
|
1630
1791
|
const BYTE* oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart;
|
1631
1792
|
if (dctx->tmpOutSize > 64 KB) copySize = 0;
|
@@ -1649,8 +1810,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1649
1810
|
}
|
1650
1811
|
}
|
1651
1812
|
|
1652
|
-
*srcSizePtr = (srcPtr - srcStart);
|
1653
|
-
*dstSizePtr = (dstPtr - dstStart);
|
1813
|
+
*srcSizePtr = (size_t)(srcPtr - srcStart);
|
1814
|
+
*dstSizePtr = (size_t)(dstPtr - dstStart);
|
1654
1815
|
return nextSrcSizeHint;
|
1655
1816
|
}
|
1656
1817
|
|