isomorfeus-ferret 0.17.2 → 0.17.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/ext/isomorfeus_ferret_ext/benchmark.c +9 -20
- data/ext/isomorfeus_ferret_ext/benchmarks_all.h +1 -2
- data/ext/isomorfeus_ferret_ext/bm_hash.c +1 -2
- data/ext/isomorfeus_ferret_ext/brotli_dec_decode.c +4 -2
- data/ext/isomorfeus_ferret_ext/brotli_enc_encode.c +3 -2
- data/ext/isomorfeus_ferret_ext/frb_analysis.c +4 -5
- data/ext/isomorfeus_ferret_ext/frb_field_info.c +3 -4
- data/ext/isomorfeus_ferret_ext/frb_index.c +118 -125
- data/ext/isomorfeus_ferret_ext/frb_lazy_doc.c +14 -16
- data/ext/isomorfeus_ferret_ext/frb_search.c +31 -23
- data/ext/isomorfeus_ferret_ext/frb_store.c +27 -13
- data/ext/isomorfeus_ferret_ext/frb_utils.c +3 -6
- data/ext/isomorfeus_ferret_ext/frt_analysis.c +39 -46
- data/ext/isomorfeus_ferret_ext/frt_analysis.h +9 -9
- data/ext/isomorfeus_ferret_ext/frt_array.c +11 -22
- data/ext/isomorfeus_ferret_ext/frt_bitvector.h +3 -6
- data/ext/isomorfeus_ferret_ext/frt_doc_field.c +87 -0
- data/ext/isomorfeus_ferret_ext/frt_doc_field.h +26 -0
- data/ext/isomorfeus_ferret_ext/frt_document.c +4 -97
- data/ext/isomorfeus_ferret_ext/frt_document.h +2 -27
- data/ext/isomorfeus_ferret_ext/frt_except.c +8 -6
- data/ext/isomorfeus_ferret_ext/frt_except.h +1 -2
- data/ext/isomorfeus_ferret_ext/frt_field_index.c +13 -32
- data/ext/isomorfeus_ferret_ext/frt_field_index.h +0 -6
- data/ext/isomorfeus_ferret_ext/frt_field_info.c +69 -0
- data/ext/isomorfeus_ferret_ext/frt_field_info.h +49 -0
- data/ext/isomorfeus_ferret_ext/frt_field_infos.c +196 -0
- data/ext/isomorfeus_ferret_ext/frt_field_infos.h +35 -0
- data/ext/isomorfeus_ferret_ext/frt_global.c +10 -4
- data/ext/isomorfeus_ferret_ext/frt_global.h +11 -15
- data/ext/isomorfeus_ferret_ext/frt_hash.c +8 -8
- data/ext/isomorfeus_ferret_ext/frt_hash.h +1 -2
- data/ext/isomorfeus_ferret_ext/frt_hashset.c +20 -40
- data/ext/isomorfeus_ferret_ext/frt_hashset.h +1 -2
- data/ext/isomorfeus_ferret_ext/frt_helper.c +7 -15
- data/ext/isomorfeus_ferret_ext/frt_in_stream.c +35 -45
- data/ext/isomorfeus_ferret_ext/frt_in_stream.h +3 -2
- data/ext/isomorfeus_ferret_ext/frt_ind.c +20 -38
- data/ext/isomorfeus_ferret_ext/frt_index.c +292 -790
- data/ext/isomorfeus_ferret_ext/frt_index.h +1 -102
- data/ext/isomorfeus_ferret_ext/frt_lang.c +5 -10
- data/ext/isomorfeus_ferret_ext/frt_lazy_doc.c +3 -3
- data/ext/isomorfeus_ferret_ext/frt_lazy_doc.h +1 -1
- data/ext/isomorfeus_ferret_ext/frt_lazy_doc_field.c +18 -25
- data/ext/isomorfeus_ferret_ext/frt_lazy_doc_field.h +5 -5
- data/ext/isomorfeus_ferret_ext/frt_mdbx_store.c +102 -70
- data/ext/isomorfeus_ferret_ext/frt_mempool.c +8 -16
- data/ext/isomorfeus_ferret_ext/frt_multimapper.c +23 -46
- data/ext/isomorfeus_ferret_ext/frt_multimapper.h +4 -8
- data/ext/isomorfeus_ferret_ext/frt_out_stream.c +31 -43
- data/ext/isomorfeus_ferret_ext/frt_out_stream.h +2 -2
- data/ext/isomorfeus_ferret_ext/frt_posh.c +6 -819
- data/ext/isomorfeus_ferret_ext/frt_posh.h +0 -57
- data/ext/isomorfeus_ferret_ext/frt_priorityqueue.c +11 -22
- data/ext/isomorfeus_ferret_ext/frt_priorityqueue.h +1 -2
- data/ext/isomorfeus_ferret_ext/frt_q_boolean.c +85 -171
- data/ext/isomorfeus_ferret_ext/frt_q_match_all.c +8 -16
- data/ext/isomorfeus_ferret_ext/frt_q_multi_term.c +1 -2
- data/ext/isomorfeus_ferret_ext/frt_q_parser.c +49 -98
- data/ext/isomorfeus_ferret_ext/frt_q_phrase.c +52 -104
- data/ext/isomorfeus_ferret_ext/frt_q_range.c +6 -12
- data/ext/isomorfeus_ferret_ext/frt_q_span.c +113 -226
- data/ext/isomorfeus_ferret_ext/frt_q_wildcard.c +1 -2
- data/ext/isomorfeus_ferret_ext/frt_ram_store.c +134 -85
- data/ext/isomorfeus_ferret_ext/frt_search.c +82 -164
- data/ext/isomorfeus_ferret_ext/frt_similarity.c +11 -22
- data/ext/isomorfeus_ferret_ext/frt_similarity.h +1 -2
- data/ext/isomorfeus_ferret_ext/frt_store.c +13 -25
- data/ext/isomorfeus_ferret_ext/frt_store.h +86 -52
- data/ext/isomorfeus_ferret_ext/frt_term_vectors.c +8 -16
- data/ext/isomorfeus_ferret_ext/frt_win32.h +5 -10
- data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.c +12 -11
- data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.h +11 -13
- data/ext/isomorfeus_ferret_ext/lz4.c +422 -195
- data/ext/isomorfeus_ferret_ext/lz4.h +114 -46
- data/ext/isomorfeus_ferret_ext/lz4frame.c +421 -242
- data/ext/isomorfeus_ferret_ext/lz4frame.h +122 -53
- data/ext/isomorfeus_ferret_ext/lz4hc.c +127 -111
- data/ext/isomorfeus_ferret_ext/lz4hc.h +14 -14
- data/ext/isomorfeus_ferret_ext/lz4xxhash.h +1 -1
- data/ext/isomorfeus_ferret_ext/mdbx.c +3762 -2526
- data/ext/isomorfeus_ferret_ext/mdbx.h +115 -70
- data/ext/isomorfeus_ferret_ext/test.c +40 -87
- data/ext/isomorfeus_ferret_ext/test.h +3 -6
- data/ext/isomorfeus_ferret_ext/test_1710.c +11 -13
- data/ext/isomorfeus_ferret_ext/test_analysis.c +32 -64
- data/ext/isomorfeus_ferret_ext/test_array.c +6 -12
- data/ext/isomorfeus_ferret_ext/test_bitvector.c +12 -24
- data/ext/isomorfeus_ferret_ext/test_document.c +23 -33
- data/ext/isomorfeus_ferret_ext/test_except.c +10 -21
- data/ext/isomorfeus_ferret_ext/test_fields.c +62 -68
- data/ext/isomorfeus_ferret_ext/test_file_deleter.c +15 -23
- data/ext/isomorfeus_ferret_ext/test_filter.c +17 -27
- data/ext/isomorfeus_ferret_ext/test_global.c +14 -29
- data/ext/isomorfeus_ferret_ext/test_hash.c +19 -38
- data/ext/isomorfeus_ferret_ext/test_hashset.c +8 -16
- data/ext/isomorfeus_ferret_ext/test_helper.c +4 -8
- data/ext/isomorfeus_ferret_ext/test_highlighter.c +16 -28
- data/ext/isomorfeus_ferret_ext/test_index.c +277 -487
- data/ext/isomorfeus_ferret_ext/test_lang.c +7 -14
- data/ext/isomorfeus_ferret_ext/test_mdbx_store.c +2 -5
- data/ext/isomorfeus_ferret_ext/test_mempool.c +5 -10
- data/ext/isomorfeus_ferret_ext/test_multimapper.c +3 -6
- data/ext/isomorfeus_ferret_ext/test_priorityqueue.c +9 -18
- data/ext/isomorfeus_ferret_ext/test_q_const_score.c +4 -6
- data/ext/isomorfeus_ferret_ext/test_q_filtered.c +3 -4
- data/ext/isomorfeus_ferret_ext/test_q_fuzzy.c +9 -15
- data/ext/isomorfeus_ferret_ext/test_q_parser.c +8 -16
- data/ext/isomorfeus_ferret_ext/test_q_span.c +19 -35
- data/ext/isomorfeus_ferret_ext/test_ram_store.c +14 -13
- data/ext/isomorfeus_ferret_ext/test_search.c +60 -109
- data/ext/isomorfeus_ferret_ext/test_segments.c +8 -13
- data/ext/isomorfeus_ferret_ext/test_similarity.c +2 -4
- data/ext/isomorfeus_ferret_ext/test_sort.c +14 -24
- data/ext/isomorfeus_ferret_ext/test_store.c +96 -115
- data/ext/isomorfeus_ferret_ext/test_term.c +9 -15
- data/ext/isomorfeus_ferret_ext/test_term_vectors.c +9 -14
- data/ext/isomorfeus_ferret_ext/test_test.c +4 -8
- data/ext/isomorfeus_ferret_ext/test_threading.c +14 -20
- data/ext/isomorfeus_ferret_ext/testhelper.c +11 -21
- data/ext/isomorfeus_ferret_ext/testhelper.h +1 -1
- data/ext/isomorfeus_ferret_ext/tests_all.h +1 -2
- data/lib/isomorfeus/ferret/index/index.rb +1 -1
- data/lib/isomorfeus/ferret/version.rb +1 -1
- metadata +24 -4
@@ -45,7 +45,7 @@
|
|
45
45
|
* Compiler Options
|
46
46
|
**************************************/
|
47
47
|
#ifdef _MSC_VER /* Visual Studio */
|
48
|
-
# pragma warning(disable : 4127)
|
48
|
+
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
49
49
|
#endif
|
50
50
|
|
51
51
|
|
@@ -62,6 +62,19 @@
|
|
62
62
|
#endif
|
63
63
|
|
64
64
|
|
65
|
+
/*-************************************
|
66
|
+
* Library declarations
|
67
|
+
**************************************/
|
68
|
+
#define LZ4F_STATIC_LINKING_ONLY
|
69
|
+
#include "lz4frame.h"
|
70
|
+
#define LZ4_STATIC_LINKING_ONLY
|
71
|
+
#include "lz4.h"
|
72
|
+
#define LZ4_HC_STATIC_LINKING_ONLY
|
73
|
+
#include "lz4hc.h"
|
74
|
+
#define XXH_STATIC_LINKING_ONLY
|
75
|
+
#include "lz4xxhash.h"
|
76
|
+
|
77
|
+
|
65
78
|
/*-************************************
|
66
79
|
* Memory routines
|
67
80
|
**************************************/
|
@@ -70,7 +83,13 @@
|
|
70
83
|
* malloc(), calloc() and free()
|
71
84
|
* towards another library or solution of their choice
|
72
85
|
* by modifying below section.
|
73
|
-
|
86
|
+
**/
|
87
|
+
|
88
|
+
#include <string.h> /* memset, memcpy, memmove */
|
89
|
+
#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */
|
90
|
+
# define MEM_INIT(p,v,s) memset((p),(v),(s))
|
91
|
+
#endif
|
92
|
+
|
74
93
|
#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */
|
75
94
|
# include <stdlib.h> /* malloc, calloc, free */
|
76
95
|
# define ALLOC(s) malloc(s)
|
@@ -78,23 +97,42 @@
|
|
78
97
|
# define FREEMEM(p) free(p)
|
79
98
|
#endif
|
80
99
|
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
100
|
+
static void* LZ4F_calloc(size_t s, LZ4F_CustomMem cmem)
|
101
|
+
{
|
102
|
+
/* custom calloc defined : use it */
|
103
|
+
if (cmem.customCalloc != NULL) {
|
104
|
+
return cmem.customCalloc(cmem.opaqueState, s);
|
105
|
+
}
|
106
|
+
/* nothing defined : use default <stdlib.h>'s calloc() */
|
107
|
+
if (cmem.customAlloc == NULL) {
|
108
|
+
return ALLOC_AND_ZERO(s);
|
109
|
+
}
|
110
|
+
/* only custom alloc defined : use it, and combine it with memset() */
|
111
|
+
{ void* const p = cmem.customAlloc(cmem.opaqueState, s);
|
112
|
+
if (p != NULL) MEM_INIT(p, 0, s);
|
113
|
+
return p;
|
114
|
+
} }
|
85
115
|
|
116
|
+
static void* LZ4F_malloc(size_t s, LZ4F_CustomMem cmem)
|
117
|
+
{
|
118
|
+
/* custom malloc defined : use it */
|
119
|
+
if (cmem.customAlloc != NULL) {
|
120
|
+
return cmem.customAlloc(cmem.opaqueState, s);
|
121
|
+
}
|
122
|
+
/* nothing defined : use default <stdlib.h>'s malloc() */
|
123
|
+
return ALLOC(s);
|
124
|
+
}
|
86
125
|
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
#include "lz4xxhash.h"
|
126
|
+
static void LZ4F_free(void* p, LZ4F_CustomMem cmem)
|
127
|
+
{
|
128
|
+
/* custom malloc defined : use it */
|
129
|
+
if (cmem.customFree != NULL) {
|
130
|
+
cmem.customFree(cmem.opaqueState, p);
|
131
|
+
return;
|
132
|
+
}
|
133
|
+
/* nothing defined : use default <stdlib.h>'s free() */
|
134
|
+
FREEMEM(p);
|
135
|
+
}
|
98
136
|
|
99
137
|
|
100
138
|
/*-************************************
|
@@ -143,7 +181,7 @@ static int g_debuglog_enable = 1;
|
|
143
181
|
#endif
|
144
182
|
|
145
183
|
|
146
|
-
/* unoptimized version; solves
|
184
|
+
/* unoptimized version; solves endianness & alignment issues */
|
147
185
|
static U32 LZ4F_readLE32 (const void* src)
|
148
186
|
{
|
149
187
|
const BYTE* const srcPtr = (const BYTE*)src;
|
@@ -206,8 +244,6 @@ static void LZ4F_writeLE64 (void* dst, U64 value64)
|
|
206
244
|
#define _4BITS 0x0F
|
207
245
|
#define _8BITS 0xFF
|
208
246
|
|
209
|
-
#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U
|
210
|
-
#define LZ4F_MAGICNUMBER 0x184D2204U
|
211
247
|
#define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U
|
212
248
|
#define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB
|
213
249
|
|
@@ -220,22 +256,27 @@ static const size_t BFSize = LZ4F_BLOCK_CHECKSUM_SIZE; /* block footer : checks
|
|
220
256
|
/*-************************************
|
221
257
|
* Structures and local types
|
222
258
|
**************************************/
|
259
|
+
|
260
|
+
typedef enum { LZ4B_COMPRESSED, LZ4B_UNCOMPRESSED} LZ4F_blockCompression_t;
|
261
|
+
|
223
262
|
typedef struct LZ4F_cctx_s
|
224
263
|
{
|
264
|
+
LZ4F_CustomMem cmem;
|
225
265
|
LZ4F_preferences_t prefs;
|
226
266
|
U32 version;
|
227
267
|
U32 cStage;
|
228
268
|
const LZ4F_CDict* cdict;
|
229
269
|
size_t maxBlockSize;
|
230
270
|
size_t maxBufferSize;
|
231
|
-
BYTE* tmpBuff;
|
232
|
-
BYTE* tmpIn;
|
233
|
-
size_t tmpInSize;
|
271
|
+
BYTE* tmpBuff; /* internal buffer, for streaming */
|
272
|
+
BYTE* tmpIn; /* starting position of data compress within internal buffer (>= tmpBuff) */
|
273
|
+
size_t tmpInSize; /* amount of data to compress after tmpIn */
|
234
274
|
U64 totalInSize;
|
235
275
|
XXH32_state_t xxh;
|
236
276
|
void* lz4CtxPtr;
|
237
277
|
U16 lz4CtxAlloc; /* sized for: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */
|
238
278
|
U16 lz4CtxState; /* in use as: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */
|
279
|
+
LZ4F_blockCompression_t blockCompression;
|
239
280
|
} LZ4F_cctx_t;
|
240
281
|
|
241
282
|
|
@@ -264,27 +305,33 @@ LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult)
|
|
264
305
|
return (LZ4F_errorCodes)(-(ptrdiff_t)functionResult);
|
265
306
|
}
|
266
307
|
|
267
|
-
static LZ4F_errorCode_t
|
308
|
+
static LZ4F_errorCode_t LZ4F_returnErrorCode(LZ4F_errorCodes code)
|
268
309
|
{
|
269
310
|
/* A compilation error here means sizeof(ptrdiff_t) is not large enough */
|
270
311
|
LZ4F_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t));
|
271
312
|
return (LZ4F_errorCode_t)-(ptrdiff_t)code;
|
272
313
|
}
|
273
314
|
|
315
|
+
#define RETURN_ERROR(e) return LZ4F_returnErrorCode(LZ4F_ERROR_ ## e)
|
316
|
+
|
317
|
+
#define RETURN_ERROR_IF(c,e) if (c) RETURN_ERROR(e)
|
318
|
+
|
319
|
+
#define FORWARD_IF_ERROR(r) if (LZ4F_isError(r)) return (r)
|
320
|
+
|
274
321
|
unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; }
|
275
322
|
|
276
323
|
int LZ4F_compressionLevel_max(void) { return LZ4HC_CLEVEL_MAX; }
|
277
324
|
|
278
|
-
size_t LZ4F_getBlockSize(
|
325
|
+
size_t LZ4F_getBlockSize(LZ4F_blockSizeID_t blockSizeID)
|
279
326
|
{
|
280
327
|
static const size_t blockSizes[4] = { 64 KB, 256 KB, 1 MB, 4 MB };
|
281
328
|
|
282
329
|
if (blockSizeID == 0) blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT;
|
283
330
|
if (blockSizeID < LZ4F_max64KB || blockSizeID > LZ4F_max4MB)
|
284
|
-
|
285
|
-
blockSizeID
|
286
|
-
|
287
|
-
}
|
331
|
+
RETURN_ERROR(maxBlockSize_invalid);
|
332
|
+
{ int const blockSizeIdx = (int)blockSizeID - (int)LZ4F_max64KB;
|
333
|
+
return blockSizes[blockSizeIdx];
|
334
|
+
} }
|
288
335
|
|
289
336
|
/*-************************************
|
290
337
|
* Private functions
|
@@ -397,21 +444,20 @@ size_t LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx,
|
|
397
444
|
MEM_INIT(&options, 0, sizeof(options));
|
398
445
|
options.stableSrc = 1;
|
399
446
|
|
400
|
-
|
401
|
-
return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
|
447
|
+
RETURN_ERROR_IF(dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs), dstMaxSize_tooSmall);
|
402
448
|
|
403
449
|
{ size_t const headerSize = LZ4F_compressBegin_usingCDict(cctx, dstBuffer, dstCapacity, cdict, &prefs); /* write header */
|
404
|
-
|
450
|
+
FORWARD_IF_ERROR(headerSize);
|
405
451
|
dstPtr += headerSize; /* header size */ }
|
406
452
|
|
407
453
|
assert(dstEnd >= dstPtr);
|
408
454
|
{ size_t const cSize = LZ4F_compressUpdate(cctx, dstPtr, (size_t)(dstEnd-dstPtr), srcBuffer, srcSize, &options);
|
409
|
-
|
455
|
+
FORWARD_IF_ERROR(cSize);
|
410
456
|
dstPtr += cSize; }
|
411
457
|
|
412
458
|
assert(dstEnd >= dstPtr);
|
413
459
|
{ size_t const tailSize = LZ4F_compressEnd(cctx, dstPtr, (size_t)(dstEnd-dstPtr), &options); /* flush last block, and generate suffix */
|
414
|
-
|
460
|
+
FORWARD_IF_ERROR(tailSize);
|
415
461
|
dstPtr += tailSize; }
|
416
462
|
|
417
463
|
assert(dstEnd >= dstStart);
|
@@ -432,27 +478,26 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
|
|
432
478
|
{
|
433
479
|
size_t result;
|
434
480
|
#if (LZ4F_HEAPMODE)
|
435
|
-
LZ4F_cctx_t
|
481
|
+
LZ4F_cctx_t* cctxPtr;
|
436
482
|
result = LZ4F_createCompressionContext(&cctxPtr, LZ4F_VERSION);
|
437
|
-
|
483
|
+
FORWARD_IF_ERROR(result);
|
438
484
|
#else
|
439
485
|
LZ4F_cctx_t cctx;
|
440
486
|
LZ4_stream_t lz4ctx;
|
441
|
-
LZ4F_cctx_t
|
487
|
+
LZ4F_cctx_t* const cctxPtr = &cctx;
|
442
488
|
|
443
|
-
DEBUGLOG(4, "LZ4F_compressFrame");
|
444
489
|
MEM_INIT(&cctx, 0, sizeof(cctx));
|
445
490
|
cctx.version = LZ4F_VERSION;
|
446
491
|
cctx.maxBufferSize = 5 MB; /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */
|
447
|
-
if (preferencesPtr == NULL
|
448
|
-
|
449
|
-
{
|
492
|
+
if ( preferencesPtr == NULL
|
493
|
+
|| preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN ) {
|
450
494
|
LZ4_initStream(&lz4ctx, sizeof(lz4ctx));
|
451
495
|
cctxPtr->lz4CtxPtr = &lz4ctx;
|
452
496
|
cctxPtr->lz4CtxAlloc = 1;
|
453
497
|
cctxPtr->lz4CtxState = 1;
|
454
498
|
}
|
455
499
|
#endif
|
500
|
+
DEBUGLOG(4, "LZ4F_compressFrame");
|
456
501
|
|
457
502
|
result = LZ4F_compressFrame_usingCDict(cctxPtr, dstBuffer, dstCapacity,
|
458
503
|
srcBuffer, srcSize,
|
@@ -461,10 +506,9 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
|
|
461
506
|
#if (LZ4F_HEAPMODE)
|
462
507
|
LZ4F_freeCompressionContext(cctxPtr);
|
463
508
|
#else
|
464
|
-
if (preferencesPtr != NULL
|
465
|
-
|
466
|
-
|
467
|
-
FREEMEM(cctxPtr->lz4CtxPtr);
|
509
|
+
if ( preferencesPtr != NULL
|
510
|
+
&& preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN ) {
|
511
|
+
LZ4F_free(cctxPtr->lz4CtxPtr, cctxPtr->cmem);
|
468
512
|
}
|
469
513
|
#endif
|
470
514
|
return result;
|
@@ -476,30 +520,31 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
|
|
476
520
|
*****************************************************/
|
477
521
|
|
478
522
|
struct LZ4F_CDict_s {
|
523
|
+
LZ4F_CustomMem cmem;
|
479
524
|
void* dictContent;
|
480
525
|
LZ4_stream_t* fastCtx;
|
481
526
|
LZ4_streamHC_t* HCCtx;
|
482
527
|
}; /* typedef'd to LZ4F_CDict within lz4frame_static.h */
|
483
528
|
|
484
|
-
|
485
|
-
|
486
|
-
* LZ4F_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
|
487
|
-
* LZ4F_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
|
488
|
-
* `dictBuffer` can be released after LZ4F_CDict creation, since its content is copied within CDict
|
489
|
-
* @return : digested dictionary for compression, or NULL if failed */
|
490
|
-
LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
|
529
|
+
LZ4F_CDict*
|
530
|
+
LZ4F_createCDict_advanced(LZ4F_CustomMem cmem, const void* dictBuffer, size_t dictSize)
|
491
531
|
{
|
492
532
|
const char* dictStart = (const char*)dictBuffer;
|
493
|
-
LZ4F_CDict* cdict = (LZ4F_CDict*)
|
494
|
-
DEBUGLOG(4, "
|
533
|
+
LZ4F_CDict* const cdict = (LZ4F_CDict*)LZ4F_malloc(sizeof(*cdict), cmem);
|
534
|
+
DEBUGLOG(4, "LZ4F_createCDict_advanced");
|
495
535
|
if (!cdict) return NULL;
|
536
|
+
cdict->cmem = cmem;
|
496
537
|
if (dictSize > 64 KB) {
|
497
538
|
dictStart += dictSize - 64 KB;
|
498
539
|
dictSize = 64 KB;
|
499
540
|
}
|
500
|
-
cdict->dictContent =
|
501
|
-
cdict->fastCtx =
|
502
|
-
cdict->
|
541
|
+
cdict->dictContent = LZ4F_malloc(dictSize, cmem);
|
542
|
+
cdict->fastCtx = (LZ4_stream_t*)LZ4F_malloc(sizeof(LZ4_stream_t), cmem);
|
543
|
+
if (cdict->fastCtx)
|
544
|
+
LZ4_initStream(cdict->fastCtx, sizeof(LZ4_stream_t));
|
545
|
+
cdict->HCCtx = (LZ4_streamHC_t*)LZ4F_malloc(sizeof(LZ4_streamHC_t), cmem);
|
546
|
+
if (cdict->HCCtx)
|
547
|
+
LZ4_initStream(cdict->HCCtx, sizeof(LZ4_streamHC_t));
|
503
548
|
if (!cdict->dictContent || !cdict->fastCtx || !cdict->HCCtx) {
|
504
549
|
LZ4F_freeCDict(cdict);
|
505
550
|
return NULL;
|
@@ -511,13 +556,25 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
|
|
511
556
|
return cdict;
|
512
557
|
}
|
513
558
|
|
559
|
+
/*! LZ4F_createCDict() :
|
560
|
+
* When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
|
561
|
+
* LZ4F_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
|
562
|
+
* LZ4F_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
|
563
|
+
* @dictBuffer can be released after LZ4F_CDict creation, since its content is copied within CDict
|
564
|
+
* @return : digested dictionary for compression, or NULL if failed */
|
565
|
+
LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
|
566
|
+
{
|
567
|
+
DEBUGLOG(4, "LZ4F_createCDict");
|
568
|
+
return LZ4F_createCDict_advanced(LZ4F_defaultCMem, dictBuffer, dictSize);
|
569
|
+
}
|
570
|
+
|
514
571
|
void LZ4F_freeCDict(LZ4F_CDict* cdict)
|
515
572
|
{
|
516
573
|
if (cdict==NULL) return; /* support free on NULL */
|
517
|
-
|
518
|
-
|
519
|
-
|
520
|
-
|
574
|
+
LZ4F_free(cdict->dictContent, cdict->cmem);
|
575
|
+
LZ4F_free(cdict->fastCtx, cdict->cmem);
|
576
|
+
LZ4F_free(cdict->HCCtx, cdict->cmem);
|
577
|
+
LZ4F_free(cdict, cdict->cmem);
|
521
578
|
}
|
522
579
|
|
523
580
|
|
@@ -525,6 +582,20 @@ void LZ4F_freeCDict(LZ4F_CDict* cdict)
|
|
525
582
|
* Advanced compression functions
|
526
583
|
***********************************/
|
527
584
|
|
585
|
+
LZ4F_cctx*
|
586
|
+
LZ4F_createCompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version)
|
587
|
+
{
|
588
|
+
LZ4F_cctx* const cctxPtr =
|
589
|
+
(LZ4F_cctx*)LZ4F_calloc(sizeof(LZ4F_cctx), customMem);
|
590
|
+
if (cctxPtr==NULL) return NULL;
|
591
|
+
|
592
|
+
cctxPtr->cmem = customMem;
|
593
|
+
cctxPtr->version = version;
|
594
|
+
cctxPtr->cStage = 0; /* Uninitialized. Next stage : init cctx */
|
595
|
+
|
596
|
+
return cctxPtr;
|
597
|
+
}
|
598
|
+
|
528
599
|
/*! LZ4F_createCompressionContext() :
|
529
600
|
* The first thing to do is to create a compressionContext object, which will be used in all compression operations.
|
530
601
|
* This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure.
|
@@ -532,17 +603,16 @@ void LZ4F_freeCDict(LZ4F_CDict* cdict)
|
|
532
603
|
* The function will provide a pointer to an allocated LZ4F_compressionContext_t object.
|
533
604
|
* If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation.
|
534
605
|
* Object can release its memory using LZ4F_freeCompressionContext();
|
535
|
-
|
536
|
-
LZ4F_errorCode_t
|
606
|
+
**/
|
607
|
+
LZ4F_errorCode_t
|
608
|
+
LZ4F_createCompressionContext(LZ4F_cctx** LZ4F_compressionContextPtr, unsigned version)
|
537
609
|
{
|
538
|
-
|
539
|
-
|
540
|
-
|
541
|
-
cctxPtr->version = version;
|
542
|
-
cctxPtr->cStage = 0; /* Next stage : init stream */
|
543
|
-
|
544
|
-
*LZ4F_compressionContextPtr = cctxPtr;
|
610
|
+
assert(LZ4F_compressionContextPtr != NULL); /* considered a violation of narrow contract */
|
611
|
+
/* in case it nonetheless happen in production */
|
612
|
+
RETURN_ERROR_IF(LZ4F_compressionContextPtr == NULL, parameter_null);
|
545
613
|
|
614
|
+
*LZ4F_compressionContextPtr = LZ4F_createCompressionContext_advanced(LZ4F_defaultCMem, version);
|
615
|
+
RETURN_ERROR_IF(*LZ4F_compressionContextPtr==NULL, allocation_failed);
|
546
616
|
return LZ4F_OK_NoError;
|
547
617
|
}
|
548
618
|
|
@@ -550,11 +620,10 @@ LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** LZ4F_compressionConte
|
|
550
620
|
LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctxPtr)
|
551
621
|
{
|
552
622
|
if (cctxPtr != NULL) { /* support free on NULL */
|
553
|
-
|
554
|
-
|
555
|
-
|
623
|
+
LZ4F_free(cctxPtr->lz4CtxPtr, cctxPtr->cmem); /* note: LZ4_streamHC_t and LZ4_stream_t are simple POD types */
|
624
|
+
LZ4F_free(cctxPtr->tmpBuff, cctxPtr->cmem);
|
625
|
+
LZ4F_free(cctxPtr, cctxPtr->cmem);
|
556
626
|
}
|
557
|
-
|
558
627
|
return LZ4F_OK_NoError;
|
559
628
|
}
|
560
629
|
|
@@ -588,11 +657,21 @@ static void LZ4F_initStream(void* ctx,
|
|
588
657
|
}
|
589
658
|
}
|
590
659
|
|
660
|
+
static int ctxTypeID_to_size(int ctxTypeID) {
|
661
|
+
switch(ctxTypeID) {
|
662
|
+
case 1:
|
663
|
+
return LZ4_sizeofState();
|
664
|
+
case 2:
|
665
|
+
return LZ4_sizeofStateHC();
|
666
|
+
default:
|
667
|
+
return 0;
|
668
|
+
}
|
669
|
+
}
|
591
670
|
|
592
671
|
/*! LZ4F_compressBegin_usingCDict() :
|
593
|
-
* init streaming compression
|
594
|
-
*
|
595
|
-
* @return : number of bytes written into dstBuffer for the header
|
672
|
+
* init streaming compression AND writes frame header into @dstBuffer.
|
673
|
+
* @dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
|
674
|
+
* @return : number of bytes written into @dstBuffer for the header
|
596
675
|
* or an error code (can be tested using LZ4F_isError())
|
597
676
|
*/
|
598
677
|
size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
|
@@ -600,41 +679,46 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
|
|
600
679
|
const LZ4F_CDict* cdict,
|
601
680
|
const LZ4F_preferences_t* preferencesPtr)
|
602
681
|
{
|
603
|
-
LZ4F_preferences_t prefNull;
|
682
|
+
LZ4F_preferences_t const prefNull = LZ4F_INIT_PREFERENCES;
|
604
683
|
BYTE* const dstStart = (BYTE*)dstBuffer;
|
605
684
|
BYTE* dstPtr = dstStart;
|
606
|
-
BYTE* headerStart;
|
607
685
|
|
608
|
-
|
609
|
-
MEM_INIT(&prefNull, 0, sizeof(prefNull));
|
686
|
+
RETURN_ERROR_IF(dstCapacity < maxFHSize, dstMaxSize_tooSmall);
|
610
687
|
if (preferencesPtr == NULL) preferencesPtr = &prefNull;
|
611
688
|
cctxPtr->prefs = *preferencesPtr;
|
612
689
|
|
613
|
-
/*
|
690
|
+
/* cctx Management */
|
614
691
|
{ U16 const ctxTypeID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2;
|
615
|
-
|
616
|
-
|
692
|
+
int requiredSize = ctxTypeID_to_size(ctxTypeID);
|
693
|
+
int allocatedSize = ctxTypeID_to_size(cctxPtr->lz4CtxAlloc);
|
694
|
+
if (allocatedSize < requiredSize) {
|
695
|
+
/* not enough space allocated */
|
696
|
+
LZ4F_free(cctxPtr->lz4CtxPtr, cctxPtr->cmem);
|
617
697
|
if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
|
618
|
-
|
698
|
+
/* must take ownership of memory allocation,
|
699
|
+
* in order to respect custom allocator contract */
|
700
|
+
cctxPtr->lz4CtxPtr = LZ4F_malloc(sizeof(LZ4_stream_t), cctxPtr->cmem);
|
701
|
+
if (cctxPtr->lz4CtxPtr)
|
702
|
+
LZ4_initStream(cctxPtr->lz4CtxPtr, sizeof(LZ4_stream_t));
|
619
703
|
} else {
|
620
|
-
cctxPtr->lz4CtxPtr =
|
704
|
+
cctxPtr->lz4CtxPtr = LZ4F_malloc(sizeof(LZ4_streamHC_t), cctxPtr->cmem);
|
705
|
+
if (cctxPtr->lz4CtxPtr)
|
706
|
+
LZ4_initStreamHC(cctxPtr->lz4CtxPtr, sizeof(LZ4_streamHC_t));
|
621
707
|
}
|
622
|
-
|
623
|
-
return err0r(LZ4F_ERROR_allocation_failed);
|
708
|
+
RETURN_ERROR_IF(cctxPtr->lz4CtxPtr == NULL, allocation_failed);
|
624
709
|
cctxPtr->lz4CtxAlloc = ctxTypeID;
|
625
710
|
cctxPtr->lz4CtxState = ctxTypeID;
|
626
711
|
} else if (cctxPtr->lz4CtxState != ctxTypeID) {
|
627
|
-
/* otherwise, a sufficient buffer is allocated,
|
628
|
-
* reset it to the correct context type */
|
712
|
+
/* otherwise, a sufficient buffer is already allocated,
|
713
|
+
* but we need to reset it to the correct context type */
|
629
714
|
if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
|
630
|
-
LZ4_initStream((LZ4_stream_t
|
715
|
+
LZ4_initStream((LZ4_stream_t*)cctxPtr->lz4CtxPtr, sizeof(LZ4_stream_t));
|
631
716
|
} else {
|
632
|
-
LZ4_initStreamHC((LZ4_streamHC_t
|
633
|
-
LZ4_setCompressionLevel((LZ4_streamHC_t
|
717
|
+
LZ4_initStreamHC((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, sizeof(LZ4_streamHC_t));
|
718
|
+
LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
|
634
719
|
}
|
635
720
|
cctxPtr->lz4CtxState = ctxTypeID;
|
636
|
-
|
637
|
-
}
|
721
|
+
} }
|
638
722
|
|
639
723
|
/* Buffer Management */
|
640
724
|
if (cctxPtr->prefs.frameInfo.blockSizeID == 0)
|
@@ -647,9 +731,9 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
|
|
647
731
|
|
648
732
|
if (cctxPtr->maxBufferSize < requiredBuffSize) {
|
649
733
|
cctxPtr->maxBufferSize = 0;
|
650
|
-
|
651
|
-
cctxPtr->tmpBuff = (BYTE*)
|
652
|
-
|
734
|
+
LZ4F_free(cctxPtr->tmpBuff, cctxPtr->cmem);
|
735
|
+
cctxPtr->tmpBuff = (BYTE*)LZ4F_calloc(requiredBuffSize, cctxPtr->cmem);
|
736
|
+
RETURN_ERROR_IF(cctxPtr->tmpBuff == NULL, allocation_failed);
|
653
737
|
cctxPtr->maxBufferSize = requiredBuffSize;
|
654
738
|
} }
|
655
739
|
cctxPtr->tmpIn = cctxPtr->tmpBuff;
|
@@ -669,31 +753,32 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
|
|
669
753
|
/* Magic Number */
|
670
754
|
LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER);
|
671
755
|
dstPtr += 4;
|
672
|
-
headerStart = dstPtr;
|
673
|
-
|
674
|
-
|
675
|
-
|
676
|
-
|
677
|
-
|
678
|
-
|
679
|
-
|
680
|
-
|
681
|
-
|
682
|
-
|
683
|
-
|
684
|
-
|
685
|
-
|
686
|
-
|
687
|
-
|
688
|
-
|
689
|
-
|
690
|
-
|
691
|
-
|
692
|
-
|
756
|
+
{ BYTE* const headerStart = dstPtr;
|
757
|
+
|
758
|
+
/* FLG Byte */
|
759
|
+
*dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */
|
760
|
+
+ ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5)
|
761
|
+
+ ((cctxPtr->prefs.frameInfo.blockChecksumFlag & _1BIT ) << 4)
|
762
|
+
+ ((unsigned)(cctxPtr->prefs.frameInfo.contentSize > 0) << 3)
|
763
|
+
+ ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2)
|
764
|
+
+ (cctxPtr->prefs.frameInfo.dictID > 0) );
|
765
|
+
/* BD Byte */
|
766
|
+
*dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4);
|
767
|
+
/* Optional Frame content size field */
|
768
|
+
if (cctxPtr->prefs.frameInfo.contentSize) {
|
769
|
+
LZ4F_writeLE64(dstPtr, cctxPtr->prefs.frameInfo.contentSize);
|
770
|
+
dstPtr += 8;
|
771
|
+
cctxPtr->totalInSize = 0;
|
772
|
+
}
|
773
|
+
/* Optional dictionary ID field */
|
774
|
+
if (cctxPtr->prefs.frameInfo.dictID) {
|
775
|
+
LZ4F_writeLE32(dstPtr, cctxPtr->prefs.frameInfo.dictID);
|
776
|
+
dstPtr += 4;
|
777
|
+
}
|
778
|
+
/* Header CRC Byte */
|
779
|
+
*dstPtr = LZ4F_headerChecksum(headerStart, (size_t)(dstPtr - headerStart));
|
780
|
+
dstPtr++;
|
693
781
|
}
|
694
|
-
/* Header CRC Byte */
|
695
|
-
*dstPtr = LZ4F_headerChecksum(headerStart, (size_t)(dstPtr - headerStart));
|
696
|
-
dstPtr++;
|
697
782
|
|
698
783
|
cctxPtr->cStage = 1; /* header written, now request input data block */
|
699
784
|
return (size_t)(dstPtr - dstStart);
|
@@ -701,9 +786,9 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
|
|
701
786
|
|
702
787
|
|
703
788
|
/*! LZ4F_compressBegin() :
|
704
|
-
* init streaming compression
|
705
|
-
*
|
706
|
-
*
|
789
|
+
* init streaming compression AND writes frame header into @dstBuffer.
|
790
|
+
* @dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
|
791
|
+
* @preferencesPtr can be NULL, in which case default parameters are selected.
|
707
792
|
* @return : number of bytes written into dstBuffer for the header
|
708
793
|
* or an error code (can be tested using LZ4F_isError())
|
709
794
|
*/
|
@@ -744,11 +829,13 @@ static size_t LZ4F_makeBlock(void* dst,
|
|
744
829
|
LZ4F_blockChecksum_t crcFlag)
|
745
830
|
{
|
746
831
|
BYTE* const cSizePtr = (BYTE*)dst;
|
747
|
-
U32 cSize
|
748
|
-
|
749
|
-
|
750
|
-
|
751
|
-
|
832
|
+
U32 cSize;
|
833
|
+
assert(compress != NULL);
|
834
|
+
cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize),
|
835
|
+
(int)(srcSize), (int)(srcSize-1),
|
836
|
+
level, cdict);
|
837
|
+
|
838
|
+
if (cSize == 0 || cSize >= srcSize) {
|
752
839
|
cSize = (U32)srcSize;
|
753
840
|
LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG);
|
754
841
|
memcpy(cSizePtr+BHSize, src, srcSize);
|
@@ -766,6 +853,7 @@ static size_t LZ4F_makeBlock(void* dst,
|
|
766
853
|
static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
|
767
854
|
{
|
768
855
|
int const acceleration = (level < 0) ? -level + 1 : 1;
|
856
|
+
DEBUGLOG(5, "LZ4F_compressBlock (srcSize=%i)", srcSize);
|
769
857
|
LZ4F_initStream(ctx, cdict, level, LZ4F_blockIndependent);
|
770
858
|
if (cdict) {
|
771
859
|
return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
|
@@ -778,6 +866,7 @@ static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, in
|
|
778
866
|
{
|
779
867
|
int const acceleration = (level < 0) ? -level + 1 : 1;
|
780
868
|
(void)cdict; /* init once at beginning of frame */
|
869
|
+
DEBUGLOG(5, "LZ4F_compressBlock_continue (srcSize=%i)", srcSize);
|
781
870
|
return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
|
782
871
|
}
|
783
872
|
|
@@ -796,8 +885,15 @@ static int LZ4F_compressBlockHC_continue(void* ctx, const char* src, char* dst,
|
|
796
885
|
return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
|
797
886
|
}
|
798
887
|
|
799
|
-
static
|
888
|
+
static int LZ4F_doNotCompressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
|
889
|
+
{
|
890
|
+
(void)ctx; (void)src; (void)dst; (void)srcSize; (void)dstCapacity; (void)level; (void)cdict;
|
891
|
+
return 0;
|
892
|
+
}
|
893
|
+
|
894
|
+
static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level, LZ4F_blockCompression_t compressMode)
|
800
895
|
{
|
896
|
+
if (compressMode == LZ4B_UNCOMPRESSED) return LZ4F_doNotCompressBlock;
|
801
897
|
if (level < LZ4HC_CLEVEL_MIN) {
|
802
898
|
if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlock;
|
803
899
|
return LZ4F_compressBlock_continue;
|
@@ -806,6 +902,7 @@ static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int lev
|
|
806
902
|
return LZ4F_compressBlockHC_continue;
|
807
903
|
}
|
808
904
|
|
905
|
+
/* Save history (up to 64KB) into @tmpBuff */
|
809
906
|
static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
|
810
907
|
{
|
811
908
|
if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
|
@@ -815,38 +912,57 @@ static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
|
|
815
912
|
|
816
913
|
typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;
|
817
914
|
|
818
|
-
|
915
|
+
static const LZ4F_compressOptions_t k_cOptionsNull = { 0, { 0, 0, 0 } };
|
916
|
+
|
917
|
+
|
918
|
+
/*! LZ4F_compressUpdateImpl() :
|
819
919
|
* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
|
820
|
-
*
|
821
|
-
*
|
920
|
+
* When successful, the function always entirely consumes @srcBuffer.
|
921
|
+
* src data is either buffered or compressed into @dstBuffer.
|
922
|
+
* If the block compression does not match the compression of the previous block, the old data is flushed
|
923
|
+
* and operations continue with the new compression mode.
|
924
|
+
* @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr) when block compression is turned on.
|
925
|
+
* @compressOptionsPtr is optional : provide NULL to mean "default".
|
822
926
|
* @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
|
823
927
|
* or an error code if it fails (which can be tested using LZ4F_isError())
|
928
|
+
* After an error, the state is left in a UB state, and must be re-initialized.
|
824
929
|
*/
|
825
|
-
size_t
|
826
|
-
|
930
|
+
static size_t LZ4F_compressUpdateImpl(LZ4F_cctx* cctxPtr,
|
931
|
+
void* dstBuffer, size_t dstCapacity,
|
827
932
|
const void* srcBuffer, size_t srcSize,
|
828
|
-
const LZ4F_compressOptions_t* compressOptionsPtr
|
829
|
-
|
830
|
-
|
933
|
+
const LZ4F_compressOptions_t* compressOptionsPtr,
|
934
|
+
LZ4F_blockCompression_t blockCompression)
|
935
|
+
{
|
831
936
|
size_t const blockSize = cctxPtr->maxBlockSize;
|
832
937
|
const BYTE* srcPtr = (const BYTE*)srcBuffer;
|
833
938
|
const BYTE* const srcEnd = srcPtr + srcSize;
|
834
939
|
BYTE* const dstStart = (BYTE*)dstBuffer;
|
835
940
|
BYTE* dstPtr = dstStart;
|
836
941
|
LZ4F_lastBlockStatus lastBlockCompressed = notDone;
|
837
|
-
compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
|
838
|
-
|
942
|
+
compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel, blockCompression);
|
943
|
+
size_t bytesWritten;
|
839
944
|
DEBUGLOG(4, "LZ4F_compressUpdate (srcSize=%zu)", srcSize);
|
840
945
|
|
841
|
-
|
946
|
+
RETURN_ERROR_IF(cctxPtr->cStage != 1, compressionState_uninitialized); /* state must be initialized and waiting for next block */
|
842
947
|
if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
|
843
|
-
|
844
|
-
|
845
|
-
if (
|
948
|
+
RETURN_ERROR(dstMaxSize_tooSmall);
|
949
|
+
|
950
|
+
if (blockCompression == LZ4B_UNCOMPRESSED && dstCapacity < srcSize)
|
951
|
+
RETURN_ERROR(dstMaxSize_tooSmall);
|
952
|
+
|
953
|
+
/* flush currently written block, to continue with new block compression */
|
954
|
+
if (cctxPtr->blockCompression != blockCompression) {
|
955
|
+
bytesWritten = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr);
|
956
|
+
dstPtr += bytesWritten;
|
957
|
+
cctxPtr->blockCompression = blockCompression;
|
958
|
+
}
|
959
|
+
|
960
|
+
if (compressOptionsPtr == NULL) compressOptionsPtr = &k_cOptionsNull;
|
846
961
|
|
847
962
|
/* complete tmp buffer */
|
848
963
|
if (cctxPtr->tmpInSize > 0) { /* some data already within tmp buffer */
|
849
964
|
size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize;
|
965
|
+
assert(blockSize > cctxPtr->tmpInSize);
|
850
966
|
if (sizeToCopy > srcSize) {
|
851
967
|
/* add src to tmpIn buffer */
|
852
968
|
memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize);
|
@@ -864,11 +980,9 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
|
|
864
980
|
compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
|
865
981
|
cctxPtr->cdict,
|
866
982
|
cctxPtr->prefs.frameInfo.blockChecksumFlag);
|
867
|
-
|
868
983
|
if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
|
869
984
|
cctxPtr->tmpInSize = 0;
|
870
|
-
|
871
|
-
}
|
985
|
+
} }
|
872
986
|
|
873
987
|
while ((size_t)(srcEnd - srcPtr) >= blockSize) {
|
874
988
|
/* compress full blocks */
|
@@ -882,33 +996,38 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
|
|
882
996
|
}
|
883
997
|
|
884
998
|
if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) {
|
885
|
-
/*
|
999
|
+
/* autoFlush : remaining input (< blockSize) is compressed */
|
886
1000
|
lastBlockCompressed = fromSrcBuffer;
|
887
1001
|
dstPtr += LZ4F_makeBlock(dstPtr,
|
888
1002
|
srcPtr, (size_t)(srcEnd - srcPtr),
|
889
1003
|
compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
|
890
1004
|
cctxPtr->cdict,
|
891
1005
|
cctxPtr->prefs.frameInfo.blockChecksumFlag);
|
892
|
-
srcPtr
|
1006
|
+
srcPtr = srcEnd;
|
893
1007
|
}
|
894
1008
|
|
895
|
-
/* preserve dictionary
|
1009
|
+
/* preserve dictionary within @tmpBuff whenever necessary */
|
896
1010
|
if ((cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) {
|
1011
|
+
/* linked blocks are only supported in compressed mode, see LZ4F_uncompressedUpdate */
|
1012
|
+
assert(blockCompression == LZ4B_COMPRESSED);
|
897
1013
|
if (compressOptionsPtr->stableSrc) {
|
898
|
-
cctxPtr->tmpIn = cctxPtr->tmpBuff;
|
1014
|
+
cctxPtr->tmpIn = cctxPtr->tmpBuff; /* src is stable : dictionary remains in src across invocations */
|
899
1015
|
} else {
|
900
1016
|
int const realDictSize = LZ4F_localSaveDict(cctxPtr);
|
901
|
-
|
1017
|
+
assert(0 <= realDictSize && realDictSize <= 64 KB);
|
902
1018
|
cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
|
903
1019
|
}
|
904
1020
|
}
|
905
1021
|
|
906
1022
|
/* keep tmpIn within limits */
|
907
|
-
if ((cctxPtr->
|
908
|
-
|
1023
|
+
if (!(cctxPtr->prefs.autoFlush) /* no autoflush : there may be some data left within internal buffer */
|
1024
|
+
&& (cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) ) /* not enough room to store next block */
|
909
1025
|
{
|
1026
|
+
/* only preserve 64KB within internal buffer. Ensures there is enough room for next block.
|
1027
|
+
* note: this situation necessarily implies lastBlockCompressed==fromTmpBuffer */
|
910
1028
|
int const realDictSize = LZ4F_localSaveDict(cctxPtr);
|
911
1029
|
cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
|
1030
|
+
assert((cctxPtr->tmpIn + blockSize) <= (cctxPtr->tmpBuff + cctxPtr->maxBufferSize));
|
912
1031
|
}
|
913
1032
|
|
914
1033
|
/* some input data left, necessarily < blockSize */
|
@@ -926,6 +1045,53 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
|
|
926
1045
|
return (size_t)(dstPtr - dstStart);
|
927
1046
|
}
|
928
1047
|
|
1048
|
+
/*! LZ4F_compressUpdate() :
|
1049
|
+
* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
|
1050
|
+
* When successful, the function always entirely consumes @srcBuffer.
|
1051
|
+
* src data is either buffered or compressed into @dstBuffer.
|
1052
|
+
* If previously an uncompressed block was written, buffered data is flushed
|
1053
|
+
* before appending compressed data is continued.
|
1054
|
+
* @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
|
1055
|
+
* @compressOptionsPtr is optional : provide NULL to mean "default".
|
1056
|
+
* @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
|
1057
|
+
* or an error code if it fails (which can be tested using LZ4F_isError())
|
1058
|
+
* After an error, the state is left in a UB state, and must be re-initialized.
|
1059
|
+
*/
|
1060
|
+
size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
|
1061
|
+
void* dstBuffer, size_t dstCapacity,
|
1062
|
+
const void* srcBuffer, size_t srcSize,
|
1063
|
+
const LZ4F_compressOptions_t* compressOptionsPtr)
|
1064
|
+
{
|
1065
|
+
return LZ4F_compressUpdateImpl(cctxPtr,
|
1066
|
+
dstBuffer, dstCapacity,
|
1067
|
+
srcBuffer, srcSize,
|
1068
|
+
compressOptionsPtr, LZ4B_COMPRESSED);
|
1069
|
+
}
|
1070
|
+
|
1071
|
+
/*! LZ4F_compressUpdate() :
|
1072
|
+
* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
|
1073
|
+
* When successful, the function always entirely consumes @srcBuffer.
|
1074
|
+
* src data is either buffered or compressed into @dstBuffer.
|
1075
|
+
* If previously an uncompressed block was written, buffered data is flushed
|
1076
|
+
* before appending compressed data is continued.
|
1077
|
+
* This is only supported when LZ4F_blockIndependent is used
|
1078
|
+
* @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
|
1079
|
+
* @compressOptionsPtr is optional : provide NULL to mean "default".
|
1080
|
+
* @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
|
1081
|
+
* or an error code if it fails (which can be tested using LZ4F_isError())
|
1082
|
+
* After an error, the state is left in a UB state, and must be re-initialized.
|
1083
|
+
*/
|
1084
|
+
size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctxPtr,
|
1085
|
+
void* dstBuffer, size_t dstCapacity,
|
1086
|
+
const void* srcBuffer, size_t srcSize,
|
1087
|
+
const LZ4F_compressOptions_t* compressOptionsPtr) {
|
1088
|
+
RETURN_ERROR_IF(cctxPtr->prefs.frameInfo.blockMode != LZ4F_blockIndependent, blockMode_invalid);
|
1089
|
+
return LZ4F_compressUpdateImpl(cctxPtr,
|
1090
|
+
dstBuffer, dstCapacity,
|
1091
|
+
srcBuffer, srcSize,
|
1092
|
+
compressOptionsPtr, LZ4B_UNCOMPRESSED);
|
1093
|
+
}
|
1094
|
+
|
929
1095
|
|
930
1096
|
/*! LZ4F_flush() :
|
931
1097
|
* When compressed data must be sent immediately, without waiting for a block to be filled,
|
@@ -944,13 +1110,12 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr,
|
|
944
1110
|
compressFunc_t compress;
|
945
1111
|
|
946
1112
|
if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */
|
947
|
-
|
948
|
-
|
949
|
-
|
950
|
-
(void)compressOptionsPtr; /* not yet useful */
|
1113
|
+
RETURN_ERROR_IF(cctxPtr->cStage != 1, compressionState_uninitialized);
|
1114
|
+
RETURN_ERROR_IF(dstCapacity < (cctxPtr->tmpInSize + BHSize + BFSize), dstMaxSize_tooSmall);
|
1115
|
+
(void)compressOptionsPtr; /* not useful (yet) */
|
951
1116
|
|
952
1117
|
/* select compression function */
|
953
|
-
compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
|
1118
|
+
compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel, cctxPtr->blockCompression);
|
954
1119
|
|
955
1120
|
/* compress tmp buffer */
|
956
1121
|
dstPtr += LZ4F_makeBlock(dstPtr,
|
@@ -992,19 +1157,19 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr,
|
|
992
1157
|
|
993
1158
|
size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr);
|
994
1159
|
DEBUGLOG(5,"LZ4F_compressEnd: dstCapacity=%u", (unsigned)dstCapacity);
|
995
|
-
|
1160
|
+
FORWARD_IF_ERROR(flushSize);
|
996
1161
|
dstPtr += flushSize;
|
997
1162
|
|
998
1163
|
assert(flushSize <= dstCapacity);
|
999
1164
|
dstCapacity -= flushSize;
|
1000
1165
|
|
1001
|
-
|
1166
|
+
RETURN_ERROR_IF(dstCapacity < 4, dstMaxSize_tooSmall);
|
1002
1167
|
LZ4F_writeLE32(dstPtr, 0);
|
1003
1168
|
dstPtr += 4; /* endMark */
|
1004
1169
|
|
1005
1170
|
if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) {
|
1006
1171
|
U32 const xxh = XXH32_digest(&(cctxPtr->xxh));
|
1007
|
-
|
1172
|
+
RETURN_ERROR_IF(dstCapacity < 8, dstMaxSize_tooSmall);
|
1008
1173
|
DEBUGLOG(5,"Writing 32-bit content checksum");
|
1009
1174
|
LZ4F_writeLE32(dstPtr, xxh);
|
1010
1175
|
dstPtr+=4; /* content Checksum */
|
@@ -1015,7 +1180,7 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr,
|
|
1015
1180
|
|
1016
1181
|
if (cctxPtr->prefs.frameInfo.contentSize) {
|
1017
1182
|
if (cctxPtr->prefs.frameInfo.contentSize != cctxPtr->totalInSize)
|
1018
|
-
|
1183
|
+
RETURN_ERROR(frameSize_wrong);
|
1019
1184
|
}
|
1020
1185
|
|
1021
1186
|
return (size_t)(dstPtr - dstStart);
|
@@ -1039,6 +1204,7 @@ typedef enum {
|
|
1039
1204
|
} dStage_t;
|
1040
1205
|
|
1041
1206
|
struct LZ4F_dctx_s {
|
1207
|
+
LZ4F_CustomMem cmem;
|
1042
1208
|
LZ4F_frameInfo_t frameInfo;
|
1043
1209
|
U32 version;
|
1044
1210
|
dStage_t dStage;
|
@@ -1056,26 +1222,37 @@ struct LZ4F_dctx_s {
|
|
1056
1222
|
size_t tmpOutStart;
|
1057
1223
|
XXH32_state_t xxh;
|
1058
1224
|
XXH32_state_t blockChecksum;
|
1225
|
+
int skipChecksum;
|
1059
1226
|
BYTE header[LZ4F_HEADER_SIZE_MAX];
|
1060
1227
|
}; /* typedef'd to LZ4F_dctx in lz4frame.h */
|
1061
1228
|
|
1062
1229
|
|
1230
|
+
LZ4F_dctx* LZ4F_createDecompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version)
|
1231
|
+
{
|
1232
|
+
LZ4F_dctx* const dctx = (LZ4F_dctx*)LZ4F_calloc(sizeof(LZ4F_dctx), customMem);
|
1233
|
+
if (dctx == NULL) return NULL;
|
1234
|
+
|
1235
|
+
dctx->cmem = customMem;
|
1236
|
+
dctx->version = version;
|
1237
|
+
return dctx;
|
1238
|
+
}
|
1239
|
+
|
1063
1240
|
/*! LZ4F_createDecompressionContext() :
|
1064
1241
|
* Create a decompressionContext object, which will track all decompression operations.
|
1065
1242
|
* Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object.
|
1066
1243
|
* Object can later be released using LZ4F_freeDecompressionContext().
|
1067
1244
|
* @return : if != 0, there was an error during context creation.
|
1068
1245
|
*/
|
1069
|
-
LZ4F_errorCode_t
|
1246
|
+
LZ4F_errorCode_t
|
1247
|
+
LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber)
|
1070
1248
|
{
|
1071
|
-
|
1072
|
-
|
1073
|
-
*LZ4F_decompressionContextPtr = NULL;
|
1074
|
-
return err0r(LZ4F_ERROR_allocation_failed);
|
1075
|
-
}
|
1249
|
+
assert(LZ4F_decompressionContextPtr != NULL); /* violation of narrow contract */
|
1250
|
+
RETURN_ERROR_IF(LZ4F_decompressionContextPtr == NULL, parameter_null); /* in case it nonetheless happen in production */
|
1076
1251
|
|
1077
|
-
|
1078
|
-
*LZ4F_decompressionContextPtr
|
1252
|
+
*LZ4F_decompressionContextPtr = LZ4F_createDecompressionContext_advanced(LZ4F_defaultCMem, versionNumber);
|
1253
|
+
if (*LZ4F_decompressionContextPtr == NULL) { /* failed allocation */
|
1254
|
+
RETURN_ERROR(allocation_failed);
|
1255
|
+
}
|
1079
1256
|
return LZ4F_OK_NoError;
|
1080
1257
|
}
|
1081
1258
|
|
@@ -1084,9 +1261,9 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx)
|
|
1084
1261
|
LZ4F_errorCode_t result = LZ4F_OK_NoError;
|
1085
1262
|
if (dctx != NULL) { /* can accept NULL input, like free() */
|
1086
1263
|
result = (LZ4F_errorCode_t)dctx->dStage;
|
1087
|
-
|
1088
|
-
|
1089
|
-
|
1264
|
+
LZ4F_free(dctx->tmpIn, dctx->cmem);
|
1265
|
+
LZ4F_free(dctx->tmpOutBuffer, dctx->cmem);
|
1266
|
+
LZ4F_free(dctx, dctx->cmem);
|
1090
1267
|
}
|
1091
1268
|
return result;
|
1092
1269
|
}
|
@@ -1099,6 +1276,7 @@ void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx)
|
|
1099
1276
|
dctx->dStage = dstage_getFrameHeader;
|
1100
1277
|
dctx->dict = NULL;
|
1101
1278
|
dctx->dictSize = 0;
|
1279
|
+
dctx->skipChecksum = 0;
|
1102
1280
|
}
|
1103
1281
|
|
1104
1282
|
|
@@ -1118,7 +1296,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
|
|
1118
1296
|
|
1119
1297
|
DEBUGLOG(5, "LZ4F_decodeHeader");
|
1120
1298
|
/* need to decode header to get frameInfo */
|
1121
|
-
|
1299
|
+
RETURN_ERROR_IF(srcSize < minFHSize, frameHeader_incomplete); /* minimal frame header size */
|
1122
1300
|
MEM_INIT(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo));
|
1123
1301
|
|
1124
1302
|
/* special case : skippable frames */
|
@@ -1132,14 +1310,13 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
|
|
1132
1310
|
} else {
|
1133
1311
|
dctx->dStage = dstage_getSFrameSize;
|
1134
1312
|
return 4;
|
1135
|
-
|
1136
|
-
}
|
1313
|
+
} }
|
1137
1314
|
|
1138
1315
|
/* control magic number */
|
1139
1316
|
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
|
1140
1317
|
if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) {
|
1141
1318
|
DEBUGLOG(4, "frame header error : unknown magic number");
|
1142
|
-
|
1319
|
+
RETURN_ERROR(frameType_unknown);
|
1143
1320
|
}
|
1144
1321
|
#endif
|
1145
1322
|
dctx->frameInfo.frameType = LZ4F_frame;
|
@@ -1153,8 +1330,8 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
|
|
1153
1330
|
contentChecksumFlag = (FLG>>2) & _1BIT;
|
1154
1331
|
dictIDFlag = FLG & _1BIT;
|
1155
1332
|
/* validate */
|
1156
|
-
if (((FLG>>1)&_1BIT) != 0)
|
1157
|
-
if (version != 1)
|
1333
|
+
if (((FLG>>1)&_1BIT) != 0) RETURN_ERROR(reservedFlag_set); /* Reserved bit */
|
1334
|
+
if (version != 1) RETURN_ERROR(headerVersion_wrong); /* Version Number, only supported value */
|
1158
1335
|
}
|
1159
1336
|
|
1160
1337
|
/* Frame Header Size */
|
@@ -1173,17 +1350,16 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
|
|
1173
1350
|
{ U32 const BD = srcPtr[5];
|
1174
1351
|
blockSizeID = (BD>>4) & _3BITS;
|
1175
1352
|
/* validate */
|
1176
|
-
if (((BD>>7)&_1BIT) != 0)
|
1177
|
-
if (blockSizeID < 4)
|
1178
|
-
if (((BD>>0)&_4BITS) != 0)
|
1353
|
+
if (((BD>>7)&_1BIT) != 0) RETURN_ERROR(reservedFlag_set); /* Reserved bit */
|
1354
|
+
if (blockSizeID < 4) RETURN_ERROR(maxBlockSize_invalid); /* 4-7 only supported values for the time being */
|
1355
|
+
if (((BD>>0)&_4BITS) != 0) RETURN_ERROR(reservedFlag_set); /* Reserved bits */
|
1179
1356
|
}
|
1180
1357
|
|
1181
1358
|
/* check header */
|
1182
1359
|
assert(frameHeaderSize > 5);
|
1183
1360
|
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
|
1184
1361
|
{ BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5);
|
1185
|
-
|
1186
|
-
return err0r(LZ4F_ERROR_headerChecksum_invalid);
|
1362
|
+
RETURN_ERROR_IF(HC != srcPtr[frameHeaderSize-1], headerChecksum_invalid);
|
1187
1363
|
}
|
1188
1364
|
#endif
|
1189
1365
|
|
@@ -1192,10 +1368,9 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
|
|
1192
1368
|
dctx->frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)blockChecksumFlag;
|
1193
1369
|
dctx->frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)contentChecksumFlag;
|
1194
1370
|
dctx->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID;
|
1195
|
-
dctx->maxBlockSize = LZ4F_getBlockSize(blockSizeID);
|
1371
|
+
dctx->maxBlockSize = LZ4F_getBlockSize((LZ4F_blockSizeID_t)blockSizeID);
|
1196
1372
|
if (contentSizeFlag)
|
1197
|
-
dctx->frameRemainingSize =
|
1198
|
-
dctx->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6);
|
1373
|
+
dctx->frameRemainingSize = dctx->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6);
|
1199
1374
|
if (dictIDFlag)
|
1200
1375
|
dctx->frameInfo.dictID = LZ4F_readLE32(srcPtr + frameHeaderSize - 5);
|
1201
1376
|
|
@@ -1211,11 +1386,11 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
|
|
1211
1386
|
*/
|
1212
1387
|
size_t LZ4F_headerSize(const void* src, size_t srcSize)
|
1213
1388
|
{
|
1214
|
-
|
1389
|
+
RETURN_ERROR_IF(src == NULL, srcPtr_wrong);
|
1215
1390
|
|
1216
1391
|
/* minimal srcSize to determine header size */
|
1217
1392
|
if (srcSize < LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH)
|
1218
|
-
|
1393
|
+
RETURN_ERROR(frameHeader_incomplete);
|
1219
1394
|
|
1220
1395
|
/* special case : skippable frames */
|
1221
1396
|
if ((LZ4F_readLE32(src) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START)
|
@@ -1224,7 +1399,7 @@ size_t LZ4F_headerSize(const void* src, size_t srcSize)
|
|
1224
1399
|
/* control magic number */
|
1225
1400
|
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
|
1226
1401
|
if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER)
|
1227
|
-
|
1402
|
+
RETURN_ERROR(frameType_unknown);
|
1228
1403
|
#endif
|
1229
1404
|
|
1230
1405
|
/* Frame Header Size */
|
@@ -1266,13 +1441,13 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
|
|
1266
1441
|
if (dctx->dStage == dstage_storeFrameHeader) {
|
1267
1442
|
/* frame decoding already started, in the middle of header => automatic fail */
|
1268
1443
|
*srcSizePtr = 0;
|
1269
|
-
|
1444
|
+
RETURN_ERROR(frameDecoding_alreadyStarted);
|
1270
1445
|
} else {
|
1271
1446
|
size_t const hSize = LZ4F_headerSize(srcBuffer, *srcSizePtr);
|
1272
1447
|
if (LZ4F_isError(hSize)) { *srcSizePtr=0; return hSize; }
|
1273
1448
|
if (*srcSizePtr < hSize) {
|
1274
1449
|
*srcSizePtr=0;
|
1275
|
-
|
1450
|
+
RETURN_ERROR(frameHeader_incomplete);
|
1276
1451
|
}
|
1277
1452
|
|
1278
1453
|
{ size_t decodeResult = LZ4F_decodeHeader(dctx, srcBuffer, hSize);
|
@@ -1290,16 +1465,14 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
|
|
1290
1465
|
|
1291
1466
|
/* LZ4F_updateDict() :
|
1292
1467
|
* only used for LZ4F_blockLinked mode
|
1293
|
-
* Condition : dstPtr != NULL
|
1468
|
+
* Condition : @dstPtr != NULL
|
1294
1469
|
*/
|
1295
1470
|
static void LZ4F_updateDict(LZ4F_dctx* dctx,
|
1296
1471
|
const BYTE* dstPtr, size_t dstSize, const BYTE* dstBufferStart,
|
1297
1472
|
unsigned withinTmp)
|
1298
1473
|
{
|
1299
1474
|
assert(dstPtr != NULL);
|
1300
|
-
if (dctx->dictSize==0)
|
1301
|
-
dctx->dict = (const BYTE*)dstPtr; /* priority to prefix mode */
|
1302
|
-
}
|
1475
|
+
if (dctx->dictSize==0) dctx->dict = (const BYTE*)dstPtr; /* will lead to prefix mode */
|
1303
1476
|
assert(dctx->dict != NULL);
|
1304
1477
|
|
1305
1478
|
if (dctx->dict + dctx->dictSize == dstPtr) { /* prefix mode, everything within dstBuffer */
|
@@ -1362,7 +1535,6 @@ static void LZ4F_updateDict(LZ4F_dctx* dctx,
|
|
1362
1535
|
}
|
1363
1536
|
|
1364
1537
|
|
1365
|
-
|
1366
1538
|
/*! LZ4F_decompress() :
|
1367
1539
|
* Call this function repetitively to regenerate compressed data in srcBuffer.
|
1368
1540
|
* The function will attempt to decode up to *srcSizePtr bytes from srcBuffer
|
@@ -1406,6 +1578,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1406
1578
|
*srcSizePtr = 0;
|
1407
1579
|
*dstSizePtr = 0;
|
1408
1580
|
assert(dctx != NULL);
|
1581
|
+
dctx->skipChecksum |= (decompressOptionsPtr->skipChecksums != 0); /* once set, disable for the remainder of the frame */
|
1409
1582
|
|
1410
1583
|
/* behaves as a state machine */
|
1411
1584
|
|
@@ -1418,7 +1591,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1418
1591
|
DEBUGLOG(6, "dstage_getFrameHeader");
|
1419
1592
|
if ((size_t)(srcEnd-srcPtr) >= maxFHSize) { /* enough to decode - shortcut */
|
1420
1593
|
size_t const hSize = LZ4F_decodeHeader(dctx, srcPtr, (size_t)(srcEnd-srcPtr)); /* will update dStage appropriately */
|
1421
|
-
|
1594
|
+
FORWARD_IF_ERROR(hSize);
|
1422
1595
|
srcPtr += hSize;
|
1423
1596
|
break;
|
1424
1597
|
}
|
@@ -1440,9 +1613,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1440
1613
|
doAnotherStage = 0; /* not enough src data, ask for some more */
|
1441
1614
|
break;
|
1442
1615
|
}
|
1443
|
-
|
1444
|
-
if (LZ4F_isError(hSize)) return hSize;
|
1445
|
-
}
|
1616
|
+
FORWARD_IF_ERROR( LZ4F_decodeHeader(dctx, dctx->header, dctx->tmpInTarget) ); /* will update dStage appropriately */
|
1446
1617
|
break;
|
1447
1618
|
|
1448
1619
|
case dstage_init:
|
@@ -1453,14 +1624,12 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1453
1624
|
+ ((dctx->frameInfo.blockMode==LZ4F_blockLinked) ? 128 KB : 0);
|
1454
1625
|
if (bufferNeeded > dctx->maxBufferSize) { /* tmp buffers too small */
|
1455
1626
|
dctx->maxBufferSize = 0; /* ensure allocation will be re-attempted on next entry*/
|
1456
|
-
|
1457
|
-
dctx->tmpIn = (BYTE*)
|
1458
|
-
|
1459
|
-
|
1460
|
-
|
1461
|
-
dctx->tmpOutBuffer
|
1462
|
-
if (dctx->tmpOutBuffer== NULL)
|
1463
|
-
return err0r(LZ4F_ERROR_allocation_failed);
|
1627
|
+
LZ4F_free(dctx->tmpIn, dctx->cmem);
|
1628
|
+
dctx->tmpIn = (BYTE*)LZ4F_malloc(dctx->maxBlockSize + BFSize /* block checksum */, dctx->cmem);
|
1629
|
+
RETURN_ERROR_IF(dctx->tmpIn == NULL, allocation_failed);
|
1630
|
+
LZ4F_free(dctx->tmpOutBuffer, dctx->cmem);
|
1631
|
+
dctx->tmpOutBuffer= (BYTE*)LZ4F_malloc(bufferNeeded, dctx->cmem);
|
1632
|
+
RETURN_ERROR_IF(dctx->tmpOutBuffer== NULL, allocation_failed);
|
1464
1633
|
dctx->maxBufferSize = bufferNeeded;
|
1465
1634
|
} }
|
1466
1635
|
dctx->tmpInSize = 0;
|
@@ -1509,7 +1678,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1509
1678
|
break;
|
1510
1679
|
}
|
1511
1680
|
if (nextCBlockSize > dctx->maxBlockSize) {
|
1512
|
-
|
1681
|
+
RETURN_ERROR(maxBlockSize_invalid);
|
1513
1682
|
}
|
1514
1683
|
if (blockHeader & LZ4F_BLOCKUNCOMPRESSED_FLAG) {
|
1515
1684
|
/* next block is uncompressed */
|
@@ -1540,11 +1709,13 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1540
1709
|
size_t const minBuffSize = MIN((size_t)(srcEnd-srcPtr), (size_t)(dstEnd-dstPtr));
|
1541
1710
|
sizeToCopy = MIN(dctx->tmpInTarget, minBuffSize);
|
1542
1711
|
memcpy(dstPtr, srcPtr, sizeToCopy);
|
1543
|
-
if (dctx->
|
1544
|
-
(
|
1712
|
+
if (!dctx->skipChecksum) {
|
1713
|
+
if (dctx->frameInfo.blockChecksumFlag) {
|
1714
|
+
(void)XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy);
|
1715
|
+
}
|
1716
|
+
if (dctx->frameInfo.contentChecksumFlag)
|
1717
|
+
(void)XXH32_update(&dctx->xxh, srcPtr, sizeToCopy);
|
1545
1718
|
}
|
1546
|
-
if (dctx->frameInfo.contentChecksumFlag)
|
1547
|
-
(void)XXH32_update(&dctx->xxh, srcPtr, sizeToCopy);
|
1548
1719
|
if (dctx->frameInfo.contentSize)
|
1549
1720
|
dctx->frameRemainingSize -= sizeToCopy;
|
1550
1721
|
|
@@ -1590,14 +1761,15 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1590
1761
|
}
|
1591
1762
|
crcSrc = dctx->header;
|
1592
1763
|
}
|
1593
|
-
|
1764
|
+
if (!dctx->skipChecksum) {
|
1765
|
+
U32 const readCRC = LZ4F_readLE32(crcSrc);
|
1594
1766
|
U32 const calcCRC = XXH32_digest(&dctx->blockChecksum);
|
1595
1767
|
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
|
1596
1768
|
DEBUGLOG(6, "compare block checksum");
|
1597
1769
|
if (readCRC != calcCRC) {
|
1598
1770
|
DEBUGLOG(4, "incorrect block checksum: %08X != %08X",
|
1599
1771
|
readCRC, calcCRC);
|
1600
|
-
|
1772
|
+
RETURN_ERROR(blockChecksum_invalid);
|
1601
1773
|
}
|
1602
1774
|
#else
|
1603
1775
|
(void)readCRC;
|
@@ -1637,37 +1809,44 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1637
1809
|
}
|
1638
1810
|
|
1639
1811
|
/* At this stage, input is large enough to decode a block */
|
1812
|
+
|
1813
|
+
/* First, decode and control block checksum if it exists */
|
1640
1814
|
if (dctx->frameInfo.blockChecksumFlag) {
|
1815
|
+
assert(dctx->tmpInTarget >= 4);
|
1641
1816
|
dctx->tmpInTarget -= 4;
|
1642
1817
|
assert(selectedIn != NULL); /* selectedIn is defined at this stage (either srcPtr, or dctx->tmpIn) */
|
1643
1818
|
{ U32 const readBlockCrc = LZ4F_readLE32(selectedIn + dctx->tmpInTarget);
|
1644
1819
|
U32 const calcBlockCrc = XXH32(selectedIn, dctx->tmpInTarget, 0);
|
1645
1820
|
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
|
1646
|
-
|
1647
|
-
return err0r(LZ4F_ERROR_blockChecksum_invalid);
|
1821
|
+
RETURN_ERROR_IF(readBlockCrc != calcBlockCrc, blockChecksum_invalid);
|
1648
1822
|
#else
|
1649
1823
|
(void)readBlockCrc;
|
1650
1824
|
(void)calcBlockCrc;
|
1651
1825
|
#endif
|
1652
1826
|
} }
|
1653
1827
|
|
1654
|
-
if
|
1828
|
+
/* decode directly into destination buffer if there is enough room */
|
1829
|
+
if ( ((size_t)(dstEnd-dstPtr) >= dctx->maxBlockSize)
|
1830
|
+
/* unless the dictionary is stored in tmpOut:
|
1831
|
+
* in which case it's faster to decode within tmpOut
|
1832
|
+
* to benefit from prefix speedup */
|
1833
|
+
&& !(dctx->dict!= NULL && (const BYTE*)dctx->dict + dctx->dictSize == dctx->tmpOut) )
|
1834
|
+
{
|
1655
1835
|
const char* dict = (const char*)dctx->dict;
|
1656
1836
|
size_t dictSize = dctx->dictSize;
|
1657
1837
|
int decodedSize;
|
1658
1838
|
assert(dstPtr != NULL);
|
1659
1839
|
if (dict && dictSize > 1 GB) {
|
1660
|
-
/*
|
1840
|
+
/* overflow control : dctx->dictSize is an int, avoid truncation / sign issues */
|
1661
1841
|
dict += dictSize - 64 KB;
|
1662
1842
|
dictSize = 64 KB;
|
1663
1843
|
}
|
1664
|
-
/* enough capacity in `dst` to decompress directly there */
|
1665
1844
|
decodedSize = LZ4_decompress_safe_usingDict(
|
1666
1845
|
(const char*)selectedIn, (char*)dstPtr,
|
1667
1846
|
(int)dctx->tmpInTarget, (int)dctx->maxBlockSize,
|
1668
1847
|
dict, (int)dictSize);
|
1669
|
-
|
1670
|
-
if (dctx->frameInfo.contentChecksumFlag)
|
1848
|
+
RETURN_ERROR_IF(decodedSize < 0, decompressionFailed);
|
1849
|
+
if ((dctx->frameInfo.contentChecksumFlag) && (!dctx->skipChecksum))
|
1671
1850
|
XXH32_update(&(dctx->xxh), dstPtr, (size_t)decodedSize);
|
1672
1851
|
if (dctx->frameInfo.contentSize)
|
1673
1852
|
dctx->frameRemainingSize -= (size_t)decodedSize;
|
@@ -1678,25 +1857,27 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1678
1857
|
}
|
1679
1858
|
|
1680
1859
|
dstPtr += decodedSize;
|
1681
|
-
dctx->dStage = dstage_getBlockHeader;
|
1860
|
+
dctx->dStage = dstage_getBlockHeader; /* end of block, let's get another one */
|
1682
1861
|
break;
|
1683
1862
|
}
|
1684
1863
|
|
1685
1864
|
/* not enough place into dst : decode into tmpOut */
|
1686
|
-
|
1865
|
+
|
1866
|
+
/* manage dictionary */
|
1687
1867
|
if (dctx->frameInfo.blockMode == LZ4F_blockLinked) {
|
1688
1868
|
if (dctx->dict == dctx->tmpOutBuffer) {
|
1869
|
+
/* truncate dictionary to 64 KB if too big */
|
1689
1870
|
if (dctx->dictSize > 128 KB) {
|
1690
1871
|
memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - 64 KB, 64 KB);
|
1691
1872
|
dctx->dictSize = 64 KB;
|
1692
1873
|
}
|
1693
1874
|
dctx->tmpOut = dctx->tmpOutBuffer + dctx->dictSize;
|
1694
|
-
} else { /* dict not within
|
1875
|
+
} else { /* dict not within tmpOut */
|
1695
1876
|
size_t const reservedDictSpace = MIN(dctx->dictSize, 64 KB);
|
1696
1877
|
dctx->tmpOut = dctx->tmpOutBuffer + reservedDictSpace;
|
1697
1878
|
} }
|
1698
1879
|
|
1699
|
-
/* Decode block */
|
1880
|
+
/* Decode block into tmpOut */
|
1700
1881
|
{ const char* dict = (const char*)dctx->dict;
|
1701
1882
|
size_t dictSize = dctx->dictSize;
|
1702
1883
|
int decodedSize;
|
@@ -1709,9 +1890,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1709
1890
|
(const char*)selectedIn, (char*)dctx->tmpOut,
|
1710
1891
|
(int)dctx->tmpInTarget, (int)dctx->maxBlockSize,
|
1711
1892
|
dict, (int)dictSize);
|
1712
|
-
|
1713
|
-
|
1714
|
-
if (dctx->frameInfo.contentChecksumFlag)
|
1893
|
+
RETURN_ERROR_IF(decodedSize < 0, decompressionFailed);
|
1894
|
+
if (dctx->frameInfo.contentChecksumFlag && !dctx->skipChecksum)
|
1715
1895
|
XXH32_update(&(dctx->xxh), dctx->tmpOut, (size_t)decodedSize);
|
1716
1896
|
if (dctx->frameInfo.contentSize)
|
1717
1897
|
dctx->frameRemainingSize -= (size_t)decodedSize;
|
@@ -1744,8 +1924,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1744
1924
|
break;
|
1745
1925
|
|
1746
1926
|
case dstage_getSuffix:
|
1747
|
-
|
1748
|
-
return err0r(LZ4F_ERROR_frameSize_wrong); /* incorrect frame size decoded */
|
1927
|
+
RETURN_ERROR_IF(dctx->frameRemainingSize, frameSize_wrong); /* incorrect frame size decoded */
|
1749
1928
|
if (!dctx->frameInfo.contentChecksumFlag) { /* no checksum, frame is completed */
|
1750
1929
|
nextSrcSizeHint = 0;
|
1751
1930
|
LZ4F_resetDecompressionContext(dctx);
|
@@ -1777,20 +1956,20 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1777
1956
|
} /* if (dctx->dStage == dstage_storeSuffix) */
|
1778
1957
|
|
1779
1958
|
/* case dstage_checkSuffix: */ /* no direct entry, avoid initialization risks */
|
1780
|
-
|
1959
|
+
if (!dctx->skipChecksum) {
|
1960
|
+
U32 const readCRC = LZ4F_readLE32(selectedIn);
|
1781
1961
|
U32 const resultCRC = XXH32_digest(&(dctx->xxh));
|
1782
1962
|
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
|
1783
|
-
|
1784
|
-
return err0r(LZ4F_ERROR_contentChecksum_invalid);
|
1963
|
+
RETURN_ERROR_IF(readCRC != resultCRC, contentChecksum_invalid);
|
1785
1964
|
#else
|
1786
1965
|
(void)readCRC;
|
1787
1966
|
(void)resultCRC;
|
1788
1967
|
#endif
|
1789
|
-
nextSrcSizeHint = 0;
|
1790
|
-
LZ4F_resetDecompressionContext(dctx);
|
1791
|
-
doAnotherStage = 0;
|
1792
|
-
break;
|
1793
1968
|
}
|
1969
|
+
nextSrcSizeHint = 0;
|
1970
|
+
LZ4F_resetDecompressionContext(dctx);
|
1971
|
+
doAnotherStage = 0;
|
1972
|
+
break;
|
1794
1973
|
|
1795
1974
|
case dstage_getSFrameSize:
|
1796
1975
|
if ((srcEnd - srcPtr) >= 4) {
|
@@ -1841,7 +2020,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1841
2020
|
} /* switch (dctx->dStage) */
|
1842
2021
|
} /* while (doAnotherStage) */
|
1843
2022
|
|
1844
|
-
/* preserve history within
|
2023
|
+
/* preserve history within tmpOut whenever necessary */
|
1845
2024
|
LZ4F_STATIC_ASSERT((unsigned)dstage_init == 2);
|
1846
2025
|
if ( (dctx->frameInfo.blockMode==LZ4F_blockLinked) /* next block will use up to 64KB from previous ones */
|
1847
2026
|
&& (dctx->dict != dctx->tmpOutBuffer) /* dictionary is not already within tmp */
|