isomorfeus-ferret 0.17.2 → 0.17.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ext/isomorfeus_ferret_ext/benchmark.c +9 -20
- data/ext/isomorfeus_ferret_ext/benchmarks_all.h +1 -2
- data/ext/isomorfeus_ferret_ext/bm_hash.c +1 -2
- data/ext/isomorfeus_ferret_ext/brotli_dec_decode.c +4 -2
- data/ext/isomorfeus_ferret_ext/brotli_enc_encode.c +3 -2
- data/ext/isomorfeus_ferret_ext/frb_analysis.c +4 -5
- data/ext/isomorfeus_ferret_ext/frb_field_info.c +3 -4
- data/ext/isomorfeus_ferret_ext/frb_index.c +118 -125
- data/ext/isomorfeus_ferret_ext/frb_lazy_doc.c +14 -16
- data/ext/isomorfeus_ferret_ext/frb_search.c +31 -23
- data/ext/isomorfeus_ferret_ext/frb_store.c +27 -13
- data/ext/isomorfeus_ferret_ext/frb_utils.c +3 -6
- data/ext/isomorfeus_ferret_ext/frt_analysis.c +39 -46
- data/ext/isomorfeus_ferret_ext/frt_analysis.h +9 -9
- data/ext/isomorfeus_ferret_ext/frt_array.c +11 -22
- data/ext/isomorfeus_ferret_ext/frt_bitvector.h +3 -6
- data/ext/isomorfeus_ferret_ext/frt_doc_field.c +87 -0
- data/ext/isomorfeus_ferret_ext/frt_doc_field.h +26 -0
- data/ext/isomorfeus_ferret_ext/frt_document.c +4 -97
- data/ext/isomorfeus_ferret_ext/frt_document.h +2 -27
- data/ext/isomorfeus_ferret_ext/frt_except.c +8 -6
- data/ext/isomorfeus_ferret_ext/frt_except.h +1 -2
- data/ext/isomorfeus_ferret_ext/frt_field_index.c +13 -32
- data/ext/isomorfeus_ferret_ext/frt_field_index.h +0 -6
- data/ext/isomorfeus_ferret_ext/frt_field_info.c +69 -0
- data/ext/isomorfeus_ferret_ext/frt_field_info.h +49 -0
- data/ext/isomorfeus_ferret_ext/frt_field_infos.c +196 -0
- data/ext/isomorfeus_ferret_ext/frt_field_infos.h +35 -0
- data/ext/isomorfeus_ferret_ext/frt_global.c +10 -4
- data/ext/isomorfeus_ferret_ext/frt_global.h +11 -15
- data/ext/isomorfeus_ferret_ext/frt_hash.c +8 -8
- data/ext/isomorfeus_ferret_ext/frt_hash.h +1 -2
- data/ext/isomorfeus_ferret_ext/frt_hashset.c +20 -40
- data/ext/isomorfeus_ferret_ext/frt_hashset.h +1 -2
- data/ext/isomorfeus_ferret_ext/frt_helper.c +7 -15
- data/ext/isomorfeus_ferret_ext/frt_in_stream.c +35 -45
- data/ext/isomorfeus_ferret_ext/frt_in_stream.h +3 -2
- data/ext/isomorfeus_ferret_ext/frt_ind.c +20 -38
- data/ext/isomorfeus_ferret_ext/frt_index.c +292 -790
- data/ext/isomorfeus_ferret_ext/frt_index.h +1 -102
- data/ext/isomorfeus_ferret_ext/frt_lang.c +5 -10
- data/ext/isomorfeus_ferret_ext/frt_lazy_doc.c +3 -3
- data/ext/isomorfeus_ferret_ext/frt_lazy_doc.h +1 -1
- data/ext/isomorfeus_ferret_ext/frt_lazy_doc_field.c +18 -25
- data/ext/isomorfeus_ferret_ext/frt_lazy_doc_field.h +5 -5
- data/ext/isomorfeus_ferret_ext/frt_mdbx_store.c +102 -70
- data/ext/isomorfeus_ferret_ext/frt_mempool.c +8 -16
- data/ext/isomorfeus_ferret_ext/frt_multimapper.c +23 -46
- data/ext/isomorfeus_ferret_ext/frt_multimapper.h +4 -8
- data/ext/isomorfeus_ferret_ext/frt_out_stream.c +31 -43
- data/ext/isomorfeus_ferret_ext/frt_out_stream.h +2 -2
- data/ext/isomorfeus_ferret_ext/frt_posh.c +6 -819
- data/ext/isomorfeus_ferret_ext/frt_posh.h +0 -57
- data/ext/isomorfeus_ferret_ext/frt_priorityqueue.c +11 -22
- data/ext/isomorfeus_ferret_ext/frt_priorityqueue.h +1 -2
- data/ext/isomorfeus_ferret_ext/frt_q_boolean.c +85 -171
- data/ext/isomorfeus_ferret_ext/frt_q_match_all.c +8 -16
- data/ext/isomorfeus_ferret_ext/frt_q_multi_term.c +1 -2
- data/ext/isomorfeus_ferret_ext/frt_q_parser.c +49 -98
- data/ext/isomorfeus_ferret_ext/frt_q_phrase.c +52 -104
- data/ext/isomorfeus_ferret_ext/frt_q_range.c +6 -12
- data/ext/isomorfeus_ferret_ext/frt_q_span.c +113 -226
- data/ext/isomorfeus_ferret_ext/frt_q_wildcard.c +1 -2
- data/ext/isomorfeus_ferret_ext/frt_ram_store.c +134 -85
- data/ext/isomorfeus_ferret_ext/frt_search.c +82 -164
- data/ext/isomorfeus_ferret_ext/frt_similarity.c +11 -22
- data/ext/isomorfeus_ferret_ext/frt_similarity.h +1 -2
- data/ext/isomorfeus_ferret_ext/frt_store.c +13 -25
- data/ext/isomorfeus_ferret_ext/frt_store.h +86 -52
- data/ext/isomorfeus_ferret_ext/frt_term_vectors.c +8 -16
- data/ext/isomorfeus_ferret_ext/frt_win32.h +5 -10
- data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.c +12 -11
- data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.h +11 -13
- data/ext/isomorfeus_ferret_ext/lz4.c +422 -195
- data/ext/isomorfeus_ferret_ext/lz4.h +114 -46
- data/ext/isomorfeus_ferret_ext/lz4frame.c +421 -242
- data/ext/isomorfeus_ferret_ext/lz4frame.h +122 -53
- data/ext/isomorfeus_ferret_ext/lz4hc.c +127 -111
- data/ext/isomorfeus_ferret_ext/lz4hc.h +14 -14
- data/ext/isomorfeus_ferret_ext/lz4xxhash.h +1 -1
- data/ext/isomorfeus_ferret_ext/mdbx.c +3762 -2526
- data/ext/isomorfeus_ferret_ext/mdbx.h +115 -70
- data/ext/isomorfeus_ferret_ext/test.c +40 -87
- data/ext/isomorfeus_ferret_ext/test.h +3 -6
- data/ext/isomorfeus_ferret_ext/test_1710.c +11 -13
- data/ext/isomorfeus_ferret_ext/test_analysis.c +32 -64
- data/ext/isomorfeus_ferret_ext/test_array.c +6 -12
- data/ext/isomorfeus_ferret_ext/test_bitvector.c +12 -24
- data/ext/isomorfeus_ferret_ext/test_document.c +23 -33
- data/ext/isomorfeus_ferret_ext/test_except.c +10 -21
- data/ext/isomorfeus_ferret_ext/test_fields.c +62 -68
- data/ext/isomorfeus_ferret_ext/test_file_deleter.c +15 -23
- data/ext/isomorfeus_ferret_ext/test_filter.c +17 -27
- data/ext/isomorfeus_ferret_ext/test_global.c +14 -29
- data/ext/isomorfeus_ferret_ext/test_hash.c +19 -38
- data/ext/isomorfeus_ferret_ext/test_hashset.c +8 -16
- data/ext/isomorfeus_ferret_ext/test_helper.c +4 -8
- data/ext/isomorfeus_ferret_ext/test_highlighter.c +16 -28
- data/ext/isomorfeus_ferret_ext/test_index.c +277 -487
- data/ext/isomorfeus_ferret_ext/test_lang.c +7 -14
- data/ext/isomorfeus_ferret_ext/test_mdbx_store.c +2 -5
- data/ext/isomorfeus_ferret_ext/test_mempool.c +5 -10
- data/ext/isomorfeus_ferret_ext/test_multimapper.c +3 -6
- data/ext/isomorfeus_ferret_ext/test_priorityqueue.c +9 -18
- data/ext/isomorfeus_ferret_ext/test_q_const_score.c +4 -6
- data/ext/isomorfeus_ferret_ext/test_q_filtered.c +3 -4
- data/ext/isomorfeus_ferret_ext/test_q_fuzzy.c +9 -15
- data/ext/isomorfeus_ferret_ext/test_q_parser.c +8 -16
- data/ext/isomorfeus_ferret_ext/test_q_span.c +19 -35
- data/ext/isomorfeus_ferret_ext/test_ram_store.c +14 -13
- data/ext/isomorfeus_ferret_ext/test_search.c +60 -109
- data/ext/isomorfeus_ferret_ext/test_segments.c +8 -13
- data/ext/isomorfeus_ferret_ext/test_similarity.c +2 -4
- data/ext/isomorfeus_ferret_ext/test_sort.c +14 -24
- data/ext/isomorfeus_ferret_ext/test_store.c +96 -115
- data/ext/isomorfeus_ferret_ext/test_term.c +9 -15
- data/ext/isomorfeus_ferret_ext/test_term_vectors.c +9 -14
- data/ext/isomorfeus_ferret_ext/test_test.c +4 -8
- data/ext/isomorfeus_ferret_ext/test_threading.c +14 -20
- data/ext/isomorfeus_ferret_ext/testhelper.c +11 -21
- data/ext/isomorfeus_ferret_ext/testhelper.h +1 -1
- data/ext/isomorfeus_ferret_ext/tests_all.h +1 -2
- data/lib/isomorfeus/ferret/index/index.rb +1 -1
- data/lib/isomorfeus/ferret/version.rb +1 -1
- metadata +24 -4
@@ -45,7 +45,7 @@
|
|
45
45
|
* Compiler Options
|
46
46
|
**************************************/
|
47
47
|
#ifdef _MSC_VER /* Visual Studio */
|
48
|
-
# pragma warning(disable : 4127)
|
48
|
+
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
49
49
|
#endif
|
50
50
|
|
51
51
|
|
@@ -62,6 +62,19 @@
|
|
62
62
|
#endif
|
63
63
|
|
64
64
|
|
65
|
+
/*-************************************
|
66
|
+
* Library declarations
|
67
|
+
**************************************/
|
68
|
+
#define LZ4F_STATIC_LINKING_ONLY
|
69
|
+
#include "lz4frame.h"
|
70
|
+
#define LZ4_STATIC_LINKING_ONLY
|
71
|
+
#include "lz4.h"
|
72
|
+
#define LZ4_HC_STATIC_LINKING_ONLY
|
73
|
+
#include "lz4hc.h"
|
74
|
+
#define XXH_STATIC_LINKING_ONLY
|
75
|
+
#include "lz4xxhash.h"
|
76
|
+
|
77
|
+
|
65
78
|
/*-************************************
|
66
79
|
* Memory routines
|
67
80
|
**************************************/
|
@@ -70,7 +83,13 @@
|
|
70
83
|
* malloc(), calloc() and free()
|
71
84
|
* towards another library or solution of their choice
|
72
85
|
* by modifying below section.
|
73
|
-
|
86
|
+
**/
|
87
|
+
|
88
|
+
#include <string.h> /* memset, memcpy, memmove */
|
89
|
+
#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */
|
90
|
+
# define MEM_INIT(p,v,s) memset((p),(v),(s))
|
91
|
+
#endif
|
92
|
+
|
74
93
|
#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */
|
75
94
|
# include <stdlib.h> /* malloc, calloc, free */
|
76
95
|
# define ALLOC(s) malloc(s)
|
@@ -78,23 +97,42 @@
|
|
78
97
|
# define FREEMEM(p) free(p)
|
79
98
|
#endif
|
80
99
|
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
100
|
+
static void* LZ4F_calloc(size_t s, LZ4F_CustomMem cmem)
|
101
|
+
{
|
102
|
+
/* custom calloc defined : use it */
|
103
|
+
if (cmem.customCalloc != NULL) {
|
104
|
+
return cmem.customCalloc(cmem.opaqueState, s);
|
105
|
+
}
|
106
|
+
/* nothing defined : use default <stdlib.h>'s calloc() */
|
107
|
+
if (cmem.customAlloc == NULL) {
|
108
|
+
return ALLOC_AND_ZERO(s);
|
109
|
+
}
|
110
|
+
/* only custom alloc defined : use it, and combine it with memset() */
|
111
|
+
{ void* const p = cmem.customAlloc(cmem.opaqueState, s);
|
112
|
+
if (p != NULL) MEM_INIT(p, 0, s);
|
113
|
+
return p;
|
114
|
+
} }
|
85
115
|
|
116
|
+
static void* LZ4F_malloc(size_t s, LZ4F_CustomMem cmem)
|
117
|
+
{
|
118
|
+
/* custom malloc defined : use it */
|
119
|
+
if (cmem.customAlloc != NULL) {
|
120
|
+
return cmem.customAlloc(cmem.opaqueState, s);
|
121
|
+
}
|
122
|
+
/* nothing defined : use default <stdlib.h>'s malloc() */
|
123
|
+
return ALLOC(s);
|
124
|
+
}
|
86
125
|
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
#include "lz4xxhash.h"
|
126
|
+
static void LZ4F_free(void* p, LZ4F_CustomMem cmem)
|
127
|
+
{
|
128
|
+
/* custom malloc defined : use it */
|
129
|
+
if (cmem.customFree != NULL) {
|
130
|
+
cmem.customFree(cmem.opaqueState, p);
|
131
|
+
return;
|
132
|
+
}
|
133
|
+
/* nothing defined : use default <stdlib.h>'s free() */
|
134
|
+
FREEMEM(p);
|
135
|
+
}
|
98
136
|
|
99
137
|
|
100
138
|
/*-************************************
|
@@ -143,7 +181,7 @@ static int g_debuglog_enable = 1;
|
|
143
181
|
#endif
|
144
182
|
|
145
183
|
|
146
|
-
/* unoptimized version; solves
|
184
|
+
/* unoptimized version; solves endianness & alignment issues */
|
147
185
|
static U32 LZ4F_readLE32 (const void* src)
|
148
186
|
{
|
149
187
|
const BYTE* const srcPtr = (const BYTE*)src;
|
@@ -206,8 +244,6 @@ static void LZ4F_writeLE64 (void* dst, U64 value64)
|
|
206
244
|
#define _4BITS 0x0F
|
207
245
|
#define _8BITS 0xFF
|
208
246
|
|
209
|
-
#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U
|
210
|
-
#define LZ4F_MAGICNUMBER 0x184D2204U
|
211
247
|
#define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U
|
212
248
|
#define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB
|
213
249
|
|
@@ -220,22 +256,27 @@ static const size_t BFSize = LZ4F_BLOCK_CHECKSUM_SIZE; /* block footer : checks
|
|
220
256
|
/*-************************************
|
221
257
|
* Structures and local types
|
222
258
|
**************************************/
|
259
|
+
|
260
|
+
typedef enum { LZ4B_COMPRESSED, LZ4B_UNCOMPRESSED} LZ4F_blockCompression_t;
|
261
|
+
|
223
262
|
typedef struct LZ4F_cctx_s
|
224
263
|
{
|
264
|
+
LZ4F_CustomMem cmem;
|
225
265
|
LZ4F_preferences_t prefs;
|
226
266
|
U32 version;
|
227
267
|
U32 cStage;
|
228
268
|
const LZ4F_CDict* cdict;
|
229
269
|
size_t maxBlockSize;
|
230
270
|
size_t maxBufferSize;
|
231
|
-
BYTE* tmpBuff;
|
232
|
-
BYTE* tmpIn;
|
233
|
-
size_t tmpInSize;
|
271
|
+
BYTE* tmpBuff; /* internal buffer, for streaming */
|
272
|
+
BYTE* tmpIn; /* starting position of data compress within internal buffer (>= tmpBuff) */
|
273
|
+
size_t tmpInSize; /* amount of data to compress after tmpIn */
|
234
274
|
U64 totalInSize;
|
235
275
|
XXH32_state_t xxh;
|
236
276
|
void* lz4CtxPtr;
|
237
277
|
U16 lz4CtxAlloc; /* sized for: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */
|
238
278
|
U16 lz4CtxState; /* in use as: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */
|
279
|
+
LZ4F_blockCompression_t blockCompression;
|
239
280
|
} LZ4F_cctx_t;
|
240
281
|
|
241
282
|
|
@@ -264,27 +305,33 @@ LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult)
|
|
264
305
|
return (LZ4F_errorCodes)(-(ptrdiff_t)functionResult);
|
265
306
|
}
|
266
307
|
|
267
|
-
static LZ4F_errorCode_t
|
308
|
+
static LZ4F_errorCode_t LZ4F_returnErrorCode(LZ4F_errorCodes code)
|
268
309
|
{
|
269
310
|
/* A compilation error here means sizeof(ptrdiff_t) is not large enough */
|
270
311
|
LZ4F_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t));
|
271
312
|
return (LZ4F_errorCode_t)-(ptrdiff_t)code;
|
272
313
|
}
|
273
314
|
|
315
|
+
#define RETURN_ERROR(e) return LZ4F_returnErrorCode(LZ4F_ERROR_ ## e)
|
316
|
+
|
317
|
+
#define RETURN_ERROR_IF(c,e) if (c) RETURN_ERROR(e)
|
318
|
+
|
319
|
+
#define FORWARD_IF_ERROR(r) if (LZ4F_isError(r)) return (r)
|
320
|
+
|
274
321
|
unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; }
|
275
322
|
|
276
323
|
int LZ4F_compressionLevel_max(void) { return LZ4HC_CLEVEL_MAX; }
|
277
324
|
|
278
|
-
size_t LZ4F_getBlockSize(
|
325
|
+
size_t LZ4F_getBlockSize(LZ4F_blockSizeID_t blockSizeID)
|
279
326
|
{
|
280
327
|
static const size_t blockSizes[4] = { 64 KB, 256 KB, 1 MB, 4 MB };
|
281
328
|
|
282
329
|
if (blockSizeID == 0) blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT;
|
283
330
|
if (blockSizeID < LZ4F_max64KB || blockSizeID > LZ4F_max4MB)
|
284
|
-
|
285
|
-
blockSizeID
|
286
|
-
|
287
|
-
}
|
331
|
+
RETURN_ERROR(maxBlockSize_invalid);
|
332
|
+
{ int const blockSizeIdx = (int)blockSizeID - (int)LZ4F_max64KB;
|
333
|
+
return blockSizes[blockSizeIdx];
|
334
|
+
} }
|
288
335
|
|
289
336
|
/*-************************************
|
290
337
|
* Private functions
|
@@ -397,21 +444,20 @@ size_t LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx,
|
|
397
444
|
MEM_INIT(&options, 0, sizeof(options));
|
398
445
|
options.stableSrc = 1;
|
399
446
|
|
400
|
-
|
401
|
-
return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
|
447
|
+
RETURN_ERROR_IF(dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs), dstMaxSize_tooSmall);
|
402
448
|
|
403
449
|
{ size_t const headerSize = LZ4F_compressBegin_usingCDict(cctx, dstBuffer, dstCapacity, cdict, &prefs); /* write header */
|
404
|
-
|
450
|
+
FORWARD_IF_ERROR(headerSize);
|
405
451
|
dstPtr += headerSize; /* header size */ }
|
406
452
|
|
407
453
|
assert(dstEnd >= dstPtr);
|
408
454
|
{ size_t const cSize = LZ4F_compressUpdate(cctx, dstPtr, (size_t)(dstEnd-dstPtr), srcBuffer, srcSize, &options);
|
409
|
-
|
455
|
+
FORWARD_IF_ERROR(cSize);
|
410
456
|
dstPtr += cSize; }
|
411
457
|
|
412
458
|
assert(dstEnd >= dstPtr);
|
413
459
|
{ size_t const tailSize = LZ4F_compressEnd(cctx, dstPtr, (size_t)(dstEnd-dstPtr), &options); /* flush last block, and generate suffix */
|
414
|
-
|
460
|
+
FORWARD_IF_ERROR(tailSize);
|
415
461
|
dstPtr += tailSize; }
|
416
462
|
|
417
463
|
assert(dstEnd >= dstStart);
|
@@ -432,27 +478,26 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
|
|
432
478
|
{
|
433
479
|
size_t result;
|
434
480
|
#if (LZ4F_HEAPMODE)
|
435
|
-
LZ4F_cctx_t
|
481
|
+
LZ4F_cctx_t* cctxPtr;
|
436
482
|
result = LZ4F_createCompressionContext(&cctxPtr, LZ4F_VERSION);
|
437
|
-
|
483
|
+
FORWARD_IF_ERROR(result);
|
438
484
|
#else
|
439
485
|
LZ4F_cctx_t cctx;
|
440
486
|
LZ4_stream_t lz4ctx;
|
441
|
-
LZ4F_cctx_t
|
487
|
+
LZ4F_cctx_t* const cctxPtr = &cctx;
|
442
488
|
|
443
|
-
DEBUGLOG(4, "LZ4F_compressFrame");
|
444
489
|
MEM_INIT(&cctx, 0, sizeof(cctx));
|
445
490
|
cctx.version = LZ4F_VERSION;
|
446
491
|
cctx.maxBufferSize = 5 MB; /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */
|
447
|
-
if (preferencesPtr == NULL
|
448
|
-
|
449
|
-
{
|
492
|
+
if ( preferencesPtr == NULL
|
493
|
+
|| preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN ) {
|
450
494
|
LZ4_initStream(&lz4ctx, sizeof(lz4ctx));
|
451
495
|
cctxPtr->lz4CtxPtr = &lz4ctx;
|
452
496
|
cctxPtr->lz4CtxAlloc = 1;
|
453
497
|
cctxPtr->lz4CtxState = 1;
|
454
498
|
}
|
455
499
|
#endif
|
500
|
+
DEBUGLOG(4, "LZ4F_compressFrame");
|
456
501
|
|
457
502
|
result = LZ4F_compressFrame_usingCDict(cctxPtr, dstBuffer, dstCapacity,
|
458
503
|
srcBuffer, srcSize,
|
@@ -461,10 +506,9 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
|
|
461
506
|
#if (LZ4F_HEAPMODE)
|
462
507
|
LZ4F_freeCompressionContext(cctxPtr);
|
463
508
|
#else
|
464
|
-
if (preferencesPtr != NULL
|
465
|
-
|
466
|
-
|
467
|
-
FREEMEM(cctxPtr->lz4CtxPtr);
|
509
|
+
if ( preferencesPtr != NULL
|
510
|
+
&& preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN ) {
|
511
|
+
LZ4F_free(cctxPtr->lz4CtxPtr, cctxPtr->cmem);
|
468
512
|
}
|
469
513
|
#endif
|
470
514
|
return result;
|
@@ -476,30 +520,31 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
|
|
476
520
|
*****************************************************/
|
477
521
|
|
478
522
|
struct LZ4F_CDict_s {
|
523
|
+
LZ4F_CustomMem cmem;
|
479
524
|
void* dictContent;
|
480
525
|
LZ4_stream_t* fastCtx;
|
481
526
|
LZ4_streamHC_t* HCCtx;
|
482
527
|
}; /* typedef'd to LZ4F_CDict within lz4frame_static.h */
|
483
528
|
|
484
|
-
|
485
|
-
|
486
|
-
* LZ4F_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
|
487
|
-
* LZ4F_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
|
488
|
-
* `dictBuffer` can be released after LZ4F_CDict creation, since its content is copied within CDict
|
489
|
-
* @return : digested dictionary for compression, or NULL if failed */
|
490
|
-
LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
|
529
|
+
LZ4F_CDict*
|
530
|
+
LZ4F_createCDict_advanced(LZ4F_CustomMem cmem, const void* dictBuffer, size_t dictSize)
|
491
531
|
{
|
492
532
|
const char* dictStart = (const char*)dictBuffer;
|
493
|
-
LZ4F_CDict* cdict = (LZ4F_CDict*)
|
494
|
-
DEBUGLOG(4, "
|
533
|
+
LZ4F_CDict* const cdict = (LZ4F_CDict*)LZ4F_malloc(sizeof(*cdict), cmem);
|
534
|
+
DEBUGLOG(4, "LZ4F_createCDict_advanced");
|
495
535
|
if (!cdict) return NULL;
|
536
|
+
cdict->cmem = cmem;
|
496
537
|
if (dictSize > 64 KB) {
|
497
538
|
dictStart += dictSize - 64 KB;
|
498
539
|
dictSize = 64 KB;
|
499
540
|
}
|
500
|
-
cdict->dictContent =
|
501
|
-
cdict->fastCtx =
|
502
|
-
cdict->
|
541
|
+
cdict->dictContent = LZ4F_malloc(dictSize, cmem);
|
542
|
+
cdict->fastCtx = (LZ4_stream_t*)LZ4F_malloc(sizeof(LZ4_stream_t), cmem);
|
543
|
+
if (cdict->fastCtx)
|
544
|
+
LZ4_initStream(cdict->fastCtx, sizeof(LZ4_stream_t));
|
545
|
+
cdict->HCCtx = (LZ4_streamHC_t*)LZ4F_malloc(sizeof(LZ4_streamHC_t), cmem);
|
546
|
+
if (cdict->HCCtx)
|
547
|
+
LZ4_initStream(cdict->HCCtx, sizeof(LZ4_streamHC_t));
|
503
548
|
if (!cdict->dictContent || !cdict->fastCtx || !cdict->HCCtx) {
|
504
549
|
LZ4F_freeCDict(cdict);
|
505
550
|
return NULL;
|
@@ -511,13 +556,25 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
|
|
511
556
|
return cdict;
|
512
557
|
}
|
513
558
|
|
559
|
+
/*! LZ4F_createCDict() :
|
560
|
+
* When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
|
561
|
+
* LZ4F_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
|
562
|
+
* LZ4F_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
|
563
|
+
* @dictBuffer can be released after LZ4F_CDict creation, since its content is copied within CDict
|
564
|
+
* @return : digested dictionary for compression, or NULL if failed */
|
565
|
+
LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
|
566
|
+
{
|
567
|
+
DEBUGLOG(4, "LZ4F_createCDict");
|
568
|
+
return LZ4F_createCDict_advanced(LZ4F_defaultCMem, dictBuffer, dictSize);
|
569
|
+
}
|
570
|
+
|
514
571
|
void LZ4F_freeCDict(LZ4F_CDict* cdict)
|
515
572
|
{
|
516
573
|
if (cdict==NULL) return; /* support free on NULL */
|
517
|
-
|
518
|
-
|
519
|
-
|
520
|
-
|
574
|
+
LZ4F_free(cdict->dictContent, cdict->cmem);
|
575
|
+
LZ4F_free(cdict->fastCtx, cdict->cmem);
|
576
|
+
LZ4F_free(cdict->HCCtx, cdict->cmem);
|
577
|
+
LZ4F_free(cdict, cdict->cmem);
|
521
578
|
}
|
522
579
|
|
523
580
|
|
@@ -525,6 +582,20 @@ void LZ4F_freeCDict(LZ4F_CDict* cdict)
|
|
525
582
|
* Advanced compression functions
|
526
583
|
***********************************/
|
527
584
|
|
585
|
+
LZ4F_cctx*
|
586
|
+
LZ4F_createCompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version)
|
587
|
+
{
|
588
|
+
LZ4F_cctx* const cctxPtr =
|
589
|
+
(LZ4F_cctx*)LZ4F_calloc(sizeof(LZ4F_cctx), customMem);
|
590
|
+
if (cctxPtr==NULL) return NULL;
|
591
|
+
|
592
|
+
cctxPtr->cmem = customMem;
|
593
|
+
cctxPtr->version = version;
|
594
|
+
cctxPtr->cStage = 0; /* Uninitialized. Next stage : init cctx */
|
595
|
+
|
596
|
+
return cctxPtr;
|
597
|
+
}
|
598
|
+
|
528
599
|
/*! LZ4F_createCompressionContext() :
|
529
600
|
* The first thing to do is to create a compressionContext object, which will be used in all compression operations.
|
530
601
|
* This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure.
|
@@ -532,17 +603,16 @@ void LZ4F_freeCDict(LZ4F_CDict* cdict)
|
|
532
603
|
* The function will provide a pointer to an allocated LZ4F_compressionContext_t object.
|
533
604
|
* If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation.
|
534
605
|
* Object can release its memory using LZ4F_freeCompressionContext();
|
535
|
-
|
536
|
-
LZ4F_errorCode_t
|
606
|
+
**/
|
607
|
+
LZ4F_errorCode_t
|
608
|
+
LZ4F_createCompressionContext(LZ4F_cctx** LZ4F_compressionContextPtr, unsigned version)
|
537
609
|
{
|
538
|
-
|
539
|
-
|
540
|
-
|
541
|
-
cctxPtr->version = version;
|
542
|
-
cctxPtr->cStage = 0; /* Next stage : init stream */
|
543
|
-
|
544
|
-
*LZ4F_compressionContextPtr = cctxPtr;
|
610
|
+
assert(LZ4F_compressionContextPtr != NULL); /* considered a violation of narrow contract */
|
611
|
+
/* in case it nonetheless happen in production */
|
612
|
+
RETURN_ERROR_IF(LZ4F_compressionContextPtr == NULL, parameter_null);
|
545
613
|
|
614
|
+
*LZ4F_compressionContextPtr = LZ4F_createCompressionContext_advanced(LZ4F_defaultCMem, version);
|
615
|
+
RETURN_ERROR_IF(*LZ4F_compressionContextPtr==NULL, allocation_failed);
|
546
616
|
return LZ4F_OK_NoError;
|
547
617
|
}
|
548
618
|
|
@@ -550,11 +620,10 @@ LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** LZ4F_compressionConte
|
|
550
620
|
LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctxPtr)
|
551
621
|
{
|
552
622
|
if (cctxPtr != NULL) { /* support free on NULL */
|
553
|
-
|
554
|
-
|
555
|
-
|
623
|
+
LZ4F_free(cctxPtr->lz4CtxPtr, cctxPtr->cmem); /* note: LZ4_streamHC_t and LZ4_stream_t are simple POD types */
|
624
|
+
LZ4F_free(cctxPtr->tmpBuff, cctxPtr->cmem);
|
625
|
+
LZ4F_free(cctxPtr, cctxPtr->cmem);
|
556
626
|
}
|
557
|
-
|
558
627
|
return LZ4F_OK_NoError;
|
559
628
|
}
|
560
629
|
|
@@ -588,11 +657,21 @@ static void LZ4F_initStream(void* ctx,
|
|
588
657
|
}
|
589
658
|
}
|
590
659
|
|
660
|
+
static int ctxTypeID_to_size(int ctxTypeID) {
|
661
|
+
switch(ctxTypeID) {
|
662
|
+
case 1:
|
663
|
+
return LZ4_sizeofState();
|
664
|
+
case 2:
|
665
|
+
return LZ4_sizeofStateHC();
|
666
|
+
default:
|
667
|
+
return 0;
|
668
|
+
}
|
669
|
+
}
|
591
670
|
|
592
671
|
/*! LZ4F_compressBegin_usingCDict() :
|
593
|
-
* init streaming compression
|
594
|
-
*
|
595
|
-
* @return : number of bytes written into dstBuffer for the header
|
672
|
+
* init streaming compression AND writes frame header into @dstBuffer.
|
673
|
+
* @dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
|
674
|
+
* @return : number of bytes written into @dstBuffer for the header
|
596
675
|
* or an error code (can be tested using LZ4F_isError())
|
597
676
|
*/
|
598
677
|
size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
|
@@ -600,41 +679,46 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
|
|
600
679
|
const LZ4F_CDict* cdict,
|
601
680
|
const LZ4F_preferences_t* preferencesPtr)
|
602
681
|
{
|
603
|
-
LZ4F_preferences_t prefNull;
|
682
|
+
LZ4F_preferences_t const prefNull = LZ4F_INIT_PREFERENCES;
|
604
683
|
BYTE* const dstStart = (BYTE*)dstBuffer;
|
605
684
|
BYTE* dstPtr = dstStart;
|
606
|
-
BYTE* headerStart;
|
607
685
|
|
608
|
-
|
609
|
-
MEM_INIT(&prefNull, 0, sizeof(prefNull));
|
686
|
+
RETURN_ERROR_IF(dstCapacity < maxFHSize, dstMaxSize_tooSmall);
|
610
687
|
if (preferencesPtr == NULL) preferencesPtr = &prefNull;
|
611
688
|
cctxPtr->prefs = *preferencesPtr;
|
612
689
|
|
613
|
-
/*
|
690
|
+
/* cctx Management */
|
614
691
|
{ U16 const ctxTypeID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2;
|
615
|
-
|
616
|
-
|
692
|
+
int requiredSize = ctxTypeID_to_size(ctxTypeID);
|
693
|
+
int allocatedSize = ctxTypeID_to_size(cctxPtr->lz4CtxAlloc);
|
694
|
+
if (allocatedSize < requiredSize) {
|
695
|
+
/* not enough space allocated */
|
696
|
+
LZ4F_free(cctxPtr->lz4CtxPtr, cctxPtr->cmem);
|
617
697
|
if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
|
618
|
-
|
698
|
+
/* must take ownership of memory allocation,
|
699
|
+
* in order to respect custom allocator contract */
|
700
|
+
cctxPtr->lz4CtxPtr = LZ4F_malloc(sizeof(LZ4_stream_t), cctxPtr->cmem);
|
701
|
+
if (cctxPtr->lz4CtxPtr)
|
702
|
+
LZ4_initStream(cctxPtr->lz4CtxPtr, sizeof(LZ4_stream_t));
|
619
703
|
} else {
|
620
|
-
cctxPtr->lz4CtxPtr =
|
704
|
+
cctxPtr->lz4CtxPtr = LZ4F_malloc(sizeof(LZ4_streamHC_t), cctxPtr->cmem);
|
705
|
+
if (cctxPtr->lz4CtxPtr)
|
706
|
+
LZ4_initStreamHC(cctxPtr->lz4CtxPtr, sizeof(LZ4_streamHC_t));
|
621
707
|
}
|
622
|
-
|
623
|
-
return err0r(LZ4F_ERROR_allocation_failed);
|
708
|
+
RETURN_ERROR_IF(cctxPtr->lz4CtxPtr == NULL, allocation_failed);
|
624
709
|
cctxPtr->lz4CtxAlloc = ctxTypeID;
|
625
710
|
cctxPtr->lz4CtxState = ctxTypeID;
|
626
711
|
} else if (cctxPtr->lz4CtxState != ctxTypeID) {
|
627
|
-
/* otherwise, a sufficient buffer is allocated,
|
628
|
-
* reset it to the correct context type */
|
712
|
+
/* otherwise, a sufficient buffer is already allocated,
|
713
|
+
* but we need to reset it to the correct context type */
|
629
714
|
if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
|
630
|
-
LZ4_initStream((LZ4_stream_t
|
715
|
+
LZ4_initStream((LZ4_stream_t*)cctxPtr->lz4CtxPtr, sizeof(LZ4_stream_t));
|
631
716
|
} else {
|
632
|
-
LZ4_initStreamHC((LZ4_streamHC_t
|
633
|
-
LZ4_setCompressionLevel((LZ4_streamHC_t
|
717
|
+
LZ4_initStreamHC((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, sizeof(LZ4_streamHC_t));
|
718
|
+
LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
|
634
719
|
}
|
635
720
|
cctxPtr->lz4CtxState = ctxTypeID;
|
636
|
-
|
637
|
-
}
|
721
|
+
} }
|
638
722
|
|
639
723
|
/* Buffer Management */
|
640
724
|
if (cctxPtr->prefs.frameInfo.blockSizeID == 0)
|
@@ -647,9 +731,9 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
|
|
647
731
|
|
648
732
|
if (cctxPtr->maxBufferSize < requiredBuffSize) {
|
649
733
|
cctxPtr->maxBufferSize = 0;
|
650
|
-
|
651
|
-
cctxPtr->tmpBuff = (BYTE*)
|
652
|
-
|
734
|
+
LZ4F_free(cctxPtr->tmpBuff, cctxPtr->cmem);
|
735
|
+
cctxPtr->tmpBuff = (BYTE*)LZ4F_calloc(requiredBuffSize, cctxPtr->cmem);
|
736
|
+
RETURN_ERROR_IF(cctxPtr->tmpBuff == NULL, allocation_failed);
|
653
737
|
cctxPtr->maxBufferSize = requiredBuffSize;
|
654
738
|
} }
|
655
739
|
cctxPtr->tmpIn = cctxPtr->tmpBuff;
|
@@ -669,31 +753,32 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
|
|
669
753
|
/* Magic Number */
|
670
754
|
LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER);
|
671
755
|
dstPtr += 4;
|
672
|
-
headerStart = dstPtr;
|
673
|
-
|
674
|
-
|
675
|
-
|
676
|
-
|
677
|
-
|
678
|
-
|
679
|
-
|
680
|
-
|
681
|
-
|
682
|
-
|
683
|
-
|
684
|
-
|
685
|
-
|
686
|
-
|
687
|
-
|
688
|
-
|
689
|
-
|
690
|
-
|
691
|
-
|
692
|
-
|
756
|
+
{ BYTE* const headerStart = dstPtr;
|
757
|
+
|
758
|
+
/* FLG Byte */
|
759
|
+
*dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */
|
760
|
+
+ ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5)
|
761
|
+
+ ((cctxPtr->prefs.frameInfo.blockChecksumFlag & _1BIT ) << 4)
|
762
|
+
+ ((unsigned)(cctxPtr->prefs.frameInfo.contentSize > 0) << 3)
|
763
|
+
+ ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2)
|
764
|
+
+ (cctxPtr->prefs.frameInfo.dictID > 0) );
|
765
|
+
/* BD Byte */
|
766
|
+
*dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4);
|
767
|
+
/* Optional Frame content size field */
|
768
|
+
if (cctxPtr->prefs.frameInfo.contentSize) {
|
769
|
+
LZ4F_writeLE64(dstPtr, cctxPtr->prefs.frameInfo.contentSize);
|
770
|
+
dstPtr += 8;
|
771
|
+
cctxPtr->totalInSize = 0;
|
772
|
+
}
|
773
|
+
/* Optional dictionary ID field */
|
774
|
+
if (cctxPtr->prefs.frameInfo.dictID) {
|
775
|
+
LZ4F_writeLE32(dstPtr, cctxPtr->prefs.frameInfo.dictID);
|
776
|
+
dstPtr += 4;
|
777
|
+
}
|
778
|
+
/* Header CRC Byte */
|
779
|
+
*dstPtr = LZ4F_headerChecksum(headerStart, (size_t)(dstPtr - headerStart));
|
780
|
+
dstPtr++;
|
693
781
|
}
|
694
|
-
/* Header CRC Byte */
|
695
|
-
*dstPtr = LZ4F_headerChecksum(headerStart, (size_t)(dstPtr - headerStart));
|
696
|
-
dstPtr++;
|
697
782
|
|
698
783
|
cctxPtr->cStage = 1; /* header written, now request input data block */
|
699
784
|
return (size_t)(dstPtr - dstStart);
|
@@ -701,9 +786,9 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
|
|
701
786
|
|
702
787
|
|
703
788
|
/*! LZ4F_compressBegin() :
|
704
|
-
* init streaming compression
|
705
|
-
*
|
706
|
-
*
|
789
|
+
* init streaming compression AND writes frame header into @dstBuffer.
|
790
|
+
* @dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
|
791
|
+
* @preferencesPtr can be NULL, in which case default parameters are selected.
|
707
792
|
* @return : number of bytes written into dstBuffer for the header
|
708
793
|
* or an error code (can be tested using LZ4F_isError())
|
709
794
|
*/
|
@@ -744,11 +829,13 @@ static size_t LZ4F_makeBlock(void* dst,
|
|
744
829
|
LZ4F_blockChecksum_t crcFlag)
|
745
830
|
{
|
746
831
|
BYTE* const cSizePtr = (BYTE*)dst;
|
747
|
-
U32 cSize
|
748
|
-
|
749
|
-
|
750
|
-
|
751
|
-
|
832
|
+
U32 cSize;
|
833
|
+
assert(compress != NULL);
|
834
|
+
cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize),
|
835
|
+
(int)(srcSize), (int)(srcSize-1),
|
836
|
+
level, cdict);
|
837
|
+
|
838
|
+
if (cSize == 0 || cSize >= srcSize) {
|
752
839
|
cSize = (U32)srcSize;
|
753
840
|
LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG);
|
754
841
|
memcpy(cSizePtr+BHSize, src, srcSize);
|
@@ -766,6 +853,7 @@ static size_t LZ4F_makeBlock(void* dst,
|
|
766
853
|
static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
|
767
854
|
{
|
768
855
|
int const acceleration = (level < 0) ? -level + 1 : 1;
|
856
|
+
DEBUGLOG(5, "LZ4F_compressBlock (srcSize=%i)", srcSize);
|
769
857
|
LZ4F_initStream(ctx, cdict, level, LZ4F_blockIndependent);
|
770
858
|
if (cdict) {
|
771
859
|
return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
|
@@ -778,6 +866,7 @@ static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, in
|
|
778
866
|
{
|
779
867
|
int const acceleration = (level < 0) ? -level + 1 : 1;
|
780
868
|
(void)cdict; /* init once at beginning of frame */
|
869
|
+
DEBUGLOG(5, "LZ4F_compressBlock_continue (srcSize=%i)", srcSize);
|
781
870
|
return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
|
782
871
|
}
|
783
872
|
|
@@ -796,8 +885,15 @@ static int LZ4F_compressBlockHC_continue(void* ctx, const char* src, char* dst,
|
|
796
885
|
return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
|
797
886
|
}
|
798
887
|
|
799
|
-
static
|
888
|
+
static int LZ4F_doNotCompressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
|
889
|
+
{
|
890
|
+
(void)ctx; (void)src; (void)dst; (void)srcSize; (void)dstCapacity; (void)level; (void)cdict;
|
891
|
+
return 0;
|
892
|
+
}
|
893
|
+
|
894
|
+
static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level, LZ4F_blockCompression_t compressMode)
|
800
895
|
{
|
896
|
+
if (compressMode == LZ4B_UNCOMPRESSED) return LZ4F_doNotCompressBlock;
|
801
897
|
if (level < LZ4HC_CLEVEL_MIN) {
|
802
898
|
if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlock;
|
803
899
|
return LZ4F_compressBlock_continue;
|
@@ -806,6 +902,7 @@ static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int lev
|
|
806
902
|
return LZ4F_compressBlockHC_continue;
|
807
903
|
}
|
808
904
|
|
905
|
+
/* Save history (up to 64KB) into @tmpBuff */
|
809
906
|
static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
|
810
907
|
{
|
811
908
|
if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
|
@@ -815,38 +912,57 @@ static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
|
|
815
912
|
|
816
913
|
typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;
|
817
914
|
|
818
|
-
|
915
|
+
static const LZ4F_compressOptions_t k_cOptionsNull = { 0, { 0, 0, 0 } };
|
916
|
+
|
917
|
+
|
918
|
+
/*! LZ4F_compressUpdateImpl() :
|
819
919
|
* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
|
820
|
-
*
|
821
|
-
*
|
920
|
+
* When successful, the function always entirely consumes @srcBuffer.
|
921
|
+
* src data is either buffered or compressed into @dstBuffer.
|
922
|
+
* If the block compression does not match the compression of the previous block, the old data is flushed
|
923
|
+
* and operations continue with the new compression mode.
|
924
|
+
* @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr) when block compression is turned on.
|
925
|
+
* @compressOptionsPtr is optional : provide NULL to mean "default".
|
822
926
|
* @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
|
823
927
|
* or an error code if it fails (which can be tested using LZ4F_isError())
|
928
|
+
* After an error, the state is left in a UB state, and must be re-initialized.
|
824
929
|
*/
|
825
|
-
size_t
|
826
|
-
|
930
|
+
static size_t LZ4F_compressUpdateImpl(LZ4F_cctx* cctxPtr,
|
931
|
+
void* dstBuffer, size_t dstCapacity,
|
827
932
|
const void* srcBuffer, size_t srcSize,
|
828
|
-
const LZ4F_compressOptions_t* compressOptionsPtr
|
829
|
-
|
830
|
-
|
933
|
+
const LZ4F_compressOptions_t* compressOptionsPtr,
|
934
|
+
LZ4F_blockCompression_t blockCompression)
|
935
|
+
{
|
831
936
|
size_t const blockSize = cctxPtr->maxBlockSize;
|
832
937
|
const BYTE* srcPtr = (const BYTE*)srcBuffer;
|
833
938
|
const BYTE* const srcEnd = srcPtr + srcSize;
|
834
939
|
BYTE* const dstStart = (BYTE*)dstBuffer;
|
835
940
|
BYTE* dstPtr = dstStart;
|
836
941
|
LZ4F_lastBlockStatus lastBlockCompressed = notDone;
|
837
|
-
compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
|
838
|
-
|
942
|
+
compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel, blockCompression);
|
943
|
+
size_t bytesWritten;
|
839
944
|
DEBUGLOG(4, "LZ4F_compressUpdate (srcSize=%zu)", srcSize);
|
840
945
|
|
841
|
-
|
946
|
+
RETURN_ERROR_IF(cctxPtr->cStage != 1, compressionState_uninitialized); /* state must be initialized and waiting for next block */
|
842
947
|
if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
|
843
|
-
|
844
|
-
|
845
|
-
if (
|
948
|
+
RETURN_ERROR(dstMaxSize_tooSmall);
|
949
|
+
|
950
|
+
if (blockCompression == LZ4B_UNCOMPRESSED && dstCapacity < srcSize)
|
951
|
+
RETURN_ERROR(dstMaxSize_tooSmall);
|
952
|
+
|
953
|
+
/* flush currently written block, to continue with new block compression */
|
954
|
+
if (cctxPtr->blockCompression != blockCompression) {
|
955
|
+
bytesWritten = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr);
|
956
|
+
dstPtr += bytesWritten;
|
957
|
+
cctxPtr->blockCompression = blockCompression;
|
958
|
+
}
|
959
|
+
|
960
|
+
if (compressOptionsPtr == NULL) compressOptionsPtr = &k_cOptionsNull;
|
846
961
|
|
847
962
|
/* complete tmp buffer */
|
848
963
|
if (cctxPtr->tmpInSize > 0) { /* some data already within tmp buffer */
|
849
964
|
size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize;
|
965
|
+
assert(blockSize > cctxPtr->tmpInSize);
|
850
966
|
if (sizeToCopy > srcSize) {
|
851
967
|
/* add src to tmpIn buffer */
|
852
968
|
memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize);
|
@@ -864,11 +980,9 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
|
|
864
980
|
compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
|
865
981
|
cctxPtr->cdict,
|
866
982
|
cctxPtr->prefs.frameInfo.blockChecksumFlag);
|
867
|
-
|
868
983
|
if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
|
869
984
|
cctxPtr->tmpInSize = 0;
|
870
|
-
|
871
|
-
}
|
985
|
+
} }
|
872
986
|
|
873
987
|
while ((size_t)(srcEnd - srcPtr) >= blockSize) {
|
874
988
|
/* compress full blocks */
|
@@ -882,33 +996,38 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
|
|
882
996
|
}
|
883
997
|
|
884
998
|
if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) {
|
885
|
-
/*
|
999
|
+
/* autoFlush : remaining input (< blockSize) is compressed */
|
886
1000
|
lastBlockCompressed = fromSrcBuffer;
|
887
1001
|
dstPtr += LZ4F_makeBlock(dstPtr,
|
888
1002
|
srcPtr, (size_t)(srcEnd - srcPtr),
|
889
1003
|
compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
|
890
1004
|
cctxPtr->cdict,
|
891
1005
|
cctxPtr->prefs.frameInfo.blockChecksumFlag);
|
892
|
-
srcPtr
|
1006
|
+
srcPtr = srcEnd;
|
893
1007
|
}
|
894
1008
|
|
895
|
-
/* preserve dictionary
|
1009
|
+
/* preserve dictionary within @tmpBuff whenever necessary */
|
896
1010
|
if ((cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) {
|
1011
|
+
/* linked blocks are only supported in compressed mode, see LZ4F_uncompressedUpdate */
|
1012
|
+
assert(blockCompression == LZ4B_COMPRESSED);
|
897
1013
|
if (compressOptionsPtr->stableSrc) {
|
898
|
-
cctxPtr->tmpIn = cctxPtr->tmpBuff;
|
1014
|
+
cctxPtr->tmpIn = cctxPtr->tmpBuff; /* src is stable : dictionary remains in src across invocations */
|
899
1015
|
} else {
|
900
1016
|
int const realDictSize = LZ4F_localSaveDict(cctxPtr);
|
901
|
-
|
1017
|
+
assert(0 <= realDictSize && realDictSize <= 64 KB);
|
902
1018
|
cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
|
903
1019
|
}
|
904
1020
|
}
|
905
1021
|
|
906
1022
|
/* keep tmpIn within limits */
|
907
|
-
if ((cctxPtr->
|
908
|
-
|
1023
|
+
if (!(cctxPtr->prefs.autoFlush) /* no autoflush : there may be some data left within internal buffer */
|
1024
|
+
&& (cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) ) /* not enough room to store next block */
|
909
1025
|
{
|
1026
|
+
/* only preserve 64KB within internal buffer. Ensures there is enough room for next block.
|
1027
|
+
* note: this situation necessarily implies lastBlockCompressed==fromTmpBuffer */
|
910
1028
|
int const realDictSize = LZ4F_localSaveDict(cctxPtr);
|
911
1029
|
cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
|
1030
|
+
assert((cctxPtr->tmpIn + blockSize) <= (cctxPtr->tmpBuff + cctxPtr->maxBufferSize));
|
912
1031
|
}
|
913
1032
|
|
914
1033
|
/* some input data left, necessarily < blockSize */
|
@@ -926,6 +1045,53 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
|
|
926
1045
|
return (size_t)(dstPtr - dstStart);
|
927
1046
|
}
|
928
1047
|
|
1048
|
+
/*! LZ4F_compressUpdate() :
|
1049
|
+
* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
|
1050
|
+
* When successful, the function always entirely consumes @srcBuffer.
|
1051
|
+
* src data is either buffered or compressed into @dstBuffer.
|
1052
|
+
* If previously an uncompressed block was written, buffered data is flushed
|
1053
|
+
* before appending compressed data is continued.
|
1054
|
+
* @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
|
1055
|
+
* @compressOptionsPtr is optional : provide NULL to mean "default".
|
1056
|
+
* @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
|
1057
|
+
* or an error code if it fails (which can be tested using LZ4F_isError())
|
1058
|
+
* After an error, the state is left in a UB state, and must be re-initialized.
|
1059
|
+
*/
|
1060
|
+
size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
|
1061
|
+
void* dstBuffer, size_t dstCapacity,
|
1062
|
+
const void* srcBuffer, size_t srcSize,
|
1063
|
+
const LZ4F_compressOptions_t* compressOptionsPtr)
|
1064
|
+
{
|
1065
|
+
return LZ4F_compressUpdateImpl(cctxPtr,
|
1066
|
+
dstBuffer, dstCapacity,
|
1067
|
+
srcBuffer, srcSize,
|
1068
|
+
compressOptionsPtr, LZ4B_COMPRESSED);
|
1069
|
+
}
|
1070
|
+
|
1071
|
+
/*! LZ4F_compressUpdate() :
|
1072
|
+
* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
|
1073
|
+
* When successful, the function always entirely consumes @srcBuffer.
|
1074
|
+
* src data is either buffered or compressed into @dstBuffer.
|
1075
|
+
* If previously an uncompressed block was written, buffered data is flushed
|
1076
|
+
* before appending compressed data is continued.
|
1077
|
+
* This is only supported when LZ4F_blockIndependent is used
|
1078
|
+
* @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
|
1079
|
+
* @compressOptionsPtr is optional : provide NULL to mean "default".
|
1080
|
+
* @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
|
1081
|
+
* or an error code if it fails (which can be tested using LZ4F_isError())
|
1082
|
+
* After an error, the state is left in a UB state, and must be re-initialized.
|
1083
|
+
*/
|
1084
|
+
size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctxPtr,
|
1085
|
+
void* dstBuffer, size_t dstCapacity,
|
1086
|
+
const void* srcBuffer, size_t srcSize,
|
1087
|
+
const LZ4F_compressOptions_t* compressOptionsPtr) {
|
1088
|
+
RETURN_ERROR_IF(cctxPtr->prefs.frameInfo.blockMode != LZ4F_blockIndependent, blockMode_invalid);
|
1089
|
+
return LZ4F_compressUpdateImpl(cctxPtr,
|
1090
|
+
dstBuffer, dstCapacity,
|
1091
|
+
srcBuffer, srcSize,
|
1092
|
+
compressOptionsPtr, LZ4B_UNCOMPRESSED);
|
1093
|
+
}
|
1094
|
+
|
929
1095
|
|
930
1096
|
/*! LZ4F_flush() :
|
931
1097
|
* When compressed data must be sent immediately, without waiting for a block to be filled,
|
@@ -944,13 +1110,12 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr,
|
|
944
1110
|
compressFunc_t compress;
|
945
1111
|
|
946
1112
|
if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */
|
947
|
-
|
948
|
-
|
949
|
-
|
950
|
-
(void)compressOptionsPtr; /* not yet useful */
|
1113
|
+
RETURN_ERROR_IF(cctxPtr->cStage != 1, compressionState_uninitialized);
|
1114
|
+
RETURN_ERROR_IF(dstCapacity < (cctxPtr->tmpInSize + BHSize + BFSize), dstMaxSize_tooSmall);
|
1115
|
+
(void)compressOptionsPtr; /* not useful (yet) */
|
951
1116
|
|
952
1117
|
/* select compression function */
|
953
|
-
compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
|
1118
|
+
compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel, cctxPtr->blockCompression);
|
954
1119
|
|
955
1120
|
/* compress tmp buffer */
|
956
1121
|
dstPtr += LZ4F_makeBlock(dstPtr,
|
@@ -992,19 +1157,19 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr,
|
|
992
1157
|
|
993
1158
|
size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr);
|
994
1159
|
DEBUGLOG(5,"LZ4F_compressEnd: dstCapacity=%u", (unsigned)dstCapacity);
|
995
|
-
|
1160
|
+
FORWARD_IF_ERROR(flushSize);
|
996
1161
|
dstPtr += flushSize;
|
997
1162
|
|
998
1163
|
assert(flushSize <= dstCapacity);
|
999
1164
|
dstCapacity -= flushSize;
|
1000
1165
|
|
1001
|
-
|
1166
|
+
RETURN_ERROR_IF(dstCapacity < 4, dstMaxSize_tooSmall);
|
1002
1167
|
LZ4F_writeLE32(dstPtr, 0);
|
1003
1168
|
dstPtr += 4; /* endMark */
|
1004
1169
|
|
1005
1170
|
if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) {
|
1006
1171
|
U32 const xxh = XXH32_digest(&(cctxPtr->xxh));
|
1007
|
-
|
1172
|
+
RETURN_ERROR_IF(dstCapacity < 8, dstMaxSize_tooSmall);
|
1008
1173
|
DEBUGLOG(5,"Writing 32-bit content checksum");
|
1009
1174
|
LZ4F_writeLE32(dstPtr, xxh);
|
1010
1175
|
dstPtr+=4; /* content Checksum */
|
@@ -1015,7 +1180,7 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr,
|
|
1015
1180
|
|
1016
1181
|
if (cctxPtr->prefs.frameInfo.contentSize) {
|
1017
1182
|
if (cctxPtr->prefs.frameInfo.contentSize != cctxPtr->totalInSize)
|
1018
|
-
|
1183
|
+
RETURN_ERROR(frameSize_wrong);
|
1019
1184
|
}
|
1020
1185
|
|
1021
1186
|
return (size_t)(dstPtr - dstStart);
|
@@ -1039,6 +1204,7 @@ typedef enum {
|
|
1039
1204
|
} dStage_t;
|
1040
1205
|
|
1041
1206
|
struct LZ4F_dctx_s {
|
1207
|
+
LZ4F_CustomMem cmem;
|
1042
1208
|
LZ4F_frameInfo_t frameInfo;
|
1043
1209
|
U32 version;
|
1044
1210
|
dStage_t dStage;
|
@@ -1056,26 +1222,37 @@ struct LZ4F_dctx_s {
|
|
1056
1222
|
size_t tmpOutStart;
|
1057
1223
|
XXH32_state_t xxh;
|
1058
1224
|
XXH32_state_t blockChecksum;
|
1225
|
+
int skipChecksum;
|
1059
1226
|
BYTE header[LZ4F_HEADER_SIZE_MAX];
|
1060
1227
|
}; /* typedef'd to LZ4F_dctx in lz4frame.h */
|
1061
1228
|
|
1062
1229
|
|
1230
|
+
LZ4F_dctx* LZ4F_createDecompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version)
|
1231
|
+
{
|
1232
|
+
LZ4F_dctx* const dctx = (LZ4F_dctx*)LZ4F_calloc(sizeof(LZ4F_dctx), customMem);
|
1233
|
+
if (dctx == NULL) return NULL;
|
1234
|
+
|
1235
|
+
dctx->cmem = customMem;
|
1236
|
+
dctx->version = version;
|
1237
|
+
return dctx;
|
1238
|
+
}
|
1239
|
+
|
1063
1240
|
/*! LZ4F_createDecompressionContext() :
|
1064
1241
|
* Create a decompressionContext object, which will track all decompression operations.
|
1065
1242
|
* Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object.
|
1066
1243
|
* Object can later be released using LZ4F_freeDecompressionContext().
|
1067
1244
|
* @return : if != 0, there was an error during context creation.
|
1068
1245
|
*/
|
1069
|
-
LZ4F_errorCode_t
|
1246
|
+
LZ4F_errorCode_t
|
1247
|
+
LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber)
|
1070
1248
|
{
|
1071
|
-
|
1072
|
-
|
1073
|
-
*LZ4F_decompressionContextPtr = NULL;
|
1074
|
-
return err0r(LZ4F_ERROR_allocation_failed);
|
1075
|
-
}
|
1249
|
+
assert(LZ4F_decompressionContextPtr != NULL); /* violation of narrow contract */
|
1250
|
+
RETURN_ERROR_IF(LZ4F_decompressionContextPtr == NULL, parameter_null); /* in case it nonetheless happen in production */
|
1076
1251
|
|
1077
|
-
|
1078
|
-
*LZ4F_decompressionContextPtr
|
1252
|
+
*LZ4F_decompressionContextPtr = LZ4F_createDecompressionContext_advanced(LZ4F_defaultCMem, versionNumber);
|
1253
|
+
if (*LZ4F_decompressionContextPtr == NULL) { /* failed allocation */
|
1254
|
+
RETURN_ERROR(allocation_failed);
|
1255
|
+
}
|
1079
1256
|
return LZ4F_OK_NoError;
|
1080
1257
|
}
|
1081
1258
|
|
@@ -1084,9 +1261,9 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx)
|
|
1084
1261
|
LZ4F_errorCode_t result = LZ4F_OK_NoError;
|
1085
1262
|
if (dctx != NULL) { /* can accept NULL input, like free() */
|
1086
1263
|
result = (LZ4F_errorCode_t)dctx->dStage;
|
1087
|
-
|
1088
|
-
|
1089
|
-
|
1264
|
+
LZ4F_free(dctx->tmpIn, dctx->cmem);
|
1265
|
+
LZ4F_free(dctx->tmpOutBuffer, dctx->cmem);
|
1266
|
+
LZ4F_free(dctx, dctx->cmem);
|
1090
1267
|
}
|
1091
1268
|
return result;
|
1092
1269
|
}
|
@@ -1099,6 +1276,7 @@ void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx)
|
|
1099
1276
|
dctx->dStage = dstage_getFrameHeader;
|
1100
1277
|
dctx->dict = NULL;
|
1101
1278
|
dctx->dictSize = 0;
|
1279
|
+
dctx->skipChecksum = 0;
|
1102
1280
|
}
|
1103
1281
|
|
1104
1282
|
|
@@ -1118,7 +1296,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
|
|
1118
1296
|
|
1119
1297
|
DEBUGLOG(5, "LZ4F_decodeHeader");
|
1120
1298
|
/* need to decode header to get frameInfo */
|
1121
|
-
|
1299
|
+
RETURN_ERROR_IF(srcSize < minFHSize, frameHeader_incomplete); /* minimal frame header size */
|
1122
1300
|
MEM_INIT(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo));
|
1123
1301
|
|
1124
1302
|
/* special case : skippable frames */
|
@@ -1132,14 +1310,13 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
|
|
1132
1310
|
} else {
|
1133
1311
|
dctx->dStage = dstage_getSFrameSize;
|
1134
1312
|
return 4;
|
1135
|
-
|
1136
|
-
}
|
1313
|
+
} }
|
1137
1314
|
|
1138
1315
|
/* control magic number */
|
1139
1316
|
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
|
1140
1317
|
if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) {
|
1141
1318
|
DEBUGLOG(4, "frame header error : unknown magic number");
|
1142
|
-
|
1319
|
+
RETURN_ERROR(frameType_unknown);
|
1143
1320
|
}
|
1144
1321
|
#endif
|
1145
1322
|
dctx->frameInfo.frameType = LZ4F_frame;
|
@@ -1153,8 +1330,8 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
|
|
1153
1330
|
contentChecksumFlag = (FLG>>2) & _1BIT;
|
1154
1331
|
dictIDFlag = FLG & _1BIT;
|
1155
1332
|
/* validate */
|
1156
|
-
if (((FLG>>1)&_1BIT) != 0)
|
1157
|
-
if (version != 1)
|
1333
|
+
if (((FLG>>1)&_1BIT) != 0) RETURN_ERROR(reservedFlag_set); /* Reserved bit */
|
1334
|
+
if (version != 1) RETURN_ERROR(headerVersion_wrong); /* Version Number, only supported value */
|
1158
1335
|
}
|
1159
1336
|
|
1160
1337
|
/* Frame Header Size */
|
@@ -1173,17 +1350,16 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
|
|
1173
1350
|
{ U32 const BD = srcPtr[5];
|
1174
1351
|
blockSizeID = (BD>>4) & _3BITS;
|
1175
1352
|
/* validate */
|
1176
|
-
if (((BD>>7)&_1BIT) != 0)
|
1177
|
-
if (blockSizeID < 4)
|
1178
|
-
if (((BD>>0)&_4BITS) != 0)
|
1353
|
+
if (((BD>>7)&_1BIT) != 0) RETURN_ERROR(reservedFlag_set); /* Reserved bit */
|
1354
|
+
if (blockSizeID < 4) RETURN_ERROR(maxBlockSize_invalid); /* 4-7 only supported values for the time being */
|
1355
|
+
if (((BD>>0)&_4BITS) != 0) RETURN_ERROR(reservedFlag_set); /* Reserved bits */
|
1179
1356
|
}
|
1180
1357
|
|
1181
1358
|
/* check header */
|
1182
1359
|
assert(frameHeaderSize > 5);
|
1183
1360
|
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
|
1184
1361
|
{ BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5);
|
1185
|
-
|
1186
|
-
return err0r(LZ4F_ERROR_headerChecksum_invalid);
|
1362
|
+
RETURN_ERROR_IF(HC != srcPtr[frameHeaderSize-1], headerChecksum_invalid);
|
1187
1363
|
}
|
1188
1364
|
#endif
|
1189
1365
|
|
@@ -1192,10 +1368,9 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
|
|
1192
1368
|
dctx->frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)blockChecksumFlag;
|
1193
1369
|
dctx->frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)contentChecksumFlag;
|
1194
1370
|
dctx->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID;
|
1195
|
-
dctx->maxBlockSize = LZ4F_getBlockSize(blockSizeID);
|
1371
|
+
dctx->maxBlockSize = LZ4F_getBlockSize((LZ4F_blockSizeID_t)blockSizeID);
|
1196
1372
|
if (contentSizeFlag)
|
1197
|
-
dctx->frameRemainingSize =
|
1198
|
-
dctx->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6);
|
1373
|
+
dctx->frameRemainingSize = dctx->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6);
|
1199
1374
|
if (dictIDFlag)
|
1200
1375
|
dctx->frameInfo.dictID = LZ4F_readLE32(srcPtr + frameHeaderSize - 5);
|
1201
1376
|
|
@@ -1211,11 +1386,11 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
|
|
1211
1386
|
*/
|
1212
1387
|
size_t LZ4F_headerSize(const void* src, size_t srcSize)
|
1213
1388
|
{
|
1214
|
-
|
1389
|
+
RETURN_ERROR_IF(src == NULL, srcPtr_wrong);
|
1215
1390
|
|
1216
1391
|
/* minimal srcSize to determine header size */
|
1217
1392
|
if (srcSize < LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH)
|
1218
|
-
|
1393
|
+
RETURN_ERROR(frameHeader_incomplete);
|
1219
1394
|
|
1220
1395
|
/* special case : skippable frames */
|
1221
1396
|
if ((LZ4F_readLE32(src) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START)
|
@@ -1224,7 +1399,7 @@ size_t LZ4F_headerSize(const void* src, size_t srcSize)
|
|
1224
1399
|
/* control magic number */
|
1225
1400
|
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
|
1226
1401
|
if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER)
|
1227
|
-
|
1402
|
+
RETURN_ERROR(frameType_unknown);
|
1228
1403
|
#endif
|
1229
1404
|
|
1230
1405
|
/* Frame Header Size */
|
@@ -1266,13 +1441,13 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
|
|
1266
1441
|
if (dctx->dStage == dstage_storeFrameHeader) {
|
1267
1442
|
/* frame decoding already started, in the middle of header => automatic fail */
|
1268
1443
|
*srcSizePtr = 0;
|
1269
|
-
|
1444
|
+
RETURN_ERROR(frameDecoding_alreadyStarted);
|
1270
1445
|
} else {
|
1271
1446
|
size_t const hSize = LZ4F_headerSize(srcBuffer, *srcSizePtr);
|
1272
1447
|
if (LZ4F_isError(hSize)) { *srcSizePtr=0; return hSize; }
|
1273
1448
|
if (*srcSizePtr < hSize) {
|
1274
1449
|
*srcSizePtr=0;
|
1275
|
-
|
1450
|
+
RETURN_ERROR(frameHeader_incomplete);
|
1276
1451
|
}
|
1277
1452
|
|
1278
1453
|
{ size_t decodeResult = LZ4F_decodeHeader(dctx, srcBuffer, hSize);
|
@@ -1290,16 +1465,14 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
|
|
1290
1465
|
|
1291
1466
|
/* LZ4F_updateDict() :
|
1292
1467
|
* only used for LZ4F_blockLinked mode
|
1293
|
-
* Condition : dstPtr != NULL
|
1468
|
+
* Condition : @dstPtr != NULL
|
1294
1469
|
*/
|
1295
1470
|
static void LZ4F_updateDict(LZ4F_dctx* dctx,
|
1296
1471
|
const BYTE* dstPtr, size_t dstSize, const BYTE* dstBufferStart,
|
1297
1472
|
unsigned withinTmp)
|
1298
1473
|
{
|
1299
1474
|
assert(dstPtr != NULL);
|
1300
|
-
if (dctx->dictSize==0)
|
1301
|
-
dctx->dict = (const BYTE*)dstPtr; /* priority to prefix mode */
|
1302
|
-
}
|
1475
|
+
if (dctx->dictSize==0) dctx->dict = (const BYTE*)dstPtr; /* will lead to prefix mode */
|
1303
1476
|
assert(dctx->dict != NULL);
|
1304
1477
|
|
1305
1478
|
if (dctx->dict + dctx->dictSize == dstPtr) { /* prefix mode, everything within dstBuffer */
|
@@ -1362,7 +1535,6 @@ static void LZ4F_updateDict(LZ4F_dctx* dctx,
|
|
1362
1535
|
}
|
1363
1536
|
|
1364
1537
|
|
1365
|
-
|
1366
1538
|
/*! LZ4F_decompress() :
|
1367
1539
|
* Call this function repetitively to regenerate compressed data in srcBuffer.
|
1368
1540
|
* The function will attempt to decode up to *srcSizePtr bytes from srcBuffer
|
@@ -1406,6 +1578,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1406
1578
|
*srcSizePtr = 0;
|
1407
1579
|
*dstSizePtr = 0;
|
1408
1580
|
assert(dctx != NULL);
|
1581
|
+
dctx->skipChecksum |= (decompressOptionsPtr->skipChecksums != 0); /* once set, disable for the remainder of the frame */
|
1409
1582
|
|
1410
1583
|
/* behaves as a state machine */
|
1411
1584
|
|
@@ -1418,7 +1591,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1418
1591
|
DEBUGLOG(6, "dstage_getFrameHeader");
|
1419
1592
|
if ((size_t)(srcEnd-srcPtr) >= maxFHSize) { /* enough to decode - shortcut */
|
1420
1593
|
size_t const hSize = LZ4F_decodeHeader(dctx, srcPtr, (size_t)(srcEnd-srcPtr)); /* will update dStage appropriately */
|
1421
|
-
|
1594
|
+
FORWARD_IF_ERROR(hSize);
|
1422
1595
|
srcPtr += hSize;
|
1423
1596
|
break;
|
1424
1597
|
}
|
@@ -1440,9 +1613,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1440
1613
|
doAnotherStage = 0; /* not enough src data, ask for some more */
|
1441
1614
|
break;
|
1442
1615
|
}
|
1443
|
-
|
1444
|
-
if (LZ4F_isError(hSize)) return hSize;
|
1445
|
-
}
|
1616
|
+
FORWARD_IF_ERROR( LZ4F_decodeHeader(dctx, dctx->header, dctx->tmpInTarget) ); /* will update dStage appropriately */
|
1446
1617
|
break;
|
1447
1618
|
|
1448
1619
|
case dstage_init:
|
@@ -1453,14 +1624,12 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1453
1624
|
+ ((dctx->frameInfo.blockMode==LZ4F_blockLinked) ? 128 KB : 0);
|
1454
1625
|
if (bufferNeeded > dctx->maxBufferSize) { /* tmp buffers too small */
|
1455
1626
|
dctx->maxBufferSize = 0; /* ensure allocation will be re-attempted on next entry*/
|
1456
|
-
|
1457
|
-
dctx->tmpIn = (BYTE*)
|
1458
|
-
|
1459
|
-
|
1460
|
-
|
1461
|
-
dctx->tmpOutBuffer
|
1462
|
-
if (dctx->tmpOutBuffer== NULL)
|
1463
|
-
return err0r(LZ4F_ERROR_allocation_failed);
|
1627
|
+
LZ4F_free(dctx->tmpIn, dctx->cmem);
|
1628
|
+
dctx->tmpIn = (BYTE*)LZ4F_malloc(dctx->maxBlockSize + BFSize /* block checksum */, dctx->cmem);
|
1629
|
+
RETURN_ERROR_IF(dctx->tmpIn == NULL, allocation_failed);
|
1630
|
+
LZ4F_free(dctx->tmpOutBuffer, dctx->cmem);
|
1631
|
+
dctx->tmpOutBuffer= (BYTE*)LZ4F_malloc(bufferNeeded, dctx->cmem);
|
1632
|
+
RETURN_ERROR_IF(dctx->tmpOutBuffer== NULL, allocation_failed);
|
1464
1633
|
dctx->maxBufferSize = bufferNeeded;
|
1465
1634
|
} }
|
1466
1635
|
dctx->tmpInSize = 0;
|
@@ -1509,7 +1678,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1509
1678
|
break;
|
1510
1679
|
}
|
1511
1680
|
if (nextCBlockSize > dctx->maxBlockSize) {
|
1512
|
-
|
1681
|
+
RETURN_ERROR(maxBlockSize_invalid);
|
1513
1682
|
}
|
1514
1683
|
if (blockHeader & LZ4F_BLOCKUNCOMPRESSED_FLAG) {
|
1515
1684
|
/* next block is uncompressed */
|
@@ -1540,11 +1709,13 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1540
1709
|
size_t const minBuffSize = MIN((size_t)(srcEnd-srcPtr), (size_t)(dstEnd-dstPtr));
|
1541
1710
|
sizeToCopy = MIN(dctx->tmpInTarget, minBuffSize);
|
1542
1711
|
memcpy(dstPtr, srcPtr, sizeToCopy);
|
1543
|
-
if (dctx->
|
1544
|
-
(
|
1712
|
+
if (!dctx->skipChecksum) {
|
1713
|
+
if (dctx->frameInfo.blockChecksumFlag) {
|
1714
|
+
(void)XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy);
|
1715
|
+
}
|
1716
|
+
if (dctx->frameInfo.contentChecksumFlag)
|
1717
|
+
(void)XXH32_update(&dctx->xxh, srcPtr, sizeToCopy);
|
1545
1718
|
}
|
1546
|
-
if (dctx->frameInfo.contentChecksumFlag)
|
1547
|
-
(void)XXH32_update(&dctx->xxh, srcPtr, sizeToCopy);
|
1548
1719
|
if (dctx->frameInfo.contentSize)
|
1549
1720
|
dctx->frameRemainingSize -= sizeToCopy;
|
1550
1721
|
|
@@ -1590,14 +1761,15 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1590
1761
|
}
|
1591
1762
|
crcSrc = dctx->header;
|
1592
1763
|
}
|
1593
|
-
|
1764
|
+
if (!dctx->skipChecksum) {
|
1765
|
+
U32 const readCRC = LZ4F_readLE32(crcSrc);
|
1594
1766
|
U32 const calcCRC = XXH32_digest(&dctx->blockChecksum);
|
1595
1767
|
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
|
1596
1768
|
DEBUGLOG(6, "compare block checksum");
|
1597
1769
|
if (readCRC != calcCRC) {
|
1598
1770
|
DEBUGLOG(4, "incorrect block checksum: %08X != %08X",
|
1599
1771
|
readCRC, calcCRC);
|
1600
|
-
|
1772
|
+
RETURN_ERROR(blockChecksum_invalid);
|
1601
1773
|
}
|
1602
1774
|
#else
|
1603
1775
|
(void)readCRC;
|
@@ -1637,37 +1809,44 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1637
1809
|
}
|
1638
1810
|
|
1639
1811
|
/* At this stage, input is large enough to decode a block */
|
1812
|
+
|
1813
|
+
/* First, decode and control block checksum if it exists */
|
1640
1814
|
if (dctx->frameInfo.blockChecksumFlag) {
|
1815
|
+
assert(dctx->tmpInTarget >= 4);
|
1641
1816
|
dctx->tmpInTarget -= 4;
|
1642
1817
|
assert(selectedIn != NULL); /* selectedIn is defined at this stage (either srcPtr, or dctx->tmpIn) */
|
1643
1818
|
{ U32 const readBlockCrc = LZ4F_readLE32(selectedIn + dctx->tmpInTarget);
|
1644
1819
|
U32 const calcBlockCrc = XXH32(selectedIn, dctx->tmpInTarget, 0);
|
1645
1820
|
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
|
1646
|
-
|
1647
|
-
return err0r(LZ4F_ERROR_blockChecksum_invalid);
|
1821
|
+
RETURN_ERROR_IF(readBlockCrc != calcBlockCrc, blockChecksum_invalid);
|
1648
1822
|
#else
|
1649
1823
|
(void)readBlockCrc;
|
1650
1824
|
(void)calcBlockCrc;
|
1651
1825
|
#endif
|
1652
1826
|
} }
|
1653
1827
|
|
1654
|
-
if
|
1828
|
+
/* decode directly into destination buffer if there is enough room */
|
1829
|
+
if ( ((size_t)(dstEnd-dstPtr) >= dctx->maxBlockSize)
|
1830
|
+
/* unless the dictionary is stored in tmpOut:
|
1831
|
+
* in which case it's faster to decode within tmpOut
|
1832
|
+
* to benefit from prefix speedup */
|
1833
|
+
&& !(dctx->dict!= NULL && (const BYTE*)dctx->dict + dctx->dictSize == dctx->tmpOut) )
|
1834
|
+
{
|
1655
1835
|
const char* dict = (const char*)dctx->dict;
|
1656
1836
|
size_t dictSize = dctx->dictSize;
|
1657
1837
|
int decodedSize;
|
1658
1838
|
assert(dstPtr != NULL);
|
1659
1839
|
if (dict && dictSize > 1 GB) {
|
1660
|
-
/*
|
1840
|
+
/* overflow control : dctx->dictSize is an int, avoid truncation / sign issues */
|
1661
1841
|
dict += dictSize - 64 KB;
|
1662
1842
|
dictSize = 64 KB;
|
1663
1843
|
}
|
1664
|
-
/* enough capacity in `dst` to decompress directly there */
|
1665
1844
|
decodedSize = LZ4_decompress_safe_usingDict(
|
1666
1845
|
(const char*)selectedIn, (char*)dstPtr,
|
1667
1846
|
(int)dctx->tmpInTarget, (int)dctx->maxBlockSize,
|
1668
1847
|
dict, (int)dictSize);
|
1669
|
-
|
1670
|
-
if (dctx->frameInfo.contentChecksumFlag)
|
1848
|
+
RETURN_ERROR_IF(decodedSize < 0, decompressionFailed);
|
1849
|
+
if ((dctx->frameInfo.contentChecksumFlag) && (!dctx->skipChecksum))
|
1671
1850
|
XXH32_update(&(dctx->xxh), dstPtr, (size_t)decodedSize);
|
1672
1851
|
if (dctx->frameInfo.contentSize)
|
1673
1852
|
dctx->frameRemainingSize -= (size_t)decodedSize;
|
@@ -1678,25 +1857,27 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1678
1857
|
}
|
1679
1858
|
|
1680
1859
|
dstPtr += decodedSize;
|
1681
|
-
dctx->dStage = dstage_getBlockHeader;
|
1860
|
+
dctx->dStage = dstage_getBlockHeader; /* end of block, let's get another one */
|
1682
1861
|
break;
|
1683
1862
|
}
|
1684
1863
|
|
1685
1864
|
/* not enough place into dst : decode into tmpOut */
|
1686
|
-
|
1865
|
+
|
1866
|
+
/* manage dictionary */
|
1687
1867
|
if (dctx->frameInfo.blockMode == LZ4F_blockLinked) {
|
1688
1868
|
if (dctx->dict == dctx->tmpOutBuffer) {
|
1869
|
+
/* truncate dictionary to 64 KB if too big */
|
1689
1870
|
if (dctx->dictSize > 128 KB) {
|
1690
1871
|
memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - 64 KB, 64 KB);
|
1691
1872
|
dctx->dictSize = 64 KB;
|
1692
1873
|
}
|
1693
1874
|
dctx->tmpOut = dctx->tmpOutBuffer + dctx->dictSize;
|
1694
|
-
} else { /* dict not within
|
1875
|
+
} else { /* dict not within tmpOut */
|
1695
1876
|
size_t const reservedDictSpace = MIN(dctx->dictSize, 64 KB);
|
1696
1877
|
dctx->tmpOut = dctx->tmpOutBuffer + reservedDictSpace;
|
1697
1878
|
} }
|
1698
1879
|
|
1699
|
-
/* Decode block */
|
1880
|
+
/* Decode block into tmpOut */
|
1700
1881
|
{ const char* dict = (const char*)dctx->dict;
|
1701
1882
|
size_t dictSize = dctx->dictSize;
|
1702
1883
|
int decodedSize;
|
@@ -1709,9 +1890,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1709
1890
|
(const char*)selectedIn, (char*)dctx->tmpOut,
|
1710
1891
|
(int)dctx->tmpInTarget, (int)dctx->maxBlockSize,
|
1711
1892
|
dict, (int)dictSize);
|
1712
|
-
|
1713
|
-
|
1714
|
-
if (dctx->frameInfo.contentChecksumFlag)
|
1893
|
+
RETURN_ERROR_IF(decodedSize < 0, decompressionFailed);
|
1894
|
+
if (dctx->frameInfo.contentChecksumFlag && !dctx->skipChecksum)
|
1715
1895
|
XXH32_update(&(dctx->xxh), dctx->tmpOut, (size_t)decodedSize);
|
1716
1896
|
if (dctx->frameInfo.contentSize)
|
1717
1897
|
dctx->frameRemainingSize -= (size_t)decodedSize;
|
@@ -1744,8 +1924,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1744
1924
|
break;
|
1745
1925
|
|
1746
1926
|
case dstage_getSuffix:
|
1747
|
-
|
1748
|
-
return err0r(LZ4F_ERROR_frameSize_wrong); /* incorrect frame size decoded */
|
1927
|
+
RETURN_ERROR_IF(dctx->frameRemainingSize, frameSize_wrong); /* incorrect frame size decoded */
|
1749
1928
|
if (!dctx->frameInfo.contentChecksumFlag) { /* no checksum, frame is completed */
|
1750
1929
|
nextSrcSizeHint = 0;
|
1751
1930
|
LZ4F_resetDecompressionContext(dctx);
|
@@ -1777,20 +1956,20 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1777
1956
|
} /* if (dctx->dStage == dstage_storeSuffix) */
|
1778
1957
|
|
1779
1958
|
/* case dstage_checkSuffix: */ /* no direct entry, avoid initialization risks */
|
1780
|
-
|
1959
|
+
if (!dctx->skipChecksum) {
|
1960
|
+
U32 const readCRC = LZ4F_readLE32(selectedIn);
|
1781
1961
|
U32 const resultCRC = XXH32_digest(&(dctx->xxh));
|
1782
1962
|
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
|
1783
|
-
|
1784
|
-
return err0r(LZ4F_ERROR_contentChecksum_invalid);
|
1963
|
+
RETURN_ERROR_IF(readCRC != resultCRC, contentChecksum_invalid);
|
1785
1964
|
#else
|
1786
1965
|
(void)readCRC;
|
1787
1966
|
(void)resultCRC;
|
1788
1967
|
#endif
|
1789
|
-
nextSrcSizeHint = 0;
|
1790
|
-
LZ4F_resetDecompressionContext(dctx);
|
1791
|
-
doAnotherStage = 0;
|
1792
|
-
break;
|
1793
1968
|
}
|
1969
|
+
nextSrcSizeHint = 0;
|
1970
|
+
LZ4F_resetDecompressionContext(dctx);
|
1971
|
+
doAnotherStage = 0;
|
1972
|
+
break;
|
1794
1973
|
|
1795
1974
|
case dstage_getSFrameSize:
|
1796
1975
|
if ((srcEnd - srcPtr) >= 4) {
|
@@ -1841,7 +2020,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
|
|
1841
2020
|
} /* switch (dctx->dStage) */
|
1842
2021
|
} /* while (doAnotherStage) */
|
1843
2022
|
|
1844
|
-
/* preserve history within
|
2023
|
+
/* preserve history within tmpOut whenever necessary */
|
1845
2024
|
LZ4F_STATIC_ASSERT((unsigned)dstage_init == 2);
|
1846
2025
|
if ( (dctx->frameInfo.blockMode==LZ4F_blockLinked) /* next block will use up to 64KB from previous ones */
|
1847
2026
|
&& (dctx->dict != dctx->tmpOutBuffer) /* dictionary is not already within tmp */
|