extlz4 0.3 → 0.3.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/HISTORY.ja.md +7 -0
  3. data/README.md +3 -3
  4. data/contrib/lz4/Makefile.inc +87 -0
  5. data/contrib/lz4/NEWS +7 -0
  6. data/contrib/lz4/README.md +1 -1
  7. data/contrib/lz4/lib/README.md +3 -5
  8. data/contrib/lz4/lib/liblz4-dll.rc.in +35 -0
  9. data/contrib/lz4/lib/lz4.c +296 -182
  10. data/contrib/lz4/lib/lz4.h +125 -40
  11. data/contrib/lz4/lib/lz4frame.c +30 -6
  12. data/contrib/lz4/lib/lz4frame.h +11 -2
  13. data/contrib/lz4/lib/lz4hc.c +93 -30
  14. data/contrib/lz4/lib/lz4hc.h +3 -0
  15. data/contrib/lz4/ossfuzz/Makefile +74 -0
  16. data/contrib/lz4/ossfuzz/compress_frame_fuzzer.c +42 -0
  17. data/contrib/lz4/ossfuzz/compress_fuzzer.c +51 -0
  18. data/contrib/lz4/ossfuzz/compress_hc_fuzzer.c +57 -0
  19. data/contrib/lz4/ossfuzz/decompress_frame_fuzzer.c +67 -0
  20. data/contrib/lz4/ossfuzz/decompress_fuzzer.c +58 -0
  21. data/contrib/lz4/ossfuzz/fuzz.h +48 -0
  22. data/contrib/lz4/ossfuzz/fuzz_helpers.h +94 -0
  23. data/contrib/lz4/ossfuzz/lz4_helpers.c +51 -0
  24. data/contrib/lz4/ossfuzz/lz4_helpers.h +13 -0
  25. data/contrib/lz4/ossfuzz/ossfuzz.sh +23 -0
  26. data/contrib/lz4/ossfuzz/round_trip_frame_fuzzer.c +39 -0
  27. data/contrib/lz4/ossfuzz/round_trip_fuzzer.c +50 -0
  28. data/contrib/lz4/ossfuzz/round_trip_hc_fuzzer.c +39 -0
  29. data/contrib/lz4/ossfuzz/round_trip_stream_fuzzer.c +302 -0
  30. data/contrib/lz4/ossfuzz/standaloneengine.c +74 -0
  31. data/contrib/lz4/ossfuzz/travisoss.sh +21 -0
  32. data/ext/blockapi.c +3 -3
  33. data/ext/hashargs.c +1 -1
  34. data/lib/extlz4.rb +5 -1
  35. data/lib/extlz4/version.rb +1 -1
  36. data/test/common.rb +2 -2
  37. metadata +22 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 738042afe5d85d9e24155d0351f9120df4ec2258ffa1f311a821c6bb399dd2a9
4
- data.tar.gz: 88e1057d4fc464ed4cef3ac93c0e979821e07c87a6fba70d3d644d07e21d0913
3
+ metadata.gz: 47cf5cc7a6110b9745b7c8fb44241bb63cec270c72a71de4d5d874f5bb054823
4
+ data.tar.gz: f2a7d26bc96f2df6ffe99db799d1c199bceb75f5e5ad01f959bca5ce48ef19ed
5
5
  SHA512:
6
- metadata.gz: d12a084e3a34c27a64cf0b236f7475760b9be86c0654062c0d89a6d148b75811122015925a75d4c9699a5b07472b8c692abb9cd427e0aba05c0e8d0a17e4384c
7
- data.tar.gz: b3902b0162ce3c5dc244c3c2975819a896aa294ec37f1012536eed07862e7dc6541ce10ba1869644fef8c3d6602c55ebfa1514a9128f4eded98863284843f6bd
6
+ metadata.gz: 3e906586be02470ae4b18fe625e15af59d958c05b71e74eb74eb5f9f767279cc04a1411bc134743c2d572d2dea4c99f7c44bfbda0618208c79c6fc3c3e2eeb64
7
+ data.tar.gz: 17962b5b4e648ef9aa4f1629c8673197ee1f9d2222ef0ddf90ce0e9513b4b5f135cc1452b36cd1d734b3dff8efc5b2318b795bb88fc0d73f05c14d75affdd508
@@ -1,3 +1,10 @@
1
+ # extlz4-0.3.1 (令和2年6月7日)
2
+
3
+ * lz4-1.9.2 に更新
4
+ * `extlz4.so` が `<extlz4-dir>/lib` へ置かれない環境に対応
5
+ * https://github.com/dearblue/ruby-extlz4/issues/2
6
+
7
+
1
8
  # extlz4-0.3 (平成31年4月)
2
9
 
3
10
  * lz4-1.9.0 に更新
data/README.md CHANGED
@@ -194,13 +194,13 @@ p src2 == data # => true
194
194
  - author: dearblue (mailto:dearblue@users.noreply.github.com)
195
195
  - project page: <https://github.com/dearblue/ruby-extlz4>
196
196
  - how to install: `gem install extlz4`
197
- - version: 0.3
197
+ - version: 0.3.1
198
198
  - product quality: technical preview
199
199
  - licensing: [2 clause BSD License](LICENSE)
200
200
  - dependency gems: none
201
201
  - dependency external c libraries: none
202
202
  - bundled external c libraries (git submodules):
203
203
  - [lz4](https://github.com/lz4/lz4)
204
- [version 1.9.0](https://github.com/lz4/lz4/tree/v1.9.0)
205
- under [2 clause BSD license](https://github.com/lz4/lz4/blob/v1.9.0/LICENSE)
204
+ [version 1.9.2](https://github.com/lz4/lz4/tree/v1.9.2)
205
+ under [2 clause BSD license](https://github.com/lz4/lz4/blob/v1.9.2/LICENSE)
206
206
  by [Yann Collet](https://github.com/Cyan4973)
@@ -0,0 +1,87 @@
1
+ ifeq ($(V), 1)
2
+ Q =
3
+ else
4
+ Q = @
5
+ endif
6
+
7
+ TARGET_OS ?= $(shell uname)
8
+ ifeq ($(TARGET_OS),)
9
+ TARGET_OS ?= $(OS)
10
+ endif
11
+
12
+ ifneq (,$(filter Windows%,$(TARGET_OS)))
13
+ LIBLZ4 = liblz4-$(LIBVER_MAJOR)
14
+ LIBLZ4_EXP = liblz4.lib
15
+ WINBASED = yes
16
+ else
17
+ LIBLZ4_EXP = liblz4.dll.a
18
+ ifneq (,$(filter MINGW%,$(TARGET_OS)))
19
+ LIBLZ4 = liblz4
20
+ WINBASED = yes
21
+ else
22
+ ifneq (,$(filter MSYS%,$(TARGET_OS)))
23
+ LIBLZ4 = msys-lz4-$(LIBVER_MAJOR)
24
+ WINBASED = yes
25
+ else
26
+ ifneq (,$(filter CYGWIN%,$(TARGET_OS)))
27
+ LIBLZ4 = cyglz4-$(LIBVER_MAJOR)
28
+ WINBASED = yes
29
+ else
30
+ LIBLZ4 = liblz4.$(SHARED_EXT_VER)
31
+ WINBASED = no
32
+ EXT =
33
+ endif
34
+ endif
35
+ endif
36
+ endif
37
+
38
+ ifeq ($(WINBASED),yes)
39
+ EXT = .exe
40
+ WINDRES = windres
41
+ endif
42
+
43
+ #determine if dev/nul based on host environment
44
+ ifneq (,$(filter MINGW% MSYS% CYGWIN%,$(shell uname)))
45
+ VOID := /dev/null
46
+ else
47
+ ifneq (,$(filter Windows%,$(OS)))
48
+ VOID := nul
49
+ else
50
+ VOID := /dev/null
51
+ endif
52
+ endif
53
+
54
+ ifneq (,$(filter Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku MidnightBSD MINGW% CYGWIN% MSYS%,$(shell uname)))
55
+ POSIX_ENV = Yes
56
+ else
57
+ POSIX_ENV = No
58
+ endif
59
+
60
+ # Avoid symlinks when targetting Windows or building on a Windows host
61
+ ifeq ($(WINBASED),yes)
62
+ LN_S = cp -p
63
+ LN_SF = cp -p
64
+ else
65
+ ifneq (,$(filter MINGW% MSYS% CYGWIN%,$(shell uname)))
66
+ LN_S = cp -p
67
+ LN_SF = cp -p
68
+ else
69
+ ifneq (,$(filter Windows%,$(OS)))
70
+ LN_S = cp -p
71
+ LN_SF = cp -p
72
+ else
73
+ LN_S = ln -s
74
+ LN_SF = ln -sf
75
+ endif
76
+ endif
77
+ endif
78
+
79
+ ifneq (,$(filter $(shell uname),SunOS))
80
+ INSTALL ?= ginstall
81
+ else
82
+ INSTALL ?= install
83
+ endif
84
+
85
+ INSTALL_PROGRAM ?= $(INSTALL) -m 755
86
+ INSTALL_DATA ?= $(INSTALL) -m 644
87
+ INSTALL_DIR ?= $(INSTALL) -d -m 755
@@ -1,3 +1,10 @@
1
+ v1.9.1
2
+ fix : decompression functions were reading a few bytes beyond input size (introduced in v1.9.0, reported by @ppodolsky and @danlark1)
3
+ api : fix : lz4frame initializers compatibility with c++, reported by @degski
4
+ cli : added command --list, based on a patch by @gabrielstedman
5
+ build: improved Windows build, by @JPeterMugaas
6
+ build: AIX, by Norman Green
7
+
1
8
  v1.9.0
2
9
  perf: large decompression speed improvement on x86/x64 (up to +20%) by @djwatson
3
10
  api : changed : _destSize() compression variants are promoted to stable API
@@ -68,7 +68,7 @@ in single-thread mode.
68
68
  | [Zstandard] 1.4.0 -1 | 2.883 | 515 MB/s | 1380 MB/s |
69
69
  | LZF v3.6 | 2.073 | 415 MB/s | 910 MB/s |
70
70
  | [zlib] deflate 1.2.11 -1| 2.730 | 100 MB/s | 415 MB/s |
71
- |**LZ4 HC -9 (v1.8.2)** |**2.721**| 41 MB/s | **4900 MB/s** |
71
+ |**LZ4 HC -9 (v1.9.0)** |**2.721**| 41 MB/s | **4900 MB/s** |
72
72
  | [zlib] deflate 1.2.11 -6| 3.099 | 36 MB/s | 445 MB/s |
73
73
 
74
74
  [zlib]: http://www.zlib.net/
@@ -56,8 +56,8 @@ The following build macro can be selected at compilation time :
56
56
  - `LZ4_DISTANCE_MAX` : control the maximum offset that the compressor will allow.
57
57
  Set to 65535 by default, which is the maximum value supported by lz4 format.
58
58
  Reducing maximum distance will reduce opportunities for LZ4 to find matches,
59
- hence will produce worse the compression ratio.
60
- However, a smaller max distance may allow compatibility with specific decoders using limited memory budget.
59
+ hence will produce a worse compression ratio.
60
+ However, a smaller max distance can allow compatibility with specific decoders using limited memory budget.
61
61
  This build macro only influences the compressed output of the compressor.
62
62
 
63
63
  - `LZ4_DISABLE_DEPRECATE_WARNINGS` : invoking a deprecated function will make the compiler generate a warning.
@@ -74,9 +74,7 @@ The following build macro can be selected at compilation time :
74
74
  lz4 source code can be amalgamated into a single file.
75
75
  One can combine all source code into `lz4_all.c` by using following command:
76
76
  ```
77
- cat lz4.c > lz4_all.c
78
- cat lz4hc.c >> lz4_all.c
79
- cat lz4frame.c >> lz4_all.c
77
+ cat lz4.c lz4hc.c lz4frame.c > lz4_all.c
80
78
  ```
81
79
  (`cat` file order is important) then compile `lz4_all.c`.
82
80
  All `*.h` files present in `/lib` remain necessary to compile `lz4_all.c`.
@@ -0,0 +1,35 @@
1
+ #include <windows.h>
2
+
3
+ // DLL version information.
4
+ 1 VERSIONINFO
5
+ FILEVERSION @LIBVER_MAJOR@,@LIBVER_MINOR@,@LIBVER_PATCH@,0
6
+ PRODUCTVERSION @LIBVER_MAJOR@,@LIBVER_MINOR@,@LIBVER_PATCH@,0
7
+ FILEFLAGSMASK VS_FFI_FILEFLAGSMASK
8
+ #ifdef _DEBUG
9
+ FILEFLAGS VS_FF_DEBUG | VS_FF_PRERELEASE
10
+ #else
11
+ FILEFLAGS 0
12
+ #endif
13
+ FILEOS VOS_NT_WINDOWS32
14
+ FILETYPE VFT_DLL
15
+ FILESUBTYPE VFT2_UNKNOWN
16
+ BEGIN
17
+ BLOCK "StringFileInfo"
18
+ BEGIN
19
+ BLOCK "040904B0"
20
+ BEGIN
21
+ VALUE "CompanyName", "Yann Collet"
22
+ VALUE "FileDescription", "Extremely fast compression"
23
+ VALUE "FileVersion", "@LIBVER_MAJOR@.@LIBVER_MINOR@.@LIBVER_PATCH@.0"
24
+ VALUE "InternalName", "@LIBLZ4@"
25
+ VALUE "LegalCopyright", "Copyright (C) 2013-2016, Yann Collet"
26
+ VALUE "OriginalFilename", "@LIBLZ4@.dll"
27
+ VALUE "ProductName", "LZ4"
28
+ VALUE "ProductVersion", "@LIBVER_MAJOR@.@LIBVER_MINOR@.@LIBVER_PATCH@.0"
29
+ END
30
+ END
31
+ BLOCK "VarFileInfo"
32
+ BEGIN
33
+ VALUE "Translation", 0x0409, 1200
34
+ END
35
+ END
@@ -98,8 +98,15 @@
98
98
  # define LZ4_SRC_INCLUDED 1
99
99
  #endif
100
100
 
101
+ #ifndef LZ4_STATIC_LINKING_ONLY
101
102
  #define LZ4_STATIC_LINKING_ONLY
103
+ #endif
104
+
105
+ #ifndef LZ4_DISABLE_DEPRECATE_WARNINGS
102
106
  #define LZ4_DISABLE_DEPRECATE_WARNINGS /* due to LZ4_decompress_safe_withPrefix64k */
107
+ #endif
108
+
109
+ #define LZ4_STATIC_LINKING_ONLY /* LZ4_DISTANCE_MAX */
103
110
  #include "lz4.h"
104
111
  /* see also "memory routines" below */
105
112
 
@@ -130,7 +137,7 @@
130
137
  #endif /* LZ4_FORCE_INLINE */
131
138
 
132
139
  /* LZ4_FORCE_O2_GCC_PPC64LE and LZ4_FORCE_O2_INLINE_GCC_PPC64LE
133
- * Gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy,
140
+ * gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy8,
134
141
  * together with a simple 8-byte copy loop as a fall-back path.
135
142
  * However, this optimization hurts the decompression speed by >30%,
136
143
  * because the execution does not go to the optimized loop
@@ -138,10 +145,10 @@
138
145
  * before going to the fall-back path become useless overhead.
139
146
  * This optimization happens only with the -O3 flag, and -O2 generates
140
147
  * a simple 8-byte copy loop.
141
- * With gcc on ppc64le, all of the LZ4_decompress_* and LZ4_wildCopy
148
+ * With gcc on ppc64le, all of the LZ4_decompress_* and LZ4_wildCopy8
142
149
  * functions are annotated with __attribute__((optimize("O2"))),
143
- * and also LZ4_wildCopy is forcibly inlined, so that the O2 attribute
144
- * of LZ4_wildCopy does not affect the compression speed.
150
+ * and also LZ4_wildCopy8 is forcibly inlined, so that the O2 attribute
151
+ * of LZ4_wildCopy8 does not affect the compression speed.
145
152
  */
146
153
  #if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__) && !defined(__clang__)
147
154
  # define LZ4_FORCE_O2_GCC_PPC64LE __attribute__((optimize("O2")))
@@ -176,6 +183,60 @@
176
183
  #define MEM_INIT(p,v,s) memset((p),(v),(s))
177
184
 
178
185
 
186
+ /*-************************************
187
+ * Common Constants
188
+ **************************************/
189
+ #define MINMATCH 4
190
+
191
+ #define WILDCOPYLENGTH 8
192
+ #define LASTLITERALS 5 /* see ../doc/lz4_Block_format.md#parsing-restrictions */
193
+ #define MFLIMIT 12 /* see ../doc/lz4_Block_format.md#parsing-restrictions */
194
+ #define MATCH_SAFEGUARD_DISTANCE ((2*WILDCOPYLENGTH) - MINMATCH) /* ensure it's possible to write 2 x wildcopyLength without overflowing output buffer */
195
+ #define FASTLOOP_SAFE_DISTANCE 64
196
+ static const int LZ4_minLength = (MFLIMIT+1);
197
+
198
+ #define KB *(1 <<10)
199
+ #define MB *(1 <<20)
200
+ #define GB *(1U<<30)
201
+
202
+ #define LZ4_DISTANCE_ABSOLUTE_MAX 65535
203
+ #if (LZ4_DISTANCE_MAX > LZ4_DISTANCE_ABSOLUTE_MAX) /* max supported by LZ4 format */
204
+ # error "LZ4_DISTANCE_MAX is too big : must be <= 65535"
205
+ #endif
206
+
207
+ #define ML_BITS 4
208
+ #define ML_MASK ((1U<<ML_BITS)-1)
209
+ #define RUN_BITS (8-ML_BITS)
210
+ #define RUN_MASK ((1U<<RUN_BITS)-1)
211
+
212
+
213
+ /*-************************************
214
+ * Error detection
215
+ **************************************/
216
+ #if defined(LZ4_DEBUG) && (LZ4_DEBUG>=1)
217
+ # include <assert.h>
218
+ #else
219
+ # ifndef assert
220
+ # define assert(condition) ((void)0)
221
+ # endif
222
+ #endif
223
+
224
+ #define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */
225
+
226
+ #if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2)
227
+ # include <stdio.h>
228
+ static int g_debuglog_enable = 1;
229
+ # define DEBUGLOG(l, ...) { \
230
+ if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \
231
+ fprintf(stderr, __FILE__ ": "); \
232
+ fprintf(stderr, __VA_ARGS__); \
233
+ fprintf(stderr, " \n"); \
234
+ } }
235
+ #else
236
+ # define DEBUGLOG(l, ...) {} /* disabled */
237
+ #endif
238
+
239
+
179
240
  /*-************************************
180
241
  * Types
181
242
  **************************************/
@@ -295,7 +356,7 @@ static void LZ4_writeLE16(void* memPtr, U16 value)
295
356
 
296
357
  /* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */
297
358
  LZ4_FORCE_O2_INLINE_GCC_PPC64LE
298
- void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd)
359
+ void LZ4_wildCopy8(void* dstPtr, const void* srcPtr, void* dstEnd)
299
360
  {
300
361
  BYTE* d = (BYTE*)dstPtr;
301
362
  const BYTE* s = (const BYTE*)srcPtr;
@@ -311,6 +372,11 @@ static const int dec64table[8] = {0, 0, 0, -1, -4, 1, 2, 3};
311
372
  #ifndef LZ4_FAST_DEC_LOOP
312
373
  # if defined(__i386__) || defined(__x86_64__)
313
374
  # define LZ4_FAST_DEC_LOOP 1
375
+ # elif defined(__aarch64__) && !defined(__clang__)
376
+ /* On aarch64, we disable this optimization for clang because on certain
377
+ * mobile chipsets and clang, it reduces performance. For more information
378
+ * refer to https://github.com/lz4/lz4/pull/707. */
379
+ # define LZ4_FAST_DEC_LOOP 1
314
380
  # else
315
381
  # define LZ4_FAST_DEC_LOOP 0
316
382
  # endif
@@ -336,7 +402,7 @@ LZ4_memcpy_using_offset_base(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, con
336
402
  srcPtr += 8;
337
403
  }
338
404
 
339
- LZ4_wildCopy(dstPtr, srcPtr, dstEnd);
405
+ LZ4_wildCopy8(dstPtr, srcPtr, dstEnd);
340
406
  }
341
407
 
342
408
  /* customized variant of memcpy, which can overwrite up to 32 bytes beyond dstEnd
@@ -352,29 +418,35 @@ LZ4_wildCopy32(void* dstPtr, const void* srcPtr, void* dstEnd)
352
418
  do { memcpy(d,s,16); memcpy(d+16,s+16,16); d+=32; s+=32; } while (d<e);
353
419
  }
354
420
 
421
+ /* LZ4_memcpy_using_offset() presumes :
422
+ * - dstEnd >= dstPtr + MINMATCH
423
+ * - there is at least 8 bytes available to write after dstEnd */
355
424
  LZ4_FORCE_O2_INLINE_GCC_PPC64LE void
356
425
  LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset)
357
426
  {
358
427
  BYTE v[8];
428
+
429
+ assert(dstEnd >= dstPtr + MINMATCH);
430
+ LZ4_write32(dstPtr, 0); /* silence an msan warning when offset==0 */
431
+
359
432
  switch(offset) {
360
433
  case 1:
361
434
  memset(v, *srcPtr, 8);
362
- goto copy_loop;
435
+ break;
363
436
  case 2:
364
437
  memcpy(v, srcPtr, 2);
365
438
  memcpy(&v[2], srcPtr, 2);
366
439
  memcpy(&v[4], &v[0], 4);
367
- goto copy_loop;
440
+ break;
368
441
  case 4:
369
442
  memcpy(v, srcPtr, 4);
370
443
  memcpy(&v[4], srcPtr, 4);
371
- goto copy_loop;
444
+ break;
372
445
  default:
373
446
  LZ4_memcpy_using_offset_base(dstPtr, srcPtr, dstEnd, offset);
374
447
  return;
375
448
  }
376
449
 
377
- copy_loop:
378
450
  memcpy(dstPtr, v, 8);
379
451
  dstPtr += 8;
380
452
  while (dstPtr < dstEnd) {
@@ -385,63 +457,6 @@ LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const si
385
457
  #endif
386
458
 
387
459
 
388
- /*-************************************
389
- * Common Constants
390
- **************************************/
391
- #define MINMATCH 4
392
-
393
- #define WILDCOPYLENGTH 8
394
- #define LASTLITERALS 5 /* see ../doc/lz4_Block_format.md#parsing-restrictions */
395
- #define MFLIMIT 12 /* see ../doc/lz4_Block_format.md#parsing-restrictions */
396
- #define MATCH_SAFEGUARD_DISTANCE ((2*WILDCOPYLENGTH) - MINMATCH) /* ensure it's possible to write 2 x wildcopyLength without overflowing output buffer */
397
- #define FASTLOOP_SAFE_DISTANCE 64
398
- static const int LZ4_minLength = (MFLIMIT+1);
399
-
400
- #define KB *(1 <<10)
401
- #define MB *(1 <<20)
402
- #define GB *(1U<<30)
403
-
404
- #ifndef LZ4_DISTANCE_MAX /* can be user - defined at compile time */
405
- # define LZ4_DISTANCE_MAX 65535
406
- #endif
407
-
408
- #if (LZ4_DISTANCE_MAX > 65535) /* max supported by LZ4 format */
409
- # error "LZ4_DISTANCE_MAX is too big : must be <= 65535"
410
- #endif
411
-
412
- #define ML_BITS 4
413
- #define ML_MASK ((1U<<ML_BITS)-1)
414
- #define RUN_BITS (8-ML_BITS)
415
- #define RUN_MASK ((1U<<RUN_BITS)-1)
416
-
417
-
418
- /*-************************************
419
- * Error detection
420
- **************************************/
421
- #if defined(LZ4_DEBUG) && (LZ4_DEBUG>=1)
422
- # include <assert.h>
423
- #else
424
- # ifndef assert
425
- # define assert(condition) ((void)0)
426
- # endif
427
- #endif
428
-
429
- #define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */
430
-
431
- #if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2)
432
- # include <stdio.h>
433
- static int g_debuglog_enable = 1;
434
- # define DEBUGLOG(l, ...) { \
435
- if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \
436
- fprintf(stderr, __FILE__ ": "); \
437
- fprintf(stderr, __VA_ARGS__); \
438
- fprintf(stderr, " \n"); \
439
- } }
440
- #else
441
- # define DEBUGLOG(l, ...) {} /* disabled */
442
- #endif
443
-
444
-
445
460
  /*-************************************
446
461
  * Common functions
447
462
  **************************************/
@@ -454,7 +469,7 @@ static unsigned LZ4_NbCommonBytes (reg_t val)
454
469
  _BitScanForward64( &r, (U64)val );
455
470
  return (int)(r>>3);
456
471
  # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)
457
- return (__builtin_ctzll((U64)val) >> 3);
472
+ return (unsigned)__builtin_ctzll((U64)val) >> 3;
458
473
  # else
459
474
  static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,
460
475
  0, 3, 1, 3, 1, 4, 2, 7,
@@ -472,7 +487,7 @@ static unsigned LZ4_NbCommonBytes (reg_t val)
472
487
  _BitScanForward( &r, (U32)val );
473
488
  return (int)(r>>3);
474
489
  # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)
475
- return (__builtin_ctz((U32)val) >> 3);
490
+ return (unsigned)__builtin_ctz((U32)val) >> 3;
476
491
  # else
477
492
  static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,
478
493
  3, 2, 2, 1, 3, 2, 0, 1,
@@ -488,7 +503,7 @@ static unsigned LZ4_NbCommonBytes (reg_t val)
488
503
  _BitScanReverse64( &r, val );
489
504
  return (unsigned)(r>>3);
490
505
  # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)
491
- return (__builtin_clzll((U64)val) >> 3);
506
+ return (unsigned)__builtin_clzll((U64)val) >> 3;
492
507
  # else
493
508
  static const U32 by32 = sizeof(val)*4; /* 32 on 64 bits (goal), 16 on 32 bits.
494
509
  Just to avoid some static analyzer complaining about shift by 32 on 32-bits target.
@@ -505,7 +520,7 @@ static unsigned LZ4_NbCommonBytes (reg_t val)
505
520
  _BitScanReverse( &r, (unsigned long)val );
506
521
  return (unsigned)(r>>3);
507
522
  # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)
508
- return (__builtin_clz((U32)val) >> 3);
523
+ return (unsigned)__builtin_clz((U32)val) >> 3;
509
524
  # else
510
525
  unsigned r;
511
526
  if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
@@ -600,9 +615,11 @@ int LZ4_sizeofState() { return LZ4_STREAMSIZE; }
600
615
  extern "C" {
601
616
  #endif
602
617
 
603
- int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize);
618
+ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize);
604
619
 
605
- int LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize, const void* dict, size_t dictSize);
620
+ int LZ4_decompress_safe_forceExtDict(const char* source, char* dest,
621
+ int compressedSize, int maxOutputSize,
622
+ const void* dictStart, size_t dictSize);
606
623
 
607
624
  #if defined (__cplusplus)
608
625
  }
@@ -637,6 +654,18 @@ LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tab
637
654
  return LZ4_hash4(LZ4_read32(p), tableType);
638
655
  }
639
656
 
657
+ static void LZ4_clearHash(U32 h, void* tableBase, tableType_t const tableType)
658
+ {
659
+ switch (tableType)
660
+ {
661
+ default: /* fallthrough */
662
+ case clearedTable: { /* illegal! */ assert(0); return; }
663
+ case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = NULL; return; }
664
+ case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = 0; return; }
665
+ case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = 0; return; }
666
+ }
667
+ }
668
+
640
669
  static void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t const tableType)
641
670
  {
642
671
  switch (tableType)
@@ -697,18 +726,19 @@ static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType
697
726
  { const U16* const hashTable = (const U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */
698
727
  }
699
728
 
700
- LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p,
701
- const void* tableBase, tableType_t tableType,
702
- const BYTE* srcBase)
729
+ LZ4_FORCE_INLINE const BYTE*
730
+ LZ4_getPosition(const BYTE* p,
731
+ const void* tableBase, tableType_t tableType,
732
+ const BYTE* srcBase)
703
733
  {
704
734
  U32 const h = LZ4_hashPosition(p, tableType);
705
735
  return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);
706
736
  }
707
737
 
708
- LZ4_FORCE_INLINE void LZ4_prepareTable(
709
- LZ4_stream_t_internal* const cctx,
710
- const int inputSize,
711
- const tableType_t tableType) {
738
+ LZ4_FORCE_INLINE void
739
+ LZ4_prepareTable(LZ4_stream_t_internal* const cctx,
740
+ const int inputSize,
741
+ const tableType_t tableType) {
712
742
  /* If compression failed during the previous step, then the context
713
743
  * is marked as dirty, therefore, it has to be fully reset.
714
744
  */
@@ -723,9 +753,10 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
723
753
  * out if it's safe to leave as is or whether it needs to be reset.
724
754
  */
725
755
  if (cctx->tableType != clearedTable) {
756
+ assert(inputSize >= 0);
726
757
  if (cctx->tableType != tableType
727
- || (tableType == byU16 && cctx->currentOffset + inputSize >= 0xFFFFU)
728
- || (tableType == byU32 && cctx->currentOffset > 1 GB)
758
+ || ((tableType == byU16) && cctx->currentOffset + (unsigned)inputSize >= 0xFFFFU)
759
+ || ((tableType == byU32) && cctx->currentOffset > 1 GB)
729
760
  || tableType == byPtr
730
761
  || inputSize >= 4 KB)
731
762
  {
@@ -805,9 +836,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
805
836
  DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, tableType=%u", inputSize, tableType);
806
837
  /* If init conditions are not met, we don't have to mark stream
807
838
  * as having dirty context, since no action was taken yet */
808
- if (outputDirective == fillOutput && maxOutputSize < 1) return 0; /* Impossible to store anything */
809
- if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */
810
- if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */
839
+ if (outputDirective == fillOutput && maxOutputSize < 1) { return 0; } /* Impossible to store anything */
840
+ if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) { return 0; } /* Unsupported inputSize, too large (or negative) */
841
+ if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) { return 0; } /* Size too large (not within 64K limit) */
811
842
  if (tableType==byPtr) assert(dictDirective==noDict); /* only supported use case with byPtr */
812
843
  assert(acceleration >= 1);
813
844
 
@@ -835,6 +866,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
835
866
  for ( ; ; ) {
836
867
  const BYTE* match;
837
868
  BYTE* token;
869
+ const BYTE* filledIp;
838
870
 
839
871
  /* Find a match */
840
872
  if (tableType == byPtr) {
@@ -903,10 +935,14 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
903
935
  forwardH = LZ4_hashPosition(forwardIp, tableType);
904
936
  LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
905
937
 
906
- if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue; /* match outside of valid area */
938
+ DEBUGLOG(7, "candidate at pos=%u (offset=%u \n", matchIndex, current - matchIndex);
939
+ if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) { continue; } /* match outside of valid area */
907
940
  assert(matchIndex < current);
908
- if ((tableType != byU16) && (matchIndex+LZ4_DISTANCE_MAX < current)) continue; /* too far */
909
- if (tableType == byU16) assert((current - matchIndex) <= LZ4_DISTANCE_MAX); /* too_far presumed impossible with byU16 */
941
+ if ( ((tableType != byU16) || (LZ4_DISTANCE_MAX < LZ4_DISTANCE_ABSOLUTE_MAX))
942
+ && (matchIndex+LZ4_DISTANCE_MAX < current)) {
943
+ continue;
944
+ } /* too far */
945
+ assert((current - matchIndex) <= LZ4_DISTANCE_MAX); /* match now expected within distance */
910
946
 
911
947
  if (LZ4_read32(match) == LZ4_read32(ip)) {
912
948
  if (maybe_extMem) offset = current - matchIndex;
@@ -917,15 +953,16 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
917
953
  }
918
954
 
919
955
  /* Catch up */
956
+ filledIp = ip;
920
957
  while (((ip>anchor) & (match > lowLimit)) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; }
921
958
 
922
959
  /* Encode Literals */
923
960
  { unsigned const litLength = (unsigned)(ip - anchor);
924
961
  token = op++;
925
962
  if ((outputDirective == limitedOutput) && /* Check output buffer overflow */
926
- (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)) )
963
+ (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)) ) {
927
964
  return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */
928
-
965
+ }
929
966
  if ((outputDirective == fillOutput) &&
930
967
  (unlikely(op + (litLength+240)/255 /* litlen */ + litLength /* literals */ + 2 /* offset */ + 1 /* token */ + MFLIMIT - MINMATCH /* min last literals so last match is <= end - MFLIMIT */ > olimit))) {
931
968
  op--;
@@ -940,7 +977,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
940
977
  else *token = (BYTE)(litLength<<ML_BITS);
941
978
 
942
979
  /* Copy Literals */
943
- LZ4_wildCopy(op, anchor, op+litLength);
980
+ LZ4_wildCopy8(op, anchor, op+litLength);
944
981
  op+=litLength;
945
982
  DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i",
946
983
  (int)(anchor-(const BYTE*)source), litLength, (int)(ip-(const BYTE*)source));
@@ -996,12 +1033,26 @@ _next_match:
996
1033
  }
997
1034
 
998
1035
  if ((outputDirective) && /* Check output buffer overflow */
999
- (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) {
1036
+ (unlikely(op + (1 + LASTLITERALS) + (matchCode+240)/255 > olimit)) ) {
1000
1037
  if (outputDirective == fillOutput) {
1001
1038
  /* Match description too long : reduce it */
1002
- U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 2 - 1 - LASTLITERALS) * 255;
1039
+ U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 1 - LASTLITERALS) * 255;
1003
1040
  ip -= matchCode - newMatchCode;
1041
+ assert(newMatchCode < matchCode);
1004
1042
  matchCode = newMatchCode;
1043
+ if (unlikely(ip <= filledIp)) {
1044
+ /* We have already filled up to filledIp so if ip ends up less than filledIp
1045
+ * we have positions in the hash table beyond the current position. This is
1046
+ * a problem if we reuse the hash table. So we have to remove these positions
1047
+ * from the hash table.
1048
+ */
1049
+ const BYTE* ptr;
1050
+ DEBUGLOG(5, "Clearing %u positions", (U32)(filledIp - ip));
1051
+ for (ptr = ip; ptr <= filledIp; ++ptr) {
1052
+ U32 const h = LZ4_hashPosition(ptr, tableType);
1053
+ LZ4_clearHash(h, cctx->hashTable, tableType);
1054
+ }
1055
+ }
1005
1056
  } else {
1006
1057
  assert(outputDirective == limitedOutput);
1007
1058
  return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */
@@ -1021,6 +1072,8 @@ _next_match:
1021
1072
  } else
1022
1073
  *token += (BYTE)(matchCode);
1023
1074
  }
1075
+ /* Ensure we have enough space for the last literals. */
1076
+ assert(!(outputDirective == fillOutput && op + 1 + LASTLITERALS > olimit));
1024
1077
 
1025
1078
  anchor = ip;
1026
1079
 
@@ -1070,7 +1123,7 @@ _next_match:
1070
1123
  LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
1071
1124
  assert(matchIndex < current);
1072
1125
  if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1)
1073
- && ((tableType==byU16) ? 1 : (matchIndex+LZ4_DISTANCE_MAX >= current))
1126
+ && (((tableType==byU16) && (LZ4_DISTANCE_MAX == LZ4_DISTANCE_ABSOLUTE_MAX)) ? 1 : (matchIndex+LZ4_DISTANCE_MAX >= current))
1074
1127
  && (LZ4_read32(match) == LZ4_read32(ip)) ) {
1075
1128
  token=op++;
1076
1129
  *token=0;
@@ -1137,7 +1190,7 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
1137
1190
  return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
1138
1191
  }
1139
1192
  } else {
1140
- if (inputSize < LZ4_64Klimit) {;
1193
+ if (inputSize < LZ4_64Klimit) {
1141
1194
  return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
1142
1195
  } else {
1143
1196
  const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32;
@@ -1300,12 +1353,12 @@ static size_t LZ4_stream_t_alignment(void)
1300
1353
  LZ4_stream_t* LZ4_initStream (void* buffer, size_t size)
1301
1354
  {
1302
1355
  DEBUGLOG(5, "LZ4_initStream");
1303
- if (buffer == NULL) return NULL;
1304
- if (size < sizeof(LZ4_stream_t)) return NULL;
1356
+ if (buffer == NULL) { return NULL; }
1357
+ if (size < sizeof(LZ4_stream_t)) { return NULL; }
1305
1358
  #ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 :
1306
1359
  it reports an aligment of 8-bytes,
1307
1360
  while actually aligning LZ4_stream_t on 4 bytes. */
1308
- if (((size_t)buffer) & (LZ4_stream_t_alignment() - 1)) return NULL; /* alignment check */
1361
+ if (((size_t)buffer) & (LZ4_stream_t_alignment() - 1)) { return NULL; } /* alignment check */
1309
1362
  #endif
1310
1363
  MEM_INIT(buffer, 0, sizeof(LZ4_stream_t));
1311
1364
  return (LZ4_stream_t*)buffer;
@@ -1355,18 +1408,18 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
1355
1408
  * there are only valid offsets in the window, which allows an optimization
1356
1409
  * in LZ4_compress_fast_continue() where it uses noDictIssue even when the
1357
1410
  * dictionary isn't a full 64k. */
1358
-
1359
- if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB;
1360
- base = dictEnd - 64 KB - dict->currentOffset;
1361
- dict->dictionary = p;
1362
- dict->dictSize = (U32)(dictEnd - p);
1363
1411
  dict->currentOffset += 64 KB;
1364
- dict->tableType = tableType;
1365
1412
 
1366
1413
  if (dictSize < (int)HASH_UNIT) {
1367
1414
  return 0;
1368
1415
  }
1369
1416
 
1417
+ if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB;
1418
+ base = dictEnd - dict->currentOffset;
1419
+ dict->dictionary = p;
1420
+ dict->dictSize = (U32)(dictEnd - p);
1421
+ dict->tableType = tableType;
1422
+
1370
1423
  while (p <= dictEnd-HASH_UNIT) {
1371
1424
  LZ4_putPosition(p, dict->hashTable, tableType, base);
1372
1425
  p+=3;
@@ -1375,26 +1428,37 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
1375
1428
  return (int)dict->dictSize;
1376
1429
  }
1377
1430
 
1378
- void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream) {
1431
+ void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream) {
1432
+ const LZ4_stream_t_internal* dictCtx = dictionaryStream == NULL ? NULL :
1433
+ &(dictionaryStream->internal_donotuse);
1434
+
1435
+ DEBUGLOG(4, "LZ4_attach_dictionary (%p, %p, size %u)",
1436
+ workingStream, dictionaryStream,
1437
+ dictCtx != NULL ? dictCtx->dictSize : 0);
1438
+
1379
1439
  /* Calling LZ4_resetStream_fast() here makes sure that changes will not be
1380
1440
  * erased by subsequent calls to LZ4_resetStream_fast() in case stream was
1381
1441
  * marked as having dirty context, e.g. requiring full reset.
1382
1442
  */
1383
- LZ4_resetStream_fast(working_stream);
1443
+ LZ4_resetStream_fast(workingStream);
1384
1444
 
1385
- if (dictionary_stream != NULL) {
1445
+ if (dictCtx != NULL) {
1386
1446
  /* If the current offset is zero, we will never look in the
1387
1447
  * external dictionary context, since there is no value a table
1388
1448
  * entry can take that indicate a miss. In that case, we need
1389
1449
  * to bump the offset to something non-zero.
1390
1450
  */
1391
- if (working_stream->internal_donotuse.currentOffset == 0) {
1392
- working_stream->internal_donotuse.currentOffset = 64 KB;
1451
+ if (workingStream->internal_donotuse.currentOffset == 0) {
1452
+ workingStream->internal_donotuse.currentOffset = 64 KB;
1453
+ }
1454
+
1455
+ /* Don't actually attach an empty dictionary.
1456
+ */
1457
+ if (dictCtx->dictSize == 0) {
1458
+ dictCtx = NULL;
1393
1459
  }
1394
- working_stream->internal_donotuse.dictCtx = &(dictionary_stream->internal_donotuse);
1395
- } else {
1396
- working_stream->internal_donotuse.dictCtx = NULL;
1397
1460
  }
1461
+ workingStream->internal_donotuse.dictCtx = dictCtx;
1398
1462
  }
1399
1463
 
1400
1464
 
@@ -1429,7 +1493,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream,
1429
1493
 
1430
1494
  DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i)", inputSize);
1431
1495
 
1432
- if (streamPtr->dirty) return 0; /* Uninitialized structure detected */
1496
+ if (streamPtr->dirty) { return 0; } /* Uninitialized structure detected */
1433
1497
  LZ4_renormDictT(streamPtr, inputSize); /* avoid index overflow */
1434
1498
  if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
1435
1499
 
@@ -1526,8 +1590,8 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize)
1526
1590
  LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse;
1527
1591
  const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize;
1528
1592
 
1529
- if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */
1530
- if ((U32)dictSize > dict->dictSize) dictSize = (int)dict->dictSize;
1593
+ if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */
1594
+ if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; }
1531
1595
 
1532
1596
  memmove(safeBuffer, previousDictEnd - dictSize, dictSize);
1533
1597
 
@@ -1601,7 +1665,7 @@ LZ4_decompress_generic(
1601
1665
  const size_t dictSize /* note : = 0 if noDict */
1602
1666
  )
1603
1667
  {
1604
- if (src == NULL) return -1;
1668
+ if (src == NULL) { return -1; }
1605
1669
 
1606
1670
  { const BYTE* ip = (const BYTE*) src;
1607
1671
  const BYTE* const iend = ip + srcSize;
@@ -1630,20 +1694,26 @@ LZ4_decompress_generic(
1630
1694
 
1631
1695
  /* Special cases */
1632
1696
  assert(lowPrefix <= op);
1633
- if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */
1634
- if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0 ? 1 : -1);
1635
- if ((endOnInput) && unlikely(srcSize==0)) return -1;
1697
+ if ((endOnInput) && (unlikely(outputSize==0))) {
1698
+ /* Empty output buffer */
1699
+ if (partialDecoding) return 0;
1700
+ return ((srcSize==1) && (*ip==0)) ? 0 : -1;
1701
+ }
1702
+ if ((!endOnInput) && (unlikely(outputSize==0))) { return (*ip==0 ? 1 : -1); }
1703
+ if ((endOnInput) && unlikely(srcSize==0)) { return -1; }
1636
1704
 
1637
1705
  /* Currently the fast loop shows a regression on qualcomm arm chips. */
1638
1706
  #if LZ4_FAST_DEC_LOOP
1639
- if ((oend - op) < FASTLOOP_SAFE_DISTANCE)
1707
+ if ((oend - op) < FASTLOOP_SAFE_DISTANCE) {
1708
+ DEBUGLOG(6, "skip fast decode loop");
1640
1709
  goto safe_decode;
1710
+ }
1641
1711
 
1642
1712
  /* Fast loop : decode sequences as long as output < iend-FASTLOOP_SAFE_DISTANCE */
1643
1713
  while (1) {
1644
1714
  /* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */
1645
1715
  assert(oend - op >= FASTLOOP_SAFE_DISTANCE);
1646
-
1716
+ if (endOnInput) { assert(ip < iend); }
1647
1717
  token = *ip++;
1648
1718
  length = token >> ML_BITS; /* literal length */
1649
1719
 
@@ -1653,46 +1723,53 @@ LZ4_decompress_generic(
1653
1723
  if (length == RUN_MASK) {
1654
1724
  variable_length_error error = ok;
1655
1725
  length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error);
1656
- if (error == initial_error) goto _output_error;
1657
- if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) goto _output_error; /* overflow detection */
1658
- if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) goto _output_error; /* overflow detection */
1726
+ if (error == initial_error) { goto _output_error; }
1727
+ if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */
1728
+ if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */
1659
1729
 
1660
1730
  /* copy literals */
1661
1731
  cpy = op+length;
1662
1732
  LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH);
1663
- if ( ((endOnInput) && ((cpy>oend-FASTLOOP_SAFE_DISTANCE) || (ip+length>iend-(2+1+LASTLITERALS))) )
1664
- || ((!endOnInput) && (cpy>oend-FASTLOOP_SAFE_DISTANCE)) )
1665
- {
1666
- goto safe_literal_copy;
1667
- }
1668
- LZ4_wildCopy32(op, ip, cpy);
1733
+ if (endOnInput) { /* LZ4_decompress_safe() */
1734
+ if ((cpy>oend-32) || (ip+length>iend-32)) { goto safe_literal_copy; }
1735
+ LZ4_wildCopy32(op, ip, cpy);
1736
+ } else { /* LZ4_decompress_fast() */
1737
+ if (cpy>oend-8) { goto safe_literal_copy; }
1738
+ LZ4_wildCopy8(op, ip, cpy); /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time :
1739
+ * it doesn't know input length, and only relies on end-of-block properties */
1740
+ }
1669
1741
  ip += length; op = cpy;
1670
1742
  } else {
1671
1743
  cpy = op+length;
1672
- /* We don't need to check oend, since we check it once for each loop below */
1673
- if ( ((endOnInput) && (ip+16>iend-(2+1+LASTLITERALS))))
1674
- {
1675
- goto safe_literal_copy;
1676
- }
1677
- /* Literals can only be 14, but hope compilers optimize if we copy by a register size */
1678
- memcpy(op, ip, 16);
1744
+ if (endOnInput) { /* LZ4_decompress_safe() */
1745
+ DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length);
1746
+ /* We don't need to check oend, since we check it once for each loop below */
1747
+ if (ip > iend-(16 + 1/*max lit + offset + nextToken*/)) { goto safe_literal_copy; }
1748
+ /* Literals can only be 14, but hope compilers optimize if we copy by a register size */
1749
+ memcpy(op, ip, 16);
1750
+ } else { /* LZ4_decompress_fast() */
1751
+ /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time :
1752
+ * it doesn't know input length, and relies on end-of-block properties */
1753
+ memcpy(op, ip, 8);
1754
+ if (length > 8) { memcpy(op+8, ip+8, 8); }
1755
+ }
1679
1756
  ip += length; op = cpy;
1680
1757
  }
1681
1758
 
1682
1759
  /* get offset */
1683
1760
  offset = LZ4_readLE16(ip); ip+=2;
1684
1761
  match = op - offset;
1762
+ assert(match <= op);
1685
1763
 
1686
1764
  /* get matchlength */
1687
1765
  length = token & ML_MASK;
1688
1766
 
1689
- if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */
1690
-
1691
1767
  if (length == ML_MASK) {
1692
1768
  variable_length_error error = ok;
1769
+ if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */
1693
1770
  length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error);
1694
- if (error != ok) goto _output_error;
1695
- if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */
1771
+ if (error != ok) { goto _output_error; }
1772
+ if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */
1696
1773
  length += MINMATCH;
1697
1774
  if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) {
1698
1775
  goto safe_match_copy;
@@ -1704,8 +1781,12 @@ LZ4_decompress_generic(
1704
1781
  }
1705
1782
 
1706
1783
  /* Fastpath check: Avoids a branch in LZ4_wildCopy32 if true */
1707
- if (!(dict == usingExtDict) || (match >= lowPrefix)) {
1784
+ if ((dict == withPrefix64k) || (match >= lowPrefix)) {
1708
1785
  if (offset >= 8) {
1786
+ assert(match >= lowPrefix);
1787
+ assert(match <= op);
1788
+ assert(op + 18 <= oend);
1789
+
1709
1790
  memcpy(op, match, 8);
1710
1791
  memcpy(op+8, match+8, 8);
1711
1792
  memcpy(op+16, match+16, 2);
@@ -1713,12 +1794,15 @@ LZ4_decompress_generic(
1713
1794
  continue;
1714
1795
  } } }
1715
1796
 
1797
+ if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */
1716
1798
  /* match starting within external dictionary */
1717
1799
  if ((dict==usingExtDict) && (match < lowPrefix)) {
1718
1800
  if (unlikely(op+length > oend-LASTLITERALS)) {
1719
- if (partialDecoding) length = MIN(length, (size_t)(oend-op));
1720
- else goto _output_error; /* doesn't respect parsing restriction */
1721
- }
1801
+ if (partialDecoding) {
1802
+ length = MIN(length, (size_t)(oend-op)); /* reach end of buffer */
1803
+ } else {
1804
+ goto _output_error; /* end-of-block condition violated */
1805
+ } }
1722
1806
 
1723
1807
  if (length <= (size_t)(lowPrefix-match)) {
1724
1808
  /* match fits entirely within external dictionary : just copy */
@@ -1733,7 +1817,7 @@ LZ4_decompress_generic(
1733
1817
  if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */
1734
1818
  BYTE* const endOfMatch = op + restSize;
1735
1819
  const BYTE* copyFrom = lowPrefix;
1736
- while (op < endOfMatch) *op++ = *copyFrom++;
1820
+ while (op < endOfMatch) { *op++ = *copyFrom++; }
1737
1821
  } else {
1738
1822
  memcpy(op, lowPrefix, restSize);
1739
1823
  op += restSize;
@@ -1806,11 +1890,11 @@ LZ4_decompress_generic(
1806
1890
 
1807
1891
  /* decode literal length */
1808
1892
  if (length == RUN_MASK) {
1809
- variable_length_error error = ok;
1810
- length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error);
1811
- if (error == initial_error) goto _output_error;
1812
- if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) goto _output_error; /* overflow detection */
1813
- if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) goto _output_error; /* overflow detection */
1893
+ variable_length_error error = ok;
1894
+ length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error);
1895
+ if (error == initial_error) { goto _output_error; }
1896
+ if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */
1897
+ if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */
1814
1898
  }
1815
1899
 
1816
1900
  /* copy literals */
@@ -1822,23 +1906,52 @@ LZ4_decompress_generic(
1822
1906
  if ( ((endOnInput) && ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) )
1823
1907
  || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) )
1824
1908
  {
1909
+ /* We've either hit the input parsing restriction or the output parsing restriction.
1910
+ * If we've hit the input parsing condition then this must be the last sequence.
1911
+ * If we've hit the output parsing condition then we are either using partialDecoding
1912
+ * or we've hit the output parsing condition.
1913
+ */
1825
1914
  if (partialDecoding) {
1826
- if (cpy > oend) { cpy = oend; assert(op<=oend); length = (size_t)(oend-op); } /* Partial decoding : stop in the middle of literal segment */
1827
- if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */
1915
+ /* Since we are partial decoding we may be in this block because of the output parsing
1916
+ * restriction, which is not valid since the output buffer is allowed to be undersized.
1917
+ */
1918
+ assert(endOnInput);
1919
+ /* If we're in this block because of the input parsing condition, then we must be on the
1920
+ * last sequence (or invalid), so we must check that we exactly consume the input.
1921
+ */
1922
+ if ((ip+length>iend-(2+1+LASTLITERALS)) && (ip+length != iend)) { goto _output_error; }
1923
+ assert(ip+length <= iend);
1924
+ /* We are finishing in the middle of a literals segment.
1925
+ * Break after the copy.
1926
+ */
1927
+ if (cpy > oend) {
1928
+ cpy = oend;
1929
+ assert(op<=oend);
1930
+ length = (size_t)(oend-op);
1931
+ }
1932
+ assert(ip+length <= iend);
1828
1933
  } else {
1829
- if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */
1830
- if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */
1934
+ /* We must be on the last sequence because of the parsing limitations so check
1935
+ * that we exactly regenerate the original size (must be exact when !endOnInput).
1936
+ */
1937
+ if ((!endOnInput) && (cpy != oend)) { goto _output_error; }
1938
+ /* We must be on the last sequence (or invalid) because of the parsing limitations
1939
+ * so check that we exactly consume the input and don't overrun the output buffer.
1940
+ */
1941
+ if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) { goto _output_error; }
1831
1942
  }
1832
- memcpy(op, ip, length);
1943
+ memmove(op, ip, length); /* supports overlapping memory regions, which only matters for in-place decompression scenarios */
1833
1944
  ip += length;
1834
1945
  op += length;
1835
- if (!partialDecoding || (cpy == oend)) {
1836
- /* Necessarily EOF, due to parsing restrictions */
1946
+ /* Necessarily EOF when !partialDecoding. When partialDecoding
1947
+ * it is EOF if we've either filled the output buffer or hit
1948
+ * the input parsing restriction.
1949
+ */
1950
+ if (!partialDecoding || (cpy == oend) || (ip == iend)) {
1837
1951
  break;
1838
1952
  }
1839
-
1840
1953
  } else {
1841
- LZ4_wildCopy(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */
1954
+ LZ4_wildCopy8(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */
1842
1955
  ip += length; op = cpy;
1843
1956
  }
1844
1957
 
@@ -1850,13 +1963,6 @@ LZ4_decompress_generic(
1850
1963
  length = token & ML_MASK;
1851
1964
 
1852
1965
  _copy_match:
1853
- if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */
1854
- if (!partialDecoding) {
1855
- assert(oend > op);
1856
- assert(oend - op >= 4);
1857
- LZ4_write32(op, 0); /* silence an msan warning when offset==0; costs <1%; */
1858
- } /* note : when partialDecoding, there is no guarantee that at least 4 bytes remain available in output buffer */
1859
-
1860
1966
  if (length == ML_MASK) {
1861
1967
  variable_length_error error = ok;
1862
1968
  length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error);
@@ -1868,6 +1974,7 @@ LZ4_decompress_generic(
1868
1974
  #if LZ4_FAST_DEC_LOOP
1869
1975
  safe_match_copy:
1870
1976
  #endif
1977
+ if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */
1871
1978
  /* match starting within external dictionary */
1872
1979
  if ((dict==usingExtDict) && (match < lowPrefix)) {
1873
1980
  if (unlikely(op+length > oend-LASTLITERALS)) {
@@ -1895,6 +2002,7 @@ LZ4_decompress_generic(
1895
2002
  } }
1896
2003
  continue;
1897
2004
  }
2005
+ assert(match >= lowPrefix);
1898
2006
 
1899
2007
  /* copy match within block */
1900
2008
  cpy = op + length;
@@ -1906,16 +2014,17 @@ LZ4_decompress_generic(
1906
2014
  const BYTE* const matchEnd = match + mlen;
1907
2015
  BYTE* const copyEnd = op + mlen;
1908
2016
  if (matchEnd > op) { /* overlap copy */
1909
- while (op < copyEnd) *op++ = *match++;
2017
+ while (op < copyEnd) { *op++ = *match++; }
1910
2018
  } else {
1911
2019
  memcpy(op, match, mlen);
1912
2020
  }
1913
2021
  op = copyEnd;
1914
- if (op==oend) break;
2022
+ if (op == oend) { break; }
1915
2023
  continue;
1916
2024
  }
1917
2025
 
1918
2026
  if (unlikely(offset<8)) {
2027
+ LZ4_write32(op, 0); /* silence msan warning when offset==0 */
1919
2028
  op[0] = match[0];
1920
2029
  op[1] = match[1];
1921
2030
  op[2] = match[2];
@@ -1931,25 +2040,26 @@ LZ4_decompress_generic(
1931
2040
 
1932
2041
  if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) {
1933
2042
  BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1);
1934
- if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */
2043
+ if (cpy > oend-LASTLITERALS) { goto _output_error; } /* Error : last LASTLITERALS bytes must be literals (uncompressed) */
1935
2044
  if (op < oCopyLimit) {
1936
- LZ4_wildCopy(op, match, oCopyLimit);
2045
+ LZ4_wildCopy8(op, match, oCopyLimit);
1937
2046
  match += oCopyLimit - op;
1938
2047
  op = oCopyLimit;
1939
2048
  }
1940
- while (op < cpy) *op++ = *match++;
2049
+ while (op < cpy) { *op++ = *match++; }
1941
2050
  } else {
1942
2051
  memcpy(op, match, 8);
1943
- if (length > 16) LZ4_wildCopy(op+8, match+8, cpy);
2052
+ if (length > 16) { LZ4_wildCopy8(op+8, match+8, cpy); }
1944
2053
  }
1945
2054
  op = cpy; /* wildcopy correction */
1946
2055
  }
1947
2056
 
1948
2057
  /* end of decoding */
1949
- if (endOnInput)
2058
+ if (endOnInput) {
1950
2059
  return (int) (((char*)op)-dst); /* Nb of output bytes decoded */
1951
- else
2060
+ } else {
1952
2061
  return (int) (((const char*)ip)-src); /* Nb of input bytes read */
2062
+ }
1953
2063
 
1954
2064
  /* Overflow error detected */
1955
2065
  _output_error:
@@ -2064,7 +2174,7 @@ LZ4_streamDecode_t* LZ4_createStreamDecode(void)
2064
2174
 
2065
2175
  int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream)
2066
2176
  {
2067
- if (LZ4_stream == NULL) return 0; /* support free on NULL */
2177
+ if (LZ4_stream == NULL) { return 0; } /* support free on NULL */
2068
2178
  FREEMEM(LZ4_stream);
2069
2179
  return 0;
2070
2180
  }
@@ -2199,18 +2309,22 @@ int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressed
2199
2309
  if (dictSize==0)
2200
2310
  return LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize);
2201
2311
  if (dictStart+dictSize == dest) {
2202
- if (dictSize >= 64 KB - 1)
2312
+ if (dictSize >= 64 KB - 1) {
2203
2313
  return LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize);
2204
- return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, dictSize);
2314
+ }
2315
+ assert(dictSize >= 0);
2316
+ return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, (size_t)dictSize);
2205
2317
  }
2206
- return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, dictStart, dictSize);
2318
+ assert(dictSize >= 0);
2319
+ return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, dictStart, (size_t)dictSize);
2207
2320
  }
2208
2321
 
2209
2322
  int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize)
2210
2323
  {
2211
2324
  if (dictSize==0 || dictStart+dictSize == dest)
2212
2325
  return LZ4_decompress_fast(source, dest, originalSize);
2213
- return LZ4_decompress_fast_extDict(source, dest, originalSize, dictStart, dictSize);
2326
+ assert(dictSize >= 0);
2327
+ return LZ4_decompress_fast_extDict(source, dest, originalSize, dictStart, (size_t)dictSize);
2214
2328
  }
2215
2329
 
2216
2330
 
@@ -2222,9 +2336,9 @@ int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, in
2222
2336
  {
2223
2337
  return LZ4_compress_default(source, dest, inputSize, maxOutputSize);
2224
2338
  }
2225
- int LZ4_compress(const char* source, char* dest, int inputSize)
2339
+ int LZ4_compress(const char* src, char* dest, int srcSize)
2226
2340
  {
2227
- return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize));
2341
+ return LZ4_compress_default(src, dest, srcSize, LZ4_compressBound(srcSize));
2228
2342
  }
2229
2343
  int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize)
2230
2344
  {