extzstd 0.3.3 → 0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +4 -3
- data/contrib/zstd/CHANGELOG +39 -2
- data/contrib/zstd/CONTRIBUTING.md +3 -3
- data/contrib/zstd/Makefile +34 -20
- data/contrib/zstd/README.md +6 -2
- data/contrib/zstd/SECURITY.md +15 -0
- data/contrib/zstd/lib/Makefile +40 -28
- data/contrib/zstd/lib/README.md +14 -1
- data/contrib/zstd/lib/common/allocations.h +1 -1
- data/contrib/zstd/lib/common/bitstream.h +49 -29
- data/contrib/zstd/lib/common/compiler.h +114 -22
- data/contrib/zstd/lib/common/cpu.h +36 -0
- data/contrib/zstd/lib/common/debug.c +6 -0
- data/contrib/zstd/lib/common/debug.h +20 -11
- data/contrib/zstd/lib/common/error_private.h +45 -36
- data/contrib/zstd/lib/common/fse.h +3 -2
- data/contrib/zstd/lib/common/fse_decompress.c +19 -17
- data/contrib/zstd/lib/common/huf.h +14 -1
- data/contrib/zstd/lib/common/mem.h +0 -9
- data/contrib/zstd/lib/common/pool.c +1 -1
- data/contrib/zstd/lib/common/pool.h +1 -1
- data/contrib/zstd/lib/common/portability_macros.h +2 -0
- data/contrib/zstd/lib/common/threading.c +8 -2
- data/contrib/zstd/lib/common/xxhash.c +5 -11
- data/contrib/zstd/lib/common/xxhash.h +2341 -1007
- data/contrib/zstd/lib/common/zstd_internal.h +5 -5
- data/contrib/zstd/lib/compress/fse_compress.c +8 -7
- data/contrib/zstd/lib/compress/huf_compress.c +54 -25
- data/contrib/zstd/lib/compress/zstd_compress.c +282 -161
- data/contrib/zstd/lib/compress/zstd_compress_internal.h +29 -27
- data/contrib/zstd/lib/compress/zstd_compress_superblock.c +224 -113
- data/contrib/zstd/lib/compress/zstd_cwksp.h +19 -13
- data/contrib/zstd/lib/compress/zstd_double_fast.c +17 -5
- data/contrib/zstd/lib/compress/zstd_double_fast.h +11 -0
- data/contrib/zstd/lib/compress/zstd_fast.c +14 -6
- data/contrib/zstd/lib/compress/zstd_lazy.c +129 -87
- data/contrib/zstd/lib/compress/zstd_lazy.h +103 -28
- data/contrib/zstd/lib/compress/zstd_ldm.c +8 -2
- data/contrib/zstd/lib/compress/zstd_opt.c +216 -112
- data/contrib/zstd/lib/compress/zstd_opt.h +31 -7
- data/contrib/zstd/lib/compress/zstdmt_compress.c +94 -79
- data/contrib/zstd/lib/decompress/huf_decompress.c +188 -126
- data/contrib/zstd/lib/decompress/huf_decompress_amd64.S +38 -19
- data/contrib/zstd/lib/decompress/zstd_decompress.c +84 -32
- data/contrib/zstd/lib/decompress/zstd_decompress_block.c +231 -208
- data/contrib/zstd/lib/decompress/zstd_decompress_block.h +1 -1
- data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +2 -0
- data/contrib/zstd/lib/dictBuilder/cover.c +16 -12
- data/contrib/zstd/lib/dictBuilder/cover.h +2 -8
- data/contrib/zstd/lib/dictBuilder/fastcover.c +2 -2
- data/contrib/zstd/lib/dictBuilder/zdict.c +12 -6
- data/contrib/zstd/lib/legacy/zstd_legacy.h +30 -0
- data/contrib/zstd/lib/legacy/zstd_v01.c +2 -0
- data/contrib/zstd/lib/legacy/zstd_v02.c +4 -16
- data/contrib/zstd/lib/legacy/zstd_v03.c +4 -16
- data/contrib/zstd/lib/legacy/zstd_v04.c +4 -11
- data/contrib/zstd/lib/legacy/zstd_v05.c +1 -0
- data/contrib/zstd/lib/legacy/zstd_v06.c +2 -9
- data/contrib/zstd/lib/legacy/zstd_v07.c +2 -10
- data/contrib/zstd/lib/libzstd.mk +34 -11
- data/contrib/zstd/lib/zstd.h +129 -60
- data/ext/extconf.rb +19 -1
- data/ext/extzstd.c +38 -14
- data/ext/extzstd.h +33 -6
- data/ext/extzstd_stream.c +74 -31
- metadata +4 -5
- data/contrib/zstd/appveyor.yml +0 -205
- data/ext/depend +0 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5f3831b1c8cc5c106619c27d9d1136868050e35d5cb997d8a06f66bcf9106070
|
4
|
+
data.tar.gz: c8824b400e0706d073553eb9a90bb445b1af9d624672d5020acf87d58fcb4524
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dd42f25011a7390206851eccb1e9c515a5245d6634844096bc35342d59d8f3c07f45564dc78efedaafd3846a9c39202ff18821e81bb1da124ace0bca88d7839f
|
7
|
+
data.tar.gz: c1f6c98bc5a1b48f4bfe5c17d3fcc04155741c90788d82ac1eec71c27b05177c02ad15bb38a2868b5194e40740e083a9133e1090dd1bb6a63910325d629204ad
|
data/README.md
CHANGED
@@ -91,7 +91,7 @@ p Ractor.new {
|
|
91
91
|
|
92
92
|
- package name: extzstd
|
93
93
|
- project page: <https://github.com/dearblue/ruby-extzstd>
|
94
|
-
- version: 0.
|
94
|
+
- version: 0.4
|
95
95
|
- product quality: TECHNICAL PREVIEW, UNSTABLE
|
96
96
|
- license: [2 clause BSD License](LICENSE)
|
97
97
|
- author: dearblue
|
@@ -99,6 +99,7 @@ p Ractor.new {
|
|
99
99
|
- dependency ruby gems: (none)
|
100
100
|
- dependency library: (none)
|
101
101
|
- bundled external C library (git submodules):
|
102
|
-
- [zstd-1.5.
|
103
|
-
under
|
102
|
+
- [zstd-1.5.6](https://github.com/facebook/zstd/blob/v1.5.6)
|
103
|
+
under dual licensed ([3 clause BSD License](https://github.com/facebook/zstd/blob/v1.5.6/LICENSE)
|
104
|
+
or [GNU General Public License, version 2](https://github.com/facebook/zstd/blob/v1.5.6/COPYING))
|
104
105
|
by [facebook](https://github.com/facebook)
|
data/contrib/zstd/CHANGELOG
CHANGED
@@ -1,3 +1,40 @@
|
|
1
|
+
V1.5.6 (Mar 2024)
|
2
|
+
api: Promote `ZSTD_c_targetCBlockSize` to Stable API by @felixhandte
|
3
|
+
api: new `ZSTD_d_maxBlockSize` experimental parameter, to reduce streaming decompression memory, by @terrelln
|
4
|
+
perf: improve performance of param `ZSTD_c_targetCBlockSize`, by @Cyan4973
|
5
|
+
perf: improved compression of arrays of integers at high compression, by @Cyan4973
|
6
|
+
lib: reduce binary size with selective built-time exclusion, by @felixhandte
|
7
|
+
lib: improved huffman speed on small data and linux kernel, by @terrelln
|
8
|
+
lib: accept dictionaries with partial literal tables, by @terrelln
|
9
|
+
lib: fix CCtx size estimation with external sequence producer, by @embg
|
10
|
+
lib: fix corner case decoder behaviors, by @Cyan4973 and @aimuz
|
11
|
+
lib: fix zdict prototype mismatch in static_only mode, by @ldv-alt
|
12
|
+
lib: fix several bugs in magicless-format decoding, by @embg
|
13
|
+
cli: add common compressed file types to `--exclude-compressed`` by @daniellerozenblit
|
14
|
+
cli: fix mixing `-c` and `-o` commands with `--rm`, by @Cyan4973
|
15
|
+
cli: fix erroneous exclusion of hidden files with `--output-dir-mirror` by @felixhandte
|
16
|
+
cli: improved time accuracy on BSD, by @felixhandte
|
17
|
+
cli: better errors on argument parsing, by @KapJI
|
18
|
+
tests: better compatibility with older versions of `grep`, by @Cyan4973
|
19
|
+
tests: lorem ipsum generator as default backup content, by @Cyan4973
|
20
|
+
build: cmake improvements by @terrelln, @sighingnow, @gjasny, @JohanMabille, @Saverio976, @gruenich, @teo-tsirpanis
|
21
|
+
build: bazel support, by @jondo2010
|
22
|
+
build: fix cross-compiling for AArch64 with lld by @jcelerier
|
23
|
+
build: fix Apple platform compatibility, by @nidhijaju
|
24
|
+
build: fix Visual 2012 and lower compatibility, by @Cyan4973
|
25
|
+
build: improve win32 support, by @DimitriPapadopoulos
|
26
|
+
build: better C90 compliance for zlibWrapper, by @emaste
|
27
|
+
port: make: fat binaries on macos, by @mredig
|
28
|
+
port: ARM64EC compatibility for Windows, by @dunhor
|
29
|
+
port: QNX support by @klausholstjacobsen
|
30
|
+
port: MSYS2 and Cygwin makefile installation and test support, by @QBos07
|
31
|
+
port: risc-v support validation in CI, by @Cyan4973
|
32
|
+
port: sparc64 support validation in CI, by @Cyan4973
|
33
|
+
port: AIX compatibility, by @likema
|
34
|
+
port: HP-UX compatibility, by @likema
|
35
|
+
doc: Improved specification accuracy, by @elasota
|
36
|
+
bug: Fix and deprecate ZSTD_generateSequences (#3981)
|
37
|
+
|
1
38
|
v1.5.5 (Apr 2023)
|
2
39
|
fix: fix rare corruption bug affecting the high compression mode, reported by @danlark1 (#3517, @terrelln)
|
3
40
|
perf: improve mid-level compression speed (#3529, #3533, #3543, @yoniko and #3552, @terrelln)
|
@@ -98,7 +135,7 @@ build: support for m68k (Motorola 68000's), by @cyan4973
|
|
98
135
|
build: improved AIX support, by @Helflym
|
99
136
|
build: improved meson unofficial build, by @eli-schwartz
|
100
137
|
cli : custom memory limit when training dictionary (#2925), by @embg
|
101
|
-
cli : report advanced parameters information when compressing in very verbose mode (
|
138
|
+
cli : report advanced parameters information when compressing in very verbose mode (`-vv`), by @Svetlitski-FB
|
102
139
|
|
103
140
|
v1.5.0 (May 11, 2021)
|
104
141
|
api: Various functions promoted from experimental to stable API: (#2579-2581, @senhuang42)
|
@@ -165,7 +202,7 @@ api: Add Function to Generate Skippable Frame (#2439, @senhuang42)
|
|
165
202
|
perf: New Algorithms for the Long Distance Matcher (#2483, @mpu)
|
166
203
|
perf: Performance Improvements for Long Distance Matcher (#2464, @mpu)
|
167
204
|
perf: Don't Shrink Window Log when Streaming with a Dictionary (#2451, @terrelln)
|
168
|
-
cli: Fix `--output-dir-mirror`
|
205
|
+
cli: Fix `--output-dir-mirror` rejection of `..` -containing paths (#2512, @felixhandte)
|
169
206
|
cli: Allow Input From Console When `-f`/`--force` is Passed (#2466, @felixhandte)
|
170
207
|
cli: Improve Help Message (#2500, @senhuang42)
|
171
208
|
tests: Remove Flaky Tests (#2455, #2486, #2445, @Cyan4973)
|
@@ -171,8 +171,8 @@ who want earlier signal.
|
|
171
171
|
| Cirrus CI | Used for testing on FreeBSD | https://github.com/marketplace/cirrus-ci/ | `.cirrus.yml` |
|
172
172
|
| Circle CI | Historically was used to provide faster signal,<br/> but we may be able to migrate these to Github Actions | https://circleci.com/docs/2.0/getting-started/#setting-up-circleci <br> https://youtu.be/Js3hMUsSZ2c <br> https://circleci.com/docs/2.0/enable-checks/ | `.circleci/config.yml` |
|
173
173
|
|
174
|
-
Note: the instructions linked above mostly cover how to set up a repository with CI from scratch.
|
175
|
-
The general idea should be the same for setting up CI on your fork of zstd, but you may have to
|
174
|
+
Note: the instructions linked above mostly cover how to set up a repository with CI from scratch.
|
175
|
+
The general idea should be the same for setting up CI on your fork of zstd, but you may have to
|
176
176
|
follow slightly different steps. In particular, please ignore any instructions related to setting up
|
177
177
|
config files (since zstd already has configs for each of these services).
|
178
178
|
|
@@ -216,7 +216,7 @@ will typically not be stable enough to obtain reliable benchmark results. If you
|
|
216
216
|
hands on a desktop, this is usually a better scenario.
|
217
217
|
|
218
218
|
Of course, benchmarking can be done on non-hyper-stable machines as well. You will just have to
|
219
|
-
do a little more work to ensure that you are in fact measuring the changes you've made not
|
219
|
+
do a little more work to ensure that you are in fact measuring the changes you've made and not
|
220
220
|
noise. Here are some things you can do to make your benchmarks more stable:
|
221
221
|
|
222
222
|
1. The most simple thing you can do to drastically improve the stability of your benchmark is
|
data/contrib/zstd/Makefile
CHANGED
@@ -145,13 +145,13 @@ clean:
|
|
145
145
|
$(Q)$(MAKE) -C contrib/largeNbDicts $@ > $(VOID)
|
146
146
|
$(Q)$(MAKE) -C contrib/externalSequenceProducer $@ > $(VOID)
|
147
147
|
$(Q)$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
|
148
|
-
$(Q)$(RM) -r lz4
|
148
|
+
$(Q)$(RM) -r lz4 cmakebuild install
|
149
149
|
@echo Cleaning completed
|
150
150
|
|
151
151
|
#------------------------------------------------------------------------------
|
152
152
|
# make install is validated only for Linux, macOS, Hurd and some BSD targets
|
153
153
|
#------------------------------------------------------------------------------
|
154
|
-
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT Haiku AIX))
|
154
|
+
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT CYGWIN_NT Haiku AIX))
|
155
155
|
|
156
156
|
HOST_OS = POSIX
|
157
157
|
|
@@ -197,6 +197,15 @@ uninstall:
|
|
197
197
|
travis-install:
|
198
198
|
$(MAKE) install PREFIX=~/install_test_dir
|
199
199
|
|
200
|
+
.PHONY: clangbuild-darwin-fat
|
201
|
+
clangbuild-darwin-fat: clean
|
202
|
+
clang -v
|
203
|
+
CXX=clang++ CC=clang CFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation -arch arm64" $(MAKE) zstd-release
|
204
|
+
mv programs/zstd programs/zstd_arm64
|
205
|
+
CXX=clang++ CC=clang CFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation -arch x86_64" $(MAKE) zstd-release
|
206
|
+
mv programs/zstd programs/zstd_x64
|
207
|
+
lipo -create programs/zstd_x64 programs/zstd_arm64 -output programs/zstd
|
208
|
+
|
200
209
|
.PHONY: gcc5build gcc6build gcc7build clangbuild m32build armbuild aarch64build ppcbuild ppc64build
|
201
210
|
gcc5build: clean
|
202
211
|
gcc-5 -v
|
@@ -308,7 +317,7 @@ update_regressionResults:
|
|
308
317
|
# run UBsan with -fsanitize-recover=pointer-overflow
|
309
318
|
# this only works with recent compilers such as gcc 8+
|
310
319
|
usan: clean
|
311
|
-
$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize
|
320
|
+
$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=undefined -Werror $(MOREFLAGS)"
|
312
321
|
|
313
322
|
asan: clean
|
314
323
|
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -Werror $(MOREFLAGS)"
|
@@ -319,17 +328,18 @@ asan-%: clean
|
|
319
328
|
msan: clean
|
320
329
|
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer -Werror $(MOREFLAGS)" HAVE_LZMA=0 # datagen.c fails this test for no obvious reason
|
321
330
|
|
322
|
-
msan-%:
|
323
|
-
|
331
|
+
msan-%:
|
332
|
+
$(MAKE) clean
|
333
|
+
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer -Werror $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -j -C $(TESTDIR) HAVE_LZMA=0 $*
|
324
334
|
|
325
335
|
asan32: clean
|
326
336
|
$(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address $(MOREFLAGS)"
|
327
337
|
|
328
338
|
uasan: clean
|
329
|
-
$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize
|
339
|
+
$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address,undefined -Werror $(MOREFLAGS)"
|
330
340
|
|
331
341
|
uasan-%: clean
|
332
|
-
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize
|
342
|
+
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address,undefined -Werror $(MOREFLAGS)" $(MAKE) -C $(TESTDIR) $*
|
333
343
|
|
334
344
|
tsan-%: clean
|
335
345
|
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread -Werror $(MOREFLAGS)" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)"
|
@@ -380,28 +390,32 @@ lz4install:
|
|
380
390
|
endif
|
381
391
|
|
382
392
|
|
383
|
-
CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release
|
384
|
-
|
385
393
|
ifneq (,$(filter MSYS%,$(shell uname)))
|
386
394
|
HOST_OS = MSYS
|
387
|
-
CMAKE_PARAMS = -G"MSYS Makefiles" -DCMAKE_BUILD_TYPE=Debug -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
|
388
395
|
endif
|
389
396
|
|
390
397
|
#------------------------------------------------------------------------
|
391
398
|
# target specific tests
|
392
399
|
#------------------------------------------------------------------------
|
393
400
|
ifneq (,$(filter $(HOST_OS),MSYS POSIX))
|
394
|
-
.PHONY: cmakebuild c89build gnu90build c99build gnu99build c11build bmix64build bmix32build bmi32build staticAnalyze
|
395
|
-
cmakebuild:
|
396
|
-
cmake --version
|
397
|
-
$(RM) -r $(BUILDIR)/cmake/build
|
398
|
-
$(MKDIR) $(BUILDIR)/cmake/build
|
399
|
-
cd $(BUILDIR)/cmake/build; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) ..
|
400
|
-
$(MAKE) -C $(BUILDIR)/cmake/build -j4;
|
401
|
-
$(MAKE) -C $(BUILDIR)/cmake/build install;
|
402
|
-
$(MAKE) -C $(BUILDIR)/cmake/build uninstall;
|
403
|
-
cd $(BUILDIR)/cmake/build; ctest -V -L Medium
|
404
401
|
|
402
|
+
CMAKE ?= cmake
|
403
|
+
CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON
|
404
|
+
|
405
|
+
ifneq (,$(filter MSYS%,$(shell uname)))
|
406
|
+
CMAKE_PARAMS = -G"MSYS Makefiles" -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
|
407
|
+
endif
|
408
|
+
|
409
|
+
.PHONY: cmakebuild
|
410
|
+
cmakebuild:
|
411
|
+
$(CMAKE) --version
|
412
|
+
$(RM) -r cmakebuild install
|
413
|
+
$(MKDIR) cmakebuild install
|
414
|
+
cd cmakebuild; $(CMAKE) -Wdev -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_FLAGS="-Werror -O0" -DCMAKE_INSTALL_PREFIX=install $(CMAKE_PARAMS) ../build/cmake
|
415
|
+
$(CMAKE) --build cmakebuild --target install -- -j V=1
|
416
|
+
cd cmakebuild; ctest -V -L Medium
|
417
|
+
|
418
|
+
.PHONY: c89build gnu90build c99build gnu99build c11build bmix64build bmix32build bmi32build staticAnalyze
|
405
419
|
c89build: clean
|
406
420
|
$(CC) -v
|
407
421
|
CFLAGS="-std=c89 -Werror -Wno-attributes -Wpedantic -Wno-long-long -Wno-variadic-macros -O0" $(MAKE) lib zstd
|
data/contrib/zstd/README.md
CHANGED
@@ -5,7 +5,7 @@ targeting real-time compression scenarios at zlib-level and better compression r
|
|
5
5
|
It's backed by a very fast entropy stage, provided by [Huff0 and FSE library](https://github.com/Cyan4973/FiniteStateEntropy).
|
6
6
|
|
7
7
|
Zstandard's format is stable and documented in [RFC8878](https://datatracker.ietf.org/doc/html/rfc8878). Multiple independent implementations are already available.
|
8
|
-
This repository represents the reference implementation, provided as an open-source dual [BSD](LICENSE)
|
8
|
+
This repository represents the reference implementation, provided as an open-source dual [BSD](LICENSE) OR [GPLv2](COPYING) licensed **C** library,
|
9
9
|
and a command line utility producing and decoding `.zst`, `.gz`, `.xz` and `.lz4` files.
|
10
10
|
Should your project require another programming language,
|
11
11
|
a list of known ports and bindings is provided on [Zstandard homepage](https://facebook.github.io/zstd/#other-languages).
|
@@ -198,6 +198,10 @@ Going into `build` directory, you will find additional possibilities:
|
|
198
198
|
You can build the zstd binary via buck by executing: `buck build programs:zstd` from the root of the repo.
|
199
199
|
The output binary will be in `buck-out/gen/programs/`.
|
200
200
|
|
201
|
+
### Bazel
|
202
|
+
|
203
|
+
You easily can integrate zstd into your Bazel project by using the module hosted on the [Bazel Central Repository](https://registry.bazel.build/modules/zstd).
|
204
|
+
|
201
205
|
## Testing
|
202
206
|
|
203
207
|
You can run quick local smoke tests by running `make check`.
|
@@ -213,7 +217,7 @@ Zstandard is considered safe for production environments.
|
|
213
217
|
|
214
218
|
## License
|
215
219
|
|
216
|
-
Zstandard is dual-licensed under [BSD](LICENSE)
|
220
|
+
Zstandard is dual-licensed under [BSD](LICENSE) OR [GPLv2](COPYING).
|
217
221
|
|
218
222
|
## Contributing
|
219
223
|
|
@@ -0,0 +1,15 @@
|
|
1
|
+
# Reporting and Fixing Security Issues
|
2
|
+
|
3
|
+
Please do not open GitHub issues or pull requests - this makes the problem immediately visible to everyone, including malicious actors. Security issues in this open source project can be safely reported via the Meta Bug Bounty program:
|
4
|
+
|
5
|
+
https://www.facebook.com/whitehat
|
6
|
+
|
7
|
+
Meta's security team will triage your report and determine whether or not is it eligible for a bounty under our program.
|
8
|
+
|
9
|
+
# Receiving Vulnerability Notifications
|
10
|
+
|
11
|
+
In the case that a significant security vulnerability is reported to us or discovered by us---without being publicly known---we will, at our discretion, notify high-profile, high-exposure users of Zstandard ahead of our public disclosure of the issue and associated fix.
|
12
|
+
|
13
|
+
If you believe your project would benefit from inclusion in this list, please reach out to one of the maintainers.
|
14
|
+
|
15
|
+
<!-- Note to maintainers: this list is kept [here](https://fburl.com/wiki/cgc1l62x). -->
|
data/contrib/zstd/lib/Makefile
CHANGED
@@ -8,6 +8,9 @@
|
|
8
8
|
# You may select, at your option, one of the above-listed licenses.
|
9
9
|
# ################################################################
|
10
10
|
|
11
|
+
# default target (when running `make` with no argument)
|
12
|
+
lib-release:
|
13
|
+
|
11
14
|
# Modules
|
12
15
|
ZSTD_LIB_COMPRESSION ?= 1
|
13
16
|
ZSTD_LIB_DECOMPRESSION ?= 1
|
@@ -54,12 +57,11 @@ VERSION := $(ZSTD_VERSION)
|
|
54
57
|
# Note: by default, the static library is built single-threaded and dynamic library is built
|
55
58
|
# multi-threaded. It is possible to force multi or single threaded builds by appending
|
56
59
|
# -mt or -nomt to the build target (like lib-mt for multi-threaded, lib-nomt for single-threaded).
|
57
|
-
|
58
|
-
default: lib-release
|
60
|
+
|
59
61
|
|
60
62
|
CPPFLAGS_DYNLIB += -DZSTD_MULTITHREAD # dynamic library build defaults to multi-threaded
|
61
63
|
LDFLAGS_DYNLIB += -pthread
|
62
|
-
|
64
|
+
CPPFLAGS_STATICLIB += # static library build defaults to single-threaded
|
63
65
|
|
64
66
|
|
65
67
|
ifeq ($(findstring GCC,$(CCVER)),GCC)
|
@@ -91,7 +93,7 @@ all: lib
|
|
91
93
|
|
92
94
|
|
93
95
|
.PHONY: libzstd.a # must be run every time
|
94
|
-
libzstd.a: CPPFLAGS += $(
|
96
|
+
libzstd.a: CPPFLAGS += $(CPPFLAGS_STATICLIB)
|
95
97
|
|
96
98
|
SET_CACHE_DIRECTORY = \
|
97
99
|
+$(MAKE) --no-print-directory $@ \
|
@@ -109,19 +111,19 @@ libzstd.a:
|
|
109
111
|
else
|
110
112
|
# BUILD_DIR is defined
|
111
113
|
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
$(
|
116
|
-
$(
|
117
|
-
$(
|
114
|
+
ZSTD_STATICLIB_DIR := $(BUILD_DIR)/static
|
115
|
+
ZSTD_STATICLIB := $(ZSTD_STATICLIB_DIR)/libzstd.a
|
116
|
+
ZSTD_STATICLIB_OBJ := $(addprefix $(ZSTD_STATICLIB_DIR)/,$(ZSTD_LOCAL_OBJ))
|
117
|
+
$(ZSTD_STATICLIB): ARFLAGS = rcs
|
118
|
+
$(ZSTD_STATICLIB): | $(ZSTD_STATICLIB_DIR)
|
119
|
+
$(ZSTD_STATICLIB): $(ZSTD_STATICLIB_OBJ)
|
118
120
|
# Check for multithread flag at target execution time
|
119
121
|
$(if $(filter -DZSTD_MULTITHREAD,$(CPPFLAGS)),\
|
120
122
|
@echo compiling multi-threaded static library $(LIBVER),\
|
121
123
|
@echo compiling single-threaded static library $(LIBVER))
|
122
124
|
$(AR) $(ARFLAGS) $@ $^
|
123
125
|
|
124
|
-
libzstd.a: $(
|
126
|
+
libzstd.a: $(ZSTD_STATICLIB)
|
125
127
|
cp -f $< $@
|
126
128
|
|
127
129
|
endif
|
@@ -182,14 +184,14 @@ lib : libzstd.a libzstd
|
|
182
184
|
# make does not consider implicit pattern rule for .PHONY target
|
183
185
|
|
184
186
|
%-mt : CPPFLAGS_DYNLIB := -DZSTD_MULTITHREAD
|
185
|
-
%-mt :
|
187
|
+
%-mt : CPPFLAGS_STATICLIB := -DZSTD_MULTITHREAD
|
186
188
|
%-mt : LDFLAGS_DYNLIB := -pthread
|
187
189
|
%-mt : %
|
188
190
|
@echo multi-threaded build completed
|
189
191
|
|
190
192
|
%-nomt : CPPFLAGS_DYNLIB :=
|
191
193
|
%-nomt : LDFLAGS_DYNLIB :=
|
192
|
-
%-nomt :
|
194
|
+
%-nomt : CPPFLAGS_STATICLIB :=
|
193
195
|
%-nomt : %
|
194
196
|
@echo single-threaded build completed
|
195
197
|
|
@@ -200,42 +202,52 @@ lib : libzstd.a libzstd
|
|
200
202
|
|
201
203
|
# Generate .h dependencies automatically
|
202
204
|
|
203
|
-
|
205
|
+
# -MMD: compiler generates dependency information as a side-effect of compilation, without system headers
|
206
|
+
# -MP: adds phony target for each dependency other than main file.
|
207
|
+
DEPFLAGS = -MMD -MP
|
204
208
|
|
205
|
-
|
209
|
+
# ensure that ZSTD_DYNLIB_DIR exists prior to generating %.o
|
210
|
+
$(ZSTD_DYNLIB_DIR)/%.o : %.c | $(ZSTD_DYNLIB_DIR)
|
206
211
|
@echo CC $@
|
207
|
-
$(COMPILE.c) $(DEPFLAGS) $(
|
212
|
+
$(COMPILE.c) $(DEPFLAGS) $(OUTPUT_OPTION) $<
|
208
213
|
|
209
|
-
$(
|
214
|
+
$(ZSTD_STATICLIB_DIR)/%.o : %.c | $(ZSTD_STATICLIB_DIR)
|
210
215
|
@echo CC $@
|
211
|
-
$(COMPILE.c) $(DEPFLAGS) $(
|
216
|
+
$(COMPILE.c) $(DEPFLAGS) $(OUTPUT_OPTION) $<
|
212
217
|
|
213
218
|
$(ZSTD_DYNLIB_DIR)/%.o : %.S | $(ZSTD_DYNLIB_DIR)
|
214
219
|
@echo AS $@
|
215
220
|
$(COMPILE.S) $(OUTPUT_OPTION) $<
|
216
221
|
|
217
|
-
$(
|
222
|
+
$(ZSTD_STATICLIB_DIR)/%.o : %.S | $(ZSTD_STATICLIB_DIR)
|
218
223
|
@echo AS $@
|
219
224
|
$(COMPILE.S) $(OUTPUT_OPTION) $<
|
220
225
|
|
221
|
-
MKDIR ?= mkdir
|
222
|
-
$(BUILD_DIR) $(ZSTD_DYNLIB_DIR) $(
|
223
|
-
$(MKDIR)
|
226
|
+
MKDIR ?= mkdir -p
|
227
|
+
$(BUILD_DIR) $(ZSTD_DYNLIB_DIR) $(ZSTD_STATICLIB_DIR):
|
228
|
+
$(MKDIR) $@
|
224
229
|
|
225
|
-
DEPFILES := $(ZSTD_DYNLIB_OBJ:.o=.d) $(
|
230
|
+
DEPFILES := $(ZSTD_DYNLIB_OBJ:.o=.d) $(ZSTD_STATICLIB_OBJ:.o=.d)
|
226
231
|
$(DEPFILES):
|
227
232
|
|
228
|
-
include
|
233
|
+
# The leading '-' means: do not fail is include fails (ex: directory does not exist yet)
|
234
|
+
-include $(wildcard $(DEPFILES))
|
229
235
|
|
230
236
|
|
231
|
-
# Special case :
|
232
|
-
|
233
|
-
|
237
|
+
# Special case : build library in single-thread mode _and_ without zstdmt_compress.c
|
238
|
+
# Note : we still need threading.c and pool.c for the dictionary builder,
|
239
|
+
# but they will correctly behave single-threaded.
|
240
|
+
ZSTDMT_FILES = zstdmt_compress.c
|
241
|
+
ZSTD_NOMT_FILES = $(filter-out $(ZSTDMT_FILES),$(notdir $(ZSTD_FILES)))
|
234
242
|
libzstd-nomt: CFLAGS += -fPIC -fvisibility=hidden
|
235
243
|
libzstd-nomt: LDFLAGS += -shared
|
236
244
|
libzstd-nomt: $(ZSTD_NOMT_FILES)
|
237
245
|
@echo compiling single-thread dynamic library $(LIBVER)
|
238
246
|
@echo files : $(ZSTD_NOMT_FILES)
|
247
|
+
@if echo "$(ZSTD_NOMT_FILES)" | tr ' ' '\n' | $(GREP) -q zstdmt; then \
|
248
|
+
echo "Error: Found zstdmt in list."; \
|
249
|
+
exit 1; \
|
250
|
+
fi
|
239
251
|
$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
|
240
252
|
|
241
253
|
.PHONY: clean
|
@@ -249,7 +261,7 @@ clean:
|
|
249
261
|
#-----------------------------------------------------------------------------
|
250
262
|
# make install is validated only for below listed environments
|
251
263
|
#-----------------------------------------------------------------------------
|
252
|
-
ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku AIX))
|
264
|
+
ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku AIX MSYS_NT CYGWIN_NT))
|
253
265
|
|
254
266
|
lib: libzstd.pc
|
255
267
|
|
data/contrib/zstd/lib/README.md
CHANGED
@@ -88,7 +88,7 @@ The file structure is designed to make this selection manually achievable for an
|
|
88
88
|
For example, advanced API for version `v0.4` is exposed in `lib/legacy/zstd_v04.h` .
|
89
89
|
|
90
90
|
- While invoking `make libzstd`, it's possible to define build macros
|
91
|
-
`ZSTD_LIB_COMPRESSION
|
91
|
+
`ZSTD_LIB_COMPRESSION`, `ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`,
|
92
92
|
and `ZSTD_LIB_DEPRECATED` as `0` to forgo compilation of the
|
93
93
|
corresponding features. This will also disable compilation of all
|
94
94
|
dependencies (e.g. `ZSTD_LIB_COMPRESSION=0` will also disable
|
@@ -119,6 +119,15 @@ The file structure is designed to make this selection manually achievable for an
|
|
119
119
|
binary is achieved by using `HUF_FORCE_DECOMPRESS_X1` and
|
120
120
|
`ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT` (implied by `ZSTD_LIB_MINIFY`).
|
121
121
|
|
122
|
+
On the compressor side, Zstd's compression levels map to several internal
|
123
|
+
strategies. In environments where the higher compression levels aren't used,
|
124
|
+
it is possible to exclude all but the fastest strategy with
|
125
|
+
`ZSTD_LIB_EXCLUDE_COMPRESSORS_DFAST_AND_UP=1`. (Note that this will change
|
126
|
+
the behavior of the default compression level.) Or if you want to retain the
|
127
|
+
default compressor as well, you can set
|
128
|
+
`ZSTD_LIB_EXCLUDE_COMPRESSORS_GREEDY_AND_UP=1`, at the cost of an additional
|
129
|
+
~20KB or so.
|
130
|
+
|
122
131
|
For squeezing the last ounce of size out, you can also define
|
123
132
|
`ZSTD_NO_INLINE`, which disables inlining, and `ZSTD_STRIP_ERROR_STRINGS`,
|
124
133
|
which removes the error messages that are otherwise returned by
|
@@ -169,6 +178,10 @@ The file structure is designed to make this selection manually achievable for an
|
|
169
178
|
`ZSTDERRORLIB_VSIBILITY`, and `ZDICTLIB_VISIBILITY` if unset, for backwards compatibility
|
170
179
|
with the old macro names.
|
171
180
|
|
181
|
+
- The C compiler macro `HUF_DISABLE_FAST_DECODE` disables the newer Huffman fast C
|
182
|
+
and assembly decoding loops. You may want to use this macro if these loops are
|
183
|
+
slower on your platform.
|
184
|
+
|
172
185
|
#### Windows : using MinGW+MSYS to create DLL
|
173
186
|
|
174
187
|
DLL can be created using MinGW+MSYS with the `make libzstd` command.
|
@@ -14,7 +14,7 @@
|
|
14
14
|
#define ZSTD_DEPS_NEED_MALLOC
|
15
15
|
#include "zstd_deps.h" /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */
|
16
16
|
|
17
|
-
#include "
|
17
|
+
#include "compiler.h" /* MEM_STATIC */
|
18
18
|
#define ZSTD_STATIC_LINKING_ONLY
|
19
19
|
#include "../zstd.h" /* ZSTD_customMem */
|
20
20
|
|
@@ -90,19 +90,20 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
|
|
90
90
|
/*-********************************************
|
91
91
|
* bitStream decoding API (read backward)
|
92
92
|
**********************************************/
|
93
|
+
typedef size_t BitContainerType;
|
93
94
|
typedef struct {
|
94
|
-
|
95
|
+
BitContainerType bitContainer;
|
95
96
|
unsigned bitsConsumed;
|
96
97
|
const char* ptr;
|
97
98
|
const char* start;
|
98
99
|
const char* limitPtr;
|
99
100
|
} BIT_DStream_t;
|
100
101
|
|
101
|
-
typedef enum { BIT_DStream_unfinished = 0,
|
102
|
-
BIT_DStream_endOfBuffer = 1,
|
103
|
-
BIT_DStream_completed = 2,
|
104
|
-
BIT_DStream_overflow = 3
|
105
|
-
|
102
|
+
typedef enum { BIT_DStream_unfinished = 0, /* fully refilled */
|
103
|
+
BIT_DStream_endOfBuffer = 1, /* still some bits left in bitstream */
|
104
|
+
BIT_DStream_completed = 2, /* bitstream entirely consumed, bit-exact */
|
105
|
+
BIT_DStream_overflow = 3 /* user requested more bits than present in bitstream */
|
106
|
+
} BIT_DStream_status; /* result of BIT_reloadDStream() */
|
106
107
|
|
107
108
|
MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize);
|
108
109
|
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits);
|
@@ -112,7 +113,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
|
|
112
113
|
|
113
114
|
/* Start by invoking BIT_initDStream().
|
114
115
|
* A chunk of the bitStream is then stored into a local register.
|
115
|
-
* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (
|
116
|
+
* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (BitContainerType).
|
116
117
|
* You can then retrieve bitFields stored into the local register, **in reverse order**.
|
117
118
|
* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method.
|
118
119
|
* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished.
|
@@ -162,7 +163,7 @@ MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC,
|
|
162
163
|
return 0;
|
163
164
|
}
|
164
165
|
|
165
|
-
|
166
|
+
FORCE_INLINE_TEMPLATE size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
|
166
167
|
{
|
167
168
|
#if defined(STATIC_BMI2) && STATIC_BMI2 == 1 && !defined(ZSTD_NO_INTRINSICS)
|
168
169
|
return _bzhi_u64(bitContainer, nbBits);
|
@@ -267,22 +268,22 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
|
267
268
|
bitD->bitContainer = *(const BYTE*)(bitD->start);
|
268
269
|
switch(srcSize)
|
269
270
|
{
|
270
|
-
case 7: bitD->bitContainer += (
|
271
|
+
case 7: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
|
271
272
|
ZSTD_FALLTHROUGH;
|
272
273
|
|
273
|
-
case 6: bitD->bitContainer += (
|
274
|
+
case 6: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
|
274
275
|
ZSTD_FALLTHROUGH;
|
275
276
|
|
276
|
-
case 5: bitD->bitContainer += (
|
277
|
+
case 5: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
|
277
278
|
ZSTD_FALLTHROUGH;
|
278
279
|
|
279
|
-
case 4: bitD->bitContainer += (
|
280
|
+
case 4: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[3]) << 24;
|
280
281
|
ZSTD_FALLTHROUGH;
|
281
282
|
|
282
|
-
case 3: bitD->bitContainer += (
|
283
|
+
case 3: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[2]) << 16;
|
283
284
|
ZSTD_FALLTHROUGH;
|
284
285
|
|
285
|
-
case 2: bitD->bitContainer += (
|
286
|
+
case 2: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[1]) << 8;
|
286
287
|
ZSTD_FALLTHROUGH;
|
287
288
|
|
288
289
|
default: break;
|
@@ -297,12 +298,12 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
|
297
298
|
return srcSize;
|
298
299
|
}
|
299
300
|
|
300
|
-
|
301
|
+
FORCE_INLINE_TEMPLATE size_t BIT_getUpperBits(BitContainerType bitContainer, U32 const start)
|
301
302
|
{
|
302
303
|
return bitContainer >> start;
|
303
304
|
}
|
304
305
|
|
305
|
-
|
306
|
+
FORCE_INLINE_TEMPLATE size_t BIT_getMiddleBits(BitContainerType bitContainer, U32 const start, U32 const nbBits)
|
306
307
|
{
|
307
308
|
U32 const regMask = sizeof(bitContainer)*8 - 1;
|
308
309
|
/* if start > regMask, bitstream is corrupted, and result is undefined */
|
@@ -325,7 +326,7 @@ MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 c
|
|
325
326
|
* On 32-bits, maxNbBits==24.
|
326
327
|
* On 64-bits, maxNbBits==56.
|
327
328
|
* @return : value extracted */
|
328
|
-
|
329
|
+
FORCE_INLINE_TEMPLATE size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
|
329
330
|
{
|
330
331
|
/* arbitrate between double-shift and shift+mask */
|
331
332
|
#if 1
|
@@ -348,7 +349,7 @@ MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
|
|
348
349
|
return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask);
|
349
350
|
}
|
350
351
|
|
351
|
-
|
352
|
+
FORCE_INLINE_TEMPLATE void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
|
352
353
|
{
|
353
354
|
bitD->bitsConsumed += nbBits;
|
354
355
|
}
|
@@ -357,7 +358,7 @@ MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
|
|
357
358
|
* Read (consume) next n bits from local register and update.
|
358
359
|
* Pay attention to not read more than nbBits contained into local register.
|
359
360
|
* @return : extracted value. */
|
360
|
-
|
361
|
+
FORCE_INLINE_TEMPLATE size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits)
|
361
362
|
{
|
362
363
|
size_t const value = BIT_lookBits(bitD, nbBits);
|
363
364
|
BIT_skipBits(bitD, nbBits);
|
@@ -374,6 +375,21 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)
|
|
374
375
|
return value;
|
375
376
|
}
|
376
377
|
|
378
|
+
/*! BIT_reloadDStream_internal() :
|
379
|
+
* Simple variant of BIT_reloadDStream(), with two conditions:
|
380
|
+
* 1. bitstream is valid : bitsConsumed <= sizeof(bitD->bitContainer)*8
|
381
|
+
* 2. look window is valid after shifted down : bitD->ptr >= bitD->start
|
382
|
+
*/
|
383
|
+
MEM_STATIC BIT_DStream_status BIT_reloadDStream_internal(BIT_DStream_t* bitD)
|
384
|
+
{
|
385
|
+
assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8);
|
386
|
+
bitD->ptr -= bitD->bitsConsumed >> 3;
|
387
|
+
assert(bitD->ptr >= bitD->start);
|
388
|
+
bitD->bitsConsumed &= 7;
|
389
|
+
bitD->bitContainer = MEM_readLEST(bitD->ptr);
|
390
|
+
return BIT_DStream_unfinished;
|
391
|
+
}
|
392
|
+
|
377
393
|
/*! BIT_reloadDStreamFast() :
|
378
394
|
* Similar to BIT_reloadDStream(), but with two differences:
|
379
395
|
* 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold!
|
@@ -384,31 +400,35 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD)
|
|
384
400
|
{
|
385
401
|
if (UNLIKELY(bitD->ptr < bitD->limitPtr))
|
386
402
|
return BIT_DStream_overflow;
|
387
|
-
|
388
|
-
bitD->ptr -= bitD->bitsConsumed >> 3;
|
389
|
-
bitD->bitsConsumed &= 7;
|
390
|
-
bitD->bitContainer = MEM_readLEST(bitD->ptr);
|
391
|
-
return BIT_DStream_unfinished;
|
403
|
+
return BIT_reloadDStream_internal(bitD);
|
392
404
|
}
|
393
405
|
|
394
406
|
/*! BIT_reloadDStream() :
|
395
407
|
* Refill `bitD` from buffer previously set in BIT_initDStream() .
|
396
|
-
* This function is safe, it guarantees it will not
|
408
|
+
* This function is safe, it guarantees it will not never beyond src buffer.
|
397
409
|
* @return : status of `BIT_DStream_t` internal register.
|
398
410
|
* when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */
|
399
|
-
|
411
|
+
FORCE_INLINE_TEMPLATE BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
|
400
412
|
{
|
401
|
-
|
413
|
+
/* note : once in overflow mode, a bitstream remains in this mode until it's reset */
|
414
|
+
if (UNLIKELY(bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))) {
|
415
|
+
static const BitContainerType zeroFilled = 0;
|
416
|
+
bitD->ptr = (const char*)&zeroFilled; /* aliasing is allowed for char */
|
417
|
+
/* overflow detected, erroneous scenario or end of stream: no update */
|
402
418
|
return BIT_DStream_overflow;
|
419
|
+
}
|
420
|
+
|
421
|
+
assert(bitD->ptr >= bitD->start);
|
403
422
|
|
404
423
|
if (bitD->ptr >= bitD->limitPtr) {
|
405
|
-
return
|
424
|
+
return BIT_reloadDStream_internal(bitD);
|
406
425
|
}
|
407
426
|
if (bitD->ptr == bitD->start) {
|
427
|
+
/* reached end of bitStream => no update */
|
408
428
|
if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer;
|
409
429
|
return BIT_DStream_completed;
|
410
430
|
}
|
411
|
-
/* start < ptr < limitPtr */
|
431
|
+
/* start < ptr < limitPtr => cautious update */
|
412
432
|
{ U32 nbBytes = bitD->bitsConsumed >> 3;
|
413
433
|
BIT_DStream_status result = BIT_DStream_unfinished;
|
414
434
|
if (bitD->ptr - nbBytes < bitD->start) {
|