extzstd 0.2 → 0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/HISTORY.ja.md +13 -0
- data/README.md +17 -14
- data/contrib/zstd/{NEWS → CHANGELOG} +115 -2
- data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
- data/contrib/zstd/Makefile +99 -53
- data/contrib/zstd/README.md +59 -39
- data/contrib/zstd/TESTING.md +1 -1
- data/contrib/zstd/appveyor.yml +17 -6
- data/contrib/zstd/lib/BUCK +29 -2
- data/contrib/zstd/lib/Makefile +118 -21
- data/contrib/zstd/lib/README.md +84 -44
- data/contrib/zstd/lib/common/bitstream.h +17 -33
- data/contrib/zstd/lib/common/compiler.h +62 -8
- data/contrib/zstd/lib/common/cpu.h +215 -0
- data/contrib/zstd/lib/common/debug.c +44 -0
- data/contrib/zstd/lib/common/debug.h +134 -0
- data/contrib/zstd/lib/common/entropy_common.c +16 -1
- data/contrib/zstd/lib/common/error_private.c +7 -0
- data/contrib/zstd/lib/common/fse.h +48 -44
- data/contrib/zstd/lib/common/fse_decompress.c +3 -3
- data/contrib/zstd/lib/common/huf.h +169 -113
- data/contrib/zstd/lib/common/mem.h +20 -2
- data/contrib/zstd/lib/common/pool.c +135 -49
- data/contrib/zstd/lib/common/pool.h +40 -21
- data/contrib/zstd/lib/common/threading.c +2 -2
- data/contrib/zstd/lib/common/threading.h +12 -12
- data/contrib/zstd/lib/common/xxhash.c +3 -2
- data/contrib/zstd/lib/common/zstd_common.c +3 -6
- data/contrib/zstd/lib/common/zstd_errors.h +17 -7
- data/contrib/zstd/lib/common/zstd_internal.h +76 -48
- data/contrib/zstd/lib/compress/fse_compress.c +89 -209
- data/contrib/zstd/lib/compress/hist.c +203 -0
- data/contrib/zstd/lib/compress/hist.h +95 -0
- data/contrib/zstd/lib/compress/huf_compress.c +188 -80
- data/contrib/zstd/lib/compress/zstd_compress.c +2500 -1203
- data/contrib/zstd/lib/compress/zstd_compress_internal.h +463 -62
- data/contrib/zstd/lib/compress/zstd_double_fast.c +321 -131
- data/contrib/zstd/lib/compress/zstd_double_fast.h +13 -4
- data/contrib/zstd/lib/compress/zstd_fast.c +335 -108
- data/contrib/zstd/lib/compress/zstd_fast.h +12 -6
- data/contrib/zstd/lib/compress/zstd_lazy.c +654 -313
- data/contrib/zstd/lib/compress/zstd_lazy.h +44 -16
- data/contrib/zstd/lib/compress/zstd_ldm.c +310 -420
- data/contrib/zstd/lib/compress/zstd_ldm.h +63 -26
- data/contrib/zstd/lib/compress/zstd_opt.c +773 -325
- data/contrib/zstd/lib/compress/zstd_opt.h +31 -5
- data/contrib/zstd/lib/compress/zstdmt_compress.c +1468 -518
- data/contrib/zstd/lib/compress/zstdmt_compress.h +96 -45
- data/contrib/zstd/lib/decompress/huf_decompress.c +518 -282
- data/contrib/zstd/lib/decompress/zstd_ddict.c +240 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
- data/contrib/zstd/lib/decompress/zstd_decompress.c +613 -1513
- data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1311 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_block.h +59 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +175 -0
- data/contrib/zstd/lib/dictBuilder/cover.c +194 -113
- data/contrib/zstd/lib/dictBuilder/cover.h +112 -0
- data/contrib/zstd/lib/dictBuilder/divsufsort.c +3 -3
- data/contrib/zstd/lib/dictBuilder/fastcover.c +740 -0
- data/contrib/zstd/lib/dictBuilder/zdict.c +142 -106
- data/contrib/zstd/lib/dictBuilder/zdict.h +115 -49
- data/contrib/zstd/lib/legacy/zstd_legacy.h +44 -12
- data/contrib/zstd/lib/legacy/zstd_v01.c +41 -10
- data/contrib/zstd/lib/legacy/zstd_v01.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v02.c +37 -12
- data/contrib/zstd/lib/legacy/zstd_v02.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v03.c +38 -12
- data/contrib/zstd/lib/legacy/zstd_v03.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v04.c +55 -174
- data/contrib/zstd/lib/legacy/zstd_v04.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v05.c +59 -31
- data/contrib/zstd/lib/legacy/zstd_v05.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v06.c +48 -20
- data/contrib/zstd/lib/legacy/zstd_v06.h +10 -5
- data/contrib/zstd/lib/legacy/zstd_v07.c +62 -29
- data/contrib/zstd/lib/legacy/zstd_v07.h +10 -5
- data/contrib/zstd/lib/zstd.h +1346 -832
- data/ext/extzstd.c +27 -19
- data/ext/extzstd_stream.c +20 -4
- data/ext/zstd_compress.c +1 -0
- data/ext/zstd_decompress.c +4 -0
- data/ext/zstd_dictbuilder.c +4 -0
- data/ext/zstd_dictbuilder_fastcover.c +5 -0
- data/lib/extzstd.rb +52 -220
- data/lib/extzstd/version.rb +1 -1
- metadata +21 -7
- data/contrib/zstd/circle.yml +0 -63
data/contrib/zstd/README.md
CHANGED
@@ -1,15 +1,20 @@
|
|
1
|
-
<p align="center"><img src="https://raw.githubusercontent.com/facebook/zstd/
|
1
|
+
<p align="center"><img src="https://raw.githubusercontent.com/facebook/zstd/dev/doc/images/zstd_logo86.png" alt="Zstandard"></p>
|
2
2
|
|
3
3
|
__Zstandard__, or `zstd` as short version, is a fast lossless compression algorithm,
|
4
4
|
targeting real-time compression scenarios at zlib-level and better compression ratios.
|
5
5
|
It's backed by a very fast entropy stage, provided by [Huff0 and FSE library](https://github.com/Cyan4973/FiniteStateEntropy).
|
6
6
|
|
7
|
-
The project is provided as an open-source BSD
|
7
|
+
The project is provided as an open-source dual [BSD](LICENSE) and [GPLv2](COPYING) licensed **C** library,
|
8
8
|
and a command line utility producing and decoding `.zst`, `.gz`, `.xz` and `.lz4` files.
|
9
9
|
Should your project require another programming language,
|
10
10
|
a list of known ports and bindings is provided on [Zstandard homepage](http://www.zstd.net/#other-languages).
|
11
11
|
|
12
|
-
Development branch status
|
12
|
+
**Development branch status:**
|
13
|
+
|
14
|
+
[![Build Status][travisDevBadge]][travisLink]
|
15
|
+
[![Build status][AppveyorDevBadge]][AppveyorLink]
|
16
|
+
[![Build status][CircleDevBadge]][CircleLink]
|
17
|
+
[![Build status][CirrusDevBadge]][CirrusLink]
|
13
18
|
|
14
19
|
[travisDevBadge]: https://travis-ci.org/facebook/zstd.svg?branch=dev "Continuous Integration test suite"
|
15
20
|
[travisLink]: https://travis-ci.org/facebook/zstd
|
@@ -17,31 +22,34 @@ Development branch status : [![Build Status][travisDevBadge]][travisLink] [![B
|
|
17
22
|
[AppveyorLink]: https://ci.appveyor.com/project/YannCollet/zstd-p0yf0
|
18
23
|
[CircleDevBadge]: https://circleci.com/gh/facebook/zstd/tree/dev.svg?style=shield "Short test suite"
|
19
24
|
[CircleLink]: https://circleci.com/gh/facebook/zstd
|
25
|
+
[CirrusDevBadge]: https://api.cirrus-ci.com/github/facebook/zstd.svg?branch=dev
|
26
|
+
[CirrusLink]: https://cirrus-ci.com/github/facebook/zstd
|
20
27
|
|
21
|
-
|
28
|
+
## Benchmarks
|
22
29
|
|
23
30
|
For reference, several fast compression algorithms were tested and compared
|
24
|
-
on a server running Linux
|
25
|
-
with a Core
|
31
|
+
on a server running Arch Linux (`Linux version 5.0.5-arch1-1`),
|
32
|
+
with a Core i9-9900K CPU @ 5.0GHz,
|
26
33
|
using [lzbench], an open-source in-memory benchmark by @inikep
|
27
|
-
compiled with
|
34
|
+
compiled with [gcc] 8.2.1,
|
28
35
|
on the [Silesia compression corpus].
|
29
36
|
|
30
37
|
[lzbench]: https://github.com/inikep/lzbench
|
31
38
|
[Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
|
39
|
+
[gcc]: https://gcc.gnu.org/
|
32
40
|
|
33
41
|
| Compressor name | Ratio | Compression| Decompress.|
|
34
42
|
| --------------- | ------| -----------| ---------- |
|
35
|
-
| **zstd 1.
|
36
|
-
| zlib 1.2.
|
37
|
-
| brotli 0.
|
38
|
-
| quicklz 1.5.0 -1 | 2.238 |
|
39
|
-
| lzo1x 2.09 -1 | 2.
|
40
|
-
| lz4 1.
|
41
|
-
| snappy 1.1.
|
42
|
-
| lzf 3.6 -1 | 2.077 |
|
43
|
-
|
44
|
-
[zlib]:http://www.zlib.net/
|
43
|
+
| **zstd 1.4.0 -1** | 2.884 | 530 MB/s | 1360 MB/s |
|
44
|
+
| zlib 1.2.11 -1 | 2.743 | 110 MB/s | 440 MB/s |
|
45
|
+
| brotli 1.0.7 -0 | 2.701 | 430 MB/s | 470 MB/s |
|
46
|
+
| quicklz 1.5.0 -1 | 2.238 | 600 MB/s | 800 MB/s |
|
47
|
+
| lzo1x 2.09 -1 | 2.106 | 680 MB/s | 950 MB/s |
|
48
|
+
| lz4 1.8.3 | 2.101 | 800 MB/s | 4220 MB/s |
|
49
|
+
| snappy 1.1.4 | 2.073 | 580 MB/s | 2020 MB/s |
|
50
|
+
| lzf 3.6 -1 | 2.077 | 440 MB/s | 930 MB/s |
|
51
|
+
|
52
|
+
[zlib]: http://www.zlib.net/
|
45
53
|
[LZ4]: http://www.lz4.org/
|
46
54
|
|
47
55
|
Zstd can also offer stronger compression ratios at the cost of compression speed.
|
@@ -50,21 +58,21 @@ Decompression speed is preserved and remains roughly the same at all settings,
|
|
50
58
|
a property shared by most LZ compression algorithms, such as [zlib] or lzma.
|
51
59
|
|
52
60
|
The following tests were run
|
53
|
-
on a server running Linux Debian (`Linux version 4.
|
61
|
+
on a server running Linux Debian (`Linux version 4.14.0-3-amd64`)
|
54
62
|
with a Core i7-6700K CPU @ 4.0GHz,
|
55
63
|
using [lzbench], an open-source in-memory benchmark by @inikep
|
56
|
-
compiled with
|
64
|
+
compiled with [gcc] 7.3.0,
|
57
65
|
on the [Silesia compression corpus].
|
58
66
|
|
59
67
|
Compression Speed vs Ratio | Decompression Speed
|
60
68
|
---------------------------|--------------------
|
61
|
-
![Compression Speed vs Ratio](doc/images/
|
69
|
+
![Compression Speed vs Ratio](doc/images/CSpeed2.png "Compression Speed vs Ratio") | ![Decompression Speed](doc/images/DSpeed3.png "Decompression Speed")
|
62
70
|
|
63
71
|
A few other algorithms can produce higher compression ratios at slower speeds, falling outside of the graph.
|
64
72
|
For a larger picture including slow modes, [click on this link](doc/images/DCspeed5.png).
|
65
73
|
|
66
74
|
|
67
|
-
|
75
|
+
## The case for Small Data compression
|
68
76
|
|
69
77
|
Previous charts provide results applicable to typical file and stream scenarios (several MB). Small data comes with different perspectives.
|
70
78
|
|
@@ -88,24 +96,24 @@ Training works if there is some correlation in a family of small data samples. T
|
|
88
96
|
Hence, deploying one dictionary per type of data will provide the greatest benefits.
|
89
97
|
Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will gradually use previously decoded content to better compress the rest of the file.
|
90
98
|
|
91
|
-
|
99
|
+
### Dictionary compression How To:
|
92
100
|
|
93
|
-
1
|
101
|
+
1. Create the dictionary
|
94
102
|
|
95
|
-
`zstd --train FullPathToTrainingSet/* -o dictionaryName`
|
103
|
+
`zstd --train FullPathToTrainingSet/* -o dictionaryName`
|
96
104
|
|
97
|
-
2
|
105
|
+
2. Compress with dictionary
|
98
106
|
|
99
|
-
`zstd -D dictionaryName FILE`
|
107
|
+
`zstd -D dictionaryName FILE`
|
100
108
|
|
101
|
-
3
|
109
|
+
3. Decompress with dictionary
|
102
110
|
|
103
|
-
`zstd -D dictionaryName --decompress FILE.zst`
|
111
|
+
`zstd -D dictionaryName --decompress FILE.zst`
|
104
112
|
|
105
113
|
|
106
|
-
|
114
|
+
## Build instructions
|
107
115
|
|
108
|
-
|
116
|
+
### Makefile
|
109
117
|
|
110
118
|
If your system is compatible with standard `make` (or `gmake`),
|
111
119
|
invoking `make` in root directory will generate `zstd` cli in root directory.
|
@@ -114,35 +122,47 @@ Other available options include:
|
|
114
122
|
- `make install` : create and install zstd cli, library and man pages
|
115
123
|
- `make check` : create and run `zstd`, tests its behavior on local platform
|
116
124
|
|
117
|
-
|
125
|
+
### cmake
|
118
126
|
|
119
127
|
A `cmake` project generator is provided within `build/cmake`.
|
120
128
|
It can generate Makefiles or other build scripts
|
121
129
|
to create `zstd` binary, and `libzstd` dynamic and static libraries.
|
122
130
|
|
123
|
-
|
131
|
+
By default, `CMAKE_BUILD_TYPE` is set to `Release`.
|
132
|
+
|
133
|
+
### Meson
|
124
134
|
|
125
|
-
A Meson project is provided within `
|
135
|
+
A Meson project is provided within [`build/meson`](build/meson). Follow
|
136
|
+
build instructions in that directory.
|
126
137
|
|
127
|
-
|
138
|
+
You can also take a look at [`.travis.yml`](.travis.yml) file for an
|
139
|
+
example about how Meson is used to build this project.
|
140
|
+
|
141
|
+
Note that default build type is **release**.
|
142
|
+
|
143
|
+
### Visual Studio (Windows)
|
128
144
|
|
129
145
|
Going into `build` directory, you will find additional possibilities:
|
130
146
|
- Projects for Visual Studio 2005, 2008 and 2010.
|
131
|
-
+ VS2010 project is compatible with VS2012, VS2013 and
|
132
|
-
- Automated build scripts for Visual compiler by @KrzysFR
|
147
|
+
+ VS2010 project is compatible with VS2012, VS2013, VS2015 and VS2017.
|
148
|
+
- Automated build scripts for Visual compiler by [@KrzysFR](https://github.com/KrzysFR), in `build/VS_scripts`,
|
133
149
|
which will build `zstd` cli and `libzstd` library without any need to open Visual Studio solution.
|
134
150
|
|
151
|
+
### Buck
|
152
|
+
|
153
|
+
You can build the zstd binary via buck by executing: `buck build programs:zstd` from the root of the repo.
|
154
|
+
The output binary will be in `buck-out/gen/programs/`.
|
135
155
|
|
136
|
-
|
156
|
+
## Status
|
137
157
|
|
138
158
|
Zstandard is currently deployed within Facebook. It is used continuously to compress large amounts of data in multiple formats and use cases.
|
139
159
|
Zstandard is considered safe for production environments.
|
140
160
|
|
141
|
-
|
161
|
+
## License
|
142
162
|
|
143
163
|
Zstandard is dual-licensed under [BSD](LICENSE) and [GPLv2](COPYING).
|
144
164
|
|
145
|
-
|
165
|
+
## Contributing
|
146
166
|
|
147
167
|
The "dev" branch is the one where all contributions are merged before reaching "master".
|
148
168
|
If you plan to propose a patch, please commit into the "dev" branch, or its own feature branch.
|
data/contrib/zstd/TESTING.md
CHANGED
@@ -41,4 +41,4 @@ They consist of the following tests:
|
|
41
41
|
- `pzstd` with asan and tsan, as well as in 32-bits mode
|
42
42
|
- Testing `zstd` with legacy mode off
|
43
43
|
- Testing `zbuff` (old streaming API)
|
44
|
-
- Entire test suite and make install on
|
44
|
+
- Entire test suite and make install on macOS
|
data/contrib/zstd/appveyor.yml
CHANGED
@@ -2,14 +2,15 @@
|
|
2
2
|
version: 1.0.{build}
|
3
3
|
branches:
|
4
4
|
only:
|
5
|
-
- dev
|
6
5
|
- master
|
6
|
+
- appveyorTest
|
7
|
+
- /visual*/
|
7
8
|
environment:
|
8
9
|
matrix:
|
9
10
|
- COMPILER: "gcc"
|
10
11
|
HOST: "mingw"
|
11
12
|
PLATFORM: "x64"
|
12
|
-
SCRIPT: "make allzstd MOREFLAGS=-static && make -C tests test-symbols fullbench-
|
13
|
+
SCRIPT: "make allzstd MOREFLAGS=-static && make -C tests test-symbols fullbench-lib"
|
13
14
|
ARTIFACT: "true"
|
14
15
|
BUILD: "true"
|
15
16
|
- COMPILER: "gcc"
|
@@ -80,12 +81,22 @@
|
|
80
81
|
SET "LDFLAGS=../../zlib/libz.a" &&
|
81
82
|
sh -c "%SCRIPT%" &&
|
82
83
|
( if [%COMPILER%]==[gcc] if [%ARTIFACT%]==[true]
|
84
|
+
ECHO Creating artifacts &&
|
85
|
+
ECHO %cd% &&
|
83
86
|
lib\dll\example\build_package.bat &&
|
84
87
|
make -C programs DEBUGFLAGS= clean zstd &&
|
85
88
|
cd programs\ && 7z a -tzip -mx9 zstd-win-binary-%PLATFORM%.zip zstd.exe &&
|
86
89
|
appveyor PushArtifact zstd-win-binary-%PLATFORM%.zip &&
|
87
90
|
cp zstd.exe ..\bin\zstd.exe &&
|
88
|
-
|
91
|
+
git clone --depth 1 --branch master https://github.com/facebook/zstd &&
|
92
|
+
cd zstd &&
|
93
|
+
git archive --format=tar master -o zstd-src.tar &&
|
94
|
+
..\zstd -19 zstd-src.tar &&
|
95
|
+
appveyor PushArtifact zstd-src.tar.zst &&
|
96
|
+
certUtil -hashfile zstd-src.tar.zst SHA256 > zstd-src.tar.zst.sha256.sig &&
|
97
|
+
appveyor PushArtifact zstd-src.tar.zst.sha256.sig &&
|
98
|
+
cd ..\..\bin\ &&
|
99
|
+
7z a -tzip -mx9 zstd-win-release-%PLATFORM%.zip * &&
|
89
100
|
appveyor PushArtifact zstd-win-release-%PLATFORM%.zip
|
90
101
|
)
|
91
102
|
)
|
@@ -172,15 +183,15 @@
|
|
172
183
|
- COMPILER: "gcc"
|
173
184
|
HOST: "mingw"
|
174
185
|
PLATFORM: "x64"
|
175
|
-
SCRIPT: "make allzstd"
|
186
|
+
SCRIPT: "CPPFLAGS=-DDEBUGLEVEL=2 CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2"
|
176
187
|
- COMPILER: "gcc"
|
177
188
|
HOST: "mingw"
|
178
189
|
PLATFORM: "x86"
|
179
|
-
SCRIPT: "make allzstd"
|
190
|
+
SCRIPT: "CFLAGS=-Werror make -j allzstd"
|
180
191
|
- COMPILER: "clang"
|
181
192
|
HOST: "mingw"
|
182
193
|
PLATFORM: "x64"
|
183
|
-
SCRIPT: "
|
194
|
+
SCRIPT: "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd"
|
184
195
|
|
185
196
|
- COMPILER: "visual"
|
186
197
|
HOST: "visual"
|
data/contrib/zstd/lib/BUCK
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
cxx_library(
|
2
2
|
name='zstd',
|
3
3
|
header_namespace='',
|
4
|
+
exported_headers=['zstd.h'],
|
4
5
|
visibility=['PUBLIC'],
|
5
6
|
deps=[
|
6
7
|
':common',
|
@@ -17,7 +18,7 @@ cxx_library(
|
|
17
18
|
exported_headers=subdir_glob([
|
18
19
|
('compress', 'zstd*.h'),
|
19
20
|
]),
|
20
|
-
srcs=glob(['compress/zstd*.c']),
|
21
|
+
srcs=glob(['compress/zstd*.c', 'compress/hist.c']),
|
21
22
|
deps=[':common'],
|
22
23
|
)
|
23
24
|
|
@@ -25,6 +26,9 @@ cxx_library(
|
|
25
26
|
name='decompress',
|
26
27
|
header_namespace='',
|
27
28
|
visibility=['PUBLIC'],
|
29
|
+
headers=subdir_glob([
|
30
|
+
('decompress', '*_impl.h'),
|
31
|
+
]),
|
28
32
|
srcs=glob(['decompress/zstd*.c']),
|
29
33
|
deps=[
|
30
34
|
':common',
|
@@ -37,7 +41,7 @@ cxx_library(
|
|
37
41
|
header_namespace='',
|
38
42
|
visibility=['PUBLIC'],
|
39
43
|
exported_headers=subdir_glob([
|
40
|
-
('
|
44
|
+
('deprecated', '*.h'),
|
41
45
|
]),
|
42
46
|
srcs=glob(['deprecated/*.c']),
|
43
47
|
deps=[':common'],
|
@@ -66,6 +70,7 @@ cxx_library(
|
|
66
70
|
]),
|
67
71
|
headers=subdir_glob([
|
68
72
|
('dictBuilder', 'divsufsort.h'),
|
73
|
+
('dictBuilder', 'cover.h'),
|
69
74
|
]),
|
70
75
|
srcs=glob(['dictBuilder/*.c']),
|
71
76
|
deps=[':common'],
|
@@ -80,6 +85,15 @@ cxx_library(
|
|
80
85
|
]),
|
81
86
|
)
|
82
87
|
|
88
|
+
cxx_library(
|
89
|
+
name='cpu',
|
90
|
+
header_namespace='',
|
91
|
+
visibility=['PUBLIC'],
|
92
|
+
exported_headers=subdir_glob([
|
93
|
+
('common', 'cpu.h'),
|
94
|
+
]),
|
95
|
+
)
|
96
|
+
|
83
97
|
cxx_library(
|
84
98
|
name='bitstream',
|
85
99
|
header_namespace='',
|
@@ -105,6 +119,7 @@ cxx_library(
|
|
105
119
|
'decompress/huf_decompress.c',
|
106
120
|
],
|
107
121
|
deps=[
|
122
|
+
':debug',
|
108
123
|
':bitstream',
|
109
124
|
':compiler',
|
110
125
|
':errors',
|
@@ -191,11 +206,23 @@ cxx_library(
|
|
191
206
|
],
|
192
207
|
)
|
193
208
|
|
209
|
+
cxx_library(
|
210
|
+
name='debug',
|
211
|
+
header_namespace='',
|
212
|
+
visibility=['PUBLIC'],
|
213
|
+
exported_headers=subdir_glob([
|
214
|
+
('common', 'debug.h'),
|
215
|
+
]),
|
216
|
+
srcs=['common/debug.c'],
|
217
|
+
)
|
218
|
+
|
194
219
|
cxx_library(
|
195
220
|
name='common',
|
196
221
|
deps=[
|
222
|
+
':debug',
|
197
223
|
':bitstream',
|
198
224
|
':compiler',
|
225
|
+
':cpu',
|
199
226
|
':entropy',
|
200
227
|
':errors',
|
201
228
|
':mem',
|
data/contrib/zstd/lib/Makefile
CHANGED
@@ -19,23 +19,102 @@ LIBVER := $(shell echo $(LIBVER_SCRIPT))
|
|
19
19
|
VERSION?= $(LIBVER)
|
20
20
|
|
21
21
|
CPPFLAGS+= -I. -I./common -DXXH_NAMESPACE=ZSTD_
|
22
|
+
ifeq ($(OS),Windows_NT) # MinGW assumed
|
23
|
+
CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting
|
24
|
+
endif
|
22
25
|
CFLAGS ?= -O3
|
23
|
-
DEBUGFLAGS
|
26
|
+
DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
|
24
27
|
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
|
25
|
-
-Wstrict-prototypes -Wundef -Wpointer-arith
|
28
|
+
-Wstrict-prototypes -Wundef -Wpointer-arith \
|
26
29
|
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
|
27
|
-
-Wredundant-decls
|
30
|
+
-Wredundant-decls -Wmissing-prototypes -Wc++-compat
|
28
31
|
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
|
29
32
|
FLAGS = $(CPPFLAGS) $(CFLAGS)
|
30
33
|
|
34
|
+
HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
|
35
|
+
GREP_OPTIONS ?=
|
36
|
+
ifeq ($HAVE_COLORNEVER, 1)
|
37
|
+
GREP_OPTIONS += --color=never
|
38
|
+
endif
|
39
|
+
GREP = grep $(GREP_OPTIONS)
|
40
|
+
|
41
|
+
ZSTDCOMMON_FILES := $(sort $(wildcard common/*.c))
|
42
|
+
ZSTDCOMP_FILES := $(sort $(wildcard compress/*.c))
|
43
|
+
ZSTDDECOMP_FILES := $(sort $(wildcard decompress/*.c))
|
44
|
+
ZDICT_FILES := $(sort $(wildcard dictBuilder/*.c))
|
45
|
+
ZDEPR_FILES := $(sort $(wildcard deprecated/*.c))
|
46
|
+
ZSTD_FILES := $(ZSTDCOMMON_FILES)
|
47
|
+
|
48
|
+
ZSTD_LEGACY_SUPPORT ?= 5
|
49
|
+
ZSTD_LIB_COMPRESSION ?= 1
|
50
|
+
ZSTD_LIB_DECOMPRESSION ?= 1
|
51
|
+
ZSTD_LIB_DICTBUILDER ?= 1
|
52
|
+
ZSTD_LIB_DEPRECATED ?= 1
|
53
|
+
HUF_FORCE_DECOMPRESS_X1 ?= 0
|
54
|
+
HUF_FORCE_DECOMPRESS_X2 ?= 0
|
55
|
+
ZSTD_FORCE_DECOMPRESS_SHORT ?= 0
|
56
|
+
ZSTD_FORCE_DECOMPRESS_LONG ?= 0
|
57
|
+
ZSTD_NO_INLINE ?= 0
|
58
|
+
ZSTD_STRIP_ERROR_STRINGS ?= 0
|
59
|
+
ZSTD_LEGACY_MULTITHREADED_API ?= 0
|
60
|
+
|
61
|
+
ifeq ($(ZSTD_LIB_COMPRESSION), 0)
|
62
|
+
ZSTD_LIB_DICTBUILDER = 0
|
63
|
+
ZSTD_LIB_DEPRECATED = 0
|
64
|
+
endif
|
65
|
+
|
66
|
+
ifeq ($(ZSTD_LIB_DECOMPRESSION), 0)
|
67
|
+
ZSTD_LEGACY_SUPPORT = 0
|
68
|
+
ZSTD_LIB_DEPRECATED = 0
|
69
|
+
endif
|
70
|
+
|
71
|
+
ifneq ($(ZSTD_LIB_COMPRESSION), 0)
|
72
|
+
ZSTD_FILES += $(ZSTDCOMP_FILES)
|
73
|
+
endif
|
74
|
+
|
75
|
+
ifneq ($(ZSTD_LIB_DECOMPRESSION), 0)
|
76
|
+
ZSTD_FILES += $(ZSTDDECOMP_FILES)
|
77
|
+
endif
|
31
78
|
|
32
|
-
|
79
|
+
ifneq ($(ZSTD_LIB_DEPRECATED), 0)
|
80
|
+
ZSTD_FILES += $(ZDEPR_FILES)
|
81
|
+
endif
|
82
|
+
|
83
|
+
ifneq ($(ZSTD_LIB_DICTBUILDER), 0)
|
84
|
+
ZSTD_FILES += $(ZDICT_FILES)
|
85
|
+
endif
|
33
86
|
|
34
|
-
|
87
|
+
ifneq ($(HUF_FORCE_DECOMPRESS_X1), 0)
|
88
|
+
CFLAGS += -DHUF_FORCE_DECOMPRESS_X1
|
89
|
+
endif
|
90
|
+
|
91
|
+
ifneq ($(HUF_FORCE_DECOMPRESS_X2), 0)
|
92
|
+
CFLAGS += -DHUF_FORCE_DECOMPRESS_X2
|
93
|
+
endif
|
94
|
+
|
95
|
+
ifneq ($(ZSTD_FORCE_DECOMPRESS_SHORT), 0)
|
96
|
+
CFLAGS += -DZSTD_FORCE_DECOMPRESS_SHORT
|
97
|
+
endif
|
98
|
+
|
99
|
+
ifneq ($(ZSTD_FORCE_DECOMPRESS_LONG), 0)
|
100
|
+
CFLAGS += -DZSTD_FORCE_DECOMPRESS_LONG
|
101
|
+
endif
|
102
|
+
|
103
|
+
ifneq ($(ZSTD_NO_INLINE), 0)
|
104
|
+
CFLAGS += -DZSTD_NO_INLINE
|
105
|
+
endif
|
106
|
+
|
107
|
+
ifneq ($(ZSTD_STRIP_ERROR_STRINGS), 0)
|
108
|
+
CFLAGS += -DZSTD_STRIP_ERROR_STRINGS
|
109
|
+
endif
|
110
|
+
|
111
|
+
ifneq ($(ZSTD_LEGACY_MULTITHREADED_API), 0)
|
112
|
+
CFLAGS += -DZSTD_LEGACY_MULTITHREADED_API
|
113
|
+
endif
|
35
114
|
|
36
115
|
ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
|
37
116
|
ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
|
38
|
-
ZSTD_FILES += $(shell ls legacy/*.c |
|
117
|
+
ZSTD_FILES += $(shell ls legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
|
39
118
|
endif
|
40
119
|
CPPFLAGS += -I./legacy
|
41
120
|
endif
|
@@ -43,7 +122,7 @@ CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
|
|
43
122
|
|
44
123
|
ZSTD_OBJ := $(patsubst %.c,%.o,$(ZSTD_FILES))
|
45
124
|
|
46
|
-
#
|
125
|
+
# macOS linker doesn't support -soname, and use different extension
|
47
126
|
# see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html
|
48
127
|
ifeq ($(shell uname), Darwin)
|
49
128
|
SHARED_EXT = dylib
|
@@ -57,8 +136,6 @@ else
|
|
57
136
|
SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
|
58
137
|
endif
|
59
138
|
|
60
|
-
LIBZSTD = libzstd.$(SHARED_EXT_VER)
|
61
|
-
|
62
139
|
|
63
140
|
.PHONY: default all clean install uninstall
|
64
141
|
|
@@ -74,19 +151,27 @@ libzstd.a: $(ZSTD_OBJ)
|
|
74
151
|
libzstd.a-mt: CPPFLAGS += -DZSTD_MULTITHREAD
|
75
152
|
libzstd.a-mt: libzstd.a
|
76
153
|
|
77
|
-
|
154
|
+
ifneq (,$(filter Windows%,$(OS)))
|
155
|
+
|
156
|
+
LIBZSTD = dll\libzstd.dll
|
78
157
|
$(LIBZSTD): $(ZSTD_FILES)
|
79
158
|
@echo compiling dynamic library $(LIBVER)
|
80
|
-
|
81
|
-
|
82
|
-
dlltool -D dll\libzstd.dll -d dll\libzstd.def -l dll\libzstd.lib
|
159
|
+
$(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll\libzstd.lib -shared $^ -o $@
|
160
|
+
|
83
161
|
else
|
162
|
+
|
163
|
+
LIBZSTD = libzstd.$(SHARED_EXT_VER)
|
164
|
+
$(LIBZSTD): LDFLAGS += -shared -fPIC -fvisibility=hidden
|
165
|
+
$(LIBZSTD): $(ZSTD_FILES)
|
166
|
+
@echo compiling dynamic library $(LIBVER)
|
84
167
|
@$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
|
85
168
|
@echo creating versioned links
|
86
169
|
@ln -sf $@ libzstd.$(SHARED_EXT_MAJOR)
|
87
170
|
@ln -sf $@ libzstd.$(SHARED_EXT)
|
171
|
+
|
88
172
|
endif
|
89
173
|
|
174
|
+
|
90
175
|
libzstd : $(LIBZSTD)
|
91
176
|
|
92
177
|
libzstd-mt : CPPFLAGS += -DZSTD_MULTITHREAD
|
@@ -111,16 +196,16 @@ libzstd-nomt: $(ZSTD_NOMT_FILES)
|
|
111
196
|
@$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
|
112
197
|
|
113
198
|
clean:
|
114
|
-
@$(RM) -r *.dSYM #
|
199
|
+
@$(RM) -r *.dSYM # macOS-specific
|
115
200
|
@$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc
|
116
201
|
@$(RM) dll/libzstd.dll dll/libzstd.lib libzstd-nomt*
|
117
202
|
@$(RM) common/*.o compress/*.o decompress/*.o dictBuilder/*.o legacy/*.o deprecated/*.o
|
118
203
|
@echo Cleaning library completed
|
119
204
|
|
120
205
|
#-----------------------------------------------------------------------------
|
121
|
-
# make install is validated only for Linux,
|
206
|
+
# make install is validated only for Linux, macOS, BSD, Hurd and Solaris targets
|
122
207
|
#-----------------------------------------------------------------------------
|
123
|
-
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
|
208
|
+
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku))
|
124
209
|
|
125
210
|
DESTDIR ?=
|
126
211
|
# directory variables : GNU conventions prefer lowercase
|
@@ -134,7 +219,7 @@ LIBDIR ?= $(libdir)
|
|
134
219
|
includedir ?= $(PREFIX)/include
|
135
220
|
INCLUDEDIR ?= $(includedir)
|
136
221
|
|
137
|
-
ifneq (,$(filter $(shell uname),
|
222
|
+
ifneq (,$(filter $(shell uname),FreeBSD NetBSD DragonFly))
|
138
223
|
PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig
|
139
224
|
else
|
140
225
|
PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig
|
@@ -159,20 +244,32 @@ libzstd.pc: libzstd.pc.in
|
|
159
244
|
-e 's|@VERSION@|$(VERSION)|' \
|
160
245
|
$< >$@
|
161
246
|
|
162
|
-
install:
|
163
|
-
|
247
|
+
install: install-pc install-static install-shared install-includes
|
248
|
+
@echo zstd static and shared library installed
|
249
|
+
|
250
|
+
install-pc: libzstd.pc
|
251
|
+
@$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/
|
164
252
|
@$(INSTALL_DATA) libzstd.pc $(DESTDIR)$(PKGCONFIGDIR)/
|
165
|
-
|
253
|
+
|
254
|
+
install-static: libzstd.a
|
255
|
+
@echo Installing static library
|
256
|
+
@$(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)/
|
166
257
|
@$(INSTALL_DATA) libzstd.a $(DESTDIR)$(LIBDIR)
|
258
|
+
|
259
|
+
install-shared: libzstd
|
260
|
+
@echo Installing shared library
|
261
|
+
@$(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)/
|
167
262
|
@$(INSTALL_PROGRAM) $(LIBZSTD) $(DESTDIR)$(LIBDIR)
|
168
263
|
@ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
|
169
264
|
@ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
|
265
|
+
|
266
|
+
install-includes:
|
170
267
|
@echo Installing includes
|
268
|
+
@$(INSTALL) -d -m 755 $(DESTDIR)$(INCLUDEDIR)/
|
171
269
|
@$(INSTALL_DATA) zstd.h $(DESTDIR)$(INCLUDEDIR)
|
172
270
|
@$(INSTALL_DATA) common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)
|
173
271
|
@$(INSTALL_DATA) deprecated/zbuff.h $(DESTDIR)$(INCLUDEDIR) # prototypes generate deprecation warnings
|
174
272
|
@$(INSTALL_DATA) dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)
|
175
|
-
@echo zstd static and shared library installed
|
176
273
|
|
177
274
|
uninstall:
|
178
275
|
@$(RM) $(DESTDIR)$(LIBDIR)/libzstd.a
|