extzstd 0.2 → 0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/HISTORY.ja.md +13 -0
- data/README.md +17 -14
- data/contrib/zstd/{NEWS → CHANGELOG} +115 -2
- data/contrib/zstd/CODE_OF_CONDUCT.md +5 -0
- data/contrib/zstd/Makefile +99 -53
- data/contrib/zstd/README.md +59 -39
- data/contrib/zstd/TESTING.md +1 -1
- data/contrib/zstd/appveyor.yml +17 -6
- data/contrib/zstd/lib/BUCK +29 -2
- data/contrib/zstd/lib/Makefile +118 -21
- data/contrib/zstd/lib/README.md +84 -44
- data/contrib/zstd/lib/common/bitstream.h +17 -33
- data/contrib/zstd/lib/common/compiler.h +62 -8
- data/contrib/zstd/lib/common/cpu.h +215 -0
- data/contrib/zstd/lib/common/debug.c +44 -0
- data/contrib/zstd/lib/common/debug.h +134 -0
- data/contrib/zstd/lib/common/entropy_common.c +16 -1
- data/contrib/zstd/lib/common/error_private.c +7 -0
- data/contrib/zstd/lib/common/fse.h +48 -44
- data/contrib/zstd/lib/common/fse_decompress.c +3 -3
- data/contrib/zstd/lib/common/huf.h +169 -113
- data/contrib/zstd/lib/common/mem.h +20 -2
- data/contrib/zstd/lib/common/pool.c +135 -49
- data/contrib/zstd/lib/common/pool.h +40 -21
- data/contrib/zstd/lib/common/threading.c +2 -2
- data/contrib/zstd/lib/common/threading.h +12 -12
- data/contrib/zstd/lib/common/xxhash.c +3 -2
- data/contrib/zstd/lib/common/zstd_common.c +3 -6
- data/contrib/zstd/lib/common/zstd_errors.h +17 -7
- data/contrib/zstd/lib/common/zstd_internal.h +76 -48
- data/contrib/zstd/lib/compress/fse_compress.c +89 -209
- data/contrib/zstd/lib/compress/hist.c +203 -0
- data/contrib/zstd/lib/compress/hist.h +95 -0
- data/contrib/zstd/lib/compress/huf_compress.c +188 -80
- data/contrib/zstd/lib/compress/zstd_compress.c +2500 -1203
- data/contrib/zstd/lib/compress/zstd_compress_internal.h +463 -62
- data/contrib/zstd/lib/compress/zstd_double_fast.c +321 -131
- data/contrib/zstd/lib/compress/zstd_double_fast.h +13 -4
- data/contrib/zstd/lib/compress/zstd_fast.c +335 -108
- data/contrib/zstd/lib/compress/zstd_fast.h +12 -6
- data/contrib/zstd/lib/compress/zstd_lazy.c +654 -313
- data/contrib/zstd/lib/compress/zstd_lazy.h +44 -16
- data/contrib/zstd/lib/compress/zstd_ldm.c +310 -420
- data/contrib/zstd/lib/compress/zstd_ldm.h +63 -26
- data/contrib/zstd/lib/compress/zstd_opt.c +773 -325
- data/contrib/zstd/lib/compress/zstd_opt.h +31 -5
- data/contrib/zstd/lib/compress/zstdmt_compress.c +1468 -518
- data/contrib/zstd/lib/compress/zstdmt_compress.h +96 -45
- data/contrib/zstd/lib/decompress/huf_decompress.c +518 -282
- data/contrib/zstd/lib/decompress/zstd_ddict.c +240 -0
- data/contrib/zstd/lib/decompress/zstd_ddict.h +44 -0
- data/contrib/zstd/lib/decompress/zstd_decompress.c +613 -1513
- data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1311 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_block.h +59 -0
- data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +175 -0
- data/contrib/zstd/lib/dictBuilder/cover.c +194 -113
- data/contrib/zstd/lib/dictBuilder/cover.h +112 -0
- data/contrib/zstd/lib/dictBuilder/divsufsort.c +3 -3
- data/contrib/zstd/lib/dictBuilder/fastcover.c +740 -0
- data/contrib/zstd/lib/dictBuilder/zdict.c +142 -106
- data/contrib/zstd/lib/dictBuilder/zdict.h +115 -49
- data/contrib/zstd/lib/legacy/zstd_legacy.h +44 -12
- data/contrib/zstd/lib/legacy/zstd_v01.c +41 -10
- data/contrib/zstd/lib/legacy/zstd_v01.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v02.c +37 -12
- data/contrib/zstd/lib/legacy/zstd_v02.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v03.c +38 -12
- data/contrib/zstd/lib/legacy/zstd_v03.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v04.c +55 -174
- data/contrib/zstd/lib/legacy/zstd_v04.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v05.c +59 -31
- data/contrib/zstd/lib/legacy/zstd_v05.h +12 -7
- data/contrib/zstd/lib/legacy/zstd_v06.c +48 -20
- data/contrib/zstd/lib/legacy/zstd_v06.h +10 -5
- data/contrib/zstd/lib/legacy/zstd_v07.c +62 -29
- data/contrib/zstd/lib/legacy/zstd_v07.h +10 -5
- data/contrib/zstd/lib/zstd.h +1346 -832
- data/ext/extzstd.c +27 -19
- data/ext/extzstd_stream.c +20 -4
- data/ext/zstd_compress.c +1 -0
- data/ext/zstd_decompress.c +4 -0
- data/ext/zstd_dictbuilder.c +4 -0
- data/ext/zstd_dictbuilder_fastcover.c +5 -0
- data/lib/extzstd.rb +52 -220
- data/lib/extzstd/version.rb +1 -1
- metadata +21 -7
- data/contrib/zstd/circle.yml +0 -63
data/contrib/zstd/README.md
CHANGED
|
@@ -1,15 +1,20 @@
|
|
|
1
|
-
<p align="center"><img src="https://raw.githubusercontent.com/facebook/zstd/
|
|
1
|
+
<p align="center"><img src="https://raw.githubusercontent.com/facebook/zstd/dev/doc/images/zstd_logo86.png" alt="Zstandard"></p>
|
|
2
2
|
|
|
3
3
|
__Zstandard__, or `zstd` as short version, is a fast lossless compression algorithm,
|
|
4
4
|
targeting real-time compression scenarios at zlib-level and better compression ratios.
|
|
5
5
|
It's backed by a very fast entropy stage, provided by [Huff0 and FSE library](https://github.com/Cyan4973/FiniteStateEntropy).
|
|
6
6
|
|
|
7
|
-
The project is provided as an open-source BSD
|
|
7
|
+
The project is provided as an open-source dual [BSD](LICENSE) and [GPLv2](COPYING) licensed **C** library,
|
|
8
8
|
and a command line utility producing and decoding `.zst`, `.gz`, `.xz` and `.lz4` files.
|
|
9
9
|
Should your project require another programming language,
|
|
10
10
|
a list of known ports and bindings is provided on [Zstandard homepage](http://www.zstd.net/#other-languages).
|
|
11
11
|
|
|
12
|
-
Development branch status
|
|
12
|
+
**Development branch status:**
|
|
13
|
+
|
|
14
|
+
[![Build Status][travisDevBadge]][travisLink]
|
|
15
|
+
[![Build status][AppveyorDevBadge]][AppveyorLink]
|
|
16
|
+
[![Build status][CircleDevBadge]][CircleLink]
|
|
17
|
+
[![Build status][CirrusDevBadge]][CirrusLink]
|
|
13
18
|
|
|
14
19
|
[travisDevBadge]: https://travis-ci.org/facebook/zstd.svg?branch=dev "Continuous Integration test suite"
|
|
15
20
|
[travisLink]: https://travis-ci.org/facebook/zstd
|
|
@@ -17,31 +22,34 @@ Development branch status : [![Build Status][travisDevBadge]][travisLink] [![B
|
|
|
17
22
|
[AppveyorLink]: https://ci.appveyor.com/project/YannCollet/zstd-p0yf0
|
|
18
23
|
[CircleDevBadge]: https://circleci.com/gh/facebook/zstd/tree/dev.svg?style=shield "Short test suite"
|
|
19
24
|
[CircleLink]: https://circleci.com/gh/facebook/zstd
|
|
25
|
+
[CirrusDevBadge]: https://api.cirrus-ci.com/github/facebook/zstd.svg?branch=dev
|
|
26
|
+
[CirrusLink]: https://cirrus-ci.com/github/facebook/zstd
|
|
20
27
|
|
|
21
|
-
|
|
28
|
+
## Benchmarks
|
|
22
29
|
|
|
23
30
|
For reference, several fast compression algorithms were tested and compared
|
|
24
|
-
on a server running Linux
|
|
25
|
-
with a Core
|
|
31
|
+
on a server running Arch Linux (`Linux version 5.0.5-arch1-1`),
|
|
32
|
+
with a Core i9-9900K CPU @ 5.0GHz,
|
|
26
33
|
using [lzbench], an open-source in-memory benchmark by @inikep
|
|
27
|
-
compiled with
|
|
34
|
+
compiled with [gcc] 8.2.1,
|
|
28
35
|
on the [Silesia compression corpus].
|
|
29
36
|
|
|
30
37
|
[lzbench]: https://github.com/inikep/lzbench
|
|
31
38
|
[Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
|
|
39
|
+
[gcc]: https://gcc.gnu.org/
|
|
32
40
|
|
|
33
41
|
| Compressor name | Ratio | Compression| Decompress.|
|
|
34
42
|
| --------------- | ------| -----------| ---------- |
|
|
35
|
-
| **zstd 1.
|
|
36
|
-
| zlib 1.2.
|
|
37
|
-
| brotli 0.
|
|
38
|
-
| quicklz 1.5.0 -1 | 2.238 |
|
|
39
|
-
| lzo1x 2.09 -1 | 2.
|
|
40
|
-
| lz4 1.
|
|
41
|
-
| snappy 1.1.
|
|
42
|
-
| lzf 3.6 -1 | 2.077 |
|
|
43
|
-
|
|
44
|
-
[zlib]:http://www.zlib.net/
|
|
43
|
+
| **zstd 1.4.0 -1** | 2.884 | 530 MB/s | 1360 MB/s |
|
|
44
|
+
| zlib 1.2.11 -1 | 2.743 | 110 MB/s | 440 MB/s |
|
|
45
|
+
| brotli 1.0.7 -0 | 2.701 | 430 MB/s | 470 MB/s |
|
|
46
|
+
| quicklz 1.5.0 -1 | 2.238 | 600 MB/s | 800 MB/s |
|
|
47
|
+
| lzo1x 2.09 -1 | 2.106 | 680 MB/s | 950 MB/s |
|
|
48
|
+
| lz4 1.8.3 | 2.101 | 800 MB/s | 4220 MB/s |
|
|
49
|
+
| snappy 1.1.4 | 2.073 | 580 MB/s | 2020 MB/s |
|
|
50
|
+
| lzf 3.6 -1 | 2.077 | 440 MB/s | 930 MB/s |
|
|
51
|
+
|
|
52
|
+
[zlib]: http://www.zlib.net/
|
|
45
53
|
[LZ4]: http://www.lz4.org/
|
|
46
54
|
|
|
47
55
|
Zstd can also offer stronger compression ratios at the cost of compression speed.
|
|
@@ -50,21 +58,21 @@ Decompression speed is preserved and remains roughly the same at all settings,
|
|
|
50
58
|
a property shared by most LZ compression algorithms, such as [zlib] or lzma.
|
|
51
59
|
|
|
52
60
|
The following tests were run
|
|
53
|
-
on a server running Linux Debian (`Linux version 4.
|
|
61
|
+
on a server running Linux Debian (`Linux version 4.14.0-3-amd64`)
|
|
54
62
|
with a Core i7-6700K CPU @ 4.0GHz,
|
|
55
63
|
using [lzbench], an open-source in-memory benchmark by @inikep
|
|
56
|
-
compiled with
|
|
64
|
+
compiled with [gcc] 7.3.0,
|
|
57
65
|
on the [Silesia compression corpus].
|
|
58
66
|
|
|
59
67
|
Compression Speed vs Ratio | Decompression Speed
|
|
60
68
|
---------------------------|--------------------
|
|
61
|
-
 | 
|
|
62
70
|
|
|
63
71
|
A few other algorithms can produce higher compression ratios at slower speeds, falling outside of the graph.
|
|
64
72
|
For a larger picture including slow modes, [click on this link](doc/images/DCspeed5.png).
|
|
65
73
|
|
|
66
74
|
|
|
67
|
-
|
|
75
|
+
## The case for Small Data compression
|
|
68
76
|
|
|
69
77
|
Previous charts provide results applicable to typical file and stream scenarios (several MB). Small data comes with different perspectives.
|
|
70
78
|
|
|
@@ -88,24 +96,24 @@ Training works if there is some correlation in a family of small data samples. T
|
|
|
88
96
|
Hence, deploying one dictionary per type of data will provide the greatest benefits.
|
|
89
97
|
Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will gradually use previously decoded content to better compress the rest of the file.
|
|
90
98
|
|
|
91
|
-
|
|
99
|
+
### Dictionary compression How To:
|
|
92
100
|
|
|
93
|
-
1
|
|
101
|
+
1. Create the dictionary
|
|
94
102
|
|
|
95
|
-
`zstd --train FullPathToTrainingSet/* -o dictionaryName`
|
|
103
|
+
`zstd --train FullPathToTrainingSet/* -o dictionaryName`
|
|
96
104
|
|
|
97
|
-
2
|
|
105
|
+
2. Compress with dictionary
|
|
98
106
|
|
|
99
|
-
`zstd -D dictionaryName FILE`
|
|
107
|
+
`zstd -D dictionaryName FILE`
|
|
100
108
|
|
|
101
|
-
3
|
|
109
|
+
3. Decompress with dictionary
|
|
102
110
|
|
|
103
|
-
`zstd -D dictionaryName --decompress FILE.zst`
|
|
111
|
+
`zstd -D dictionaryName --decompress FILE.zst`
|
|
104
112
|
|
|
105
113
|
|
|
106
|
-
|
|
114
|
+
## Build instructions
|
|
107
115
|
|
|
108
|
-
|
|
116
|
+
### Makefile
|
|
109
117
|
|
|
110
118
|
If your system is compatible with standard `make` (or `gmake`),
|
|
111
119
|
invoking `make` in root directory will generate `zstd` cli in root directory.
|
|
@@ -114,35 +122,47 @@ Other available options include:
|
|
|
114
122
|
- `make install` : create and install zstd cli, library and man pages
|
|
115
123
|
- `make check` : create and run `zstd`, tests its behavior on local platform
|
|
116
124
|
|
|
117
|
-
|
|
125
|
+
### cmake
|
|
118
126
|
|
|
119
127
|
A `cmake` project generator is provided within `build/cmake`.
|
|
120
128
|
It can generate Makefiles or other build scripts
|
|
121
129
|
to create `zstd` binary, and `libzstd` dynamic and static libraries.
|
|
122
130
|
|
|
123
|
-
|
|
131
|
+
By default, `CMAKE_BUILD_TYPE` is set to `Release`.
|
|
132
|
+
|
|
133
|
+
### Meson
|
|
124
134
|
|
|
125
|
-
A Meson project is provided within `
|
|
135
|
+
A Meson project is provided within [`build/meson`](build/meson). Follow
|
|
136
|
+
build instructions in that directory.
|
|
126
137
|
|
|
127
|
-
|
|
138
|
+
You can also take a look at [`.travis.yml`](.travis.yml) file for an
|
|
139
|
+
example about how Meson is used to build this project.
|
|
140
|
+
|
|
141
|
+
Note that default build type is **release**.
|
|
142
|
+
|
|
143
|
+
### Visual Studio (Windows)
|
|
128
144
|
|
|
129
145
|
Going into `build` directory, you will find additional possibilities:
|
|
130
146
|
- Projects for Visual Studio 2005, 2008 and 2010.
|
|
131
|
-
+ VS2010 project is compatible with VS2012, VS2013 and
|
|
132
|
-
- Automated build scripts for Visual compiler by @KrzysFR
|
|
147
|
+
+ VS2010 project is compatible with VS2012, VS2013, VS2015 and VS2017.
|
|
148
|
+
- Automated build scripts for Visual compiler by [@KrzysFR](https://github.com/KrzysFR), in `build/VS_scripts`,
|
|
133
149
|
which will build `zstd` cli and `libzstd` library without any need to open Visual Studio solution.
|
|
134
150
|
|
|
151
|
+
### Buck
|
|
152
|
+
|
|
153
|
+
You can build the zstd binary via buck by executing: `buck build programs:zstd` from the root of the repo.
|
|
154
|
+
The output binary will be in `buck-out/gen/programs/`.
|
|
135
155
|
|
|
136
|
-
|
|
156
|
+
## Status
|
|
137
157
|
|
|
138
158
|
Zstandard is currently deployed within Facebook. It is used continuously to compress large amounts of data in multiple formats and use cases.
|
|
139
159
|
Zstandard is considered safe for production environments.
|
|
140
160
|
|
|
141
|
-
|
|
161
|
+
## License
|
|
142
162
|
|
|
143
163
|
Zstandard is dual-licensed under [BSD](LICENSE) and [GPLv2](COPYING).
|
|
144
164
|
|
|
145
|
-
|
|
165
|
+
## Contributing
|
|
146
166
|
|
|
147
167
|
The "dev" branch is the one where all contributions are merged before reaching "master".
|
|
148
168
|
If you plan to propose a patch, please commit into the "dev" branch, or its own feature branch.
|
data/contrib/zstd/TESTING.md
CHANGED
|
@@ -41,4 +41,4 @@ They consist of the following tests:
|
|
|
41
41
|
- `pzstd` with asan and tsan, as well as in 32-bits mode
|
|
42
42
|
- Testing `zstd` with legacy mode off
|
|
43
43
|
- Testing `zbuff` (old streaming API)
|
|
44
|
-
- Entire test suite and make install on
|
|
44
|
+
- Entire test suite and make install on macOS
|
data/contrib/zstd/appveyor.yml
CHANGED
|
@@ -2,14 +2,15 @@
|
|
|
2
2
|
version: 1.0.{build}
|
|
3
3
|
branches:
|
|
4
4
|
only:
|
|
5
|
-
- dev
|
|
6
5
|
- master
|
|
6
|
+
- appveyorTest
|
|
7
|
+
- /visual*/
|
|
7
8
|
environment:
|
|
8
9
|
matrix:
|
|
9
10
|
- COMPILER: "gcc"
|
|
10
11
|
HOST: "mingw"
|
|
11
12
|
PLATFORM: "x64"
|
|
12
|
-
SCRIPT: "make allzstd MOREFLAGS=-static && make -C tests test-symbols fullbench-
|
|
13
|
+
SCRIPT: "make allzstd MOREFLAGS=-static && make -C tests test-symbols fullbench-lib"
|
|
13
14
|
ARTIFACT: "true"
|
|
14
15
|
BUILD: "true"
|
|
15
16
|
- COMPILER: "gcc"
|
|
@@ -80,12 +81,22 @@
|
|
|
80
81
|
SET "LDFLAGS=../../zlib/libz.a" &&
|
|
81
82
|
sh -c "%SCRIPT%" &&
|
|
82
83
|
( if [%COMPILER%]==[gcc] if [%ARTIFACT%]==[true]
|
|
84
|
+
ECHO Creating artifacts &&
|
|
85
|
+
ECHO %cd% &&
|
|
83
86
|
lib\dll\example\build_package.bat &&
|
|
84
87
|
make -C programs DEBUGFLAGS= clean zstd &&
|
|
85
88
|
cd programs\ && 7z a -tzip -mx9 zstd-win-binary-%PLATFORM%.zip zstd.exe &&
|
|
86
89
|
appveyor PushArtifact zstd-win-binary-%PLATFORM%.zip &&
|
|
87
90
|
cp zstd.exe ..\bin\zstd.exe &&
|
|
88
|
-
|
|
91
|
+
git clone --depth 1 --branch master https://github.com/facebook/zstd &&
|
|
92
|
+
cd zstd &&
|
|
93
|
+
git archive --format=tar master -o zstd-src.tar &&
|
|
94
|
+
..\zstd -19 zstd-src.tar &&
|
|
95
|
+
appveyor PushArtifact zstd-src.tar.zst &&
|
|
96
|
+
certUtil -hashfile zstd-src.tar.zst SHA256 > zstd-src.tar.zst.sha256.sig &&
|
|
97
|
+
appveyor PushArtifact zstd-src.tar.zst.sha256.sig &&
|
|
98
|
+
cd ..\..\bin\ &&
|
|
99
|
+
7z a -tzip -mx9 zstd-win-release-%PLATFORM%.zip * &&
|
|
89
100
|
appveyor PushArtifact zstd-win-release-%PLATFORM%.zip
|
|
90
101
|
)
|
|
91
102
|
)
|
|
@@ -172,15 +183,15 @@
|
|
|
172
183
|
- COMPILER: "gcc"
|
|
173
184
|
HOST: "mingw"
|
|
174
185
|
PLATFORM: "x64"
|
|
175
|
-
SCRIPT: "make allzstd"
|
|
186
|
+
SCRIPT: "CPPFLAGS=-DDEBUGLEVEL=2 CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2"
|
|
176
187
|
- COMPILER: "gcc"
|
|
177
188
|
HOST: "mingw"
|
|
178
189
|
PLATFORM: "x86"
|
|
179
|
-
SCRIPT: "make allzstd"
|
|
190
|
+
SCRIPT: "CFLAGS=-Werror make -j allzstd"
|
|
180
191
|
- COMPILER: "clang"
|
|
181
192
|
HOST: "mingw"
|
|
182
193
|
PLATFORM: "x64"
|
|
183
|
-
SCRIPT: "
|
|
194
|
+
SCRIPT: "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd"
|
|
184
195
|
|
|
185
196
|
- COMPILER: "visual"
|
|
186
197
|
HOST: "visual"
|
data/contrib/zstd/lib/BUCK
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
cxx_library(
|
|
2
2
|
name='zstd',
|
|
3
3
|
header_namespace='',
|
|
4
|
+
exported_headers=['zstd.h'],
|
|
4
5
|
visibility=['PUBLIC'],
|
|
5
6
|
deps=[
|
|
6
7
|
':common',
|
|
@@ -17,7 +18,7 @@ cxx_library(
|
|
|
17
18
|
exported_headers=subdir_glob([
|
|
18
19
|
('compress', 'zstd*.h'),
|
|
19
20
|
]),
|
|
20
|
-
srcs=glob(['compress/zstd*.c']),
|
|
21
|
+
srcs=glob(['compress/zstd*.c', 'compress/hist.c']),
|
|
21
22
|
deps=[':common'],
|
|
22
23
|
)
|
|
23
24
|
|
|
@@ -25,6 +26,9 @@ cxx_library(
|
|
|
25
26
|
name='decompress',
|
|
26
27
|
header_namespace='',
|
|
27
28
|
visibility=['PUBLIC'],
|
|
29
|
+
headers=subdir_glob([
|
|
30
|
+
('decompress', '*_impl.h'),
|
|
31
|
+
]),
|
|
28
32
|
srcs=glob(['decompress/zstd*.c']),
|
|
29
33
|
deps=[
|
|
30
34
|
':common',
|
|
@@ -37,7 +41,7 @@ cxx_library(
|
|
|
37
41
|
header_namespace='',
|
|
38
42
|
visibility=['PUBLIC'],
|
|
39
43
|
exported_headers=subdir_glob([
|
|
40
|
-
('
|
|
44
|
+
('deprecated', '*.h'),
|
|
41
45
|
]),
|
|
42
46
|
srcs=glob(['deprecated/*.c']),
|
|
43
47
|
deps=[':common'],
|
|
@@ -66,6 +70,7 @@ cxx_library(
|
|
|
66
70
|
]),
|
|
67
71
|
headers=subdir_glob([
|
|
68
72
|
('dictBuilder', 'divsufsort.h'),
|
|
73
|
+
('dictBuilder', 'cover.h'),
|
|
69
74
|
]),
|
|
70
75
|
srcs=glob(['dictBuilder/*.c']),
|
|
71
76
|
deps=[':common'],
|
|
@@ -80,6 +85,15 @@ cxx_library(
|
|
|
80
85
|
]),
|
|
81
86
|
)
|
|
82
87
|
|
|
88
|
+
cxx_library(
|
|
89
|
+
name='cpu',
|
|
90
|
+
header_namespace='',
|
|
91
|
+
visibility=['PUBLIC'],
|
|
92
|
+
exported_headers=subdir_glob([
|
|
93
|
+
('common', 'cpu.h'),
|
|
94
|
+
]),
|
|
95
|
+
)
|
|
96
|
+
|
|
83
97
|
cxx_library(
|
|
84
98
|
name='bitstream',
|
|
85
99
|
header_namespace='',
|
|
@@ -105,6 +119,7 @@ cxx_library(
|
|
|
105
119
|
'decompress/huf_decompress.c',
|
|
106
120
|
],
|
|
107
121
|
deps=[
|
|
122
|
+
':debug',
|
|
108
123
|
':bitstream',
|
|
109
124
|
':compiler',
|
|
110
125
|
':errors',
|
|
@@ -191,11 +206,23 @@ cxx_library(
|
|
|
191
206
|
],
|
|
192
207
|
)
|
|
193
208
|
|
|
209
|
+
cxx_library(
|
|
210
|
+
name='debug',
|
|
211
|
+
header_namespace='',
|
|
212
|
+
visibility=['PUBLIC'],
|
|
213
|
+
exported_headers=subdir_glob([
|
|
214
|
+
('common', 'debug.h'),
|
|
215
|
+
]),
|
|
216
|
+
srcs=['common/debug.c'],
|
|
217
|
+
)
|
|
218
|
+
|
|
194
219
|
cxx_library(
|
|
195
220
|
name='common',
|
|
196
221
|
deps=[
|
|
222
|
+
':debug',
|
|
197
223
|
':bitstream',
|
|
198
224
|
':compiler',
|
|
225
|
+
':cpu',
|
|
199
226
|
':entropy',
|
|
200
227
|
':errors',
|
|
201
228
|
':mem',
|
data/contrib/zstd/lib/Makefile
CHANGED
|
@@ -19,23 +19,102 @@ LIBVER := $(shell echo $(LIBVER_SCRIPT))
|
|
|
19
19
|
VERSION?= $(LIBVER)
|
|
20
20
|
|
|
21
21
|
CPPFLAGS+= -I. -I./common -DXXH_NAMESPACE=ZSTD_
|
|
22
|
+
ifeq ($(OS),Windows_NT) # MinGW assumed
|
|
23
|
+
CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting
|
|
24
|
+
endif
|
|
22
25
|
CFLAGS ?= -O3
|
|
23
|
-
DEBUGFLAGS
|
|
26
|
+
DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
|
|
24
27
|
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
|
|
25
|
-
-Wstrict-prototypes -Wundef -Wpointer-arith
|
|
28
|
+
-Wstrict-prototypes -Wundef -Wpointer-arith \
|
|
26
29
|
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
|
|
27
|
-
-Wredundant-decls
|
|
30
|
+
-Wredundant-decls -Wmissing-prototypes -Wc++-compat
|
|
28
31
|
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
|
|
29
32
|
FLAGS = $(CPPFLAGS) $(CFLAGS)
|
|
30
33
|
|
|
34
|
+
HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
|
|
35
|
+
GREP_OPTIONS ?=
|
|
36
|
+
ifeq ($HAVE_COLORNEVER, 1)
|
|
37
|
+
GREP_OPTIONS += --color=never
|
|
38
|
+
endif
|
|
39
|
+
GREP = grep $(GREP_OPTIONS)
|
|
40
|
+
|
|
41
|
+
ZSTDCOMMON_FILES := $(sort $(wildcard common/*.c))
|
|
42
|
+
ZSTDCOMP_FILES := $(sort $(wildcard compress/*.c))
|
|
43
|
+
ZSTDDECOMP_FILES := $(sort $(wildcard decompress/*.c))
|
|
44
|
+
ZDICT_FILES := $(sort $(wildcard dictBuilder/*.c))
|
|
45
|
+
ZDEPR_FILES := $(sort $(wildcard deprecated/*.c))
|
|
46
|
+
ZSTD_FILES := $(ZSTDCOMMON_FILES)
|
|
47
|
+
|
|
48
|
+
ZSTD_LEGACY_SUPPORT ?= 5
|
|
49
|
+
ZSTD_LIB_COMPRESSION ?= 1
|
|
50
|
+
ZSTD_LIB_DECOMPRESSION ?= 1
|
|
51
|
+
ZSTD_LIB_DICTBUILDER ?= 1
|
|
52
|
+
ZSTD_LIB_DEPRECATED ?= 1
|
|
53
|
+
HUF_FORCE_DECOMPRESS_X1 ?= 0
|
|
54
|
+
HUF_FORCE_DECOMPRESS_X2 ?= 0
|
|
55
|
+
ZSTD_FORCE_DECOMPRESS_SHORT ?= 0
|
|
56
|
+
ZSTD_FORCE_DECOMPRESS_LONG ?= 0
|
|
57
|
+
ZSTD_NO_INLINE ?= 0
|
|
58
|
+
ZSTD_STRIP_ERROR_STRINGS ?= 0
|
|
59
|
+
ZSTD_LEGACY_MULTITHREADED_API ?= 0
|
|
60
|
+
|
|
61
|
+
ifeq ($(ZSTD_LIB_COMPRESSION), 0)
|
|
62
|
+
ZSTD_LIB_DICTBUILDER = 0
|
|
63
|
+
ZSTD_LIB_DEPRECATED = 0
|
|
64
|
+
endif
|
|
65
|
+
|
|
66
|
+
ifeq ($(ZSTD_LIB_DECOMPRESSION), 0)
|
|
67
|
+
ZSTD_LEGACY_SUPPORT = 0
|
|
68
|
+
ZSTD_LIB_DEPRECATED = 0
|
|
69
|
+
endif
|
|
70
|
+
|
|
71
|
+
ifneq ($(ZSTD_LIB_COMPRESSION), 0)
|
|
72
|
+
ZSTD_FILES += $(ZSTDCOMP_FILES)
|
|
73
|
+
endif
|
|
74
|
+
|
|
75
|
+
ifneq ($(ZSTD_LIB_DECOMPRESSION), 0)
|
|
76
|
+
ZSTD_FILES += $(ZSTDDECOMP_FILES)
|
|
77
|
+
endif
|
|
31
78
|
|
|
32
|
-
|
|
79
|
+
ifneq ($(ZSTD_LIB_DEPRECATED), 0)
|
|
80
|
+
ZSTD_FILES += $(ZDEPR_FILES)
|
|
81
|
+
endif
|
|
82
|
+
|
|
83
|
+
ifneq ($(ZSTD_LIB_DICTBUILDER), 0)
|
|
84
|
+
ZSTD_FILES += $(ZDICT_FILES)
|
|
85
|
+
endif
|
|
33
86
|
|
|
34
|
-
|
|
87
|
+
ifneq ($(HUF_FORCE_DECOMPRESS_X1), 0)
|
|
88
|
+
CFLAGS += -DHUF_FORCE_DECOMPRESS_X1
|
|
89
|
+
endif
|
|
90
|
+
|
|
91
|
+
ifneq ($(HUF_FORCE_DECOMPRESS_X2), 0)
|
|
92
|
+
CFLAGS += -DHUF_FORCE_DECOMPRESS_X2
|
|
93
|
+
endif
|
|
94
|
+
|
|
95
|
+
ifneq ($(ZSTD_FORCE_DECOMPRESS_SHORT), 0)
|
|
96
|
+
CFLAGS += -DZSTD_FORCE_DECOMPRESS_SHORT
|
|
97
|
+
endif
|
|
98
|
+
|
|
99
|
+
ifneq ($(ZSTD_FORCE_DECOMPRESS_LONG), 0)
|
|
100
|
+
CFLAGS += -DZSTD_FORCE_DECOMPRESS_LONG
|
|
101
|
+
endif
|
|
102
|
+
|
|
103
|
+
ifneq ($(ZSTD_NO_INLINE), 0)
|
|
104
|
+
CFLAGS += -DZSTD_NO_INLINE
|
|
105
|
+
endif
|
|
106
|
+
|
|
107
|
+
ifneq ($(ZSTD_STRIP_ERROR_STRINGS), 0)
|
|
108
|
+
CFLAGS += -DZSTD_STRIP_ERROR_STRINGS
|
|
109
|
+
endif
|
|
110
|
+
|
|
111
|
+
ifneq ($(ZSTD_LEGACY_MULTITHREADED_API), 0)
|
|
112
|
+
CFLAGS += -DZSTD_LEGACY_MULTITHREADED_API
|
|
113
|
+
endif
|
|
35
114
|
|
|
36
115
|
ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
|
|
37
116
|
ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
|
|
38
|
-
ZSTD_FILES += $(shell ls legacy/*.c |
|
|
117
|
+
ZSTD_FILES += $(shell ls legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
|
|
39
118
|
endif
|
|
40
119
|
CPPFLAGS += -I./legacy
|
|
41
120
|
endif
|
|
@@ -43,7 +122,7 @@ CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
|
|
|
43
122
|
|
|
44
123
|
ZSTD_OBJ := $(patsubst %.c,%.o,$(ZSTD_FILES))
|
|
45
124
|
|
|
46
|
-
#
|
|
125
|
+
# macOS linker doesn't support -soname, and use different extension
|
|
47
126
|
# see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html
|
|
48
127
|
ifeq ($(shell uname), Darwin)
|
|
49
128
|
SHARED_EXT = dylib
|
|
@@ -57,8 +136,6 @@ else
|
|
|
57
136
|
SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
|
|
58
137
|
endif
|
|
59
138
|
|
|
60
|
-
LIBZSTD = libzstd.$(SHARED_EXT_VER)
|
|
61
|
-
|
|
62
139
|
|
|
63
140
|
.PHONY: default all clean install uninstall
|
|
64
141
|
|
|
@@ -74,19 +151,27 @@ libzstd.a: $(ZSTD_OBJ)
|
|
|
74
151
|
libzstd.a-mt: CPPFLAGS += -DZSTD_MULTITHREAD
|
|
75
152
|
libzstd.a-mt: libzstd.a
|
|
76
153
|
|
|
77
|
-
|
|
154
|
+
ifneq (,$(filter Windows%,$(OS)))
|
|
155
|
+
|
|
156
|
+
LIBZSTD = dll\libzstd.dll
|
|
78
157
|
$(LIBZSTD): $(ZSTD_FILES)
|
|
79
158
|
@echo compiling dynamic library $(LIBVER)
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
dlltool -D dll\libzstd.dll -d dll\libzstd.def -l dll\libzstd.lib
|
|
159
|
+
$(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll\libzstd.lib -shared $^ -o $@
|
|
160
|
+
|
|
83
161
|
else
|
|
162
|
+
|
|
163
|
+
LIBZSTD = libzstd.$(SHARED_EXT_VER)
|
|
164
|
+
$(LIBZSTD): LDFLAGS += -shared -fPIC -fvisibility=hidden
|
|
165
|
+
$(LIBZSTD): $(ZSTD_FILES)
|
|
166
|
+
@echo compiling dynamic library $(LIBVER)
|
|
84
167
|
@$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
|
|
85
168
|
@echo creating versioned links
|
|
86
169
|
@ln -sf $@ libzstd.$(SHARED_EXT_MAJOR)
|
|
87
170
|
@ln -sf $@ libzstd.$(SHARED_EXT)
|
|
171
|
+
|
|
88
172
|
endif
|
|
89
173
|
|
|
174
|
+
|
|
90
175
|
libzstd : $(LIBZSTD)
|
|
91
176
|
|
|
92
177
|
libzstd-mt : CPPFLAGS += -DZSTD_MULTITHREAD
|
|
@@ -111,16 +196,16 @@ libzstd-nomt: $(ZSTD_NOMT_FILES)
|
|
|
111
196
|
@$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
|
|
112
197
|
|
|
113
198
|
clean:
|
|
114
|
-
@$(RM) -r *.dSYM #
|
|
199
|
+
@$(RM) -r *.dSYM # macOS-specific
|
|
115
200
|
@$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc
|
|
116
201
|
@$(RM) dll/libzstd.dll dll/libzstd.lib libzstd-nomt*
|
|
117
202
|
@$(RM) common/*.o compress/*.o decompress/*.o dictBuilder/*.o legacy/*.o deprecated/*.o
|
|
118
203
|
@echo Cleaning library completed
|
|
119
204
|
|
|
120
205
|
#-----------------------------------------------------------------------------
|
|
121
|
-
# make install is validated only for Linux,
|
|
206
|
+
# make install is validated only for Linux, macOS, BSD, Hurd and Solaris targets
|
|
122
207
|
#-----------------------------------------------------------------------------
|
|
123
|
-
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
|
|
208
|
+
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku))
|
|
124
209
|
|
|
125
210
|
DESTDIR ?=
|
|
126
211
|
# directory variables : GNU conventions prefer lowercase
|
|
@@ -134,7 +219,7 @@ LIBDIR ?= $(libdir)
|
|
|
134
219
|
includedir ?= $(PREFIX)/include
|
|
135
220
|
INCLUDEDIR ?= $(includedir)
|
|
136
221
|
|
|
137
|
-
ifneq (,$(filter $(shell uname),
|
|
222
|
+
ifneq (,$(filter $(shell uname),FreeBSD NetBSD DragonFly))
|
|
138
223
|
PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig
|
|
139
224
|
else
|
|
140
225
|
PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig
|
|
@@ -159,20 +244,32 @@ libzstd.pc: libzstd.pc.in
|
|
|
159
244
|
-e 's|@VERSION@|$(VERSION)|' \
|
|
160
245
|
$< >$@
|
|
161
246
|
|
|
162
|
-
install:
|
|
163
|
-
|
|
247
|
+
install: install-pc install-static install-shared install-includes
|
|
248
|
+
@echo zstd static and shared library installed
|
|
249
|
+
|
|
250
|
+
install-pc: libzstd.pc
|
|
251
|
+
@$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/
|
|
164
252
|
@$(INSTALL_DATA) libzstd.pc $(DESTDIR)$(PKGCONFIGDIR)/
|
|
165
|
-
|
|
253
|
+
|
|
254
|
+
install-static: libzstd.a
|
|
255
|
+
@echo Installing static library
|
|
256
|
+
@$(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)/
|
|
166
257
|
@$(INSTALL_DATA) libzstd.a $(DESTDIR)$(LIBDIR)
|
|
258
|
+
|
|
259
|
+
install-shared: libzstd
|
|
260
|
+
@echo Installing shared library
|
|
261
|
+
@$(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)/
|
|
167
262
|
@$(INSTALL_PROGRAM) $(LIBZSTD) $(DESTDIR)$(LIBDIR)
|
|
168
263
|
@ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
|
|
169
264
|
@ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
|
|
265
|
+
|
|
266
|
+
install-includes:
|
|
170
267
|
@echo Installing includes
|
|
268
|
+
@$(INSTALL) -d -m 755 $(DESTDIR)$(INCLUDEDIR)/
|
|
171
269
|
@$(INSTALL_DATA) zstd.h $(DESTDIR)$(INCLUDEDIR)
|
|
172
270
|
@$(INSTALL_DATA) common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)
|
|
173
271
|
@$(INSTALL_DATA) deprecated/zbuff.h $(DESTDIR)$(INCLUDEDIR) # prototypes generate deprecation warnings
|
|
174
272
|
@$(INSTALL_DATA) dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)
|
|
175
|
-
@echo zstd static and shared library installed
|
|
176
273
|
|
|
177
274
|
uninstall:
|
|
178
275
|
@$(RM) $(DESTDIR)$(LIBDIR)/libzstd.a
|