zstdlib 0.1.0-x64-mingw32
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/CHANGES.md +9 -0
- data/Gemfile +3 -0
- data/README.md +96 -0
- data/Rakefile +29 -0
- data/ext/zstdlib/extconf.rb +53 -0
- data/ext/zstdlib/ruby/zlib-2.2/zlib.c +4659 -0
- data/ext/zstdlib/ruby/zlib-2.3/zlib.c +4686 -0
- data/ext/zstdlib/ruby/zlib-2.4/zlib.c +4843 -0
- data/ext/zstdlib/ruby/zlib-2.5/zlib.c +4848 -0
- data/ext/zstdlib/ruby/zlib-2.6/zlib.c +4890 -0
- data/ext/zstdlib/zlib-1.2.11/adler32.c +186 -0
- data/ext/zstdlib/zlib-1.2.11/compress.c +86 -0
- data/ext/zstdlib/zlib-1.2.11/crc32.c +442 -0
- data/ext/zstdlib/zlib-1.2.11/crc32.h +441 -0
- data/ext/zstdlib/zlib-1.2.11/deflate.c +2163 -0
- data/ext/zstdlib/zlib-1.2.11/deflate.h +349 -0
- data/ext/zstdlib/zlib-1.2.11/gzclose.c +25 -0
- data/ext/zstdlib/zlib-1.2.11/gzguts.h +218 -0
- data/ext/zstdlib/zlib-1.2.11/gzlib.c +637 -0
- data/ext/zstdlib/zlib-1.2.11/gzread.c +654 -0
- data/ext/zstdlib/zlib-1.2.11/gzwrite.c +665 -0
- data/ext/zstdlib/zlib-1.2.11/infback.c +640 -0
- data/ext/zstdlib/zlib-1.2.11/inffast.c +323 -0
- data/ext/zstdlib/zlib-1.2.11/inffast.h +11 -0
- data/ext/zstdlib/zlib-1.2.11/inffixed.h +94 -0
- data/ext/zstdlib/zlib-1.2.11/inflate.c +1561 -0
- data/ext/zstdlib/zlib-1.2.11/inflate.h +125 -0
- data/ext/zstdlib/zlib-1.2.11/inftrees.c +304 -0
- data/ext/zstdlib/zlib-1.2.11/inftrees.h +62 -0
- data/ext/zstdlib/zlib-1.2.11/trees.c +1203 -0
- data/ext/zstdlib/zlib-1.2.11/trees.h +128 -0
- data/ext/zstdlib/zlib-1.2.11/uncompr.c +93 -0
- data/ext/zstdlib/zlib-1.2.11/zconf.h +534 -0
- data/ext/zstdlib/zlib-1.2.11/zlib.h +1912 -0
- data/ext/zstdlib/zlib-1.2.11/zutil.c +325 -0
- data/ext/zstdlib/zlib-1.2.11/zutil.h +271 -0
- data/ext/zstdlib/zlib.mk +14 -0
- data/ext/zstdlib/zlibwrapper.mk +14 -0
- data/ext/zstdlib/zlibwrapper/zlibwrapper.c +14 -0
- data/ext/zstdlib/zstd-1.3.8/lib/common/bitstream.h +455 -0
- data/ext/zstdlib/zstd-1.3.8/lib/common/compiler.h +140 -0
- data/ext/zstdlib/zstd-1.3.8/lib/common/cpu.h +215 -0
- data/ext/zstdlib/zstd-1.3.8/lib/common/debug.c +44 -0
- data/ext/zstdlib/zstd-1.3.8/lib/common/debug.h +134 -0
- data/ext/zstdlib/zstd-1.3.8/lib/common/entropy_common.c +236 -0
- data/ext/zstdlib/zstd-1.3.8/lib/common/error_private.c +54 -0
- data/ext/zstdlib/zstd-1.3.8/lib/common/error_private.h +76 -0
- data/ext/zstdlib/zstd-1.3.8/lib/common/fse.h +708 -0
- data/ext/zstdlib/zstd-1.3.8/lib/common/fse_decompress.c +309 -0
- data/ext/zstdlib/zstd-1.3.8/lib/common/huf.h +358 -0
- data/ext/zstdlib/zstd-1.3.8/lib/common/mem.h +380 -0
- data/ext/zstdlib/zstd-1.3.8/lib/common/pool.c +340 -0
- data/ext/zstdlib/zstd-1.3.8/lib/common/pool.h +84 -0
- data/ext/zstdlib/zstd-1.3.8/lib/common/threading.c +75 -0
- data/ext/zstdlib/zstd-1.3.8/lib/common/threading.h +123 -0
- data/ext/zstdlib/zstd-1.3.8/lib/common/xxhash.c +876 -0
- data/ext/zstdlib/zstd-1.3.8/lib/common/xxhash.h +305 -0
- data/ext/zstdlib/zstd-1.3.8/lib/common/zstd_common.c +83 -0
- data/ext/zstdlib/zstd-1.3.8/lib/common/zstd_errors.h +93 -0
- data/ext/zstdlib/zstd-1.3.8/lib/common/zstd_internal.h +266 -0
- data/ext/zstdlib/zstd-1.3.8/lib/compress/fse_compress.c +721 -0
- data/ext/zstdlib/zstd-1.3.8/lib/compress/hist.c +203 -0
- data/ext/zstdlib/zstd-1.3.8/lib/compress/hist.h +95 -0
- data/ext/zstdlib/zstd-1.3.8/lib/compress/huf_compress.c +798 -0
- data/ext/zstdlib/zstd-1.3.8/lib/compress/zstd_compress.c +4290 -0
- data/ext/zstdlib/zstd-1.3.8/lib/compress/zstd_compress_internal.h +860 -0
- data/ext/zstdlib/zstd-1.3.8/lib/compress/zstd_double_fast.c +499 -0
- data/ext/zstdlib/zstd-1.3.8/lib/compress/zstd_double_fast.h +38 -0
- data/ext/zstdlib/zstd-1.3.8/lib/compress/zstd_fast.c +391 -0
- data/ext/zstdlib/zstd-1.3.8/lib/compress/zstd_fast.h +37 -0
- data/ext/zstdlib/zstd-1.3.8/lib/compress/zstd_lazy.c +1106 -0
- data/ext/zstdlib/zstd-1.3.8/lib/compress/zstd_lazy.h +67 -0
- data/ext/zstdlib/zstd-1.3.8/lib/compress/zstd_ldm.c +597 -0
- data/ext/zstdlib/zstd-1.3.8/lib/compress/zstd_ldm.h +105 -0
- data/ext/zstdlib/zstd-1.3.8/lib/compress/zstd_opt.c +1217 -0
- data/ext/zstdlib/zstd-1.3.8/lib/compress/zstd_opt.h +56 -0
- data/ext/zstdlib/zstd-1.3.8/lib/compress/zstdmt_compress.c +2107 -0
- data/ext/zstdlib/zstd-1.3.8/lib/compress/zstdmt_compress.h +174 -0
- data/ext/zstdlib/zstd-1.3.8/lib/decompress/huf_decompress.c +1232 -0
- data/ext/zstdlib/zstd-1.3.8/lib/decompress/zstd_ddict.c +240 -0
- data/ext/zstdlib/zstd-1.3.8/lib/decompress/zstd_ddict.h +44 -0
- data/ext/zstdlib/zstd-1.3.8/lib/decompress/zstd_decompress.c +1672 -0
- data/ext/zstdlib/zstd-1.3.8/lib/decompress/zstd_decompress_block.c +1307 -0
- data/ext/zstdlib/zstd-1.3.8/lib/decompress/zstd_decompress_block.h +59 -0
- data/ext/zstdlib/zstd-1.3.8/lib/decompress/zstd_decompress_internal.h +168 -0
- data/ext/zstdlib/zstd-1.3.8/lib/zstd.h +1766 -0
- data/ext/zstdlib/zstd-1.3.8/zlibWrapper/gzclose.c +28 -0
- data/ext/zstdlib/zstd-1.3.8/zlibWrapper/gzcompatibility.h +68 -0
- data/ext/zstdlib/zstd-1.3.8/zlibWrapper/gzguts.h +227 -0
- data/ext/zstdlib/zstd-1.3.8/zlibWrapper/gzlib.c +640 -0
- data/ext/zstdlib/zstd-1.3.8/zlibWrapper/gzread.c +670 -0
- data/ext/zstdlib/zstd-1.3.8/zlibWrapper/gzwrite.c +671 -0
- data/ext/zstdlib/zstd-1.3.8/zlibWrapper/zstd_zlibwrapper.c +1105 -0
- data/ext/zstdlib/zstd-1.3.8/zlibWrapper/zstd_zlibwrapper.h +88 -0
- data/ext/zstdlib/zstd.mk +14 -0
- data/lib/2.2/zstdlib.so +0 -0
- data/lib/2.3/zstdlib.so +0 -0
- data/lib/2.4/zstdlib.so +0 -0
- data/lib/2.5/zstdlib.so +0 -0
- data/lib/2.6/zstdlib.so +0 -0
- data/lib/zstdlib.rb +6 -0
- data/test/zstdlib_test.rb +21 -0
- metadata +205 -0
@@ -0,0 +1,380 @@
|
|
1
|
+
/*
|
2
|
+
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
3
|
+
* All rights reserved.
|
4
|
+
*
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
9
|
+
*/
|
10
|
+
|
11
|
+
#ifndef MEM_H_MODULE
|
12
|
+
#define MEM_H_MODULE
|
13
|
+
|
14
|
+
#if defined (__cplusplus)
|
15
|
+
extern "C" {
|
16
|
+
#endif
|
17
|
+
|
18
|
+
/*-****************************************
|
19
|
+
* Dependencies
|
20
|
+
******************************************/
|
21
|
+
#include <stddef.h> /* size_t, ptrdiff_t */
|
22
|
+
#include <string.h> /* memcpy */
|
23
|
+
|
24
|
+
|
25
|
+
/*-****************************************
|
26
|
+
* Compiler specifics
|
27
|
+
******************************************/
|
28
|
+
#if defined(_MSC_VER) /* Visual Studio */
|
29
|
+
# include <stdlib.h> /* _byteswap_ulong */
|
30
|
+
# include <intrin.h> /* _byteswap_* */
|
31
|
+
#endif
|
32
|
+
#if defined(__GNUC__)
|
33
|
+
# define MEM_STATIC static __inline __attribute__((unused))
|
34
|
+
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
35
|
+
# define MEM_STATIC static inline
|
36
|
+
#elif defined(_MSC_VER)
|
37
|
+
# define MEM_STATIC static __inline
|
38
|
+
#else
|
39
|
+
# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
|
40
|
+
#endif
|
41
|
+
|
42
|
+
#ifndef __has_builtin
|
43
|
+
# define __has_builtin(x) 0 /* compat. with non-clang compilers */
|
44
|
+
#endif
|
45
|
+
|
46
|
+
/* code only tested on 32 and 64 bits systems */
|
47
|
+
#define MEM_STATIC_ASSERT(c) { enum { MEM_static_assert = 1/(int)(!!(c)) }; }
|
48
|
+
MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
|
49
|
+
|
50
|
+
|
51
|
+
/*-**************************************************************
|
52
|
+
* Basic Types
|
53
|
+
*****************************************************************/
|
54
|
+
#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
|
55
|
+
# include <stdint.h>
|
56
|
+
typedef uint8_t BYTE;
|
57
|
+
typedef uint16_t U16;
|
58
|
+
typedef int16_t S16;
|
59
|
+
typedef uint32_t U32;
|
60
|
+
typedef int32_t S32;
|
61
|
+
typedef uint64_t U64;
|
62
|
+
typedef int64_t S64;
|
63
|
+
#else
|
64
|
+
# include <limits.h>
|
65
|
+
#if CHAR_BIT != 8
|
66
|
+
# error "this implementation requires char to be exactly 8-bit type"
|
67
|
+
#endif
|
68
|
+
typedef unsigned char BYTE;
|
69
|
+
#if USHRT_MAX != 65535
|
70
|
+
# error "this implementation requires short to be exactly 16-bit type"
|
71
|
+
#endif
|
72
|
+
typedef unsigned short U16;
|
73
|
+
typedef signed short S16;
|
74
|
+
#if UINT_MAX != 4294967295
|
75
|
+
# error "this implementation requires int to be exactly 32-bit type"
|
76
|
+
#endif
|
77
|
+
typedef unsigned int U32;
|
78
|
+
typedef signed int S32;
|
79
|
+
/* note : there are no limits defined for long long type in C90.
|
80
|
+
* limits exist in C99, however, in such case, <stdint.h> is preferred */
|
81
|
+
typedef unsigned long long U64;
|
82
|
+
typedef signed long long S64;
|
83
|
+
#endif
|
84
|
+
|
85
|
+
|
86
|
+
/*-**************************************************************
|
87
|
+
* Memory I/O
|
88
|
+
*****************************************************************/
|
89
|
+
/* MEM_FORCE_MEMORY_ACCESS :
|
90
|
+
* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
|
91
|
+
* Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
|
92
|
+
* The below switch allow to select different access method for improved performance.
|
93
|
+
* Method 0 (default) : use `memcpy()`. Safe and portable.
|
94
|
+
* Method 1 : `__packed` statement. It depends on compiler extension (i.e., not portable).
|
95
|
+
* This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
|
96
|
+
* Method 2 : direct access. This method is portable but violate C standard.
|
97
|
+
* It can generate buggy code on targets depending on alignment.
|
98
|
+
* In some circumstances, it's the only known way to get the most performance (i.e. GCC + ARMv6)
|
99
|
+
* See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
|
100
|
+
* Prefer these methods in priority order (0 > 1 > 2)
|
101
|
+
*/
|
102
|
+
#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
|
103
|
+
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
|
104
|
+
# define MEM_FORCE_MEMORY_ACCESS 2
|
105
|
+
# elif defined(__INTEL_COMPILER) || defined(__GNUC__)
|
106
|
+
# define MEM_FORCE_MEMORY_ACCESS 1
|
107
|
+
# endif
|
108
|
+
#endif
|
109
|
+
|
110
|
+
MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; }
|
111
|
+
MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; }
|
112
|
+
|
113
|
+
MEM_STATIC unsigned MEM_isLittleEndian(void)
|
114
|
+
{
|
115
|
+
const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */
|
116
|
+
return one.c[0];
|
117
|
+
}
|
118
|
+
|
119
|
+
#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
|
120
|
+
|
121
|
+
/* violates C standard, by lying on structure alignment.
|
122
|
+
Only use if no other choice to achieve best performance on target platform */
|
123
|
+
MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
|
124
|
+
MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
|
125
|
+
MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
|
126
|
+
MEM_STATIC size_t MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; }
|
127
|
+
|
128
|
+
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
|
129
|
+
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
|
130
|
+
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
|
131
|
+
|
132
|
+
#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
|
133
|
+
|
134
|
+
/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
|
135
|
+
/* currently only defined for gcc and icc */
|
136
|
+
#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32))
|
137
|
+
__pragma( pack(push, 1) )
|
138
|
+
typedef struct { U16 v; } unalign16;
|
139
|
+
typedef struct { U32 v; } unalign32;
|
140
|
+
typedef struct { U64 v; } unalign64;
|
141
|
+
typedef struct { size_t v; } unalignArch;
|
142
|
+
__pragma( pack(pop) )
|
143
|
+
#else
|
144
|
+
typedef struct { U16 v; } __attribute__((packed)) unalign16;
|
145
|
+
typedef struct { U32 v; } __attribute__((packed)) unalign32;
|
146
|
+
typedef struct { U64 v; } __attribute__((packed)) unalign64;
|
147
|
+
typedef struct { size_t v; } __attribute__((packed)) unalignArch;
|
148
|
+
#endif
|
149
|
+
|
150
|
+
MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign16*)ptr)->v; }
|
151
|
+
MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign32*)ptr)->v; }
|
152
|
+
MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign64*)ptr)->v; }
|
153
|
+
MEM_STATIC size_t MEM_readST(const void* ptr) { return ((const unalignArch*)ptr)->v; }
|
154
|
+
|
155
|
+
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign16*)memPtr)->v = value; }
|
156
|
+
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign32*)memPtr)->v = value; }
|
157
|
+
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign64*)memPtr)->v = value; }
|
158
|
+
|
159
|
+
#else
|
160
|
+
|
161
|
+
/* default method, safe and standard.
|
162
|
+
can sometimes prove slower */
|
163
|
+
|
164
|
+
MEM_STATIC U16 MEM_read16(const void* memPtr)
|
165
|
+
{
|
166
|
+
U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
167
|
+
}
|
168
|
+
|
169
|
+
MEM_STATIC U32 MEM_read32(const void* memPtr)
|
170
|
+
{
|
171
|
+
U32 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
172
|
+
}
|
173
|
+
|
174
|
+
MEM_STATIC U64 MEM_read64(const void* memPtr)
|
175
|
+
{
|
176
|
+
U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
177
|
+
}
|
178
|
+
|
179
|
+
MEM_STATIC size_t MEM_readST(const void* memPtr)
|
180
|
+
{
|
181
|
+
size_t val; memcpy(&val, memPtr, sizeof(val)); return val;
|
182
|
+
}
|
183
|
+
|
184
|
+
MEM_STATIC void MEM_write16(void* memPtr, U16 value)
|
185
|
+
{
|
186
|
+
memcpy(memPtr, &value, sizeof(value));
|
187
|
+
}
|
188
|
+
|
189
|
+
MEM_STATIC void MEM_write32(void* memPtr, U32 value)
|
190
|
+
{
|
191
|
+
memcpy(memPtr, &value, sizeof(value));
|
192
|
+
}
|
193
|
+
|
194
|
+
MEM_STATIC void MEM_write64(void* memPtr, U64 value)
|
195
|
+
{
|
196
|
+
memcpy(memPtr, &value, sizeof(value));
|
197
|
+
}
|
198
|
+
|
199
|
+
#endif /* MEM_FORCE_MEMORY_ACCESS */
|
200
|
+
|
201
|
+
MEM_STATIC U32 MEM_swap32(U32 in)
|
202
|
+
{
|
203
|
+
#if defined(_MSC_VER) /* Visual Studio */
|
204
|
+
return _byteswap_ulong(in);
|
205
|
+
#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \
|
206
|
+
|| (defined(__clang__) && __has_builtin(__builtin_bswap32))
|
207
|
+
return __builtin_bswap32(in);
|
208
|
+
#else
|
209
|
+
return ((in << 24) & 0xff000000 ) |
|
210
|
+
((in << 8) & 0x00ff0000 ) |
|
211
|
+
((in >> 8) & 0x0000ff00 ) |
|
212
|
+
((in >> 24) & 0x000000ff );
|
213
|
+
#endif
|
214
|
+
}
|
215
|
+
|
216
|
+
MEM_STATIC U64 MEM_swap64(U64 in)
|
217
|
+
{
|
218
|
+
#if defined(_MSC_VER) /* Visual Studio */
|
219
|
+
return _byteswap_uint64(in);
|
220
|
+
#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \
|
221
|
+
|| (defined(__clang__) && __has_builtin(__builtin_bswap64))
|
222
|
+
return __builtin_bswap64(in);
|
223
|
+
#else
|
224
|
+
return ((in << 56) & 0xff00000000000000ULL) |
|
225
|
+
((in << 40) & 0x00ff000000000000ULL) |
|
226
|
+
((in << 24) & 0x0000ff0000000000ULL) |
|
227
|
+
((in << 8) & 0x000000ff00000000ULL) |
|
228
|
+
((in >> 8) & 0x00000000ff000000ULL) |
|
229
|
+
((in >> 24) & 0x0000000000ff0000ULL) |
|
230
|
+
((in >> 40) & 0x000000000000ff00ULL) |
|
231
|
+
((in >> 56) & 0x00000000000000ffULL);
|
232
|
+
#endif
|
233
|
+
}
|
234
|
+
|
235
|
+
MEM_STATIC size_t MEM_swapST(size_t in)
|
236
|
+
{
|
237
|
+
if (MEM_32bits())
|
238
|
+
return (size_t)MEM_swap32((U32)in);
|
239
|
+
else
|
240
|
+
return (size_t)MEM_swap64((U64)in);
|
241
|
+
}
|
242
|
+
|
243
|
+
/*=== Little endian r/w ===*/
|
244
|
+
|
245
|
+
MEM_STATIC U16 MEM_readLE16(const void* memPtr)
|
246
|
+
{
|
247
|
+
if (MEM_isLittleEndian())
|
248
|
+
return MEM_read16(memPtr);
|
249
|
+
else {
|
250
|
+
const BYTE* p = (const BYTE*)memPtr;
|
251
|
+
return (U16)(p[0] + (p[1]<<8));
|
252
|
+
}
|
253
|
+
}
|
254
|
+
|
255
|
+
MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
|
256
|
+
{
|
257
|
+
if (MEM_isLittleEndian()) {
|
258
|
+
MEM_write16(memPtr, val);
|
259
|
+
} else {
|
260
|
+
BYTE* p = (BYTE*)memPtr;
|
261
|
+
p[0] = (BYTE)val;
|
262
|
+
p[1] = (BYTE)(val>>8);
|
263
|
+
}
|
264
|
+
}
|
265
|
+
|
266
|
+
MEM_STATIC U32 MEM_readLE24(const void* memPtr)
|
267
|
+
{
|
268
|
+
return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
|
269
|
+
}
|
270
|
+
|
271
|
+
MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val)
|
272
|
+
{
|
273
|
+
MEM_writeLE16(memPtr, (U16)val);
|
274
|
+
((BYTE*)memPtr)[2] = (BYTE)(val>>16);
|
275
|
+
}
|
276
|
+
|
277
|
+
MEM_STATIC U32 MEM_readLE32(const void* memPtr)
|
278
|
+
{
|
279
|
+
if (MEM_isLittleEndian())
|
280
|
+
return MEM_read32(memPtr);
|
281
|
+
else
|
282
|
+
return MEM_swap32(MEM_read32(memPtr));
|
283
|
+
}
|
284
|
+
|
285
|
+
MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
|
286
|
+
{
|
287
|
+
if (MEM_isLittleEndian())
|
288
|
+
MEM_write32(memPtr, val32);
|
289
|
+
else
|
290
|
+
MEM_write32(memPtr, MEM_swap32(val32));
|
291
|
+
}
|
292
|
+
|
293
|
+
MEM_STATIC U64 MEM_readLE64(const void* memPtr)
|
294
|
+
{
|
295
|
+
if (MEM_isLittleEndian())
|
296
|
+
return MEM_read64(memPtr);
|
297
|
+
else
|
298
|
+
return MEM_swap64(MEM_read64(memPtr));
|
299
|
+
}
|
300
|
+
|
301
|
+
MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
|
302
|
+
{
|
303
|
+
if (MEM_isLittleEndian())
|
304
|
+
MEM_write64(memPtr, val64);
|
305
|
+
else
|
306
|
+
MEM_write64(memPtr, MEM_swap64(val64));
|
307
|
+
}
|
308
|
+
|
309
|
+
MEM_STATIC size_t MEM_readLEST(const void* memPtr)
|
310
|
+
{
|
311
|
+
if (MEM_32bits())
|
312
|
+
return (size_t)MEM_readLE32(memPtr);
|
313
|
+
else
|
314
|
+
return (size_t)MEM_readLE64(memPtr);
|
315
|
+
}
|
316
|
+
|
317
|
+
MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
|
318
|
+
{
|
319
|
+
if (MEM_32bits())
|
320
|
+
MEM_writeLE32(memPtr, (U32)val);
|
321
|
+
else
|
322
|
+
MEM_writeLE64(memPtr, (U64)val);
|
323
|
+
}
|
324
|
+
|
325
|
+
/*=== Big endian r/w ===*/
|
326
|
+
|
327
|
+
MEM_STATIC U32 MEM_readBE32(const void* memPtr)
|
328
|
+
{
|
329
|
+
if (MEM_isLittleEndian())
|
330
|
+
return MEM_swap32(MEM_read32(memPtr));
|
331
|
+
else
|
332
|
+
return MEM_read32(memPtr);
|
333
|
+
}
|
334
|
+
|
335
|
+
MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32)
|
336
|
+
{
|
337
|
+
if (MEM_isLittleEndian())
|
338
|
+
MEM_write32(memPtr, MEM_swap32(val32));
|
339
|
+
else
|
340
|
+
MEM_write32(memPtr, val32);
|
341
|
+
}
|
342
|
+
|
343
|
+
MEM_STATIC U64 MEM_readBE64(const void* memPtr)
|
344
|
+
{
|
345
|
+
if (MEM_isLittleEndian())
|
346
|
+
return MEM_swap64(MEM_read64(memPtr));
|
347
|
+
else
|
348
|
+
return MEM_read64(memPtr);
|
349
|
+
}
|
350
|
+
|
351
|
+
MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64)
|
352
|
+
{
|
353
|
+
if (MEM_isLittleEndian())
|
354
|
+
MEM_write64(memPtr, MEM_swap64(val64));
|
355
|
+
else
|
356
|
+
MEM_write64(memPtr, val64);
|
357
|
+
}
|
358
|
+
|
359
|
+
MEM_STATIC size_t MEM_readBEST(const void* memPtr)
|
360
|
+
{
|
361
|
+
if (MEM_32bits())
|
362
|
+
return (size_t)MEM_readBE32(memPtr);
|
363
|
+
else
|
364
|
+
return (size_t)MEM_readBE64(memPtr);
|
365
|
+
}
|
366
|
+
|
367
|
+
MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
|
368
|
+
{
|
369
|
+
if (MEM_32bits())
|
370
|
+
MEM_writeBE32(memPtr, (U32)val);
|
371
|
+
else
|
372
|
+
MEM_writeBE64(memPtr, (U64)val);
|
373
|
+
}
|
374
|
+
|
375
|
+
|
376
|
+
#if defined (__cplusplus)
|
377
|
+
}
|
378
|
+
#endif
|
379
|
+
|
380
|
+
#endif /* MEM_H_MODULE */
|
@@ -0,0 +1,340 @@
|
|
1
|
+
/*
|
2
|
+
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
3
|
+
* All rights reserved.
|
4
|
+
*
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
9
|
+
*/
|
10
|
+
|
11
|
+
|
12
|
+
/* ====== Dependencies ======= */
|
13
|
+
#include <stddef.h> /* size_t */
|
14
|
+
#include "debug.h" /* assert */
|
15
|
+
#include "zstd_internal.h" /* ZSTD_malloc, ZSTD_free */
|
16
|
+
#include "pool.h"
|
17
|
+
|
18
|
+
/* ====== Compiler specifics ====== */
|
19
|
+
#if defined(_MSC_VER)
|
20
|
+
# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
|
21
|
+
#endif
|
22
|
+
|
23
|
+
|
24
|
+
#ifdef ZSTD_MULTITHREAD
|
25
|
+
|
26
|
+
#include "threading.h" /* pthread adaptation */
|
27
|
+
|
28
|
+
/* A job is a function and an opaque argument */
|
29
|
+
typedef struct POOL_job_s {
|
30
|
+
POOL_function function;
|
31
|
+
void *opaque;
|
32
|
+
} POOL_job;
|
33
|
+
|
34
|
+
struct POOL_ctx_s {
|
35
|
+
ZSTD_customMem customMem;
|
36
|
+
/* Keep track of the threads */
|
37
|
+
ZSTD_pthread_t* threads;
|
38
|
+
size_t threadCapacity;
|
39
|
+
size_t threadLimit;
|
40
|
+
|
41
|
+
/* The queue is a circular buffer */
|
42
|
+
POOL_job *queue;
|
43
|
+
size_t queueHead;
|
44
|
+
size_t queueTail;
|
45
|
+
size_t queueSize;
|
46
|
+
|
47
|
+
/* The number of threads working on jobs */
|
48
|
+
size_t numThreadsBusy;
|
49
|
+
/* Indicates if the queue is empty */
|
50
|
+
int queueEmpty;
|
51
|
+
|
52
|
+
/* The mutex protects the queue */
|
53
|
+
ZSTD_pthread_mutex_t queueMutex;
|
54
|
+
/* Condition variable for pushers to wait on when the queue is full */
|
55
|
+
ZSTD_pthread_cond_t queuePushCond;
|
56
|
+
/* Condition variables for poppers to wait on when the queue is empty */
|
57
|
+
ZSTD_pthread_cond_t queuePopCond;
|
58
|
+
/* Indicates if the queue is shutting down */
|
59
|
+
int shutdown;
|
60
|
+
};
|
61
|
+
|
62
|
+
/* POOL_thread() :
|
63
|
+
* Work thread for the thread pool.
|
64
|
+
* Waits for jobs and executes them.
|
65
|
+
* @returns : NULL on failure else non-null.
|
66
|
+
*/
|
67
|
+
static void* POOL_thread(void* opaque) {
|
68
|
+
POOL_ctx* const ctx = (POOL_ctx*)opaque;
|
69
|
+
if (!ctx) { return NULL; }
|
70
|
+
for (;;) {
|
71
|
+
/* Lock the mutex and wait for a non-empty queue or until shutdown */
|
72
|
+
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
73
|
+
|
74
|
+
while ( ctx->queueEmpty
|
75
|
+
|| (ctx->numThreadsBusy >= ctx->threadLimit) ) {
|
76
|
+
if (ctx->shutdown) {
|
77
|
+
/* even if !queueEmpty, (possible if numThreadsBusy >= threadLimit),
|
78
|
+
* a few threads will be shutdown while !queueEmpty,
|
79
|
+
* but enough threads will remain active to finish the queue */
|
80
|
+
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
81
|
+
return opaque;
|
82
|
+
}
|
83
|
+
ZSTD_pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex);
|
84
|
+
}
|
85
|
+
/* Pop a job off the queue */
|
86
|
+
{ POOL_job const job = ctx->queue[ctx->queueHead];
|
87
|
+
ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize;
|
88
|
+
ctx->numThreadsBusy++;
|
89
|
+
ctx->queueEmpty = ctx->queueHead == ctx->queueTail;
|
90
|
+
/* Unlock the mutex, signal a pusher, and run the job */
|
91
|
+
ZSTD_pthread_cond_signal(&ctx->queuePushCond);
|
92
|
+
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
93
|
+
|
94
|
+
job.function(job.opaque);
|
95
|
+
|
96
|
+
/* If the intended queue size was 0, signal after finishing job */
|
97
|
+
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
98
|
+
ctx->numThreadsBusy--;
|
99
|
+
if (ctx->queueSize == 1) {
|
100
|
+
ZSTD_pthread_cond_signal(&ctx->queuePushCond);
|
101
|
+
}
|
102
|
+
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
103
|
+
}
|
104
|
+
} /* for (;;) */
|
105
|
+
assert(0); /* Unreachable */
|
106
|
+
}
|
107
|
+
|
108
|
+
POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
|
109
|
+
return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem);
|
110
|
+
}
|
111
|
+
|
112
|
+
POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
|
113
|
+
ZSTD_customMem customMem) {
|
114
|
+
POOL_ctx* ctx;
|
115
|
+
/* Check parameters */
|
116
|
+
if (!numThreads) { return NULL; }
|
117
|
+
/* Allocate the context and zero initialize */
|
118
|
+
ctx = (POOL_ctx*)ZSTD_calloc(sizeof(POOL_ctx), customMem);
|
119
|
+
if (!ctx) { return NULL; }
|
120
|
+
/* Initialize the job queue.
|
121
|
+
* It needs one extra space since one space is wasted to differentiate
|
122
|
+
* empty and full queues.
|
123
|
+
*/
|
124
|
+
ctx->queueSize = queueSize + 1;
|
125
|
+
ctx->queue = (POOL_job*)ZSTD_malloc(ctx->queueSize * sizeof(POOL_job), customMem);
|
126
|
+
ctx->queueHead = 0;
|
127
|
+
ctx->queueTail = 0;
|
128
|
+
ctx->numThreadsBusy = 0;
|
129
|
+
ctx->queueEmpty = 1;
|
130
|
+
(void)ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL);
|
131
|
+
(void)ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL);
|
132
|
+
(void)ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL);
|
133
|
+
ctx->shutdown = 0;
|
134
|
+
/* Allocate space for the thread handles */
|
135
|
+
ctx->threads = (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), customMem);
|
136
|
+
ctx->threadCapacity = 0;
|
137
|
+
ctx->customMem = customMem;
|
138
|
+
/* Check for errors */
|
139
|
+
if (!ctx->threads || !ctx->queue) { POOL_free(ctx); return NULL; }
|
140
|
+
/* Initialize the threads */
|
141
|
+
{ size_t i;
|
142
|
+
for (i = 0; i < numThreads; ++i) {
|
143
|
+
if (ZSTD_pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) {
|
144
|
+
ctx->threadCapacity = i;
|
145
|
+
POOL_free(ctx);
|
146
|
+
return NULL;
|
147
|
+
} }
|
148
|
+
ctx->threadCapacity = numThreads;
|
149
|
+
ctx->threadLimit = numThreads;
|
150
|
+
}
|
151
|
+
return ctx;
|
152
|
+
}
|
153
|
+
|
154
|
+
/*! POOL_join() :
|
155
|
+
Shutdown the queue, wake any sleeping threads, and join all of the threads.
|
156
|
+
*/
|
157
|
+
static void POOL_join(POOL_ctx* ctx) {
|
158
|
+
/* Shut down the queue */
|
159
|
+
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
160
|
+
ctx->shutdown = 1;
|
161
|
+
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
162
|
+
/* Wake up sleeping threads */
|
163
|
+
ZSTD_pthread_cond_broadcast(&ctx->queuePushCond);
|
164
|
+
ZSTD_pthread_cond_broadcast(&ctx->queuePopCond);
|
165
|
+
/* Join all of the threads */
|
166
|
+
{ size_t i;
|
167
|
+
for (i = 0; i < ctx->threadCapacity; ++i) {
|
168
|
+
ZSTD_pthread_join(ctx->threads[i], NULL); /* note : could fail */
|
169
|
+
} }
|
170
|
+
}
|
171
|
+
|
172
|
+
void POOL_free(POOL_ctx *ctx) {
|
173
|
+
if (!ctx) { return; }
|
174
|
+
POOL_join(ctx);
|
175
|
+
ZSTD_pthread_mutex_destroy(&ctx->queueMutex);
|
176
|
+
ZSTD_pthread_cond_destroy(&ctx->queuePushCond);
|
177
|
+
ZSTD_pthread_cond_destroy(&ctx->queuePopCond);
|
178
|
+
ZSTD_free(ctx->queue, ctx->customMem);
|
179
|
+
ZSTD_free(ctx->threads, ctx->customMem);
|
180
|
+
ZSTD_free(ctx, ctx->customMem);
|
181
|
+
}
|
182
|
+
|
183
|
+
|
184
|
+
|
185
|
+
size_t POOL_sizeof(POOL_ctx *ctx) {
|
186
|
+
if (ctx==NULL) return 0; /* supports sizeof NULL */
|
187
|
+
return sizeof(*ctx)
|
188
|
+
+ ctx->queueSize * sizeof(POOL_job)
|
189
|
+
+ ctx->threadCapacity * sizeof(ZSTD_pthread_t);
|
190
|
+
}
|
191
|
+
|
192
|
+
|
193
|
+
/* @return : 0 on success, 1 on error */
|
194
|
+
static int POOL_resize_internal(POOL_ctx* ctx, size_t numThreads)
|
195
|
+
{
|
196
|
+
if (numThreads <= ctx->threadCapacity) {
|
197
|
+
if (!numThreads) return 1;
|
198
|
+
ctx->threadLimit = numThreads;
|
199
|
+
return 0;
|
200
|
+
}
|
201
|
+
/* numThreads > threadCapacity */
|
202
|
+
{ ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem);
|
203
|
+
if (!threadPool) return 1;
|
204
|
+
/* replace existing thread pool */
|
205
|
+
memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(*threadPool));
|
206
|
+
ZSTD_free(ctx->threads, ctx->customMem);
|
207
|
+
ctx->threads = threadPool;
|
208
|
+
/* Initialize additional threads */
|
209
|
+
{ size_t threadId;
|
210
|
+
for (threadId = ctx->threadCapacity; threadId < numThreads; ++threadId) {
|
211
|
+
if (ZSTD_pthread_create(&threadPool[threadId], NULL, &POOL_thread, ctx)) {
|
212
|
+
ctx->threadCapacity = threadId;
|
213
|
+
return 1;
|
214
|
+
} }
|
215
|
+
} }
|
216
|
+
/* successfully expanded */
|
217
|
+
ctx->threadCapacity = numThreads;
|
218
|
+
ctx->threadLimit = numThreads;
|
219
|
+
return 0;
|
220
|
+
}
|
221
|
+
|
222
|
+
/* @return : 0 on success, 1 on error */
|
223
|
+
int POOL_resize(POOL_ctx* ctx, size_t numThreads)
|
224
|
+
{
|
225
|
+
int result;
|
226
|
+
if (ctx==NULL) return 1;
|
227
|
+
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
228
|
+
result = POOL_resize_internal(ctx, numThreads);
|
229
|
+
ZSTD_pthread_cond_broadcast(&ctx->queuePopCond);
|
230
|
+
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
231
|
+
return result;
|
232
|
+
}
|
233
|
+
|
234
|
+
/**
|
235
|
+
* Returns 1 if the queue is full and 0 otherwise.
|
236
|
+
*
|
237
|
+
* When queueSize is 1 (pool was created with an intended queueSize of 0),
|
238
|
+
* then a queue is empty if there is a thread free _and_ no job is waiting.
|
239
|
+
*/
|
240
|
+
static int isQueueFull(POOL_ctx const* ctx) {
|
241
|
+
if (ctx->queueSize > 1) {
|
242
|
+
return ctx->queueHead == ((ctx->queueTail + 1) % ctx->queueSize);
|
243
|
+
} else {
|
244
|
+
return (ctx->numThreadsBusy == ctx->threadLimit) ||
|
245
|
+
!ctx->queueEmpty;
|
246
|
+
}
|
247
|
+
}
|
248
|
+
|
249
|
+
|
250
|
+
static void POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque)
|
251
|
+
{
|
252
|
+
POOL_job const job = {function, opaque};
|
253
|
+
assert(ctx != NULL);
|
254
|
+
if (ctx->shutdown) return;
|
255
|
+
|
256
|
+
ctx->queueEmpty = 0;
|
257
|
+
ctx->queue[ctx->queueTail] = job;
|
258
|
+
ctx->queueTail = (ctx->queueTail + 1) % ctx->queueSize;
|
259
|
+
ZSTD_pthread_cond_signal(&ctx->queuePopCond);
|
260
|
+
}
|
261
|
+
|
262
|
+
void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque)
|
263
|
+
{
|
264
|
+
assert(ctx != NULL);
|
265
|
+
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
266
|
+
/* Wait until there is space in the queue for the new job */
|
267
|
+
while (isQueueFull(ctx) && (!ctx->shutdown)) {
|
268
|
+
ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex);
|
269
|
+
}
|
270
|
+
POOL_add_internal(ctx, function, opaque);
|
271
|
+
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
272
|
+
}
|
273
|
+
|
274
|
+
|
275
|
+
int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque)
|
276
|
+
{
|
277
|
+
assert(ctx != NULL);
|
278
|
+
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
279
|
+
if (isQueueFull(ctx)) {
|
280
|
+
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
281
|
+
return 0;
|
282
|
+
}
|
283
|
+
POOL_add_internal(ctx, function, opaque);
|
284
|
+
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
285
|
+
return 1;
|
286
|
+
}
|
287
|
+
|
288
|
+
|
289
|
+
#else /* ZSTD_MULTITHREAD not defined */
|
290
|
+
|
291
|
+
/* ========================== */
|
292
|
+
/* No multi-threading support */
|
293
|
+
/* ========================== */
|
294
|
+
|
295
|
+
|
296
|
+
/* We don't need any data, but if it is empty, malloc() might return NULL. */
|
297
|
+
struct POOL_ctx_s {
|
298
|
+
int dummy;
|
299
|
+
};
|
300
|
+
static POOL_ctx g_ctx;
|
301
|
+
|
302
|
+
POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
|
303
|
+
return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem);
|
304
|
+
}
|
305
|
+
|
306
|
+
POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) {
|
307
|
+
(void)numThreads;
|
308
|
+
(void)queueSize;
|
309
|
+
(void)customMem;
|
310
|
+
return &g_ctx;
|
311
|
+
}
|
312
|
+
|
313
|
+
void POOL_free(POOL_ctx* ctx) {
|
314
|
+
assert(!ctx || ctx == &g_ctx);
|
315
|
+
(void)ctx;
|
316
|
+
}
|
317
|
+
|
318
|
+
int POOL_resize(POOL_ctx* ctx, size_t numThreads) {
|
319
|
+
(void)ctx; (void)numThreads;
|
320
|
+
return 0;
|
321
|
+
}
|
322
|
+
|
323
|
+
void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) {
|
324
|
+
(void)ctx;
|
325
|
+
function(opaque);
|
326
|
+
}
|
327
|
+
|
328
|
+
int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) {
|
329
|
+
(void)ctx;
|
330
|
+
function(opaque);
|
331
|
+
return 1;
|
332
|
+
}
|
333
|
+
|
334
|
+
size_t POOL_sizeof(POOL_ctx* ctx) {
|
335
|
+
if (ctx==NULL) return 0; /* supports sizeof NULL */
|
336
|
+
assert(ctx == &g_ctx);
|
337
|
+
return sizeof(*ctx);
|
338
|
+
}
|
339
|
+
|
340
|
+
#endif /* ZSTD_MULTITHREAD */
|