zstdlib 0.9.0-x86_64-darwin
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.yardopts +6 -0
- data/CHANGES.md +78 -0
- data/Gemfile +3 -0
- data/README.md +107 -0
- data/Rakefile +59 -0
- data/ext/zstdlib_c/extconf.rb +54 -0
- data/ext/zstdlib_c/ruby/zlib-2.2/zstdlib.c +4675 -0
- data/ext/zstdlib_c/ruby/zlib-2.3/zstdlib.c +4702 -0
- data/ext/zstdlib_c/ruby/zlib-2.4/zstdlib.c +4859 -0
- data/ext/zstdlib_c/ruby/zlib-2.5/zstdlib.c +4864 -0
- data/ext/zstdlib_c/ruby/zlib-2.6/zstdlib.c +4906 -0
- data/ext/zstdlib_c/ruby/zlib-2.7/zstdlib.c +4895 -0
- data/ext/zstdlib_c/ruby/zlib-3.0/zstdlib.c +4994 -0
- data/ext/zstdlib_c/ruby/zlib-3.1/zstdlib.c +5076 -0
- data/ext/zstdlib_c/zlib-1.2.11/adler32.c +186 -0
- data/ext/zstdlib_c/zlib-1.2.11/compress.c +86 -0
- data/ext/zstdlib_c/zlib-1.2.11/crc32.c +442 -0
- data/ext/zstdlib_c/zlib-1.2.11/crc32.h +441 -0
- data/ext/zstdlib_c/zlib-1.2.11/deflate.c +2163 -0
- data/ext/zstdlib_c/zlib-1.2.11/deflate.h +349 -0
- data/ext/zstdlib_c/zlib-1.2.11/gzclose.c +25 -0
- data/ext/zstdlib_c/zlib-1.2.11/gzguts.h +218 -0
- data/ext/zstdlib_c/zlib-1.2.11/gzlib.c +637 -0
- data/ext/zstdlib_c/zlib-1.2.11/gzread.c +654 -0
- data/ext/zstdlib_c/zlib-1.2.11/gzwrite.c +665 -0
- data/ext/zstdlib_c/zlib-1.2.11/infback.c +640 -0
- data/ext/zstdlib_c/zlib-1.2.11/inffast.c +323 -0
- data/ext/zstdlib_c/zlib-1.2.11/inffast.h +11 -0
- data/ext/zstdlib_c/zlib-1.2.11/inffixed.h +94 -0
- data/ext/zstdlib_c/zlib-1.2.11/inflate.c +1561 -0
- data/ext/zstdlib_c/zlib-1.2.11/inflate.h +125 -0
- data/ext/zstdlib_c/zlib-1.2.11/inftrees.c +304 -0
- data/ext/zstdlib_c/zlib-1.2.11/inftrees.h +62 -0
- data/ext/zstdlib_c/zlib-1.2.11/trees.c +1203 -0
- data/ext/zstdlib_c/zlib-1.2.11/trees.h +128 -0
- data/ext/zstdlib_c/zlib-1.2.11/uncompr.c +93 -0
- data/ext/zstdlib_c/zlib-1.2.11/zconf.h +534 -0
- data/ext/zstdlib_c/zlib-1.2.11/zlib.h +1912 -0
- data/ext/zstdlib_c/zlib-1.2.11/zutil.c +325 -0
- data/ext/zstdlib_c/zlib-1.2.11/zutil.h +271 -0
- data/ext/zstdlib_c/zlib.mk +14 -0
- data/ext/zstdlib_c/zlibwrapper/zlibwrapper.c +10 -0
- data/ext/zstdlib_c/zlibwrapper.mk +14 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/bitstream.h +478 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/compiler.h +335 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/cpu.h +213 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/debug.c +24 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/debug.h +107 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/entropy_common.c +368 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/error_private.c +56 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/error_private.h +159 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/fse.h +717 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/fse_decompress.c +403 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/huf.h +364 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/mem.h +442 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/pool.c +355 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/pool.h +84 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/portability_macros.h +137 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/threading.c +122 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/threading.h +155 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/xxhash.c +24 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/xxhash.h +5686 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/zstd_common.c +83 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/zstd_deps.h +111 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/zstd_internal.h +493 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/common/zstd_trace.h +163 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/clevels.h +134 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/fse_compress.c +741 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/hist.c +181 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/hist.h +75 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/huf_compress.c +1370 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_compress.c +6327 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_compress_internal.h +1458 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_compress_literals.c +159 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_compress_literals.h +31 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_compress_sequences.c +442 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_compress_sequences.h +54 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_compress_superblock.c +573 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_compress_superblock.h +32 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_cwksp.h +676 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_double_fast.c +696 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_double_fast.h +38 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_fast.c +675 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_fast.h +37 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_lazy.c +2104 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_lazy.h +125 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_ldm.c +724 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_ldm.h +117 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_ldm_geartab.h +106 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_opt.c +1446 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_opt.h +56 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstdmt_compress.c +1859 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstdmt_compress.h +113 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/decompress/huf_decompress.c +1889 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/decompress/huf_decompress_amd64.S +585 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/decompress/zstd_ddict.c +244 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/decompress/zstd_ddict.h +44 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/decompress/zstd_decompress.c +2230 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/decompress/zstd_decompress_block.c +2072 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/decompress/zstd_decompress_block.h +68 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/decompress/zstd_decompress_internal.h +236 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/zdict.h +452 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/zstd.h +2575 -0
- data/ext/zstdlib_c/zstd-1.5.2/lib/zstd_errors.h +95 -0
- data/ext/zstdlib_c/zstd-1.5.2/zlibWrapper/gzclose.c +28 -0
- data/ext/zstdlib_c/zstd-1.5.2/zlibWrapper/gzcompatibility.h +68 -0
- data/ext/zstdlib_c/zstd-1.5.2/zlibWrapper/gzguts.h +229 -0
- data/ext/zstdlib_c/zstd-1.5.2/zlibWrapper/gzlib.c +640 -0
- data/ext/zstdlib_c/zstd-1.5.2/zlibWrapper/gzread.c +678 -0
- data/ext/zstdlib_c/zstd-1.5.2/zlibWrapper/gzwrite.c +671 -0
- data/ext/zstdlib_c/zstd-1.5.2/zlibWrapper/zstd_zlibwrapper.c +1198 -0
- data/ext/zstdlib_c/zstd-1.5.2/zlibWrapper/zstd_zlibwrapper.h +88 -0
- data/ext/zstdlib_c/zstd.mk +15 -0
- data/lib/2.4/zstdlib_c.bundle +0 -0
- data/lib/2.5/zstdlib_c.bundle +0 -0
- data/lib/2.6/zstdlib_c.bundle +0 -0
- data/lib/2.7/zstdlib_c.bundle +0 -0
- data/lib/3.0/zstdlib_c.bundle +0 -0
- data/lib/3.1/zstdlib_c.bundle +0 -0
- data/lib/zstdlib.rb +6 -0
- data/test/zstdlib_test.rb +21 -0
- metadata +237 -0
@@ -0,0 +1,355 @@
|
|
1
|
+
/*
|
2
|
+
* Copyright (c) Yann Collet, Facebook, Inc.
|
3
|
+
* All rights reserved.
|
4
|
+
*
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
9
|
+
*/
|
10
|
+
|
11
|
+
|
12
|
+
/* ====== Dependencies ======= */
|
13
|
+
#include "zstd_deps.h" /* size_t */
|
14
|
+
#include "debug.h" /* assert */
|
15
|
+
#include "zstd_internal.h" /* ZSTD_customMalloc, ZSTD_customFree */
|
16
|
+
#include "pool.h"
|
17
|
+
|
18
|
+
/* ====== Compiler specifics ====== */
|
19
|
+
#if defined(_MSC_VER)
|
20
|
+
# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
|
21
|
+
#endif
|
22
|
+
|
23
|
+
|
24
|
+
#ifdef ZSTD_MULTITHREAD
|
25
|
+
|
26
|
+
#include "threading.h" /* pthread adaptation */
|
27
|
+
|
28
|
+
/* A job is a function and an opaque argument */
|
29
|
+
typedef struct POOL_job_s {
|
30
|
+
POOL_function function;
|
31
|
+
void *opaque;
|
32
|
+
} POOL_job;
|
33
|
+
|
34
|
+
struct POOL_ctx_s {
|
35
|
+
ZSTD_customMem customMem;
|
36
|
+
/* Keep track of the threads */
|
37
|
+
ZSTD_pthread_t* threads;
|
38
|
+
size_t threadCapacity;
|
39
|
+
size_t threadLimit;
|
40
|
+
|
41
|
+
/* The queue is a circular buffer */
|
42
|
+
POOL_job *queue;
|
43
|
+
size_t queueHead;
|
44
|
+
size_t queueTail;
|
45
|
+
size_t queueSize;
|
46
|
+
|
47
|
+
/* The number of threads working on jobs */
|
48
|
+
size_t numThreadsBusy;
|
49
|
+
/* Indicates if the queue is empty */
|
50
|
+
int queueEmpty;
|
51
|
+
|
52
|
+
/* The mutex protects the queue */
|
53
|
+
ZSTD_pthread_mutex_t queueMutex;
|
54
|
+
/* Condition variable for pushers to wait on when the queue is full */
|
55
|
+
ZSTD_pthread_cond_t queuePushCond;
|
56
|
+
/* Condition variables for poppers to wait on when the queue is empty */
|
57
|
+
ZSTD_pthread_cond_t queuePopCond;
|
58
|
+
/* Indicates if the queue is shutting down */
|
59
|
+
int shutdown;
|
60
|
+
};
|
61
|
+
|
62
|
+
/* POOL_thread() :
|
63
|
+
* Work thread for the thread pool.
|
64
|
+
* Waits for jobs and executes them.
|
65
|
+
* @returns : NULL on failure else non-null.
|
66
|
+
*/
|
67
|
+
static void* POOL_thread(void* opaque) {
|
68
|
+
POOL_ctx* const ctx = (POOL_ctx*)opaque;
|
69
|
+
if (!ctx) { return NULL; }
|
70
|
+
for (;;) {
|
71
|
+
/* Lock the mutex and wait for a non-empty queue or until shutdown */
|
72
|
+
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
73
|
+
|
74
|
+
while ( ctx->queueEmpty
|
75
|
+
|| (ctx->numThreadsBusy >= ctx->threadLimit) ) {
|
76
|
+
if (ctx->shutdown) {
|
77
|
+
/* even if !queueEmpty, (possible if numThreadsBusy >= threadLimit),
|
78
|
+
* a few threads will be shutdown while !queueEmpty,
|
79
|
+
* but enough threads will remain active to finish the queue */
|
80
|
+
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
81
|
+
return opaque;
|
82
|
+
}
|
83
|
+
ZSTD_pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex);
|
84
|
+
}
|
85
|
+
/* Pop a job off the queue */
|
86
|
+
{ POOL_job const job = ctx->queue[ctx->queueHead];
|
87
|
+
ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize;
|
88
|
+
ctx->numThreadsBusy++;
|
89
|
+
ctx->queueEmpty = (ctx->queueHead == ctx->queueTail);
|
90
|
+
/* Unlock the mutex, signal a pusher, and run the job */
|
91
|
+
ZSTD_pthread_cond_signal(&ctx->queuePushCond);
|
92
|
+
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
93
|
+
|
94
|
+
job.function(job.opaque);
|
95
|
+
|
96
|
+
/* If the intended queue size was 0, signal after finishing job */
|
97
|
+
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
98
|
+
ctx->numThreadsBusy--;
|
99
|
+
if (ctx->queueSize == 1) {
|
100
|
+
ZSTD_pthread_cond_signal(&ctx->queuePushCond);
|
101
|
+
}
|
102
|
+
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
103
|
+
}
|
104
|
+
} /* for (;;) */
|
105
|
+
assert(0); /* Unreachable */
|
106
|
+
}
|
107
|
+
|
108
|
+
/* ZSTD_createThreadPool() : public access point */
|
109
|
+
POOL_ctx* ZSTD_createThreadPool(size_t numThreads) {
|
110
|
+
return POOL_create (numThreads, 0);
|
111
|
+
}
|
112
|
+
|
113
|
+
POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
|
114
|
+
return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem);
|
115
|
+
}
|
116
|
+
|
117
|
+
POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
|
118
|
+
ZSTD_customMem customMem)
|
119
|
+
{
|
120
|
+
POOL_ctx* ctx;
|
121
|
+
/* Check parameters */
|
122
|
+
if (!numThreads) { return NULL; }
|
123
|
+
/* Allocate the context and zero initialize */
|
124
|
+
ctx = (POOL_ctx*)ZSTD_customCalloc(sizeof(POOL_ctx), customMem);
|
125
|
+
if (!ctx) { return NULL; }
|
126
|
+
/* Initialize the job queue.
|
127
|
+
* It needs one extra space since one space is wasted to differentiate
|
128
|
+
* empty and full queues.
|
129
|
+
*/
|
130
|
+
ctx->queueSize = queueSize + 1;
|
131
|
+
ctx->queue = (POOL_job*)ZSTD_customMalloc(ctx->queueSize * sizeof(POOL_job), customMem);
|
132
|
+
ctx->queueHead = 0;
|
133
|
+
ctx->queueTail = 0;
|
134
|
+
ctx->numThreadsBusy = 0;
|
135
|
+
ctx->queueEmpty = 1;
|
136
|
+
{
|
137
|
+
int error = 0;
|
138
|
+
error |= ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL);
|
139
|
+
error |= ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL);
|
140
|
+
error |= ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL);
|
141
|
+
if (error) { POOL_free(ctx); return NULL; }
|
142
|
+
}
|
143
|
+
ctx->shutdown = 0;
|
144
|
+
/* Allocate space for the thread handles */
|
145
|
+
ctx->threads = (ZSTD_pthread_t*)ZSTD_customMalloc(numThreads * sizeof(ZSTD_pthread_t), customMem);
|
146
|
+
ctx->threadCapacity = 0;
|
147
|
+
ctx->customMem = customMem;
|
148
|
+
/* Check for errors */
|
149
|
+
if (!ctx->threads || !ctx->queue) { POOL_free(ctx); return NULL; }
|
150
|
+
/* Initialize the threads */
|
151
|
+
{ size_t i;
|
152
|
+
for (i = 0; i < numThreads; ++i) {
|
153
|
+
if (ZSTD_pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) {
|
154
|
+
ctx->threadCapacity = i;
|
155
|
+
POOL_free(ctx);
|
156
|
+
return NULL;
|
157
|
+
} }
|
158
|
+
ctx->threadCapacity = numThreads;
|
159
|
+
ctx->threadLimit = numThreads;
|
160
|
+
}
|
161
|
+
return ctx;
|
162
|
+
}
|
163
|
+
|
164
|
+
/*! POOL_join() :
|
165
|
+
Shutdown the queue, wake any sleeping threads, and join all of the threads.
|
166
|
+
*/
|
167
|
+
static void POOL_join(POOL_ctx* ctx) {
|
168
|
+
/* Shut down the queue */
|
169
|
+
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
170
|
+
ctx->shutdown = 1;
|
171
|
+
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
172
|
+
/* Wake up sleeping threads */
|
173
|
+
ZSTD_pthread_cond_broadcast(&ctx->queuePushCond);
|
174
|
+
ZSTD_pthread_cond_broadcast(&ctx->queuePopCond);
|
175
|
+
/* Join all of the threads */
|
176
|
+
{ size_t i;
|
177
|
+
for (i = 0; i < ctx->threadCapacity; ++i) {
|
178
|
+
ZSTD_pthread_join(ctx->threads[i], NULL); /* note : could fail */
|
179
|
+
} }
|
180
|
+
}
|
181
|
+
|
182
|
+
void POOL_free(POOL_ctx *ctx) {
|
183
|
+
if (!ctx) { return; }
|
184
|
+
POOL_join(ctx);
|
185
|
+
ZSTD_pthread_mutex_destroy(&ctx->queueMutex);
|
186
|
+
ZSTD_pthread_cond_destroy(&ctx->queuePushCond);
|
187
|
+
ZSTD_pthread_cond_destroy(&ctx->queuePopCond);
|
188
|
+
ZSTD_customFree(ctx->queue, ctx->customMem);
|
189
|
+
ZSTD_customFree(ctx->threads, ctx->customMem);
|
190
|
+
ZSTD_customFree(ctx, ctx->customMem);
|
191
|
+
}
|
192
|
+
|
193
|
+
void ZSTD_freeThreadPool (ZSTD_threadPool* pool) {
|
194
|
+
POOL_free (pool);
|
195
|
+
}
|
196
|
+
|
197
|
+
size_t POOL_sizeof(const POOL_ctx* ctx) {
|
198
|
+
if (ctx==NULL) return 0; /* supports sizeof NULL */
|
199
|
+
return sizeof(*ctx)
|
200
|
+
+ ctx->queueSize * sizeof(POOL_job)
|
201
|
+
+ ctx->threadCapacity * sizeof(ZSTD_pthread_t);
|
202
|
+
}
|
203
|
+
|
204
|
+
|
205
|
+
/* @return : 0 on success, 1 on error */
|
206
|
+
static int POOL_resize_internal(POOL_ctx* ctx, size_t numThreads)
|
207
|
+
{
|
208
|
+
if (numThreads <= ctx->threadCapacity) {
|
209
|
+
if (!numThreads) return 1;
|
210
|
+
ctx->threadLimit = numThreads;
|
211
|
+
return 0;
|
212
|
+
}
|
213
|
+
/* numThreads > threadCapacity */
|
214
|
+
{ ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_customMalloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem);
|
215
|
+
if (!threadPool) return 1;
|
216
|
+
/* replace existing thread pool */
|
217
|
+
ZSTD_memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(*threadPool));
|
218
|
+
ZSTD_customFree(ctx->threads, ctx->customMem);
|
219
|
+
ctx->threads = threadPool;
|
220
|
+
/* Initialize additional threads */
|
221
|
+
{ size_t threadId;
|
222
|
+
for (threadId = ctx->threadCapacity; threadId < numThreads; ++threadId) {
|
223
|
+
if (ZSTD_pthread_create(&threadPool[threadId], NULL, &POOL_thread, ctx)) {
|
224
|
+
ctx->threadCapacity = threadId;
|
225
|
+
return 1;
|
226
|
+
} }
|
227
|
+
} }
|
228
|
+
/* successfully expanded */
|
229
|
+
ctx->threadCapacity = numThreads;
|
230
|
+
ctx->threadLimit = numThreads;
|
231
|
+
return 0;
|
232
|
+
}
|
233
|
+
|
234
|
+
/* @return : 0 on success, 1 on error */
|
235
|
+
int POOL_resize(POOL_ctx* ctx, size_t numThreads)
|
236
|
+
{
|
237
|
+
int result;
|
238
|
+
if (ctx==NULL) return 1;
|
239
|
+
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
240
|
+
result = POOL_resize_internal(ctx, numThreads);
|
241
|
+
ZSTD_pthread_cond_broadcast(&ctx->queuePopCond);
|
242
|
+
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
243
|
+
return result;
|
244
|
+
}
|
245
|
+
|
246
|
+
/**
|
247
|
+
* Returns 1 if the queue is full and 0 otherwise.
|
248
|
+
*
|
249
|
+
* When queueSize is 1 (pool was created with an intended queueSize of 0),
|
250
|
+
* then a queue is empty if there is a thread free _and_ no job is waiting.
|
251
|
+
*/
|
252
|
+
static int isQueueFull(POOL_ctx const* ctx) {
|
253
|
+
if (ctx->queueSize > 1) {
|
254
|
+
return ctx->queueHead == ((ctx->queueTail + 1) % ctx->queueSize);
|
255
|
+
} else {
|
256
|
+
return (ctx->numThreadsBusy == ctx->threadLimit) ||
|
257
|
+
!ctx->queueEmpty;
|
258
|
+
}
|
259
|
+
}
|
260
|
+
|
261
|
+
|
262
|
+
static void
|
263
|
+
POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque)
|
264
|
+
{
|
265
|
+
POOL_job const job = {function, opaque};
|
266
|
+
assert(ctx != NULL);
|
267
|
+
if (ctx->shutdown) return;
|
268
|
+
|
269
|
+
ctx->queueEmpty = 0;
|
270
|
+
ctx->queue[ctx->queueTail] = job;
|
271
|
+
ctx->queueTail = (ctx->queueTail + 1) % ctx->queueSize;
|
272
|
+
ZSTD_pthread_cond_signal(&ctx->queuePopCond);
|
273
|
+
}
|
274
|
+
|
275
|
+
void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque)
|
276
|
+
{
|
277
|
+
assert(ctx != NULL);
|
278
|
+
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
279
|
+
/* Wait until there is space in the queue for the new job */
|
280
|
+
while (isQueueFull(ctx) && (!ctx->shutdown)) {
|
281
|
+
ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex);
|
282
|
+
}
|
283
|
+
POOL_add_internal(ctx, function, opaque);
|
284
|
+
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
285
|
+
}
|
286
|
+
|
287
|
+
|
288
|
+
int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque)
|
289
|
+
{
|
290
|
+
assert(ctx != NULL);
|
291
|
+
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
292
|
+
if (isQueueFull(ctx)) {
|
293
|
+
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
294
|
+
return 0;
|
295
|
+
}
|
296
|
+
POOL_add_internal(ctx, function, opaque);
|
297
|
+
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
298
|
+
return 1;
|
299
|
+
}
|
300
|
+
|
301
|
+
|
302
|
+
#else /* ZSTD_MULTITHREAD not defined */
|
303
|
+
|
304
|
+
/* ========================== */
|
305
|
+
/* No multi-threading support */
|
306
|
+
/* ========================== */
|
307
|
+
|
308
|
+
|
309
|
+
/* We don't need any data, but if it is empty, malloc() might return NULL. */
|
310
|
+
struct POOL_ctx_s {
|
311
|
+
int dummy;
|
312
|
+
};
|
313
|
+
static POOL_ctx g_poolCtx;
|
314
|
+
|
315
|
+
POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
|
316
|
+
return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem);
|
317
|
+
}
|
318
|
+
|
319
|
+
POOL_ctx*
|
320
|
+
POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem)
|
321
|
+
{
|
322
|
+
(void)numThreads;
|
323
|
+
(void)queueSize;
|
324
|
+
(void)customMem;
|
325
|
+
return &g_poolCtx;
|
326
|
+
}
|
327
|
+
|
328
|
+
void POOL_free(POOL_ctx* ctx) {
|
329
|
+
assert(!ctx || ctx == &g_poolCtx);
|
330
|
+
(void)ctx;
|
331
|
+
}
|
332
|
+
|
333
|
+
int POOL_resize(POOL_ctx* ctx, size_t numThreads) {
|
334
|
+
(void)ctx; (void)numThreads;
|
335
|
+
return 0;
|
336
|
+
}
|
337
|
+
|
338
|
+
void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) {
|
339
|
+
(void)ctx;
|
340
|
+
function(opaque);
|
341
|
+
}
|
342
|
+
|
343
|
+
int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) {
|
344
|
+
(void)ctx;
|
345
|
+
function(opaque);
|
346
|
+
return 1;
|
347
|
+
}
|
348
|
+
|
349
|
+
size_t POOL_sizeof(const POOL_ctx* ctx) {
|
350
|
+
if (ctx==NULL) return 0; /* supports sizeof NULL */
|
351
|
+
assert(ctx == &g_poolCtx);
|
352
|
+
return sizeof(*ctx);
|
353
|
+
}
|
354
|
+
|
355
|
+
#endif /* ZSTD_MULTITHREAD */
|
@@ -0,0 +1,84 @@
|
|
1
|
+
/*
|
2
|
+
* Copyright (c) Yann Collet, Facebook, Inc.
|
3
|
+
* All rights reserved.
|
4
|
+
*
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
9
|
+
*/
|
10
|
+
|
11
|
+
#ifndef POOL_H
|
12
|
+
#define POOL_H
|
13
|
+
|
14
|
+
#if defined (__cplusplus)
|
15
|
+
extern "C" {
|
16
|
+
#endif
|
17
|
+
|
18
|
+
|
19
|
+
#include "zstd_deps.h"
|
20
|
+
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_customMem */
|
21
|
+
#include "../zstd.h"
|
22
|
+
|
23
|
+
typedef struct POOL_ctx_s POOL_ctx;
|
24
|
+
|
25
|
+
/*! POOL_create() :
|
26
|
+
* Create a thread pool with at most `numThreads` threads.
|
27
|
+
* `numThreads` must be at least 1.
|
28
|
+
* The maximum number of queued jobs before blocking is `queueSize`.
|
29
|
+
* @return : POOL_ctx pointer on success, else NULL.
|
30
|
+
*/
|
31
|
+
POOL_ctx* POOL_create(size_t numThreads, size_t queueSize);
|
32
|
+
|
33
|
+
POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
|
34
|
+
ZSTD_customMem customMem);
|
35
|
+
|
36
|
+
/*! POOL_free() :
|
37
|
+
* Free a thread pool returned by POOL_create().
|
38
|
+
*/
|
39
|
+
void POOL_free(POOL_ctx* ctx);
|
40
|
+
|
41
|
+
/*! POOL_resize() :
|
42
|
+
* Expands or shrinks pool's number of threads.
|
43
|
+
* This is more efficient than releasing + creating a new context,
|
44
|
+
* since it tries to preserve and re-use existing threads.
|
45
|
+
* `numThreads` must be at least 1.
|
46
|
+
* @return : 0 when resize was successful,
|
47
|
+
* !0 (typically 1) if there is an error.
|
48
|
+
* note : only numThreads can be resized, queueSize remains unchanged.
|
49
|
+
*/
|
50
|
+
int POOL_resize(POOL_ctx* ctx, size_t numThreads);
|
51
|
+
|
52
|
+
/*! POOL_sizeof() :
|
53
|
+
* @return threadpool memory usage
|
54
|
+
* note : compatible with NULL (returns 0 in this case)
|
55
|
+
*/
|
56
|
+
size_t POOL_sizeof(const POOL_ctx* ctx);
|
57
|
+
|
58
|
+
/*! POOL_function :
|
59
|
+
* The function type that can be added to a thread pool.
|
60
|
+
*/
|
61
|
+
typedef void (*POOL_function)(void*);
|
62
|
+
|
63
|
+
/*! POOL_add() :
|
64
|
+
* Add the job `function(opaque)` to the thread pool. `ctx` must be valid.
|
65
|
+
* Possibly blocks until there is room in the queue.
|
66
|
+
* Note : The function may be executed asynchronously,
|
67
|
+
* therefore, `opaque` must live until function has been completed.
|
68
|
+
*/
|
69
|
+
void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque);
|
70
|
+
|
71
|
+
|
72
|
+
/*! POOL_tryAdd() :
|
73
|
+
* Add the job `function(opaque)` to thread pool _if_ a queue slot is available.
|
74
|
+
* Returns immediately even if not (does not block).
|
75
|
+
* @return : 1 if successful, 0 if not.
|
76
|
+
*/
|
77
|
+
int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque);
|
78
|
+
|
79
|
+
|
80
|
+
#if defined (__cplusplus)
|
81
|
+
}
|
82
|
+
#endif
|
83
|
+
|
84
|
+
#endif
|
@@ -0,0 +1,137 @@
|
|
1
|
+
/*
|
2
|
+
* Copyright (c) Facebook, Inc.
|
3
|
+
* All rights reserved.
|
4
|
+
*
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
9
|
+
*/
|
10
|
+
|
11
|
+
#ifndef ZSTD_PORTABILITY_MACROS_H
|
12
|
+
#define ZSTD_PORTABILITY_MACROS_H
|
13
|
+
|
14
|
+
/**
|
15
|
+
* This header file contains macro defintions to support portability.
|
16
|
+
* This header is shared between C and ASM code, so it MUST only
|
17
|
+
* contain macro definitions. It MUST not contain any C code.
|
18
|
+
*
|
19
|
+
* This header ONLY defines macros to detect platforms/feature support.
|
20
|
+
*
|
21
|
+
*/
|
22
|
+
|
23
|
+
|
24
|
+
/* compat. with non-clang compilers */
|
25
|
+
#ifndef __has_attribute
|
26
|
+
#define __has_attribute(x) 0
|
27
|
+
#endif
|
28
|
+
|
29
|
+
/* compat. with non-clang compilers */
|
30
|
+
#ifndef __has_builtin
|
31
|
+
# define __has_builtin(x) 0
|
32
|
+
#endif
|
33
|
+
|
34
|
+
/* compat. with non-clang compilers */
|
35
|
+
#ifndef __has_feature
|
36
|
+
# define __has_feature(x) 0
|
37
|
+
#endif
|
38
|
+
|
39
|
+
/* detects whether we are being compiled under msan */
|
40
|
+
#ifndef ZSTD_MEMORY_SANITIZER
|
41
|
+
# if __has_feature(memory_sanitizer)
|
42
|
+
# define ZSTD_MEMORY_SANITIZER 1
|
43
|
+
# else
|
44
|
+
# define ZSTD_MEMORY_SANITIZER 0
|
45
|
+
# endif
|
46
|
+
#endif
|
47
|
+
|
48
|
+
/* detects whether we are being compiled under asan */
|
49
|
+
#ifndef ZSTD_ADDRESS_SANITIZER
|
50
|
+
# if __has_feature(address_sanitizer)
|
51
|
+
# define ZSTD_ADDRESS_SANITIZER 1
|
52
|
+
# elif defined(__SANITIZE_ADDRESS__)
|
53
|
+
# define ZSTD_ADDRESS_SANITIZER 1
|
54
|
+
# else
|
55
|
+
# define ZSTD_ADDRESS_SANITIZER 0
|
56
|
+
# endif
|
57
|
+
#endif
|
58
|
+
|
59
|
+
/* detects whether we are being compiled under dfsan */
|
60
|
+
#ifndef ZSTD_DATAFLOW_SANITIZER
|
61
|
+
# if __has_feature(dataflow_sanitizer)
|
62
|
+
# define ZSTD_DATAFLOW_SANITIZER 1
|
63
|
+
# else
|
64
|
+
# define ZSTD_DATAFLOW_SANITIZER 0
|
65
|
+
# endif
|
66
|
+
#endif
|
67
|
+
|
68
|
+
/* Mark the internal assembly functions as hidden */
|
69
|
+
#ifdef __ELF__
|
70
|
+
# define ZSTD_HIDE_ASM_FUNCTION(func) .hidden func
|
71
|
+
#else
|
72
|
+
# define ZSTD_HIDE_ASM_FUNCTION(func)
|
73
|
+
#endif
|
74
|
+
|
75
|
+
/* Enable runtime BMI2 dispatch based on the CPU.
|
76
|
+
* Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default.
|
77
|
+
*/
|
78
|
+
#ifndef DYNAMIC_BMI2
|
79
|
+
#if ((defined(__clang__) && __has_attribute(__target__)) \
|
80
|
+
|| (defined(__GNUC__) \
|
81
|
+
&& (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \
|
82
|
+
&& (defined(__x86_64__) || defined(_M_X64)) \
|
83
|
+
&& !defined(__BMI2__)
|
84
|
+
# define DYNAMIC_BMI2 1
|
85
|
+
#else
|
86
|
+
# define DYNAMIC_BMI2 0
|
87
|
+
#endif
|
88
|
+
#endif
|
89
|
+
|
90
|
+
/**
|
91
|
+
* Only enable assembly for GNUC comptabile compilers,
|
92
|
+
* because other platforms may not support GAS assembly syntax.
|
93
|
+
*
|
94
|
+
* Only enable assembly for Linux / MacOS, other platforms may
|
95
|
+
* work, but they haven't been tested. This could likely be
|
96
|
+
* extended to BSD systems.
|
97
|
+
*
|
98
|
+
* Disable assembly when MSAN is enabled, because MSAN requires
|
99
|
+
* 100% of code to be instrumented to work.
|
100
|
+
*/
|
101
|
+
#if defined(__GNUC__)
|
102
|
+
# if defined(__linux__) || defined(__linux) || defined(__APPLE__)
|
103
|
+
# if ZSTD_MEMORY_SANITIZER
|
104
|
+
# define ZSTD_ASM_SUPPORTED 0
|
105
|
+
# elif ZSTD_DATAFLOW_SANITIZER
|
106
|
+
# define ZSTD_ASM_SUPPORTED 0
|
107
|
+
# else
|
108
|
+
# define ZSTD_ASM_SUPPORTED 1
|
109
|
+
# endif
|
110
|
+
# else
|
111
|
+
# define ZSTD_ASM_SUPPORTED 0
|
112
|
+
# endif
|
113
|
+
#else
|
114
|
+
# define ZSTD_ASM_SUPPORTED 0
|
115
|
+
#endif
|
116
|
+
|
117
|
+
/**
|
118
|
+
* Determines whether we should enable assembly for x86-64
|
119
|
+
* with BMI2.
|
120
|
+
*
|
121
|
+
* Enable if all of the following conditions hold:
|
122
|
+
* - ASM hasn't been explicitly disabled by defining ZSTD_DISABLE_ASM
|
123
|
+
* - Assembly is supported
|
124
|
+
* - We are compiling for x86-64 and either:
|
125
|
+
* - DYNAMIC_BMI2 is enabled
|
126
|
+
* - BMI2 is supported at compile time
|
127
|
+
*/
|
128
|
+
#if !defined(ZSTD_DISABLE_ASM) && \
|
129
|
+
ZSTD_ASM_SUPPORTED && \
|
130
|
+
defined(__x86_64__) && \
|
131
|
+
(DYNAMIC_BMI2 || defined(__BMI2__))
|
132
|
+
# define ZSTD_ENABLE_ASM_X86_64_BMI2 1
|
133
|
+
#else
|
134
|
+
# define ZSTD_ENABLE_ASM_X86_64_BMI2 0
|
135
|
+
#endif
|
136
|
+
|
137
|
+
#endif /* ZSTD_PORTABILITY_MACROS_H */
|
@@ -0,0 +1,122 @@
|
|
1
|
+
/**
|
2
|
+
* Copyright (c) 2016 Tino Reichardt
|
3
|
+
* All rights reserved.
|
4
|
+
*
|
5
|
+
* You can contact the author at:
|
6
|
+
* - zstdmt source repository: https://github.com/mcmilk/zstdmt
|
7
|
+
*
|
8
|
+
* This source code is licensed under both the BSD-style license (found in the
|
9
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
10
|
+
* in the COPYING file in the root directory of this source tree).
|
11
|
+
* You may select, at your option, one of the above-listed licenses.
|
12
|
+
*/
|
13
|
+
|
14
|
+
/**
|
15
|
+
* This file will hold wrapper for systems, which do not support pthreads
|
16
|
+
*/
|
17
|
+
|
18
|
+
#include "threading.h"
|
19
|
+
|
20
|
+
/* create fake symbol to avoid empty translation unit warning */
|
21
|
+
int g_ZSTD_threading_useless_symbol;
|
22
|
+
|
23
|
+
#if defined(ZSTD_MULTITHREAD) && defined(_WIN32)
|
24
|
+
|
25
|
+
/**
|
26
|
+
* Windows minimalist Pthread Wrapper, based on :
|
27
|
+
* http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
|
28
|
+
*/
|
29
|
+
|
30
|
+
|
31
|
+
/* === Dependencies === */
|
32
|
+
#include <process.h>
|
33
|
+
#include <errno.h>
|
34
|
+
|
35
|
+
|
36
|
+
/* === Implementation === */
|
37
|
+
|
38
|
+
static unsigned __stdcall worker(void *arg)
|
39
|
+
{
|
40
|
+
ZSTD_pthread_t* const thread = (ZSTD_pthread_t*) arg;
|
41
|
+
thread->arg = thread->start_routine(thread->arg);
|
42
|
+
return 0;
|
43
|
+
}
|
44
|
+
|
45
|
+
int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
|
46
|
+
void* (*start_routine) (void*), void* arg)
|
47
|
+
{
|
48
|
+
(void)unused;
|
49
|
+
thread->arg = arg;
|
50
|
+
thread->start_routine = start_routine;
|
51
|
+
thread->handle = (HANDLE) _beginthreadex(NULL, 0, worker, thread, 0, NULL);
|
52
|
+
|
53
|
+
if (!thread->handle)
|
54
|
+
return errno;
|
55
|
+
else
|
56
|
+
return 0;
|
57
|
+
}
|
58
|
+
|
59
|
+
int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr)
|
60
|
+
{
|
61
|
+
DWORD result;
|
62
|
+
|
63
|
+
if (!thread.handle) return 0;
|
64
|
+
|
65
|
+
result = WaitForSingleObject(thread.handle, INFINITE);
|
66
|
+
switch (result) {
|
67
|
+
case WAIT_OBJECT_0:
|
68
|
+
if (value_ptr) *value_ptr = thread.arg;
|
69
|
+
return 0;
|
70
|
+
case WAIT_ABANDONED:
|
71
|
+
return EINVAL;
|
72
|
+
default:
|
73
|
+
return GetLastError();
|
74
|
+
}
|
75
|
+
}
|
76
|
+
|
77
|
+
#endif /* ZSTD_MULTITHREAD */
|
78
|
+
|
79
|
+
#if defined(ZSTD_MULTITHREAD) && DEBUGLEVEL >= 1 && !defined(_WIN32)
|
80
|
+
|
81
|
+
#define ZSTD_DEPS_NEED_MALLOC
|
82
|
+
#include "zstd_deps.h"
|
83
|
+
|
84
|
+
int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr)
|
85
|
+
{
|
86
|
+
*mutex = (pthread_mutex_t*)ZSTD_malloc(sizeof(pthread_mutex_t));
|
87
|
+
if (!*mutex)
|
88
|
+
return 1;
|
89
|
+
return pthread_mutex_init(*mutex, attr);
|
90
|
+
}
|
91
|
+
|
92
|
+
int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex)
|
93
|
+
{
|
94
|
+
if (!*mutex)
|
95
|
+
return 0;
|
96
|
+
{
|
97
|
+
int const ret = pthread_mutex_destroy(*mutex);
|
98
|
+
ZSTD_free(*mutex);
|
99
|
+
return ret;
|
100
|
+
}
|
101
|
+
}
|
102
|
+
|
103
|
+
int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr)
|
104
|
+
{
|
105
|
+
*cond = (pthread_cond_t*)ZSTD_malloc(sizeof(pthread_cond_t));
|
106
|
+
if (!*cond)
|
107
|
+
return 1;
|
108
|
+
return pthread_cond_init(*cond, attr);
|
109
|
+
}
|
110
|
+
|
111
|
+
int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond)
|
112
|
+
{
|
113
|
+
if (!*cond)
|
114
|
+
return 0;
|
115
|
+
{
|
116
|
+
int const ret = pthread_cond_destroy(*cond);
|
117
|
+
ZSTD_free(*cond);
|
118
|
+
return ret;
|
119
|
+
}
|
120
|
+
}
|
121
|
+
|
122
|
+
#endif
|