zstd-ruby 1.3.4.0 → 1.3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/ext/zstdruby/libzstd/Makefile +56 -10
- data/ext/zstdruby/libzstd/README.md +4 -0
- data/ext/zstdruby/libzstd/common/bitstream.h +6 -19
- data/ext/zstdruby/libzstd/common/compiler.h +3 -3
- data/ext/zstdruby/libzstd/common/cpu.h +1 -2
- data/ext/zstdruby/libzstd/common/debug.c +44 -0
- data/ext/zstdruby/libzstd/common/debug.h +123 -0
- data/ext/zstdruby/libzstd/common/entropy_common.c +16 -1
- data/ext/zstdruby/libzstd/common/fse.h +45 -41
- data/ext/zstdruby/libzstd/common/fse_decompress.c +1 -1
- data/ext/zstdruby/libzstd/common/huf.h +34 -27
- data/ext/zstdruby/libzstd/common/pool.c +89 -32
- data/ext/zstdruby/libzstd/common/pool.h +29 -19
- data/ext/zstdruby/libzstd/common/zstd_common.c +0 -5
- data/ext/zstdruby/libzstd/common/zstd_internal.h +3 -37
- data/ext/zstdruby/libzstd/compress/fse_compress.c +28 -163
- data/ext/zstdruby/libzstd/compress/hist.c +195 -0
- data/ext/zstdruby/libzstd/compress/hist.h +92 -0
- data/ext/zstdruby/libzstd/compress/huf_compress.c +14 -6
- data/ext/zstdruby/libzstd/compress/zstd_compress.c +798 -350
- data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +120 -34
- data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +247 -87
- data/ext/zstdruby/libzstd/compress/zstd_double_fast.h +4 -1
- data/ext/zstdruby/libzstd/compress/zstd_fast.c +177 -56
- data/ext/zstdruby/libzstd/compress/zstd_fast.h +4 -1
- data/ext/zstdruby/libzstd/compress/zstd_lazy.c +331 -65
- data/ext/zstdruby/libzstd/compress/zstd_lazy.h +13 -0
- data/ext/zstdruby/libzstd/compress/zstd_ldm.c +15 -20
- data/ext/zstdruby/libzstd/compress/zstd_ldm.h +1 -2
- data/ext/zstdruby/libzstd/compress/zstd_opt.c +503 -300
- data/ext/zstdruby/libzstd/compress/zstd_opt.h +7 -0
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +122 -47
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +5 -5
- data/ext/zstdruby/libzstd/decompress/huf_decompress.c +325 -325
- data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +80 -43
- data/ext/zstdruby/libzstd/dictBuilder/cover.c +9 -2
- data/ext/zstdruby/libzstd/dictBuilder/zdict.c +5 -5
- data/ext/zstdruby/libzstd/legacy/zstd_v04.c +12 -61
- data/ext/zstdruby/libzstd/zstd.h +137 -69
- data/lib/zstd-ruby/version.rb +1 -1
- metadata +7 -3
@@ -49,7 +49,7 @@
|
|
49
49
|
* Error Management
|
50
50
|
****************************************************************/
|
51
51
|
#define FSE_isError ERR_isError
|
52
|
-
#define FSE_STATIC_ASSERT(c)
|
52
|
+
#define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */
|
53
53
|
|
54
54
|
/* check and forward error code */
|
55
55
|
#define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; }
|
@@ -1,7 +1,7 @@
|
|
1
1
|
/* ******************************************************************
|
2
|
-
|
3
|
-
|
4
|
-
Copyright (C) 2013-
|
2
|
+
huff0 huffman codec,
|
3
|
+
part of Finite State Entropy library
|
4
|
+
Copyright (C) 2013-present, Yann Collet.
|
5
5
|
|
6
6
|
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
7
7
|
|
@@ -163,25 +163,25 @@ HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity,
|
|
163
163
|
/* static allocation of HUF's DTable */
|
164
164
|
typedef U32 HUF_DTable;
|
165
165
|
#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog)))
|
166
|
-
#define
|
166
|
+
#define HUF_CREATE_STATIC_DTABLEX1(DTable, maxTableLog) \
|
167
167
|
HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) }
|
168
|
-
#define
|
168
|
+
#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
|
169
169
|
HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) }
|
170
170
|
|
171
171
|
|
172
172
|
/* ****************************************
|
173
173
|
* Advanced decompression functions
|
174
174
|
******************************************/
|
175
|
-
size_t
|
176
|
-
size_t
|
175
|
+
size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
176
|
+
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
177
177
|
|
178
178
|
size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */
|
179
179
|
size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
|
180
180
|
size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */
|
181
|
-
size_t
|
182
|
-
size_t
|
183
|
-
size_t
|
184
|
-
size_t
|
181
|
+
size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
182
|
+
size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
|
183
|
+
size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
184
|
+
size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
|
185
185
|
|
186
186
|
|
187
187
|
/* ****************************************
|
@@ -208,7 +208,7 @@ size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, si
|
|
208
208
|
typedef enum {
|
209
209
|
HUF_repeat_none, /**< Cannot use the previous table */
|
210
210
|
HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */
|
211
|
-
HUF_repeat_valid /**< Can use the previous table and it is
|
211
|
+
HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */
|
212
212
|
} HUF_repeat;
|
213
213
|
/** HUF_compress4X_repeat() :
|
214
214
|
* Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
|
@@ -227,7 +227,9 @@ size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
|
|
227
227
|
*/
|
228
228
|
#define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1)
|
229
229
|
#define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned))
|
230
|
-
size_t HUF_buildCTable_wksp (HUF_CElt* tree,
|
230
|
+
size_t HUF_buildCTable_wksp (HUF_CElt* tree,
|
231
|
+
const U32* count, U32 maxSymbolValue, U32 maxNbBits,
|
232
|
+
void* workSpace, size_t wkspSize);
|
231
233
|
|
232
234
|
/*! HUF_readStats() :
|
233
235
|
* Read compact Huffman tree, saved by HUF_writeCTable().
|
@@ -242,10 +244,15 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize,
|
|
242
244
|
* Loading a CTable saved with HUF_writeCTable() */
|
243
245
|
size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
|
244
246
|
|
247
|
+
/** HUF_getNbBits() :
|
248
|
+
* Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX
|
249
|
+
* Note 1 : is not inlined, as HUF_CElt definition is private
|
250
|
+
* Note 2 : const void* used, so that it can provide a statically allocated table as argument (which uses type U32) */
|
251
|
+
U32 HUF_getNbBits(const void* symbolTable, U32 symbolValue);
|
245
252
|
|
246
253
|
/*
|
247
254
|
* HUF_decompress() does the following:
|
248
|
-
* 1. select the decompression algorithm (
|
255
|
+
* 1. select the decompression algorithm (X1, X2) based on pre-computed heuristics
|
249
256
|
* 2. build Huffman table from save, using HUF_readDTableX?()
|
250
257
|
* 3. decode 1 or 4 segments in parallel using HUF_decompress?X?_usingDTable()
|
251
258
|
*/
|
@@ -253,13 +260,13 @@ size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void
|
|
253
260
|
/** HUF_selectDecoder() :
|
254
261
|
* Tells which decoder is likely to decode faster,
|
255
262
|
* based on a set of pre-computed metrics.
|
256
|
-
* @return : 0==
|
263
|
+
* @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 .
|
257
264
|
* Assumption : 0 < dstSize <= 128 KB */
|
258
265
|
U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
|
259
266
|
|
260
267
|
/**
|
261
268
|
* The minimum workspace size for the `workSpace` used in
|
262
|
-
*
|
269
|
+
* HUF_readDTableX1_wksp() and HUF_readDTableX2_wksp().
|
263
270
|
*
|
264
271
|
* The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when
|
265
272
|
* HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15.
|
@@ -270,14 +277,14 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
|
|
270
277
|
#define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10)
|
271
278
|
#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
|
272
279
|
|
280
|
+
size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize);
|
281
|
+
size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
|
273
282
|
size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize);
|
274
283
|
size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
|
275
|
-
size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize);
|
276
|
-
size_t HUF_readDTableX4_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
|
277
284
|
|
278
285
|
size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
286
|
+
size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
279
287
|
size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
280
|
-
size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
281
288
|
|
282
289
|
|
283
290
|
/* ====================== */
|
@@ -298,25 +305,25 @@ size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
|
|
298
305
|
void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
|
299
306
|
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2);
|
300
307
|
|
301
|
-
size_t
|
302
|
-
size_t
|
308
|
+
size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
|
309
|
+
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
|
303
310
|
|
304
311
|
size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
|
305
312
|
size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);
|
306
|
-
size_t
|
307
|
-
size_t
|
308
|
-
size_t
|
309
|
-
size_t
|
313
|
+
size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
314
|
+
size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
|
315
|
+
size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
316
|
+
size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
|
310
317
|
|
311
318
|
size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */
|
319
|
+
size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
312
320
|
size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
313
|
-
size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
314
321
|
|
315
322
|
/* BMI2 variants.
|
316
323
|
* If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
|
317
324
|
*/
|
318
325
|
size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
|
319
|
-
size_t
|
326
|
+
size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
|
320
327
|
size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
|
321
328
|
size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
|
322
329
|
|
@@ -10,9 +10,10 @@
|
|
10
10
|
|
11
11
|
|
12
12
|
/* ====== Dependencies ======= */
|
13
|
-
#include <stddef.h>
|
14
|
-
#include "
|
13
|
+
#include <stddef.h> /* size_t */
|
14
|
+
#include "debug.h" /* assert */
|
15
15
|
#include "zstd_internal.h" /* ZSTD_malloc, ZSTD_free */
|
16
|
+
#include "pool.h"
|
16
17
|
|
17
18
|
/* ====== Compiler specifics ====== */
|
18
19
|
#if defined(_MSC_VER)
|
@@ -33,8 +34,9 @@ typedef struct POOL_job_s {
|
|
33
34
|
struct POOL_ctx_s {
|
34
35
|
ZSTD_customMem customMem;
|
35
36
|
/* Keep track of the threads */
|
36
|
-
ZSTD_pthread_t
|
37
|
-
size_t
|
37
|
+
ZSTD_pthread_t* threads;
|
38
|
+
size_t threadCapacity;
|
39
|
+
size_t threadLimit;
|
38
40
|
|
39
41
|
/* The queue is a circular buffer */
|
40
42
|
POOL_job *queue;
|
@@ -58,10 +60,10 @@ struct POOL_ctx_s {
|
|
58
60
|
};
|
59
61
|
|
60
62
|
/* POOL_thread() :
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
*/
|
63
|
+
* Work thread for the thread pool.
|
64
|
+
* Waits for jobs and executes them.
|
65
|
+
* @returns : NULL on failure else non-null.
|
66
|
+
*/
|
65
67
|
static void* POOL_thread(void* opaque) {
|
66
68
|
POOL_ctx* const ctx = (POOL_ctx*)opaque;
|
67
69
|
if (!ctx) { return NULL; }
|
@@ -69,14 +71,17 @@ static void* POOL_thread(void* opaque) {
|
|
69
71
|
/* Lock the mutex and wait for a non-empty queue or until shutdown */
|
70
72
|
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
71
73
|
|
72
|
-
while (ctx->queueEmpty
|
74
|
+
while ( ctx->queueEmpty
|
75
|
+
|| (ctx->numThreadsBusy >= ctx->threadLimit) ) {
|
76
|
+
if (ctx->shutdown) {
|
77
|
+
/* even if !queueEmpty, (possible if numThreadsBusy >= threadLimit),
|
78
|
+
* a few threads will be shutdown while !queueEmpty,
|
79
|
+
* but enough threads will remain active to finish the queue */
|
80
|
+
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
81
|
+
return opaque;
|
82
|
+
}
|
73
83
|
ZSTD_pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex);
|
74
84
|
}
|
75
|
-
/* empty => shutting down: so stop */
|
76
|
-
if (ctx->queueEmpty) {
|
77
|
-
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
78
|
-
return opaque;
|
79
|
-
}
|
80
85
|
/* Pop a job off the queue */
|
81
86
|
{ POOL_job const job = ctx->queue[ctx->queueHead];
|
82
87
|
ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize;
|
@@ -89,30 +94,32 @@ static void* POOL_thread(void* opaque) {
|
|
89
94
|
job.function(job.opaque);
|
90
95
|
|
91
96
|
/* If the intended queue size was 0, signal after finishing job */
|
97
|
+
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
98
|
+
ctx->numThreadsBusy--;
|
92
99
|
if (ctx->queueSize == 1) {
|
93
|
-
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
94
|
-
ctx->numThreadsBusy--;
|
95
|
-
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
96
100
|
ZSTD_pthread_cond_signal(&ctx->queuePushCond);
|
97
|
-
|
101
|
+
}
|
102
|
+
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
103
|
+
}
|
98
104
|
} /* for (;;) */
|
99
|
-
/* Unreachable */
|
105
|
+
assert(0); /* Unreachable */
|
100
106
|
}
|
101
107
|
|
102
108
|
POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
|
103
109
|
return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem);
|
104
110
|
}
|
105
111
|
|
106
|
-
POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
|
112
|
+
POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
|
113
|
+
ZSTD_customMem customMem) {
|
107
114
|
POOL_ctx* ctx;
|
108
|
-
/* Check
|
115
|
+
/* Check parameters */
|
109
116
|
if (!numThreads) { return NULL; }
|
110
117
|
/* Allocate the context and zero initialize */
|
111
118
|
ctx = (POOL_ctx*)ZSTD_calloc(sizeof(POOL_ctx), customMem);
|
112
119
|
if (!ctx) { return NULL; }
|
113
120
|
/* Initialize the job queue.
|
114
|
-
* It needs one extra space since one space is wasted to differentiate
|
115
|
-
* and full queues.
|
121
|
+
* It needs one extra space since one space is wasted to differentiate
|
122
|
+
* empty and full queues.
|
116
123
|
*/
|
117
124
|
ctx->queueSize = queueSize + 1;
|
118
125
|
ctx->queue = (POOL_job*)ZSTD_malloc(ctx->queueSize * sizeof(POOL_job), customMem);
|
@@ -126,7 +133,7 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customM
|
|
126
133
|
ctx->shutdown = 0;
|
127
134
|
/* Allocate space for the thread handles */
|
128
135
|
ctx->threads = (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), customMem);
|
129
|
-
ctx->
|
136
|
+
ctx->threadCapacity = 0;
|
130
137
|
ctx->customMem = customMem;
|
131
138
|
/* Check for errors */
|
132
139
|
if (!ctx->threads || !ctx->queue) { POOL_free(ctx); return NULL; }
|
@@ -134,11 +141,12 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customM
|
|
134
141
|
{ size_t i;
|
135
142
|
for (i = 0; i < numThreads; ++i) {
|
136
143
|
if (ZSTD_pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) {
|
137
|
-
ctx->
|
144
|
+
ctx->threadCapacity = i;
|
138
145
|
POOL_free(ctx);
|
139
146
|
return NULL;
|
140
147
|
} }
|
141
|
-
ctx->
|
148
|
+
ctx->threadCapacity = numThreads;
|
149
|
+
ctx->threadLimit = numThreads;
|
142
150
|
}
|
143
151
|
return ctx;
|
144
152
|
}
|
@@ -156,8 +164,8 @@ static void POOL_join(POOL_ctx* ctx) {
|
|
156
164
|
ZSTD_pthread_cond_broadcast(&ctx->queuePopCond);
|
157
165
|
/* Join all of the threads */
|
158
166
|
{ size_t i;
|
159
|
-
for (i = 0; i < ctx->
|
160
|
-
ZSTD_pthread_join(ctx->threads[i], NULL);
|
167
|
+
for (i = 0; i < ctx->threadCapacity; ++i) {
|
168
|
+
ZSTD_pthread_join(ctx->threads[i], NULL); /* note : could fail */
|
161
169
|
} }
|
162
170
|
}
|
163
171
|
|
@@ -172,24 +180,68 @@ void POOL_free(POOL_ctx *ctx) {
|
|
172
180
|
ZSTD_free(ctx, ctx->customMem);
|
173
181
|
}
|
174
182
|
|
183
|
+
|
184
|
+
|
175
185
|
size_t POOL_sizeof(POOL_ctx *ctx) {
|
176
186
|
if (ctx==NULL) return 0; /* supports sizeof NULL */
|
177
187
|
return sizeof(*ctx)
|
178
188
|
+ ctx->queueSize * sizeof(POOL_job)
|
179
|
-
+ ctx->
|
189
|
+
+ ctx->threadCapacity * sizeof(ZSTD_pthread_t);
|
190
|
+
}
|
191
|
+
|
192
|
+
|
193
|
+
/* @return : 0 on success, 1 on error */
|
194
|
+
static int POOL_resize_internal(POOL_ctx* ctx, size_t numThreads)
|
195
|
+
{
|
196
|
+
if (numThreads <= ctx->threadCapacity) {
|
197
|
+
if (!numThreads) return 1;
|
198
|
+
ctx->threadLimit = numThreads;
|
199
|
+
return 0;
|
200
|
+
}
|
201
|
+
/* numThreads > threadCapacity */
|
202
|
+
{ ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem);
|
203
|
+
if (!threadPool) return 1;
|
204
|
+
/* replace existing thread pool */
|
205
|
+
memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(*threadPool));
|
206
|
+
ZSTD_free(ctx->threads, ctx->customMem);
|
207
|
+
ctx->threads = threadPool;
|
208
|
+
/* Initialize additional threads */
|
209
|
+
{ size_t threadId;
|
210
|
+
for (threadId = ctx->threadCapacity; threadId < numThreads; ++threadId) {
|
211
|
+
if (ZSTD_pthread_create(&threadPool[threadId], NULL, &POOL_thread, ctx)) {
|
212
|
+
ctx->threadCapacity = threadId;
|
213
|
+
return 1;
|
214
|
+
} }
|
215
|
+
} }
|
216
|
+
/* successfully expanded */
|
217
|
+
ctx->threadCapacity = numThreads;
|
218
|
+
ctx->threadLimit = numThreads;
|
219
|
+
return 0;
|
220
|
+
}
|
221
|
+
|
222
|
+
/* @return : 0 on success, 1 on error */
|
223
|
+
int POOL_resize(POOL_ctx* ctx, size_t numThreads)
|
224
|
+
{
|
225
|
+
int result;
|
226
|
+
if (ctx==NULL) return 1;
|
227
|
+
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
228
|
+
result = POOL_resize_internal(ctx, numThreads);
|
229
|
+
ZSTD_pthread_cond_broadcast(&ctx->queuePopCond);
|
230
|
+
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
231
|
+
return result;
|
180
232
|
}
|
181
233
|
|
182
234
|
/**
|
183
235
|
* Returns 1 if the queue is full and 0 otherwise.
|
184
236
|
*
|
185
|
-
*
|
186
|
-
* then a queue is empty if there is a thread free
|
237
|
+
* When queueSize is 1 (pool was created with an intended queueSize of 0),
|
238
|
+
* then a queue is empty if there is a thread free _and_ no job is waiting.
|
187
239
|
*/
|
188
240
|
static int isQueueFull(POOL_ctx const* ctx) {
|
189
241
|
if (ctx->queueSize > 1) {
|
190
242
|
return ctx->queueHead == ((ctx->queueTail + 1) % ctx->queueSize);
|
191
243
|
} else {
|
192
|
-
return ctx->numThreadsBusy == ctx->
|
244
|
+
return (ctx->numThreadsBusy == ctx->threadLimit) ||
|
193
245
|
!ctx->queueEmpty;
|
194
246
|
}
|
195
247
|
}
|
@@ -263,6 +315,11 @@ void POOL_free(POOL_ctx* ctx) {
|
|
263
315
|
(void)ctx;
|
264
316
|
}
|
265
317
|
|
318
|
+
int POOL_resize(POOL_ctx* ctx, size_t numThreads) {
|
319
|
+
(void)ctx; (void)numThreads;
|
320
|
+
return 0;
|
321
|
+
}
|
322
|
+
|
266
323
|
void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) {
|
267
324
|
(void)ctx;
|
268
325
|
function(opaque);
|
@@ -30,40 +30,50 @@ typedef struct POOL_ctx_s POOL_ctx;
|
|
30
30
|
*/
|
31
31
|
POOL_ctx* POOL_create(size_t numThreads, size_t queueSize);
|
32
32
|
|
33
|
-
POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
|
33
|
+
POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
|
34
|
+
ZSTD_customMem customMem);
|
34
35
|
|
35
36
|
/*! POOL_free() :
|
36
|
-
|
37
|
-
*/
|
37
|
+
* Free a thread pool returned by POOL_create().
|
38
|
+
*/
|
38
39
|
void POOL_free(POOL_ctx* ctx);
|
39
40
|
|
41
|
+
/*! POOL_resize() :
|
42
|
+
* Expands or shrinks pool's number of threads.
|
43
|
+
* This is more efficient than releasing + creating a new context,
|
44
|
+
* since it tries to preserve and re-use existing threads.
|
45
|
+
* `numThreads` must be at least 1.
|
46
|
+
* @return : 0 when resize was successful,
|
47
|
+
* !0 (typically 1) if there is an error.
|
48
|
+
* note : only numThreads can be resized, queueSize remains unchanged.
|
49
|
+
*/
|
50
|
+
int POOL_resize(POOL_ctx* ctx, size_t numThreads);
|
51
|
+
|
40
52
|
/*! POOL_sizeof() :
|
41
|
-
|
42
|
-
|
53
|
+
* @return threadpool memory usage
|
54
|
+
* note : compatible with NULL (returns 0 in this case)
|
55
|
+
*/
|
43
56
|
size_t POOL_sizeof(POOL_ctx* ctx);
|
44
57
|
|
45
58
|
/*! POOL_function :
|
46
|
-
|
47
|
-
*/
|
59
|
+
* The function type that can be added to a thread pool.
|
60
|
+
*/
|
48
61
|
typedef void (*POOL_function)(void*);
|
49
|
-
/*! POOL_add_function :
|
50
|
-
The function type for a generic thread pool add function.
|
51
|
-
*/
|
52
|
-
typedef void (*POOL_add_function)(void*, POOL_function, void*);
|
53
62
|
|
54
63
|
/*! POOL_add() :
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
64
|
+
* Add the job `function(opaque)` to the thread pool. `ctx` must be valid.
|
65
|
+
* Possibly blocks until there is room in the queue.
|
66
|
+
* Note : The function may be executed asynchronously,
|
67
|
+
* therefore, `opaque` must live until function has been completed.
|
68
|
+
*/
|
59
69
|
void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque);
|
60
70
|
|
61
71
|
|
62
72
|
/*! POOL_tryAdd() :
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
*/
|
73
|
+
* Add the job `function(opaque)` to thread pool _if_ a worker is available.
|
74
|
+
* Returns immediately even if not (does not block).
|
75
|
+
* @return : 1 if successful, 0 if not.
|
76
|
+
*/
|
67
77
|
int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque);
|
68
78
|
|
69
79
|
|
@@ -46,11 +46,6 @@ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
|
|
46
46
|
* provides error code string from enum */
|
47
47
|
const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); }
|
48
48
|
|
49
|
-
/*! g_debuglog_enable :
|
50
|
-
* turn on/off debug traces (global switch) */
|
51
|
-
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG >= 2)
|
52
|
-
int g_debuglog_enable = 1;
|
53
|
-
#endif
|
54
49
|
|
55
50
|
|
56
51
|
/*=**************************************************************
|