zstd-ruby 1.3.4.0 → 1.3.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -1
  3. data/ext/zstdruby/libzstd/Makefile +56 -10
  4. data/ext/zstdruby/libzstd/README.md +4 -0
  5. data/ext/zstdruby/libzstd/common/bitstream.h +6 -19
  6. data/ext/zstdruby/libzstd/common/compiler.h +3 -3
  7. data/ext/zstdruby/libzstd/common/cpu.h +1 -2
  8. data/ext/zstdruby/libzstd/common/debug.c +44 -0
  9. data/ext/zstdruby/libzstd/common/debug.h +123 -0
  10. data/ext/zstdruby/libzstd/common/entropy_common.c +16 -1
  11. data/ext/zstdruby/libzstd/common/fse.h +45 -41
  12. data/ext/zstdruby/libzstd/common/fse_decompress.c +1 -1
  13. data/ext/zstdruby/libzstd/common/huf.h +34 -27
  14. data/ext/zstdruby/libzstd/common/pool.c +89 -32
  15. data/ext/zstdruby/libzstd/common/pool.h +29 -19
  16. data/ext/zstdruby/libzstd/common/zstd_common.c +0 -5
  17. data/ext/zstdruby/libzstd/common/zstd_internal.h +3 -37
  18. data/ext/zstdruby/libzstd/compress/fse_compress.c +28 -163
  19. data/ext/zstdruby/libzstd/compress/hist.c +195 -0
  20. data/ext/zstdruby/libzstd/compress/hist.h +92 -0
  21. data/ext/zstdruby/libzstd/compress/huf_compress.c +14 -6
  22. data/ext/zstdruby/libzstd/compress/zstd_compress.c +798 -350
  23. data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +120 -34
  24. data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +247 -87
  25. data/ext/zstdruby/libzstd/compress/zstd_double_fast.h +4 -1
  26. data/ext/zstdruby/libzstd/compress/zstd_fast.c +177 -56
  27. data/ext/zstdruby/libzstd/compress/zstd_fast.h +4 -1
  28. data/ext/zstdruby/libzstd/compress/zstd_lazy.c +331 -65
  29. data/ext/zstdruby/libzstd/compress/zstd_lazy.h +13 -0
  30. data/ext/zstdruby/libzstd/compress/zstd_ldm.c +15 -20
  31. data/ext/zstdruby/libzstd/compress/zstd_ldm.h +1 -2
  32. data/ext/zstdruby/libzstd/compress/zstd_opt.c +503 -300
  33. data/ext/zstdruby/libzstd/compress/zstd_opt.h +7 -0
  34. data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +122 -47
  35. data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +5 -5
  36. data/ext/zstdruby/libzstd/decompress/huf_decompress.c +325 -325
  37. data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +80 -43
  38. data/ext/zstdruby/libzstd/dictBuilder/cover.c +9 -2
  39. data/ext/zstdruby/libzstd/dictBuilder/zdict.c +5 -5
  40. data/ext/zstdruby/libzstd/legacy/zstd_v04.c +12 -61
  41. data/ext/zstdruby/libzstd/zstd.h +137 -69
  42. data/lib/zstd-ruby/version.rb +1 -1
  43. metadata +7 -3
@@ -49,7 +49,7 @@
49
49
  * Error Management
50
50
  ****************************************************************/
51
51
  #define FSE_isError ERR_isError
52
- #define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
52
+ #define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */
53
53
 
54
54
  /* check and forward error code */
55
55
  #define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; }
@@ -1,7 +1,7 @@
1
1
  /* ******************************************************************
2
- Huffman coder, part of New Generation Entropy library
3
- header file
4
- Copyright (C) 2013-2016, Yann Collet.
2
+ huff0 huffman codec,
3
+ part of Finite State Entropy library
4
+ Copyright (C) 2013-present, Yann Collet.
5
5
 
6
6
  BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
7
7
 
@@ -163,25 +163,25 @@ HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity,
163
163
  /* static allocation of HUF's DTable */
164
164
  typedef U32 HUF_DTable;
165
165
  #define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog)))
166
- #define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
166
+ #define HUF_CREATE_STATIC_DTABLEX1(DTable, maxTableLog) \
167
167
  HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) }
168
- #define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \
168
+ #define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
169
169
  HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) }
170
170
 
171
171
 
172
172
  /* ****************************************
173
173
  * Advanced decompression functions
174
174
  ******************************************/
175
- size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
176
- size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
175
+ size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
176
+ size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
177
177
 
178
178
  size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */
179
179
  size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
180
180
  size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */
181
- size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
182
- size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
183
- size_t HUF_decompress4X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
184
- size_t HUF_decompress4X4_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
181
+ size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
182
+ size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
183
+ size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
184
+ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
185
185
 
186
186
 
187
187
  /* ****************************************
@@ -208,7 +208,7 @@ size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, si
208
208
  typedef enum {
209
209
  HUF_repeat_none, /**< Cannot use the previous table */
210
210
  HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */
211
- HUF_repeat_valid /**< Can use the previous table and it is asumed to be valid */
211
+ HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */
212
212
  } HUF_repeat;
213
213
  /** HUF_compress4X_repeat() :
214
214
  * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
@@ -227,7 +227,9 @@ size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
227
227
  */
228
228
  #define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1)
229
229
  #define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned))
230
- size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize);
230
+ size_t HUF_buildCTable_wksp (HUF_CElt* tree,
231
+ const U32* count, U32 maxSymbolValue, U32 maxNbBits,
232
+ void* workSpace, size_t wkspSize);
231
233
 
232
234
  /*! HUF_readStats() :
233
235
  * Read compact Huffman tree, saved by HUF_writeCTable().
@@ -242,10 +244,15 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize,
242
244
  * Loading a CTable saved with HUF_writeCTable() */
243
245
  size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
244
246
 
247
+ /** HUF_getNbBits() :
248
+ * Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX
249
+ * Note 1 : is not inlined, as HUF_CElt definition is private
250
+ * Note 2 : const void* used, so that it can provide a statically allocated table as argument (which uses type U32) */
251
+ U32 HUF_getNbBits(const void* symbolTable, U32 symbolValue);
245
252
 
246
253
  /*
247
254
  * HUF_decompress() does the following:
248
- * 1. select the decompression algorithm (X2, X4) based on pre-computed heuristics
255
+ * 1. select the decompression algorithm (X1, X2) based on pre-computed heuristics
249
256
  * 2. build Huffman table from save, using HUF_readDTableX?()
250
257
  * 3. decode 1 or 4 segments in parallel using HUF_decompress?X?_usingDTable()
251
258
  */
@@ -253,13 +260,13 @@ size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void
253
260
  /** HUF_selectDecoder() :
254
261
  * Tells which decoder is likely to decode faster,
255
262
  * based on a set of pre-computed metrics.
256
- * @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
263
+ * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 .
257
264
  * Assumption : 0 < dstSize <= 128 KB */
258
265
  U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
259
266
 
260
267
  /**
261
268
  * The minimum workspace size for the `workSpace` used in
262
- * HUF_readDTableX2_wksp() and HUF_readDTableX4_wksp().
269
+ * HUF_readDTableX1_wksp() and HUF_readDTableX2_wksp().
263
270
  *
264
271
  * The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when
265
272
  * HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15.
@@ -270,14 +277,14 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
270
277
  #define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10)
271
278
  #define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
272
279
 
280
+ size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize);
281
+ size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
273
282
  size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize);
274
283
  size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
275
- size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize);
276
- size_t HUF_readDTableX4_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
277
284
 
278
285
  size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
286
+ size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
279
287
  size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
280
- size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
281
288
 
282
289
 
283
290
  /* ====================== */
@@ -298,25 +305,25 @@ size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
298
305
  void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
299
306
  HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2);
300
307
 
301
- size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
302
- size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
308
+ size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
309
+ size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
303
310
 
304
311
  size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
305
312
  size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);
306
- size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
307
- size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
308
- size_t HUF_decompress1X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
309
- size_t HUF_decompress1X4_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
313
+ size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
314
+ size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
315
+ size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
316
+ size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
310
317
 
311
318
  size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */
319
+ size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
312
320
  size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
313
- size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
314
321
 
315
322
  /* BMI2 variants.
316
323
  * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
317
324
  */
318
325
  size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
319
- size_t HUF_decompress1X2_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
326
+ size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
320
327
  size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
321
328
  size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
322
329
 
@@ -10,9 +10,10 @@
10
10
 
11
11
 
12
12
  /* ====== Dependencies ======= */
13
- #include <stddef.h> /* size_t */
14
- #include "pool.h"
13
+ #include <stddef.h> /* size_t */
14
+ #include "debug.h" /* assert */
15
15
  #include "zstd_internal.h" /* ZSTD_malloc, ZSTD_free */
16
+ #include "pool.h"
16
17
 
17
18
  /* ====== Compiler specifics ====== */
18
19
  #if defined(_MSC_VER)
@@ -33,8 +34,9 @@ typedef struct POOL_job_s {
33
34
  struct POOL_ctx_s {
34
35
  ZSTD_customMem customMem;
35
36
  /* Keep track of the threads */
36
- ZSTD_pthread_t *threads;
37
- size_t numThreads;
37
+ ZSTD_pthread_t* threads;
38
+ size_t threadCapacity;
39
+ size_t threadLimit;
38
40
 
39
41
  /* The queue is a circular buffer */
40
42
  POOL_job *queue;
@@ -58,10 +60,10 @@ struct POOL_ctx_s {
58
60
  };
59
61
 
60
62
  /* POOL_thread() :
61
- Work thread for the thread pool.
62
- Waits for jobs and executes them.
63
- @returns : NULL on failure else non-null.
64
- */
63
+ * Work thread for the thread pool.
64
+ * Waits for jobs and executes them.
65
+ * @returns : NULL on failure else non-null.
66
+ */
65
67
  static void* POOL_thread(void* opaque) {
66
68
  POOL_ctx* const ctx = (POOL_ctx*)opaque;
67
69
  if (!ctx) { return NULL; }
@@ -69,14 +71,17 @@ static void* POOL_thread(void* opaque) {
69
71
  /* Lock the mutex and wait for a non-empty queue or until shutdown */
70
72
  ZSTD_pthread_mutex_lock(&ctx->queueMutex);
71
73
 
72
- while (ctx->queueEmpty && !ctx->shutdown) {
74
+ while ( ctx->queueEmpty
75
+ || (ctx->numThreadsBusy >= ctx->threadLimit) ) {
76
+ if (ctx->shutdown) {
77
+ /* even if !queueEmpty, (possible if numThreadsBusy >= threadLimit),
78
+ * a few threads will be shutdown while !queueEmpty,
79
+ * but enough threads will remain active to finish the queue */
80
+ ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
81
+ return opaque;
82
+ }
73
83
  ZSTD_pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex);
74
84
  }
75
- /* empty => shutting down: so stop */
76
- if (ctx->queueEmpty) {
77
- ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
78
- return opaque;
79
- }
80
85
  /* Pop a job off the queue */
81
86
  { POOL_job const job = ctx->queue[ctx->queueHead];
82
87
  ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize;
@@ -89,30 +94,32 @@ static void* POOL_thread(void* opaque) {
89
94
  job.function(job.opaque);
90
95
 
91
96
  /* If the intended queue size was 0, signal after finishing job */
97
+ ZSTD_pthread_mutex_lock(&ctx->queueMutex);
98
+ ctx->numThreadsBusy--;
92
99
  if (ctx->queueSize == 1) {
93
- ZSTD_pthread_mutex_lock(&ctx->queueMutex);
94
- ctx->numThreadsBusy--;
95
- ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
96
100
  ZSTD_pthread_cond_signal(&ctx->queuePushCond);
97
- } }
101
+ }
102
+ ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
103
+ }
98
104
  } /* for (;;) */
99
- /* Unreachable */
105
+ assert(0); /* Unreachable */
100
106
  }
101
107
 
102
108
  POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
103
109
  return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem);
104
110
  }
105
111
 
106
- POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) {
112
+ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
113
+ ZSTD_customMem customMem) {
107
114
  POOL_ctx* ctx;
108
- /* Check the parameters */
115
+ /* Check parameters */
109
116
  if (!numThreads) { return NULL; }
110
117
  /* Allocate the context and zero initialize */
111
118
  ctx = (POOL_ctx*)ZSTD_calloc(sizeof(POOL_ctx), customMem);
112
119
  if (!ctx) { return NULL; }
113
120
  /* Initialize the job queue.
114
- * It needs one extra space since one space is wasted to differentiate empty
115
- * and full queues.
121
+ * It needs one extra space since one space is wasted to differentiate
122
+ * empty and full queues.
116
123
  */
117
124
  ctx->queueSize = queueSize + 1;
118
125
  ctx->queue = (POOL_job*)ZSTD_malloc(ctx->queueSize * sizeof(POOL_job), customMem);
@@ -126,7 +133,7 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customM
126
133
  ctx->shutdown = 0;
127
134
  /* Allocate space for the thread handles */
128
135
  ctx->threads = (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), customMem);
129
- ctx->numThreads = 0;
136
+ ctx->threadCapacity = 0;
130
137
  ctx->customMem = customMem;
131
138
  /* Check for errors */
132
139
  if (!ctx->threads || !ctx->queue) { POOL_free(ctx); return NULL; }
@@ -134,11 +141,12 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customM
134
141
  { size_t i;
135
142
  for (i = 0; i < numThreads; ++i) {
136
143
  if (ZSTD_pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) {
137
- ctx->numThreads = i;
144
+ ctx->threadCapacity = i;
138
145
  POOL_free(ctx);
139
146
  return NULL;
140
147
  } }
141
- ctx->numThreads = numThreads;
148
+ ctx->threadCapacity = numThreads;
149
+ ctx->threadLimit = numThreads;
142
150
  }
143
151
  return ctx;
144
152
  }
@@ -156,8 +164,8 @@ static void POOL_join(POOL_ctx* ctx) {
156
164
  ZSTD_pthread_cond_broadcast(&ctx->queuePopCond);
157
165
  /* Join all of the threads */
158
166
  { size_t i;
159
- for (i = 0; i < ctx->numThreads; ++i) {
160
- ZSTD_pthread_join(ctx->threads[i], NULL);
167
+ for (i = 0; i < ctx->threadCapacity; ++i) {
168
+ ZSTD_pthread_join(ctx->threads[i], NULL); /* note : could fail */
161
169
  } }
162
170
  }
163
171
 
@@ -172,24 +180,68 @@ void POOL_free(POOL_ctx *ctx) {
172
180
  ZSTD_free(ctx, ctx->customMem);
173
181
  }
174
182
 
183
+
184
+
175
185
  size_t POOL_sizeof(POOL_ctx *ctx) {
176
186
  if (ctx==NULL) return 0; /* supports sizeof NULL */
177
187
  return sizeof(*ctx)
178
188
  + ctx->queueSize * sizeof(POOL_job)
179
- + ctx->numThreads * sizeof(ZSTD_pthread_t);
189
+ + ctx->threadCapacity * sizeof(ZSTD_pthread_t);
190
+ }
191
+
192
+
193
+ /* @return : 0 on success, 1 on error */
194
+ static int POOL_resize_internal(POOL_ctx* ctx, size_t numThreads)
195
+ {
196
+ if (numThreads <= ctx->threadCapacity) {
197
+ if (!numThreads) return 1;
198
+ ctx->threadLimit = numThreads;
199
+ return 0;
200
+ }
201
+ /* numThreads > threadCapacity */
202
+ { ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem);
203
+ if (!threadPool) return 1;
204
+ /* replace existing thread pool */
205
+ memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(*threadPool));
206
+ ZSTD_free(ctx->threads, ctx->customMem);
207
+ ctx->threads = threadPool;
208
+ /* Initialize additional threads */
209
+ { size_t threadId;
210
+ for (threadId = ctx->threadCapacity; threadId < numThreads; ++threadId) {
211
+ if (ZSTD_pthread_create(&threadPool[threadId], NULL, &POOL_thread, ctx)) {
212
+ ctx->threadCapacity = threadId;
213
+ return 1;
214
+ } }
215
+ } }
216
+ /* successfully expanded */
217
+ ctx->threadCapacity = numThreads;
218
+ ctx->threadLimit = numThreads;
219
+ return 0;
220
+ }
221
+
222
+ /* @return : 0 on success, 1 on error */
223
+ int POOL_resize(POOL_ctx* ctx, size_t numThreads)
224
+ {
225
+ int result;
226
+ if (ctx==NULL) return 1;
227
+ ZSTD_pthread_mutex_lock(&ctx->queueMutex);
228
+ result = POOL_resize_internal(ctx, numThreads);
229
+ ZSTD_pthread_cond_broadcast(&ctx->queuePopCond);
230
+ ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
231
+ return result;
180
232
  }
181
233
 
182
234
  /**
183
235
  * Returns 1 if the queue is full and 0 otherwise.
184
236
  *
185
- * If the queueSize is 1 (the pool was created with an intended queueSize of 0),
186
- * then a queue is empty if there is a thread free and no job is waiting.
237
+ * When queueSize is 1 (pool was created with an intended queueSize of 0),
238
+ * then a queue is empty if there is a thread free _and_ no job is waiting.
187
239
  */
188
240
  static int isQueueFull(POOL_ctx const* ctx) {
189
241
  if (ctx->queueSize > 1) {
190
242
  return ctx->queueHead == ((ctx->queueTail + 1) % ctx->queueSize);
191
243
  } else {
192
- return ctx->numThreadsBusy == ctx->numThreads ||
244
+ return (ctx->numThreadsBusy == ctx->threadLimit) ||
193
245
  !ctx->queueEmpty;
194
246
  }
195
247
  }
@@ -263,6 +315,11 @@ void POOL_free(POOL_ctx* ctx) {
263
315
  (void)ctx;
264
316
  }
265
317
 
318
+ int POOL_resize(POOL_ctx* ctx, size_t numThreads) {
319
+ (void)ctx; (void)numThreads;
320
+ return 0;
321
+ }
322
+
266
323
  void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) {
267
324
  (void)ctx;
268
325
  function(opaque);
@@ -30,40 +30,50 @@ typedef struct POOL_ctx_s POOL_ctx;
30
30
  */
31
31
  POOL_ctx* POOL_create(size_t numThreads, size_t queueSize);
32
32
 
33
- POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem);
33
+ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
34
+ ZSTD_customMem customMem);
34
35
 
35
36
  /*! POOL_free() :
36
- Free a thread pool returned by POOL_create().
37
- */
37
+ * Free a thread pool returned by POOL_create().
38
+ */
38
39
  void POOL_free(POOL_ctx* ctx);
39
40
 
41
+ /*! POOL_resize() :
42
+ * Expands or shrinks pool's number of threads.
43
+ * This is more efficient than releasing + creating a new context,
44
+ * since it tries to preserve and re-use existing threads.
45
+ * `numThreads` must be at least 1.
46
+ * @return : 0 when resize was successful,
47
+ * !0 (typically 1) if there is an error.
48
+ * note : only numThreads can be resized, queueSize remains unchanged.
49
+ */
50
+ int POOL_resize(POOL_ctx* ctx, size_t numThreads);
51
+
40
52
  /*! POOL_sizeof() :
41
- return memory usage of pool returned by POOL_create().
42
- */
53
+ * @return threadpool memory usage
54
+ * note : compatible with NULL (returns 0 in this case)
55
+ */
43
56
  size_t POOL_sizeof(POOL_ctx* ctx);
44
57
 
45
58
  /*! POOL_function :
46
- The function type that can be added to a thread pool.
47
- */
59
+ * The function type that can be added to a thread pool.
60
+ */
48
61
  typedef void (*POOL_function)(void*);
49
- /*! POOL_add_function :
50
- The function type for a generic thread pool add function.
51
- */
52
- typedef void (*POOL_add_function)(void*, POOL_function, void*);
53
62
 
54
63
  /*! POOL_add() :
55
- Add the job `function(opaque)` to the thread pool. `ctx` must be valid.
56
- Possibly blocks until there is room in the queue.
57
- Note : The function may be executed asynchronously, so `opaque` must live until the function has been completed.
58
- */
64
+ * Add the job `function(opaque)` to the thread pool. `ctx` must be valid.
65
+ * Possibly blocks until there is room in the queue.
66
+ * Note : The function may be executed asynchronously,
67
+ * therefore, `opaque` must live until function has been completed.
68
+ */
59
69
  void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque);
60
70
 
61
71
 
62
72
  /*! POOL_tryAdd() :
63
- Add the job `function(opaque)` to the thread pool if a worker is available.
64
- return immediately otherwise.
65
- @return : 1 if successful, 0 if not.
66
- */
73
+ * Add the job `function(opaque)` to thread pool _if_ a worker is available.
74
+ * Returns immediately even if not (does not block).
75
+ * @return : 1 if successful, 0 if not.
76
+ */
67
77
  int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque);
68
78
 
69
79
 
@@ -46,11 +46,6 @@ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
46
46
  * provides error code string from enum */
47
47
  const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); }
48
48
 
49
- /*! g_debuglog_enable :
50
- * turn on/off debug traces (global switch) */
51
- #if defined(ZSTD_DEBUG) && (ZSTD_DEBUG >= 2)
52
- int g_debuglog_enable = 1;
53
- #endif
54
49
 
55
50
 
56
51
  /*=**************************************************************