zstdlib 0.4.0 → 0.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (65) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGES.md +6 -1
  3. data/README.md +1 -1
  4. data/ext/zstdlib/extconf.rb +2 -2
  5. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/bitstream.h +3 -2
  6. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/compiler.h +14 -2
  7. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/cpu.h +0 -0
  8. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/debug.c +0 -0
  9. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/debug.h +0 -0
  10. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/entropy_common.c +0 -0
  11. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/error_private.c +0 -0
  12. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/error_private.h +0 -0
  13. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/fse.h +1 -1
  14. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/fse_decompress.c +2 -0
  15. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/huf.h +0 -0
  16. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/mem.h +73 -0
  17. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/pool.c +7 -3
  18. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/pool.h +0 -0
  19. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/threading.c +46 -1
  20. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/threading.h +32 -1
  21. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/xxhash.c +0 -0
  22. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/xxhash.h +0 -0
  23. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/zstd_common.c +0 -0
  24. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/zstd_errors.h +0 -0
  25. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/common/zstd_internal.h +32 -55
  26. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/fse_compress.c +0 -0
  27. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/hist.c +0 -0
  28. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/hist.h +0 -0
  29. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/huf_compress.c +0 -0
  30. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_compress.c +633 -436
  31. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_compress_internal.h +54 -12
  32. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_compress_literals.c +10 -5
  33. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_compress_literals.h +1 -1
  34. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_compress_sequences.c +3 -3
  35. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_compress_sequences.h +1 -1
  36. data/ext/zstdlib/zstd-1.4.4/lib/compress/zstd_cwksp.h +535 -0
  37. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_double_fast.c +9 -9
  38. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_double_fast.h +0 -0
  39. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_fast.c +30 -39
  40. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_fast.h +0 -0
  41. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_lazy.c +5 -5
  42. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_lazy.h +0 -0
  43. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_ldm.c +4 -4
  44. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_ldm.h +0 -0
  45. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_opt.c +1 -1
  46. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstd_opt.h +0 -0
  47. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstdmt_compress.c +32 -26
  48. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/compress/zstdmt_compress.h +0 -0
  49. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/decompress/huf_decompress.c +2 -0
  50. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/decompress/zstd_ddict.c +0 -0
  51. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/decompress/zstd_ddict.h +0 -0
  52. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/decompress/zstd_decompress.c +14 -16
  53. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/decompress/zstd_decompress_block.c +144 -146
  54. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/decompress/zstd_decompress_block.h +0 -0
  55. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/decompress/zstd_decompress_internal.h +0 -0
  56. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/lib/zstd.h +161 -59
  57. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/zlibWrapper/gzclose.c +1 -1
  58. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/zlibWrapper/gzcompatibility.h +0 -0
  59. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/zlibWrapper/gzguts.h +0 -0
  60. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/zlibWrapper/gzlib.c +9 -9
  61. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/zlibWrapper/gzread.c +16 -8
  62. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/zlibWrapper/gzwrite.c +8 -8
  63. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/zlibWrapper/zstd_zlibwrapper.c +15 -11
  64. data/ext/zstdlib/{zstd-1.4.3 → zstd-1.4.4}/zlibWrapper/zstd_zlibwrapper.h +0 -0
  65. metadata +62 -61
@@ -19,6 +19,7 @@
19
19
  * Dependencies
20
20
  ***************************************/
21
21
  #include "zstd_internal.h"
22
+ #include "zstd_cwksp.h"
22
23
  #ifdef ZSTD_MULTITHREAD
23
24
  # include "zstdmt_compress.h"
24
25
  #endif
@@ -192,6 +193,13 @@ typedef struct {
192
193
  size_t capacity; /* The capacity starting from `seq` pointer */
193
194
  } rawSeqStore_t;
194
195
 
196
+ typedef struct {
197
+ int collectSequences;
198
+ ZSTD_Sequence* seqStart;
199
+ size_t seqIndex;
200
+ size_t maxSequences;
201
+ } SeqCollector;
202
+
195
203
  struct ZSTD_CCtx_params_s {
196
204
  ZSTD_format_e format;
197
205
  ZSTD_compressionParameters cParams;
@@ -203,6 +211,9 @@ struct ZSTD_CCtx_params_s {
203
211
  size_t targetCBlockSize; /* Tries to fit compressed block size to be around targetCBlockSize.
204
212
  * No target when targetCBlockSize == 0.
205
213
  * There is no guarantee on compressed block size */
214
+ int srcSizeHint; /* User's best guess of source size.
215
+ * Hint is not valid when srcSizeHint == 0.
216
+ * There is no guarantee that hint is close to actual source size */
206
217
 
207
218
  ZSTD_dictAttachPref_e attachDictPref;
208
219
  ZSTD_literalCompressionMode_e literalCompressionMode;
@@ -228,9 +239,7 @@ struct ZSTD_CCtx_s {
228
239
  ZSTD_CCtx_params appliedParams;
229
240
  U32 dictID;
230
241
 
231
- int workSpaceOversizedDuration;
232
- void* workSpace;
233
- size_t workSpaceSize;
242
+ ZSTD_cwksp workspace; /* manages buffer for dynamic allocations */
234
243
  size_t blockSize;
235
244
  unsigned long long pledgedSrcSizePlusOne; /* this way, 0 (default) == unknown */
236
245
  unsigned long long consumedSrcSize;
@@ -238,6 +247,8 @@ struct ZSTD_CCtx_s {
238
247
  XXH64_state_t xxhState;
239
248
  ZSTD_customMem customMem;
240
249
  size_t staticSize;
250
+ SeqCollector seqCollector;
251
+ int isFirstBlock;
241
252
 
242
253
  seqStore_t seqStore; /* sequences storage ptrs */
243
254
  ldmState_t ldmState; /* long distance matching state */
@@ -337,26 +348,57 @@ MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat)
337
348
  return (srcSize >> minlog) + 2;
338
349
  }
339
350
 
351
+ /*! ZSTD_safecopyLiterals() :
352
+ * memcpy() function that won't read beyond more than WILDCOPY_OVERLENGTH bytes past ilimit_w.
353
+ * Only called when the sequence ends past ilimit_w, so it only needs to be optimized for single
354
+ * large copies.
355
+ */
356
+ static void ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const iend, BYTE const* ilimit_w) {
357
+ assert(iend > ilimit_w);
358
+ if (ip <= ilimit_w) {
359
+ ZSTD_wildcopy(op, ip, ilimit_w - ip, ZSTD_no_overlap);
360
+ op += ilimit_w - ip;
361
+ ip = ilimit_w;
362
+ }
363
+ while (ip < iend) *op++ = *ip++;
364
+ }
365
+
340
366
  /*! ZSTD_storeSeq() :
341
- * Store a sequence (literal length, literals, offset code and match length code) into seqStore_t.
342
- * `offsetCode` : distance to match + 3 (values 1-3 are repCodes).
367
+ * Store a sequence (litlen, litPtr, offCode and mlBase) into seqStore_t.
368
+ * `offCode` : distance to match + ZSTD_REP_MOVE (values <= ZSTD_REP_MOVE are repCodes).
343
369
  * `mlBase` : matchLength - MINMATCH
370
+ * Allowed to overread literals up to litLimit.
344
371
  */
345
- MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t mlBase)
372
+ HINT_INLINE UNUSED_ATTR
373
+ void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, const BYTE* litLimit, U32 offCode, size_t mlBase)
346
374
  {
375
+ BYTE const* const litLimit_w = litLimit - WILDCOPY_OVERLENGTH;
376
+ BYTE const* const litEnd = literals + litLength;
347
377
  #if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6)
348
378
  static const BYTE* g_start = NULL;
349
379
  if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */
350
380
  { U32 const pos = (U32)((const BYTE*)literals - g_start);
351
381
  DEBUGLOG(6, "Cpos%7u :%3u literals, match%4u bytes at offCode%7u",
352
- pos, (U32)litLength, (U32)mlBase+MINMATCH, (U32)offsetCode);
382
+ pos, (U32)litLength, (U32)mlBase+MINMATCH, (U32)offCode);
353
383
  }
354
384
  #endif
355
385
  assert((size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart) < seqStorePtr->maxNbSeq);
356
386
  /* copy Literals */
357
387
  assert(seqStorePtr->maxNbLit <= 128 KB);
358
388
  assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit);
359
- ZSTD_wildcopy(seqStorePtr->lit, literals, (ptrdiff_t)litLength, ZSTD_no_overlap);
389
+ assert(literals + litLength <= litLimit);
390
+ if (litEnd <= litLimit_w) {
391
+ /* Common case we can use wildcopy.
392
+ * First copy 16 bytes, because literals are likely short.
393
+ */
394
+ assert(WILDCOPY_OVERLENGTH >= 16);
395
+ ZSTD_copy16(seqStorePtr->lit, literals);
396
+ if (litLength > 16) {
397
+ ZSTD_wildcopy(seqStorePtr->lit+16, literals+16, (ptrdiff_t)litLength-16, ZSTD_no_overlap);
398
+ }
399
+ } else {
400
+ ZSTD_safecopyLiterals(seqStorePtr->lit, literals, litEnd, litLimit_w);
401
+ }
360
402
  seqStorePtr->lit += litLength;
361
403
 
362
404
  /* literal Length */
@@ -368,7 +410,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v
368
410
  seqStorePtr->sequences[0].litLength = (U16)litLength;
369
411
 
370
412
  /* match offset */
371
- seqStorePtr->sequences[0].offset = offsetCode + 1;
413
+ seqStorePtr->sequences[0].offset = offCode + 1;
372
414
 
373
415
  /* match Length */
374
416
  if (mlBase>0xFFFF) {
@@ -910,7 +952,7 @@ ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
910
952
  size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
911
953
  const void* dict, size_t dictSize,
912
954
  const ZSTD_CDict* cdict,
913
- ZSTD_CCtx_params params, unsigned long long pledgedSrcSize);
955
+ const ZSTD_CCtx_params* params, unsigned long long pledgedSrcSize);
914
956
 
915
957
  void ZSTD_resetSeqStore(seqStore_t* ssPtr);
916
958
 
@@ -925,7 +967,7 @@ size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx,
925
967
  ZSTD_dictContentType_e dictContentType,
926
968
  ZSTD_dictTableLoadMethod_e dtlm,
927
969
  const ZSTD_CDict* cdict,
928
- ZSTD_CCtx_params params,
970
+ const ZSTD_CCtx_params* params,
929
971
  unsigned long long pledgedSrcSize);
930
972
 
931
973
  /* ZSTD_compress_advanced_internal() :
@@ -934,7 +976,7 @@ size_t ZSTD_compress_advanced_internal(ZSTD_CCtx* cctx,
934
976
  void* dst, size_t dstCapacity,
935
977
  const void* src, size_t srcSize,
936
978
  const void* dict,size_t dictSize,
937
- ZSTD_CCtx_params params);
979
+ const ZSTD_CCtx_params* params);
938
980
 
939
981
 
940
982
  /* ZSTD_writeLastEmptyBlock() :
@@ -70,7 +70,7 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
70
70
  ZSTD_strategy strategy, int disableLiteralCompression,
71
71
  void* dst, size_t dstCapacity,
72
72
  const void* src, size_t srcSize,
73
- void* workspace, size_t wkspSize,
73
+ void* entropyWorkspace, size_t entropyWorkspaceSize,
74
74
  const int bmi2)
75
75
  {
76
76
  size_t const minGain = ZSTD_minGain(srcSize, strategy);
@@ -99,10 +99,15 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
99
99
  { HUF_repeat repeat = prevHuf->repeatMode;
100
100
  int const preferRepeat = strategy < ZSTD_lazy ? srcSize <= 1024 : 0;
101
101
  if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
102
- cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11,
103
- workspace, wkspSize, (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2)
104
- : HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11,
105
- workspace, wkspSize, (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2);
102
+ cLitSize = singleStream ?
103
+ HUF_compress1X_repeat(
104
+ ostart+lhSize, dstCapacity-lhSize, src, srcSize,
105
+ 255, 11, entropyWorkspace, entropyWorkspaceSize,
106
+ (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2) :
107
+ HUF_compress4X_repeat(
108
+ ostart+lhSize, dstCapacity-lhSize, src, srcSize,
109
+ 255, 11, entropyWorkspace, entropyWorkspaceSize,
110
+ (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2);
106
111
  if (repeat != HUF_repeat_none) {
107
112
  /* reused the existing table */
108
113
  hType = set_repeat;
@@ -23,7 +23,7 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
23
23
  ZSTD_strategy strategy, int disableLiteralCompression,
24
24
  void* dst, size_t dstCapacity,
25
25
  const void* src, size_t srcSize,
26
- void* workspace, size_t wkspSize,
26
+ void* entropyWorkspace, size_t entropyWorkspaceSize,
27
27
  const int bmi2);
28
28
 
29
29
  #endif /* ZSTD_COMPRESS_LITERALS_H */
@@ -222,7 +222,7 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
222
222
  const BYTE* codeTable, size_t nbSeq,
223
223
  const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax,
224
224
  const FSE_CTable* prevCTable, size_t prevCTableSize,
225
- void* workspace, size_t workspaceSize)
225
+ void* entropyWorkspace, size_t entropyWorkspaceSize)
226
226
  {
227
227
  BYTE* op = (BYTE*)dst;
228
228
  const BYTE* const oend = op + dstCapacity;
@@ -238,7 +238,7 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
238
238
  memcpy(nextCTable, prevCTable, prevCTableSize);
239
239
  return 0;
240
240
  case set_basic:
241
- FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, workspace, workspaceSize)); /* note : could be pre-calculated */
241
+ FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize)); /* note : could be pre-calculated */
242
242
  return 0;
243
243
  case set_compressed: {
244
244
  S16 norm[MaxSeq + 1];
@@ -252,7 +252,7 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
252
252
  FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max));
253
253
  { size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */
254
254
  FORWARD_IF_ERROR(NCountSize);
255
- FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, workspace, workspaceSize));
255
+ FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, entropyWorkspace, entropyWorkspaceSize));
256
256
  return NCountSize;
257
257
  }
258
258
  }
@@ -35,7 +35,7 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity,
35
35
  const BYTE* codeTable, size_t nbSeq,
36
36
  const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax,
37
37
  const FSE_CTable* prevCTable, size_t prevCTableSize,
38
- void* workspace, size_t workspaceSize);
38
+ void* entropyWorkspace, size_t entropyWorkspaceSize);
39
39
 
40
40
  size_t ZSTD_encodeSequences(
41
41
  void* dst, size_t dstCapacity,
@@ -0,0 +1,535 @@
1
+ /*
2
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
3
+ * All rights reserved.
4
+ *
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
9
+ */
10
+
11
+ #ifndef ZSTD_CWKSP_H
12
+ #define ZSTD_CWKSP_H
13
+
14
+ /*-*************************************
15
+ * Dependencies
16
+ ***************************************/
17
+ #include "zstd_internal.h"
18
+
19
+ #if defined (__cplusplus)
20
+ extern "C" {
21
+ #endif
22
+
23
+ /*-*************************************
24
+ * Constants
25
+ ***************************************/
26
+
27
+ /* define "workspace is too large" as this number of times larger than needed */
28
+ #define ZSTD_WORKSPACETOOLARGE_FACTOR 3
29
+
30
+ /* when workspace is continuously too large
31
+ * during at least this number of times,
32
+ * context's memory usage is considered wasteful,
33
+ * because it's sized to handle a worst case scenario which rarely happens.
34
+ * In which case, resize it down to free some memory */
35
+ #define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128
36
+
37
+ /* Since the workspace is effectively its own little malloc implementation /
38
+ * arena, when we run under ASAN, we should similarly insert redzones between
39
+ * each internal element of the workspace, so ASAN will catch overruns that
40
+ * reach outside an object but that stay inside the workspace.
41
+ *
42
+ * This defines the size of that redzone.
43
+ */
44
+ #ifndef ZSTD_CWKSP_ASAN_REDZONE_SIZE
45
+ #define ZSTD_CWKSP_ASAN_REDZONE_SIZE 128
46
+ #endif
47
+
48
+ /*-*************************************
49
+ * Structures
50
+ ***************************************/
51
+ typedef enum {
52
+ ZSTD_cwksp_alloc_objects,
53
+ ZSTD_cwksp_alloc_buffers,
54
+ ZSTD_cwksp_alloc_aligned
55
+ } ZSTD_cwksp_alloc_phase_e;
56
+
57
+ /**
58
+ * Zstd fits all its internal datastructures into a single continuous buffer,
59
+ * so that it only needs to perform a single OS allocation (or so that a buffer
60
+ * can be provided to it and it can perform no allocations at all). This buffer
61
+ * is called the workspace.
62
+ *
63
+ * Several optimizations complicate that process of allocating memory ranges
64
+ * from this workspace for each internal datastructure:
65
+ *
66
+ * - These different internal datastructures have different setup requirements:
67
+ *
68
+ * - The static objects need to be cleared once and can then be trivially
69
+ * reused for each compression.
70
+ *
71
+ * - Various buffers don't need to be initialized at all--they are always
72
+ * written into before they're read.
73
+ *
74
+ * - The matchstate tables have a unique requirement that they don't need
75
+ * their memory to be totally cleared, but they do need the memory to have
76
+ * some bound, i.e., a guarantee that all values in the memory they've been
77
+ * allocated is less than some maximum value (which is the starting value
78
+ * for the indices that they will then use for compression). When this
79
+ * guarantee is provided to them, they can use the memory without any setup
80
+ * work. When it can't, they have to clear the area.
81
+ *
82
+ * - These buffers also have different alignment requirements.
83
+ *
84
+ * - We would like to reuse the objects in the workspace for multiple
85
+ * compressions without having to perform any expensive reallocation or
86
+ * reinitialization work.
87
+ *
88
+ * - We would like to be able to efficiently reuse the workspace across
89
+ * multiple compressions **even when the compression parameters change** and
90
+ * we need to resize some of the objects (where possible).
91
+ *
92
+ * To attempt to manage this buffer, given these constraints, the ZSTD_cwksp
93
+ * abstraction was created. It works as follows:
94
+ *
95
+ * Workspace Layout:
96
+ *
97
+ * [ ... workspace ... ]
98
+ * [objects][tables ... ->] free space [<- ... aligned][<- ... buffers]
99
+ *
100
+ * The various objects that live in the workspace are divided into the
101
+ * following categories, and are allocated separately:
102
+ *
103
+ * - Static objects: this is optionally the enclosing ZSTD_CCtx or ZSTD_CDict,
104
+ * so that literally everything fits in a single buffer. Note: if present,
105
+ * this must be the first object in the workspace, since ZSTD_free{CCtx,
106
+ * CDict}() rely on a pointer comparison to see whether one or two frees are
107
+ * required.
108
+ *
109
+ * - Fixed size objects: these are fixed-size, fixed-count objects that are
110
+ * nonetheless "dynamically" allocated in the workspace so that we can
111
+ * control how they're initialized separately from the broader ZSTD_CCtx.
112
+ * Examples:
113
+ * - Entropy Workspace
114
+ * - 2 x ZSTD_compressedBlockState_t
115
+ * - CDict dictionary contents
116
+ *
117
+ * - Tables: these are any of several different datastructures (hash tables,
118
+ * chain tables, binary trees) that all respect a common format: they are
119
+ * uint32_t arrays, all of whose values are between 0 and (nextSrc - base).
120
+ * Their sizes depend on the cparams.
121
+ *
122
+ * - Aligned: these buffers are used for various purposes that require 4 byte
123
+ * alignment, but don't require any initialization before they're used.
124
+ *
125
+ * - Buffers: these buffers are used for various purposes that don't require
126
+ * any alignment or initialization before they're used. This means they can
127
+ * be moved around at no cost for a new compression.
128
+ *
129
+ * Allocating Memory:
130
+ *
131
+ * The various types of objects must be allocated in order, so they can be
132
+ * correctly packed into the workspace buffer. That order is:
133
+ *
134
+ * 1. Objects
135
+ * 2. Buffers
136
+ * 3. Aligned
137
+ * 4. Tables
138
+ *
139
+ * Attempts to reserve objects of different types out of order will fail.
140
+ */
141
+ typedef struct {
142
+ void* workspace;
143
+ void* workspaceEnd;
144
+
145
+ void* objectEnd;
146
+ void* tableEnd;
147
+ void* tableValidEnd;
148
+ void* allocStart;
149
+
150
+ int allocFailed;
151
+ int workspaceOversizedDuration;
152
+ ZSTD_cwksp_alloc_phase_e phase;
153
+ } ZSTD_cwksp;
154
+
155
+ /*-*************************************
156
+ * Functions
157
+ ***************************************/
158
+
159
+ MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws);
160
+
161
+ MEM_STATIC void ZSTD_cwksp_assert_internal_consistency(ZSTD_cwksp* ws) {
162
+ (void)ws;
163
+ assert(ws->workspace <= ws->objectEnd);
164
+ assert(ws->objectEnd <= ws->tableEnd);
165
+ assert(ws->objectEnd <= ws->tableValidEnd);
166
+ assert(ws->tableEnd <= ws->allocStart);
167
+ assert(ws->tableValidEnd <= ws->allocStart);
168
+ assert(ws->allocStart <= ws->workspaceEnd);
169
+ }
170
+
171
+ /**
172
+ * Align must be a power of 2.
173
+ */
174
+ MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) {
175
+ size_t const mask = align - 1;
176
+ assert((align & mask) == 0);
177
+ return (size + mask) & ~mask;
178
+ }
179
+
180
+ /**
181
+ * Use this to determine how much space in the workspace we will consume to
182
+ * allocate this object. (Normally it should be exactly the size of the object,
183
+ * but under special conditions, like ASAN, where we pad each object, it might
184
+ * be larger.)
185
+ *
186
+ * Since tables aren't currently redzoned, you don't need to call through this
187
+ * to figure out how much space you need for the matchState tables. Everything
188
+ * else is though.
189
+ */
190
+ MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) {
191
+ #if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
192
+ return size + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE;
193
+ #else
194
+ return size;
195
+ #endif
196
+ }
197
+
198
+ MEM_STATIC void ZSTD_cwksp_internal_advance_phase(
199
+ ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase) {
200
+ assert(phase >= ws->phase);
201
+ if (phase > ws->phase) {
202
+ if (ws->phase < ZSTD_cwksp_alloc_buffers &&
203
+ phase >= ZSTD_cwksp_alloc_buffers) {
204
+ ws->tableValidEnd = ws->objectEnd;
205
+ }
206
+ if (ws->phase < ZSTD_cwksp_alloc_aligned &&
207
+ phase >= ZSTD_cwksp_alloc_aligned) {
208
+ /* If unaligned allocations down from a too-large top have left us
209
+ * unaligned, we need to realign our alloc ptr. Technically, this
210
+ * can consume space that is unaccounted for in the neededSpace
211
+ * calculation. However, I believe this can only happen when the
212
+ * workspace is too large, and specifically when it is too large
213
+ * by a larger margin than the space that will be consumed. */
214
+ /* TODO: cleaner, compiler warning friendly way to do this??? */
215
+ ws->allocStart = (BYTE*)ws->allocStart - ((size_t)ws->allocStart & (sizeof(U32)-1));
216
+ if (ws->allocStart < ws->tableValidEnd) {
217
+ ws->tableValidEnd = ws->allocStart;
218
+ }
219
+ }
220
+ ws->phase = phase;
221
+ }
222
+ }
223
+
224
+ /**
225
+ * Returns whether this object/buffer/etc was allocated in this workspace.
226
+ */
227
+ MEM_STATIC int ZSTD_cwksp_owns_buffer(const ZSTD_cwksp* ws, const void* ptr) {
228
+ return (ptr != NULL) && (ws->workspace <= ptr) && (ptr <= ws->workspaceEnd);
229
+ }
230
+
231
+ /**
232
+ * Internal function. Do not use directly.
233
+ */
234
+ MEM_STATIC void* ZSTD_cwksp_reserve_internal(
235
+ ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) {
236
+ void* alloc;
237
+ void* bottom = ws->tableEnd;
238
+ ZSTD_cwksp_internal_advance_phase(ws, phase);
239
+ alloc = (BYTE *)ws->allocStart - bytes;
240
+
241
+ #if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
242
+ /* over-reserve space */
243
+ alloc = (BYTE *)alloc - 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE;
244
+ #endif
245
+
246
+ DEBUGLOG(5, "cwksp: reserving %p %zd bytes, %zd bytes remaining",
247
+ alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes);
248
+ ZSTD_cwksp_assert_internal_consistency(ws);
249
+ assert(alloc >= bottom);
250
+ if (alloc < bottom) {
251
+ DEBUGLOG(4, "cwksp: alloc failed!");
252
+ ws->allocFailed = 1;
253
+ return NULL;
254
+ }
255
+ if (alloc < ws->tableValidEnd) {
256
+ ws->tableValidEnd = alloc;
257
+ }
258
+ ws->allocStart = alloc;
259
+
260
+ #if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
261
+ /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on
262
+ * either size. */
263
+ alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE;
264
+ __asan_unpoison_memory_region(alloc, bytes);
265
+ #endif
266
+
267
+ return alloc;
268
+ }
269
+
270
+ /**
271
+ * Reserves and returns unaligned memory.
272
+ */
273
+ MEM_STATIC BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) {
274
+ return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers);
275
+ }
276
+
277
+ /**
278
+ * Reserves and returns memory sized on and aligned on sizeof(unsigned).
279
+ */
280
+ MEM_STATIC void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) {
281
+ assert((bytes & (sizeof(U32)-1)) == 0);
282
+ return ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, sizeof(U32)), ZSTD_cwksp_alloc_aligned);
283
+ }
284
+
285
+ /**
286
+ * Aligned on sizeof(unsigned). These buffers have the special property that
287
+ * their values remain constrained, allowing us to re-use them without
288
+ * memset()-ing them.
289
+ */
290
+ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) {
291
+ const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned;
292
+ void* alloc = ws->tableEnd;
293
+ void* end = (BYTE *)alloc + bytes;
294
+ void* top = ws->allocStart;
295
+
296
+ DEBUGLOG(5, "cwksp: reserving %p table %zd bytes, %zd bytes remaining",
297
+ alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes);
298
+ assert((bytes & (sizeof(U32)-1)) == 0);
299
+ ZSTD_cwksp_internal_advance_phase(ws, phase);
300
+ ZSTD_cwksp_assert_internal_consistency(ws);
301
+ assert(end <= top);
302
+ if (end > top) {
303
+ DEBUGLOG(4, "cwksp: table alloc failed!");
304
+ ws->allocFailed = 1;
305
+ return NULL;
306
+ }
307
+ ws->tableEnd = end;
308
+
309
+ #if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
310
+ __asan_unpoison_memory_region(alloc, bytes);
311
+ #endif
312
+
313
+ return alloc;
314
+ }
315
+
316
+ /**
317
+ * Aligned on sizeof(void*).
318
+ */
319
+ MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) {
320
+ size_t roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*));
321
+ void* alloc = ws->objectEnd;
322
+ void* end = (BYTE*)alloc + roundedBytes;
323
+
324
+ #if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
325
+ /* over-reserve space */
326
+ end = (BYTE *)end + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE;
327
+ #endif
328
+
329
+ DEBUGLOG(5,
330
+ "cwksp: reserving %p object %zd bytes (rounded to %zd), %zd bytes remaining",
331
+ alloc, bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes);
332
+ assert(((size_t)alloc & (sizeof(void*)-1)) == 0);
333
+ assert((bytes & (sizeof(void*)-1)) == 0);
334
+ ZSTD_cwksp_assert_internal_consistency(ws);
335
+ /* we must be in the first phase, no advance is possible */
336
+ if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) {
337
+ DEBUGLOG(4, "cwksp: object alloc failed!");
338
+ ws->allocFailed = 1;
339
+ return NULL;
340
+ }
341
+ ws->objectEnd = end;
342
+ ws->tableEnd = end;
343
+ ws->tableValidEnd = end;
344
+
345
+ #if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
346
+ /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on
347
+ * either size. */
348
+ alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE;
349
+ __asan_unpoison_memory_region(alloc, bytes);
350
+ #endif
351
+
352
+ return alloc;
353
+ }
354
+
355
+ MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws) {
356
+ DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_dirty");
357
+
358
+ #if defined (MEMORY_SANITIZER) && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE)
359
+ /* To validate that the table re-use logic is sound, and that we don't
360
+ * access table space that we haven't cleaned, we re-"poison" the table
361
+ * space every time we mark it dirty. */
362
+ {
363
+ size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd;
364
+ assert(__msan_test_shadow(ws->objectEnd, size) == -1);
365
+ __msan_poison(ws->objectEnd, size);
366
+ }
367
+ #endif
368
+
369
+ assert(ws->tableValidEnd >= ws->objectEnd);
370
+ assert(ws->tableValidEnd <= ws->allocStart);
371
+ ws->tableValidEnd = ws->objectEnd;
372
+ ZSTD_cwksp_assert_internal_consistency(ws);
373
+ }
374
+
375
+ MEM_STATIC void ZSTD_cwksp_mark_tables_clean(ZSTD_cwksp* ws) {
376
+ DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_clean");
377
+ assert(ws->tableValidEnd >= ws->objectEnd);
378
+ assert(ws->tableValidEnd <= ws->allocStart);
379
+ if (ws->tableValidEnd < ws->tableEnd) {
380
+ ws->tableValidEnd = ws->tableEnd;
381
+ }
382
+ ZSTD_cwksp_assert_internal_consistency(ws);
383
+ }
384
+
385
+ /**
386
+ * Zero the part of the allocated tables not already marked clean.
387
+ */
388
+ MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) {
389
+ DEBUGLOG(4, "cwksp: ZSTD_cwksp_clean_tables");
390
+ assert(ws->tableValidEnd >= ws->objectEnd);
391
+ assert(ws->tableValidEnd <= ws->allocStart);
392
+ if (ws->tableValidEnd < ws->tableEnd) {
393
+ memset(ws->tableValidEnd, 0, (BYTE*)ws->tableEnd - (BYTE*)ws->tableValidEnd);
394
+ }
395
+ ZSTD_cwksp_mark_tables_clean(ws);
396
+ }
397
+
398
+ /**
399
+ * Invalidates table allocations.
400
+ * All other allocations remain valid.
401
+ */
402
+ MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) {
403
+ DEBUGLOG(4, "cwksp: clearing tables!");
404
+
405
+ #if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
406
+ {
407
+ size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd;
408
+ __asan_poison_memory_region(ws->objectEnd, size);
409
+ }
410
+ #endif
411
+
412
+ ws->tableEnd = ws->objectEnd;
413
+ ZSTD_cwksp_assert_internal_consistency(ws);
414
+ }
415
+
416
+ /**
417
+ * Invalidates all buffer, aligned, and table allocations.
418
+ * Object allocations remain valid.
419
+ */
420
+ MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) {
421
+ DEBUGLOG(4, "cwksp: clearing!");
422
+
423
+ #if defined (MEMORY_SANITIZER) && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE)
424
+ /* To validate that the context re-use logic is sound, and that we don't
425
+ * access stuff that this compression hasn't initialized, we re-"poison"
426
+ * the workspace (or at least the non-static, non-table parts of it)
427
+ * every time we start a new compression. */
428
+ {
429
+ size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->tableValidEnd;
430
+ __msan_poison(ws->tableValidEnd, size);
431
+ }
432
+ #endif
433
+
434
+ #if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
435
+ {
436
+ size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->objectEnd;
437
+ __asan_poison_memory_region(ws->objectEnd, size);
438
+ }
439
+ #endif
440
+
441
+ ws->tableEnd = ws->objectEnd;
442
+ ws->allocStart = ws->workspaceEnd;
443
+ ws->allocFailed = 0;
444
+ if (ws->phase > ZSTD_cwksp_alloc_buffers) {
445
+ ws->phase = ZSTD_cwksp_alloc_buffers;
446
+ }
447
+ ZSTD_cwksp_assert_internal_consistency(ws);
448
+ }
449
+
450
+ /**
451
+ * The provided workspace takes ownership of the buffer [start, start+size).
452
+ * Any existing values in the workspace are ignored (the previously managed
453
+ * buffer, if present, must be separately freed).
454
+ */
455
+ MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size) {
456
+ DEBUGLOG(4, "cwksp: init'ing workspace with %zd bytes", size);
457
+ assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */
458
+ ws->workspace = start;
459
+ ws->workspaceEnd = (BYTE*)start + size;
460
+ ws->objectEnd = ws->workspace;
461
+ ws->tableValidEnd = ws->objectEnd;
462
+ ws->phase = ZSTD_cwksp_alloc_objects;
463
+ ZSTD_cwksp_clear(ws);
464
+ ws->workspaceOversizedDuration = 0;
465
+ ZSTD_cwksp_assert_internal_consistency(ws);
466
+ }
467
+
468
+ MEM_STATIC size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) {
469
+ void* workspace = ZSTD_malloc(size, customMem);
470
+ DEBUGLOG(4, "cwksp: creating new workspace with %zd bytes", size);
471
+ RETURN_ERROR_IF(workspace == NULL, memory_allocation);
472
+ ZSTD_cwksp_init(ws, workspace, size);
473
+ return 0;
474
+ }
475
+
476
+ MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) {
477
+ void *ptr = ws->workspace;
478
+ DEBUGLOG(4, "cwksp: freeing workspace");
479
+ memset(ws, 0, sizeof(ZSTD_cwksp));
480
+ ZSTD_free(ptr, customMem);
481
+ }
482
+
483
+ /**
484
+ * Moves the management of a workspace from one cwksp to another. The src cwksp
485
+ * is left in an invalid state (src must be re-init()'ed before its used again).
486
+ */
487
+ MEM_STATIC void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) {
488
+ *dst = *src;
489
+ memset(src, 0, sizeof(ZSTD_cwksp));
490
+ }
491
+
492
+ MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) {
493
+ return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace);
494
+ }
495
+
496
+ MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) {
497
+ return ws->allocFailed;
498
+ }
499
+
500
+ /*-*************************************
501
+ * Functions Checking Free Space
502
+ ***************************************/
503
+
504
+ MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) {
505
+ return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd);
506
+ }
507
+
508
+ MEM_STATIC int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t additionalNeededSpace) {
509
+ return ZSTD_cwksp_available_space(ws) >= additionalNeededSpace;
510
+ }
511
+
512
+ MEM_STATIC int ZSTD_cwksp_check_too_large(ZSTD_cwksp* ws, size_t additionalNeededSpace) {
513
+ return ZSTD_cwksp_check_available(
514
+ ws, additionalNeededSpace * ZSTD_WORKSPACETOOLARGE_FACTOR);
515
+ }
516
+
517
+ MEM_STATIC int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t additionalNeededSpace) {
518
+ return ZSTD_cwksp_check_too_large(ws, additionalNeededSpace)
519
+ && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION;
520
+ }
521
+
522
+ MEM_STATIC void ZSTD_cwksp_bump_oversized_duration(
523
+ ZSTD_cwksp* ws, size_t additionalNeededSpace) {
524
+ if (ZSTD_cwksp_check_too_large(ws, additionalNeededSpace)) {
525
+ ws->workspaceOversizedDuration++;
526
+ } else {
527
+ ws->workspaceOversizedDuration = 0;
528
+ }
529
+ }
530
+
531
+ #if defined (__cplusplus)
532
+ }
533
+ #endif
534
+
535
+ #endif /* ZSTD_CWKSP_H */