extzstd 0.3.1 → 0.3.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (113) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +28 -14
  3. data/contrib/zstd/CHANGELOG +301 -56
  4. data/contrib/zstd/CONTRIBUTING.md +169 -72
  5. data/contrib/zstd/LICENSE +4 -4
  6. data/contrib/zstd/Makefile +116 -87
  7. data/contrib/zstd/Package.swift +36 -0
  8. data/contrib/zstd/README.md +62 -32
  9. data/contrib/zstd/TESTING.md +2 -3
  10. data/contrib/zstd/appveyor.yml +52 -136
  11. data/contrib/zstd/lib/BUCK +5 -7
  12. data/contrib/zstd/lib/Makefile +225 -222
  13. data/contrib/zstd/lib/README.md +51 -6
  14. data/contrib/zstd/lib/common/allocations.h +55 -0
  15. data/contrib/zstd/lib/common/bits.h +200 -0
  16. data/contrib/zstd/lib/common/bitstream.h +45 -62
  17. data/contrib/zstd/lib/common/compiler.h +205 -22
  18. data/contrib/zstd/lib/common/cpu.h +1 -3
  19. data/contrib/zstd/lib/common/debug.c +1 -1
  20. data/contrib/zstd/lib/common/debug.h +12 -19
  21. data/contrib/zstd/lib/common/entropy_common.c +172 -48
  22. data/contrib/zstd/lib/common/error_private.c +10 -2
  23. data/contrib/zstd/lib/common/error_private.h +82 -3
  24. data/contrib/zstd/lib/common/fse.h +37 -86
  25. data/contrib/zstd/lib/common/fse_decompress.c +117 -92
  26. data/contrib/zstd/lib/common/huf.h +99 -166
  27. data/contrib/zstd/lib/common/mem.h +124 -142
  28. data/contrib/zstd/lib/common/pool.c +54 -27
  29. data/contrib/zstd/lib/common/pool.h +10 -4
  30. data/contrib/zstd/lib/common/portability_macros.h +156 -0
  31. data/contrib/zstd/lib/common/threading.c +74 -19
  32. data/contrib/zstd/lib/common/threading.h +5 -10
  33. data/contrib/zstd/lib/common/xxhash.c +7 -847
  34. data/contrib/zstd/lib/common/xxhash.h +5568 -167
  35. data/contrib/zstd/lib/common/zstd_common.c +2 -37
  36. data/contrib/zstd/lib/common/zstd_deps.h +111 -0
  37. data/contrib/zstd/lib/common/zstd_internal.h +132 -187
  38. data/contrib/zstd/lib/common/zstd_trace.h +163 -0
  39. data/contrib/zstd/lib/compress/clevels.h +134 -0
  40. data/contrib/zstd/lib/compress/fse_compress.c +83 -157
  41. data/contrib/zstd/lib/compress/hist.c +27 -29
  42. data/contrib/zstd/lib/compress/hist.h +2 -2
  43. data/contrib/zstd/lib/compress/huf_compress.c +916 -279
  44. data/contrib/zstd/lib/compress/zstd_compress.c +3773 -1019
  45. data/contrib/zstd/lib/compress/zstd_compress_internal.h +610 -203
  46. data/contrib/zstd/lib/compress/zstd_compress_literals.c +119 -42
  47. data/contrib/zstd/lib/compress/zstd_compress_literals.h +16 -6
  48. data/contrib/zstd/lib/compress/zstd_compress_sequences.c +42 -19
  49. data/contrib/zstd/lib/compress/zstd_compress_sequences.h +1 -1
  50. data/contrib/zstd/lib/compress/zstd_compress_superblock.c +49 -317
  51. data/contrib/zstd/lib/compress/zstd_compress_superblock.h +1 -1
  52. data/contrib/zstd/lib/compress/zstd_cwksp.h +320 -103
  53. data/contrib/zstd/lib/compress/zstd_double_fast.c +388 -151
  54. data/contrib/zstd/lib/compress/zstd_double_fast.h +3 -2
  55. data/contrib/zstd/lib/compress/zstd_fast.c +729 -265
  56. data/contrib/zstd/lib/compress/zstd_fast.h +3 -2
  57. data/contrib/zstd/lib/compress/zstd_lazy.c +1270 -251
  58. data/contrib/zstd/lib/compress/zstd_lazy.h +61 -1
  59. data/contrib/zstd/lib/compress/zstd_ldm.c +324 -219
  60. data/contrib/zstd/lib/compress/zstd_ldm.h +9 -2
  61. data/contrib/zstd/lib/compress/zstd_ldm_geartab.h +106 -0
  62. data/contrib/zstd/lib/compress/zstd_opt.c +481 -209
  63. data/contrib/zstd/lib/compress/zstd_opt.h +1 -1
  64. data/contrib/zstd/lib/compress/zstdmt_compress.c +181 -457
  65. data/contrib/zstd/lib/compress/zstdmt_compress.h +34 -113
  66. data/contrib/zstd/lib/decompress/huf_decompress.c +1199 -565
  67. data/contrib/zstd/lib/decompress/huf_decompress_amd64.S +576 -0
  68. data/contrib/zstd/lib/decompress/zstd_ddict.c +12 -12
  69. data/contrib/zstd/lib/decompress/zstd_ddict.h +2 -2
  70. data/contrib/zstd/lib/decompress/zstd_decompress.c +627 -157
  71. data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1086 -326
  72. data/contrib/zstd/lib/decompress/zstd_decompress_block.h +19 -5
  73. data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +62 -13
  74. data/contrib/zstd/lib/deprecated/zbuff.h +1 -1
  75. data/contrib/zstd/lib/deprecated/zbuff_common.c +1 -1
  76. data/contrib/zstd/lib/deprecated/zbuff_compress.c +24 -4
  77. data/contrib/zstd/lib/deprecated/zbuff_decompress.c +3 -1
  78. data/contrib/zstd/lib/dictBuilder/cover.c +73 -52
  79. data/contrib/zstd/lib/dictBuilder/cover.h +7 -6
  80. data/contrib/zstd/lib/dictBuilder/divsufsort.c +1 -1
  81. data/contrib/zstd/lib/dictBuilder/fastcover.c +44 -35
  82. data/contrib/zstd/lib/dictBuilder/zdict.c +103 -111
  83. data/contrib/zstd/lib/legacy/zstd_legacy.h +8 -1
  84. data/contrib/zstd/lib/legacy/zstd_v01.c +21 -54
  85. data/contrib/zstd/lib/legacy/zstd_v01.h +1 -1
  86. data/contrib/zstd/lib/legacy/zstd_v02.c +29 -70
  87. data/contrib/zstd/lib/legacy/zstd_v02.h +1 -1
  88. data/contrib/zstd/lib/legacy/zstd_v03.c +30 -73
  89. data/contrib/zstd/lib/legacy/zstd_v03.h +1 -1
  90. data/contrib/zstd/lib/legacy/zstd_v04.c +29 -71
  91. data/contrib/zstd/lib/legacy/zstd_v04.h +1 -1
  92. data/contrib/zstd/lib/legacy/zstd_v05.c +40 -86
  93. data/contrib/zstd/lib/legacy/zstd_v05.h +1 -1
  94. data/contrib/zstd/lib/legacy/zstd_v06.c +47 -88
  95. data/contrib/zstd/lib/legacy/zstd_v06.h +1 -1
  96. data/contrib/zstd/lib/legacy/zstd_v07.c +40 -83
  97. data/contrib/zstd/lib/legacy/zstd_v07.h +1 -1
  98. data/contrib/zstd/lib/libzstd.mk +214 -0
  99. data/contrib/zstd/lib/libzstd.pc.in +7 -6
  100. data/contrib/zstd/lib/module.modulemap +35 -0
  101. data/contrib/zstd/lib/{dictBuilder/zdict.h → zdict.h} +203 -34
  102. data/contrib/zstd/lib/zstd.h +1217 -287
  103. data/contrib/zstd/lib/{common/zstd_errors.h → zstd_errors.h} +28 -8
  104. data/ext/extconf.rb +7 -6
  105. data/ext/extzstd.c +19 -10
  106. data/ext/extzstd.h +6 -0
  107. data/ext/libzstd_conf.h +0 -1
  108. data/ext/zstd_decompress_asm.S +1 -0
  109. data/gemstub.rb +3 -21
  110. data/lib/extzstd/version.rb +6 -1
  111. data/lib/extzstd.rb +0 -2
  112. data/test/test_basic.rb +0 -5
  113. metadata +18 -6
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -19,113 +19,60 @@
19
19
  /* Note : This is an internal API.
20
20
  * These APIs used to be exposed with ZSTDLIB_API,
21
21
  * because it used to be the only way to invoke MT compression.
22
- * Now, it's recommended to use ZSTD_compress2 and ZSTD_compressStream2()
23
- * instead.
24
- *
25
- * If you depend on these APIs and can't switch, then define
26
- * ZSTD_LEGACY_MULTITHREADED_API when making the dynamic library.
27
- * However, we may completely remove these functions in a future
28
- * release, so please switch soon.
22
+ * Now, you must use ZSTD_compress2 and ZSTD_compressStream2() instead.
29
23
  *
30
24
  * This API requires ZSTD_MULTITHREAD to be defined during compilation,
31
25
  * otherwise ZSTDMT_createCCtx*() will fail.
32
26
  */
33
27
 
34
- #ifdef ZSTD_LEGACY_MULTITHREADED_API
35
- # define ZSTDMT_API ZSTDLIB_API
36
- #else
37
- # define ZSTDMT_API
38
- #endif
39
-
40
28
  /* === Dependencies === */
41
- #include <stddef.h> /* size_t */
29
+ #include "../common/zstd_deps.h" /* size_t */
42
30
  #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */
43
31
  #include "../zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
44
32
 
45
33
 
46
34
  /* === Constants === */
47
- #ifndef ZSTDMT_NBWORKERS_MAX
48
- # define ZSTDMT_NBWORKERS_MAX 200
35
+ #ifndef ZSTDMT_NBWORKERS_MAX /* a different value can be selected at compile time */
36
+ # define ZSTDMT_NBWORKERS_MAX ((sizeof(void*)==4) /*32-bit*/ ? 64 : 256)
49
37
  #endif
50
- #ifndef ZSTDMT_JOBSIZE_MIN
51
- # define ZSTDMT_JOBSIZE_MIN (1 MB)
38
+ #ifndef ZSTDMT_JOBSIZE_MIN /* a different value can be selected at compile time */
39
+ # define ZSTDMT_JOBSIZE_MIN (512 KB)
52
40
  #endif
53
41
  #define ZSTDMT_JOBLOG_MAX (MEM_32bits() ? 29 : 30)
54
42
  #define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1024 MB))
55
43
 
56
44
 
45
+ /* ========================================================
46
+ * === Private interface, for use by ZSTD_compress.c ===
47
+ * === Not exposed in libzstd. Never invoke directly ===
48
+ * ======================================================== */
49
+
57
50
  /* === Memory management === */
58
51
  typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx;
59
52
  /* Requires ZSTD_MULTITHREAD to be defined during compilation, otherwise it will return NULL. */
60
- ZSTDMT_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbWorkers);
61
- /* Requires ZSTD_MULTITHREAD to be defined during compilation, otherwise it will return NULL. */
62
- ZSTDMT_API ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers,
63
- ZSTD_customMem cMem);
64
- ZSTDMT_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx);
65
-
66
- ZSTDMT_API size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx);
67
-
68
-
69
- /* === Simple one-pass compression function === */
70
-
71
- ZSTDMT_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
72
- void* dst, size_t dstCapacity,
73
- const void* src, size_t srcSize,
74
- int compressionLevel);
75
-
53
+ ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers,
54
+ ZSTD_customMem cMem,
55
+ ZSTD_threadPool *pool);
56
+ size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx);
76
57
 
58
+ size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx);
77
59
 
78
60
  /* === Streaming functions === */
79
61
 
80
- ZSTDMT_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel);
81
- ZSTDMT_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< if srcSize is not known at reset time, use ZSTD_CONTENTSIZE_UNKNOWN. Note: for compatibility with older programs, 0 means the same as ZSTD_CONTENTSIZE_UNKNOWN, but it will change in the future to mean "empty" */
82
-
83
- ZSTDMT_API size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx);
84
- ZSTDMT_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
85
-
86
- ZSTDMT_API size_t ZSTDMT_flushStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */
87
- ZSTDMT_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */
88
-
89
-
90
- /* === Advanced functions and parameters === */
91
-
92
- ZSTDMT_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
93
- void* dst, size_t dstCapacity,
94
- const void* src, size_t srcSize,
95
- const ZSTD_CDict* cdict,
96
- ZSTD_parameters params,
97
- int overlapLog);
98
-
99
- ZSTDMT_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
100
- const void* dict, size_t dictSize, /* dict can be released after init, a local copy is preserved within zcs */
101
- ZSTD_parameters params,
102
- unsigned long long pledgedSrcSize); /* pledgedSrcSize is optional and can be zero == unknown */
103
-
104
- ZSTDMT_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
105
- const ZSTD_CDict* cdict,
106
- ZSTD_frameParameters fparams,
107
- unsigned long long pledgedSrcSize); /* note : zero means empty */
108
-
109
- /* ZSTDMT_parameter :
110
- * List of parameters that can be set using ZSTDMT_setMTCtxParameter() */
111
- typedef enum {
112
- ZSTDMT_p_jobSize, /* Each job is compressed in parallel. By default, this value is dynamically determined depending on compression parameters. Can be set explicitly here. */
113
- ZSTDMT_p_overlapLog, /* Each job may reload a part of previous job to enhance compression ratio; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window. This is a "sticky" parameter : its value will be re-used on next compression job */
114
- ZSTDMT_p_rsyncable /* Enables rsyncable mode. */
115
- } ZSTDMT_parameter;
116
-
117
- /* ZSTDMT_setMTCtxParameter() :
118
- * allow setting individual parameters, one at a time, among a list of enums defined in ZSTDMT_parameter.
119
- * The function must be called typically after ZSTD_createCCtx() but __before ZSTDMT_init*() !__
120
- * Parameters not explicitly reset by ZSTDMT_init*() remain the same in consecutive compression sessions.
121
- * @return : 0, or an error code (which can be tested using ZSTD_isError()) */
122
- ZSTDMT_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int value);
123
-
124
- /* ZSTDMT_getMTCtxParameter() :
125
- * Query the ZSTDMT_CCtx for a parameter value.
126
- * @return : 0, or an error code (which can be tested using ZSTD_isError()) */
127
- ZSTDMT_API size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int* value);
62
+ size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx);
128
63
 
64
+ /*! ZSTDMT_initCStream_internal() :
65
+ * Private use only. Init streaming operation.
66
+ * expects params to be valid.
67
+ * must receive dict, or cdict, or none, but not both.
68
+ * mtctx can be freshly constructed or reused from a prior compression.
69
+ * If mtctx is reused, memory allocations from the prior compression may not be freed,
70
+ * even if they are not needed for the current compression.
71
+ * @return : 0, or an error code */
72
+ size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* mtctx,
73
+ const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType,
74
+ const ZSTD_CDict* cdict,
75
+ ZSTD_CCtx_params params, unsigned long long pledgedSrcSize);
129
76
 
130
77
  /*! ZSTDMT_compressStream_generic() :
131
78
  * Combines ZSTDMT_compressStream() with optional ZSTDMT_flushStream() or ZSTDMT_endStream()
@@ -134,16 +81,10 @@ ZSTDMT_API size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter
134
81
  * 0 if fully flushed
135
82
  * or an error code
136
83
  * note : needs to be init using any ZSTD_initCStream*() variant */
137
- ZSTDMT_API size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
138
- ZSTD_outBuffer* output,
139
- ZSTD_inBuffer* input,
140
- ZSTD_EndDirective endOp);
141
-
142
-
143
- /* ========================================================
144
- * === Private interface, for use by ZSTD_compress.c ===
145
- * === Not exposed in libzstd. Never invoke directly ===
146
- * ======================================================== */
84
+ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
85
+ ZSTD_outBuffer* output,
86
+ ZSTD_inBuffer* input,
87
+ ZSTD_EndDirective endOp);
147
88
 
148
89
  /*! ZSTDMT_toFlushNow()
149
90
  * Tell how many bytes are ready to be flushed immediately.
@@ -153,15 +94,6 @@ ZSTDMT_API size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
153
94
  * therefore flushing is limited by speed of oldest job. */
154
95
  size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx);
155
96
 
156
- /*! ZSTDMT_CCtxParam_setMTCtxParameter()
157
- * like ZSTDMT_setMTCtxParameter(), but into a ZSTD_CCtx_Params */
158
- size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, int value);
159
-
160
- /*! ZSTDMT_CCtxParam_setNbWorkers()
161
- * Set nbWorkers, and clamp it.
162
- * Also reset jobSize and overlapLog */
163
- size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorkers);
164
-
165
97
  /*! ZSTDMT_updateCParams_whileCompressing() :
166
98
  * Updates only a selected set of compression parameters, to remain compatible with current frame.
167
99
  * New parameters will be applied to next compression job. */
@@ -174,17 +106,6 @@ void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_p
174
106
  ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx);
175
107
 
176
108
 
177
- /*! ZSTDMT_initCStream_internal() :
178
- * Private use only. Init streaming operation.
179
- * expects params to be valid.
180
- * must receive dict, or cdict, or none, but not both.
181
- * @return : 0, or an error code */
182
- size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
183
- const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType,
184
- const ZSTD_CDict* cdict,
185
- ZSTD_CCtx_params params, unsigned long long pledgedSrcSize);
186
-
187
-
188
109
  #if defined (__cplusplus)
189
110
  }
190
111
  #endif