extzstd 0.3.1 → 0.3.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (113) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +28 -14
  3. data/contrib/zstd/CHANGELOG +301 -56
  4. data/contrib/zstd/CONTRIBUTING.md +169 -72
  5. data/contrib/zstd/LICENSE +4 -4
  6. data/contrib/zstd/Makefile +116 -87
  7. data/contrib/zstd/Package.swift +36 -0
  8. data/contrib/zstd/README.md +62 -32
  9. data/contrib/zstd/TESTING.md +2 -3
  10. data/contrib/zstd/appveyor.yml +52 -136
  11. data/contrib/zstd/lib/BUCK +5 -7
  12. data/contrib/zstd/lib/Makefile +225 -222
  13. data/contrib/zstd/lib/README.md +51 -6
  14. data/contrib/zstd/lib/common/allocations.h +55 -0
  15. data/contrib/zstd/lib/common/bits.h +200 -0
  16. data/contrib/zstd/lib/common/bitstream.h +45 -62
  17. data/contrib/zstd/lib/common/compiler.h +205 -22
  18. data/contrib/zstd/lib/common/cpu.h +1 -3
  19. data/contrib/zstd/lib/common/debug.c +1 -1
  20. data/contrib/zstd/lib/common/debug.h +12 -19
  21. data/contrib/zstd/lib/common/entropy_common.c +172 -48
  22. data/contrib/zstd/lib/common/error_private.c +10 -2
  23. data/contrib/zstd/lib/common/error_private.h +82 -3
  24. data/contrib/zstd/lib/common/fse.h +37 -86
  25. data/contrib/zstd/lib/common/fse_decompress.c +117 -92
  26. data/contrib/zstd/lib/common/huf.h +99 -166
  27. data/contrib/zstd/lib/common/mem.h +124 -142
  28. data/contrib/zstd/lib/common/pool.c +54 -27
  29. data/contrib/zstd/lib/common/pool.h +10 -4
  30. data/contrib/zstd/lib/common/portability_macros.h +156 -0
  31. data/contrib/zstd/lib/common/threading.c +74 -19
  32. data/contrib/zstd/lib/common/threading.h +5 -10
  33. data/contrib/zstd/lib/common/xxhash.c +7 -847
  34. data/contrib/zstd/lib/common/xxhash.h +5568 -167
  35. data/contrib/zstd/lib/common/zstd_common.c +2 -37
  36. data/contrib/zstd/lib/common/zstd_deps.h +111 -0
  37. data/contrib/zstd/lib/common/zstd_internal.h +132 -187
  38. data/contrib/zstd/lib/common/zstd_trace.h +163 -0
  39. data/contrib/zstd/lib/compress/clevels.h +134 -0
  40. data/contrib/zstd/lib/compress/fse_compress.c +83 -157
  41. data/contrib/zstd/lib/compress/hist.c +27 -29
  42. data/contrib/zstd/lib/compress/hist.h +2 -2
  43. data/contrib/zstd/lib/compress/huf_compress.c +916 -279
  44. data/contrib/zstd/lib/compress/zstd_compress.c +3773 -1019
  45. data/contrib/zstd/lib/compress/zstd_compress_internal.h +610 -203
  46. data/contrib/zstd/lib/compress/zstd_compress_literals.c +119 -42
  47. data/contrib/zstd/lib/compress/zstd_compress_literals.h +16 -6
  48. data/contrib/zstd/lib/compress/zstd_compress_sequences.c +42 -19
  49. data/contrib/zstd/lib/compress/zstd_compress_sequences.h +1 -1
  50. data/contrib/zstd/lib/compress/zstd_compress_superblock.c +49 -317
  51. data/contrib/zstd/lib/compress/zstd_compress_superblock.h +1 -1
  52. data/contrib/zstd/lib/compress/zstd_cwksp.h +320 -103
  53. data/contrib/zstd/lib/compress/zstd_double_fast.c +388 -151
  54. data/contrib/zstd/lib/compress/zstd_double_fast.h +3 -2
  55. data/contrib/zstd/lib/compress/zstd_fast.c +729 -265
  56. data/contrib/zstd/lib/compress/zstd_fast.h +3 -2
  57. data/contrib/zstd/lib/compress/zstd_lazy.c +1270 -251
  58. data/contrib/zstd/lib/compress/zstd_lazy.h +61 -1
  59. data/contrib/zstd/lib/compress/zstd_ldm.c +324 -219
  60. data/contrib/zstd/lib/compress/zstd_ldm.h +9 -2
  61. data/contrib/zstd/lib/compress/zstd_ldm_geartab.h +106 -0
  62. data/contrib/zstd/lib/compress/zstd_opt.c +481 -209
  63. data/contrib/zstd/lib/compress/zstd_opt.h +1 -1
  64. data/contrib/zstd/lib/compress/zstdmt_compress.c +181 -457
  65. data/contrib/zstd/lib/compress/zstdmt_compress.h +34 -113
  66. data/contrib/zstd/lib/decompress/huf_decompress.c +1199 -565
  67. data/contrib/zstd/lib/decompress/huf_decompress_amd64.S +576 -0
  68. data/contrib/zstd/lib/decompress/zstd_ddict.c +12 -12
  69. data/contrib/zstd/lib/decompress/zstd_ddict.h +2 -2
  70. data/contrib/zstd/lib/decompress/zstd_decompress.c +627 -157
  71. data/contrib/zstd/lib/decompress/zstd_decompress_block.c +1086 -326
  72. data/contrib/zstd/lib/decompress/zstd_decompress_block.h +19 -5
  73. data/contrib/zstd/lib/decompress/zstd_decompress_internal.h +62 -13
  74. data/contrib/zstd/lib/deprecated/zbuff.h +1 -1
  75. data/contrib/zstd/lib/deprecated/zbuff_common.c +1 -1
  76. data/contrib/zstd/lib/deprecated/zbuff_compress.c +24 -4
  77. data/contrib/zstd/lib/deprecated/zbuff_decompress.c +3 -1
  78. data/contrib/zstd/lib/dictBuilder/cover.c +73 -52
  79. data/contrib/zstd/lib/dictBuilder/cover.h +7 -6
  80. data/contrib/zstd/lib/dictBuilder/divsufsort.c +1 -1
  81. data/contrib/zstd/lib/dictBuilder/fastcover.c +44 -35
  82. data/contrib/zstd/lib/dictBuilder/zdict.c +103 -111
  83. data/contrib/zstd/lib/legacy/zstd_legacy.h +8 -1
  84. data/contrib/zstd/lib/legacy/zstd_v01.c +21 -54
  85. data/contrib/zstd/lib/legacy/zstd_v01.h +1 -1
  86. data/contrib/zstd/lib/legacy/zstd_v02.c +29 -70
  87. data/contrib/zstd/lib/legacy/zstd_v02.h +1 -1
  88. data/contrib/zstd/lib/legacy/zstd_v03.c +30 -73
  89. data/contrib/zstd/lib/legacy/zstd_v03.h +1 -1
  90. data/contrib/zstd/lib/legacy/zstd_v04.c +29 -71
  91. data/contrib/zstd/lib/legacy/zstd_v04.h +1 -1
  92. data/contrib/zstd/lib/legacy/zstd_v05.c +40 -86
  93. data/contrib/zstd/lib/legacy/zstd_v05.h +1 -1
  94. data/contrib/zstd/lib/legacy/zstd_v06.c +47 -88
  95. data/contrib/zstd/lib/legacy/zstd_v06.h +1 -1
  96. data/contrib/zstd/lib/legacy/zstd_v07.c +40 -83
  97. data/contrib/zstd/lib/legacy/zstd_v07.h +1 -1
  98. data/contrib/zstd/lib/libzstd.mk +214 -0
  99. data/contrib/zstd/lib/libzstd.pc.in +7 -6
  100. data/contrib/zstd/lib/module.modulemap +35 -0
  101. data/contrib/zstd/lib/{dictBuilder/zdict.h → zdict.h} +203 -34
  102. data/contrib/zstd/lib/zstd.h +1217 -287
  103. data/contrib/zstd/lib/{common/zstd_errors.h → zstd_errors.h} +28 -8
  104. data/ext/extconf.rb +7 -6
  105. data/ext/extzstd.c +19 -10
  106. data/ext/extzstd.h +6 -0
  107. data/ext/libzstd_conf.h +0 -1
  108. data/ext/zstd_decompress_asm.S +1 -0
  109. data/gemstub.rb +3 -21
  110. data/lib/extzstd/version.rb +6 -1
  111. data/lib/extzstd.rb +0 -2
  112. data/test/test_basic.rb +0 -5
  113. metadata +18 -6
@@ -0,0 +1,156 @@
1
+ /*
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
+ * All rights reserved.
4
+ *
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
9
+ */
10
+
11
+ #ifndef ZSTD_PORTABILITY_MACROS_H
12
+ #define ZSTD_PORTABILITY_MACROS_H
13
+
14
+ /**
15
+ * This header file contains macro definitions to support portability.
16
+ * This header is shared between C and ASM code, so it MUST only
17
+ * contain macro definitions. It MUST not contain any C code.
18
+ *
19
+ * This header ONLY defines macros to detect platforms/feature support.
20
+ *
21
+ */
22
+
23
+
24
+ /* compat. with non-clang compilers */
25
+ #ifndef __has_attribute
26
+ #define __has_attribute(x) 0
27
+ #endif
28
+
29
+ /* compat. with non-clang compilers */
30
+ #ifndef __has_builtin
31
+ # define __has_builtin(x) 0
32
+ #endif
33
+
34
+ /* compat. with non-clang compilers */
35
+ #ifndef __has_feature
36
+ # define __has_feature(x) 0
37
+ #endif
38
+
39
+ /* detects whether we are being compiled under msan */
40
+ #ifndef ZSTD_MEMORY_SANITIZER
41
+ # if __has_feature(memory_sanitizer)
42
+ # define ZSTD_MEMORY_SANITIZER 1
43
+ # else
44
+ # define ZSTD_MEMORY_SANITIZER 0
45
+ # endif
46
+ #endif
47
+
48
+ /* detects whether we are being compiled under asan */
49
+ #ifndef ZSTD_ADDRESS_SANITIZER
50
+ # if __has_feature(address_sanitizer)
51
+ # define ZSTD_ADDRESS_SANITIZER 1
52
+ # elif defined(__SANITIZE_ADDRESS__)
53
+ # define ZSTD_ADDRESS_SANITIZER 1
54
+ # else
55
+ # define ZSTD_ADDRESS_SANITIZER 0
56
+ # endif
57
+ #endif
58
+
59
+ /* detects whether we are being compiled under dfsan */
60
+ #ifndef ZSTD_DATAFLOW_SANITIZER
61
+ # if __has_feature(dataflow_sanitizer)
62
+ # define ZSTD_DATAFLOW_SANITIZER 1
63
+ # else
64
+ # define ZSTD_DATAFLOW_SANITIZER 0
65
+ # endif
66
+ #endif
67
+
68
+ /* Mark the internal assembly functions as hidden */
69
+ #ifdef __ELF__
70
+ # define ZSTD_HIDE_ASM_FUNCTION(func) .hidden func
71
+ #else
72
+ # define ZSTD_HIDE_ASM_FUNCTION(func)
73
+ #endif
74
+
75
+ /* Enable runtime BMI2 dispatch based on the CPU.
76
+ * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default.
77
+ */
78
+ #ifndef DYNAMIC_BMI2
79
+ #if ((defined(__clang__) && __has_attribute(__target__)) \
80
+ || (defined(__GNUC__) \
81
+ && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \
82
+ && (defined(__x86_64__) || defined(_M_X64)) \
83
+ && !defined(__BMI2__)
84
+ # define DYNAMIC_BMI2 1
85
+ #else
86
+ # define DYNAMIC_BMI2 0
87
+ #endif
88
+ #endif
89
+
90
+ /**
91
+ * Only enable assembly for GNUC compatible compilers,
92
+ * because other platforms may not support GAS assembly syntax.
93
+ *
94
+ * Only enable assembly for Linux / MacOS, other platforms may
95
+ * work, but they haven't been tested. This could likely be
96
+ * extended to BSD systems.
97
+ *
98
+ * Disable assembly when MSAN is enabled, because MSAN requires
99
+ * 100% of code to be instrumented to work.
100
+ */
101
+ #if defined(__GNUC__)
102
+ # if defined(__linux__) || defined(__linux) || defined(__APPLE__)
103
+ # if ZSTD_MEMORY_SANITIZER
104
+ # define ZSTD_ASM_SUPPORTED 0
105
+ # elif ZSTD_DATAFLOW_SANITIZER
106
+ # define ZSTD_ASM_SUPPORTED 0
107
+ # else
108
+ # define ZSTD_ASM_SUPPORTED 1
109
+ # endif
110
+ # else
111
+ # define ZSTD_ASM_SUPPORTED 0
112
+ # endif
113
+ #else
114
+ # define ZSTD_ASM_SUPPORTED 0
115
+ #endif
116
+
117
+ /**
118
+ * Determines whether we should enable assembly for x86-64
119
+ * with BMI2.
120
+ *
121
+ * Enable if all of the following conditions hold:
122
+ * - ASM hasn't been explicitly disabled by defining ZSTD_DISABLE_ASM
123
+ * - Assembly is supported
124
+ * - We are compiling for x86-64 and either:
125
+ * - DYNAMIC_BMI2 is enabled
126
+ * - BMI2 is supported at compile time
127
+ */
128
+ #if !defined(ZSTD_DISABLE_ASM) && \
129
+ ZSTD_ASM_SUPPORTED && \
130
+ defined(__x86_64__) && \
131
+ (DYNAMIC_BMI2 || defined(__BMI2__))
132
+ # define ZSTD_ENABLE_ASM_X86_64_BMI2 1
133
+ #else
134
+ # define ZSTD_ENABLE_ASM_X86_64_BMI2 0
135
+ #endif
136
+
137
+ /*
138
+ * For x86 ELF targets, add .note.gnu.property section for Intel CET in
139
+ * assembly sources when CET is enabled.
140
+ *
141
+ * Additionally, any function that may be called indirectly must begin
142
+ * with ZSTD_CET_ENDBRANCH.
143
+ */
144
+ #if defined(__ELF__) && (defined(__x86_64__) || defined(__i386__)) \
145
+ && defined(__has_include)
146
+ # if __has_include(<cet.h>)
147
+ # include <cet.h>
148
+ # define ZSTD_CET_ENDBRANCH _CET_ENDBR
149
+ # endif
150
+ #endif
151
+
152
+ #ifndef ZSTD_CET_ENDBRANCH
153
+ # define ZSTD_CET_ENDBRANCH
154
+ #endif
155
+
156
+ #endif /* ZSTD_PORTABILITY_MACROS_H */
@@ -23,8 +23,7 @@ int g_ZSTD_threading_useless_symbol;
23
23
  #if defined(ZSTD_MULTITHREAD) && defined(_WIN32)
24
24
 
25
25
  /**
26
- * Windows minimalist Pthread Wrapper, based on :
27
- * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
26
+ * Windows minimalist Pthread Wrapper
28
27
  */
29
28
 
30
29
 
@@ -35,37 +34,92 @@ int g_ZSTD_threading_useless_symbol;
35
34
 
36
35
  /* === Implementation === */
37
36
 
37
+ typedef struct {
38
+ void* (*start_routine)(void*);
39
+ void* arg;
40
+ int initialized;
41
+ ZSTD_pthread_cond_t initialized_cond;
42
+ ZSTD_pthread_mutex_t initialized_mutex;
43
+ } ZSTD_thread_params_t;
44
+
38
45
  static unsigned __stdcall worker(void *arg)
39
46
  {
40
- ZSTD_pthread_t* const thread = (ZSTD_pthread_t*) arg;
41
- thread->arg = thread->start_routine(thread->arg);
47
+ void* (*start_routine)(void*);
48
+ void* thread_arg;
49
+
50
+ /* Initialized thread_arg and start_routine and signal main thread that we don't need it
51
+ * to wait any longer.
52
+ */
53
+ {
54
+ ZSTD_thread_params_t* thread_param = (ZSTD_thread_params_t*)arg;
55
+ thread_arg = thread_param->arg;
56
+ start_routine = thread_param->start_routine;
57
+
58
+ /* Signal main thread that we are running and do not depend on its memory anymore */
59
+ ZSTD_pthread_mutex_lock(&thread_param->initialized_mutex);
60
+ thread_param->initialized = 1;
61
+ ZSTD_pthread_cond_signal(&thread_param->initialized_cond);
62
+ ZSTD_pthread_mutex_unlock(&thread_param->initialized_mutex);
63
+ }
64
+
65
+ start_routine(thread_arg);
66
+
42
67
  return 0;
43
68
  }
44
69
 
45
70
  int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
46
71
  void* (*start_routine) (void*), void* arg)
47
72
  {
73
+ ZSTD_thread_params_t thread_param;
48
74
  (void)unused;
49
- thread->arg = arg;
50
- thread->start_routine = start_routine;
51
- thread->handle = (HANDLE) _beginthreadex(NULL, 0, worker, thread, 0, NULL);
52
75
 
53
- if (!thread->handle)
76
+ thread_param.start_routine = start_routine;
77
+ thread_param.arg = arg;
78
+ thread_param.initialized = 0;
79
+ *thread = NULL;
80
+
81
+ /* Setup thread initialization synchronization */
82
+ if(ZSTD_pthread_cond_init(&thread_param.initialized_cond, NULL)) {
83
+ /* Should never happen on Windows */
84
+ return -1;
85
+ }
86
+ if(ZSTD_pthread_mutex_init(&thread_param.initialized_mutex, NULL)) {
87
+ /* Should never happen on Windows */
88
+ ZSTD_pthread_cond_destroy(&thread_param.initialized_cond);
89
+ return -1;
90
+ }
91
+
92
+ /* Spawn thread */
93
+ *thread = (HANDLE)_beginthreadex(NULL, 0, worker, &thread_param, 0, NULL);
94
+ if (!thread) {
95
+ ZSTD_pthread_mutex_destroy(&thread_param.initialized_mutex);
96
+ ZSTD_pthread_cond_destroy(&thread_param.initialized_cond);
54
97
  return errno;
55
- else
56
- return 0;
98
+ }
99
+
100
+ /* Wait for thread to be initialized */
101
+ ZSTD_pthread_mutex_lock(&thread_param.initialized_mutex);
102
+ while(!thread_param.initialized) {
103
+ ZSTD_pthread_cond_wait(&thread_param.initialized_cond, &thread_param.initialized_mutex);
104
+ }
105
+ ZSTD_pthread_mutex_unlock(&thread_param.initialized_mutex);
106
+ ZSTD_pthread_mutex_destroy(&thread_param.initialized_mutex);
107
+ ZSTD_pthread_cond_destroy(&thread_param.initialized_cond);
108
+
109
+ return 0;
57
110
  }
58
111
 
59
- int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr)
112
+ int ZSTD_pthread_join(ZSTD_pthread_t thread)
60
113
  {
61
114
  DWORD result;
62
115
 
63
- if (!thread.handle) return 0;
116
+ if (!thread) return 0;
117
+
118
+ result = WaitForSingleObject(thread, INFINITE);
119
+ CloseHandle(thread);
64
120
 
65
- result = WaitForSingleObject(thread.handle, INFINITE);
66
121
  switch (result) {
67
122
  case WAIT_OBJECT_0:
68
- if (value_ptr) *value_ptr = thread.arg;
69
123
  return 0;
70
124
  case WAIT_ABANDONED:
71
125
  return EINVAL;
@@ -78,11 +132,12 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr)
78
132
 
79
133
  #if defined(ZSTD_MULTITHREAD) && DEBUGLEVEL >= 1 && !defined(_WIN32)
80
134
 
81
- #include <stdlib.h>
135
+ #define ZSTD_DEPS_NEED_MALLOC
136
+ #include "zstd_deps.h"
82
137
 
83
138
  int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr)
84
139
  {
85
- *mutex = (pthread_mutex_t*)malloc(sizeof(pthread_mutex_t));
140
+ *mutex = (pthread_mutex_t*)ZSTD_malloc(sizeof(pthread_mutex_t));
86
141
  if (!*mutex)
87
142
  return 1;
88
143
  return pthread_mutex_init(*mutex, attr);
@@ -94,14 +149,14 @@ int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex)
94
149
  return 0;
95
150
  {
96
151
  int const ret = pthread_mutex_destroy(*mutex);
97
- free(*mutex);
152
+ ZSTD_free(*mutex);
98
153
  return ret;
99
154
  }
100
155
  }
101
156
 
102
157
  int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr)
103
158
  {
104
- *cond = (pthread_cond_t*)malloc(sizeof(pthread_cond_t));
159
+ *cond = (pthread_cond_t*)ZSTD_malloc(sizeof(pthread_cond_t));
105
160
  if (!*cond)
106
161
  return 1;
107
162
  return pthread_cond_init(*cond, attr);
@@ -113,7 +168,7 @@ int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond)
113
168
  return 0;
114
169
  {
115
170
  int const ret = pthread_cond_destroy(*cond);
116
- free(*cond);
171
+ ZSTD_free(*cond);
117
172
  return ret;
118
173
  }
119
174
  }
@@ -23,8 +23,7 @@ extern "C" {
23
23
  #if defined(ZSTD_MULTITHREAD) && defined(_WIN32)
24
24
 
25
25
  /**
26
- * Windows minimalist Pthread Wrapper, based on :
27
- * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
26
+ * Windows minimalist Pthread Wrapper
28
27
  */
29
28
  #ifdef WINVER
30
29
  # undef WINVER
@@ -62,16 +61,12 @@ extern "C" {
62
61
  #define ZSTD_pthread_cond_broadcast(a) WakeAllConditionVariable((a))
63
62
 
64
63
  /* ZSTD_pthread_create() and ZSTD_pthread_join() */
65
- typedef struct {
66
- HANDLE handle;
67
- void* (*start_routine)(void*);
68
- void* arg;
69
- } ZSTD_pthread_t;
64
+ typedef HANDLE ZSTD_pthread_t;
70
65
 
71
66
  int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
72
67
  void* (*start_routine) (void*), void* arg);
73
68
 
74
- int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
69
+ int ZSTD_pthread_join(ZSTD_pthread_t thread);
75
70
 
76
71
  /**
77
72
  * add here more wrappers as required
@@ -99,7 +94,7 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
99
94
 
100
95
  #define ZSTD_pthread_t pthread_t
101
96
  #define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
102
- #define ZSTD_pthread_join(a, b) pthread_join((a),(b))
97
+ #define ZSTD_pthread_join(a) pthread_join((a),NULL)
103
98
 
104
99
  #else /* DEBUGLEVEL >= 1 */
105
100
 
@@ -124,7 +119,7 @@ int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond);
124
119
 
125
120
  #define ZSTD_pthread_t pthread_t
126
121
  #define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
127
- #define ZSTD_pthread_join(a, b) pthread_join((a),(b))
122
+ #define ZSTD_pthread_join(a) pthread_join((a),NULL)
128
123
 
129
124
  #endif
130
125