vibe_zstd 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. checksums.yaml +7 -0
  2. data/.standard.yml +3 -0
  3. data/CHANGELOG.md +22 -0
  4. data/LICENSE.txt +21 -0
  5. data/README.md +978 -0
  6. data/Rakefile +20 -0
  7. data/benchmark/README.md +198 -0
  8. data/benchmark/compression_levels.rb +99 -0
  9. data/benchmark/context_reuse.rb +174 -0
  10. data/benchmark/decompression_speed_by_level.rb +65 -0
  11. data/benchmark/dictionary_training.rb +182 -0
  12. data/benchmark/dictionary_usage.rb +121 -0
  13. data/benchmark/for_readme.rb +157 -0
  14. data/benchmark/generate_fixture.rb +82 -0
  15. data/benchmark/helpers.rb +237 -0
  16. data/benchmark/multithreading.rb +105 -0
  17. data/benchmark/run_all.rb +150 -0
  18. data/benchmark/streaming.rb +154 -0
  19. data/ext/vibe_zstd/Makefile +270 -0
  20. data/ext/vibe_zstd/cctx.c +565 -0
  21. data/ext/vibe_zstd/dctx.c +493 -0
  22. data/ext/vibe_zstd/dict.c +587 -0
  23. data/ext/vibe_zstd/extconf.rb +52 -0
  24. data/ext/vibe_zstd/frames.c +132 -0
  25. data/ext/vibe_zstd/libzstd/LICENSE +30 -0
  26. data/ext/vibe_zstd/libzstd/common/allocations.h +55 -0
  27. data/ext/vibe_zstd/libzstd/common/bits.h +205 -0
  28. data/ext/vibe_zstd/libzstd/common/bitstream.h +454 -0
  29. data/ext/vibe_zstd/libzstd/common/compiler.h +464 -0
  30. data/ext/vibe_zstd/libzstd/common/cpu.h +249 -0
  31. data/ext/vibe_zstd/libzstd/common/debug.c +30 -0
  32. data/ext/vibe_zstd/libzstd/common/debug.h +107 -0
  33. data/ext/vibe_zstd/libzstd/common/entropy_common.c +340 -0
  34. data/ext/vibe_zstd/libzstd/common/error_private.c +64 -0
  35. data/ext/vibe_zstd/libzstd/common/error_private.h +158 -0
  36. data/ext/vibe_zstd/libzstd/common/fse.h +625 -0
  37. data/ext/vibe_zstd/libzstd/common/fse_decompress.c +315 -0
  38. data/ext/vibe_zstd/libzstd/common/huf.h +277 -0
  39. data/ext/vibe_zstd/libzstd/common/mem.h +422 -0
  40. data/ext/vibe_zstd/libzstd/common/pool.c +371 -0
  41. data/ext/vibe_zstd/libzstd/common/pool.h +81 -0
  42. data/ext/vibe_zstd/libzstd/common/portability_macros.h +171 -0
  43. data/ext/vibe_zstd/libzstd/common/threading.c +182 -0
  44. data/ext/vibe_zstd/libzstd/common/threading.h +142 -0
  45. data/ext/vibe_zstd/libzstd/common/xxhash.c +18 -0
  46. data/ext/vibe_zstd/libzstd/common/xxhash.h +7094 -0
  47. data/ext/vibe_zstd/libzstd/common/zstd_common.c +48 -0
  48. data/ext/vibe_zstd/libzstd/common/zstd_deps.h +123 -0
  49. data/ext/vibe_zstd/libzstd/common/zstd_internal.h +324 -0
  50. data/ext/vibe_zstd/libzstd/common/zstd_trace.h +156 -0
  51. data/ext/vibe_zstd/libzstd/compress/clevels.h +134 -0
  52. data/ext/vibe_zstd/libzstd/compress/fse_compress.c +625 -0
  53. data/ext/vibe_zstd/libzstd/compress/hist.c +191 -0
  54. data/ext/vibe_zstd/libzstd/compress/hist.h +82 -0
  55. data/ext/vibe_zstd/libzstd/compress/huf_compress.c +1464 -0
  56. data/ext/vibe_zstd/libzstd/compress/zstd_compress.c +7843 -0
  57. data/ext/vibe_zstd/libzstd/compress/zstd_compress_internal.h +1636 -0
  58. data/ext/vibe_zstd/libzstd/compress/zstd_compress_literals.c +235 -0
  59. data/ext/vibe_zstd/libzstd/compress/zstd_compress_literals.h +39 -0
  60. data/ext/vibe_zstd/libzstd/compress/zstd_compress_sequences.c +442 -0
  61. data/ext/vibe_zstd/libzstd/compress/zstd_compress_sequences.h +55 -0
  62. data/ext/vibe_zstd/libzstd/compress/zstd_compress_superblock.c +688 -0
  63. data/ext/vibe_zstd/libzstd/compress/zstd_compress_superblock.h +32 -0
  64. data/ext/vibe_zstd/libzstd/compress/zstd_cwksp.h +765 -0
  65. data/ext/vibe_zstd/libzstd/compress/zstd_double_fast.c +778 -0
  66. data/ext/vibe_zstd/libzstd/compress/zstd_double_fast.h +42 -0
  67. data/ext/vibe_zstd/libzstd/compress/zstd_fast.c +985 -0
  68. data/ext/vibe_zstd/libzstd/compress/zstd_fast.h +30 -0
  69. data/ext/vibe_zstd/libzstd/compress/zstd_lazy.c +2199 -0
  70. data/ext/vibe_zstd/libzstd/compress/zstd_lazy.h +193 -0
  71. data/ext/vibe_zstd/libzstd/compress/zstd_ldm.c +745 -0
  72. data/ext/vibe_zstd/libzstd/compress/zstd_ldm.h +109 -0
  73. data/ext/vibe_zstd/libzstd/compress/zstd_ldm_geartab.h +106 -0
  74. data/ext/vibe_zstd/libzstd/compress/zstd_opt.c +1580 -0
  75. data/ext/vibe_zstd/libzstd/compress/zstd_opt.h +72 -0
  76. data/ext/vibe_zstd/libzstd/compress/zstd_preSplit.c +238 -0
  77. data/ext/vibe_zstd/libzstd/compress/zstd_preSplit.h +33 -0
  78. data/ext/vibe_zstd/libzstd/compress/zstdmt_compress.c +1923 -0
  79. data/ext/vibe_zstd/libzstd/compress/zstdmt_compress.h +102 -0
  80. data/ext/vibe_zstd/libzstd/decompress/huf_decompress.c +1944 -0
  81. data/ext/vibe_zstd/libzstd/decompress/huf_decompress_amd64.S +602 -0
  82. data/ext/vibe_zstd/libzstd/decompress/zstd_ddict.c +244 -0
  83. data/ext/vibe_zstd/libzstd/decompress/zstd_ddict.h +44 -0
  84. data/ext/vibe_zstd/libzstd/decompress/zstd_decompress.c +2410 -0
  85. data/ext/vibe_zstd/libzstd/decompress/zstd_decompress_block.c +2209 -0
  86. data/ext/vibe_zstd/libzstd/decompress/zstd_decompress_block.h +73 -0
  87. data/ext/vibe_zstd/libzstd/decompress/zstd_decompress_internal.h +240 -0
  88. data/ext/vibe_zstd/libzstd/deprecated/zbuff.h +214 -0
  89. data/ext/vibe_zstd/libzstd/deprecated/zbuff_common.c +26 -0
  90. data/ext/vibe_zstd/libzstd/deprecated/zbuff_compress.c +167 -0
  91. data/ext/vibe_zstd/libzstd/deprecated/zbuff_decompress.c +77 -0
  92. data/ext/vibe_zstd/libzstd/dictBuilder/cover.c +1302 -0
  93. data/ext/vibe_zstd/libzstd/dictBuilder/cover.h +152 -0
  94. data/ext/vibe_zstd/libzstd/dictBuilder/divsufsort.c +1913 -0
  95. data/ext/vibe_zstd/libzstd/dictBuilder/divsufsort.h +57 -0
  96. data/ext/vibe_zstd/libzstd/dictBuilder/fastcover.c +766 -0
  97. data/ext/vibe_zstd/libzstd/dictBuilder/zdict.c +1133 -0
  98. data/ext/vibe_zstd/libzstd/zdict.h +481 -0
  99. data/ext/vibe_zstd/libzstd/zstd.h +3198 -0
  100. data/ext/vibe_zstd/libzstd/zstd_errors.h +107 -0
  101. data/ext/vibe_zstd/streaming.c +410 -0
  102. data/ext/vibe_zstd/vibe_zstd.c +293 -0
  103. data/ext/vibe_zstd/vibe_zstd.h +56 -0
  104. data/ext/vibe_zstd/vibe_zstd_internal.h +27 -0
  105. data/lib/vibe_zstd/constants.rb +67 -0
  106. data/lib/vibe_zstd/version.rb +5 -0
  107. data/lib/vibe_zstd.rb +255 -0
  108. data/sig/vibe_zstd.rbs +76 -0
  109. metadata +179 -0
@@ -0,0 +1,48 @@
1
+ /*
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
+ * All rights reserved.
4
+ *
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
9
+ */
10
+
11
+
12
+
13
+ /*-*************************************
14
+ * Dependencies
15
+ ***************************************/
16
+ #define ZSTD_DEPS_NEED_MALLOC
17
+ #include "error_private.h"
18
+ #include "zstd_internal.h"
19
+
20
+
21
+ /*-****************************************
22
+ * Version
23
+ ******************************************/
24
+ unsigned ZSTD_versionNumber(void) { return ZSTD_VERSION_NUMBER; }
25
+
26
+ const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; }
27
+
28
+
29
+ /*-****************************************
30
+ * ZSTD Error Management
31
+ ******************************************/
32
+ #undef ZSTD_isError /* defined within zstd_internal.h */
33
+ /*! ZSTD_isError() :
34
+ * tells if a return value is an error code
35
+ * symbol is required for external callers */
36
+ unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
37
+
38
+ /*! ZSTD_getErrorName() :
39
+ * provides error code string from function result (useful for debugging) */
40
+ const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); }
41
+
42
+ /*! ZSTD_getError() :
43
+ * convert a `size_t` function result into a proper ZSTD_errorCode enum */
44
+ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
45
+
46
+ /*! ZSTD_getErrorString() :
47
+ * provides error code string from enum */
48
+ const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); }
@@ -0,0 +1,123 @@
1
+ /*
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
+ * All rights reserved.
4
+ *
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
9
+ */
10
+
11
+ /* This file provides common libc dependencies that zstd requires.
12
+ * The purpose is to allow replacing this file with a custom implementation
13
+ * to compile zstd without libc support.
14
+ */
15
+
16
+ /* Need:
17
+ * NULL
18
+ * INT_MAX
19
+ * UINT_MAX
20
+ * ZSTD_memcpy()
21
+ * ZSTD_memset()
22
+ * ZSTD_memmove()
23
+ */
24
+ #ifndef ZSTD_DEPS_COMMON
25
+ #define ZSTD_DEPS_COMMON
26
+
27
+ /* Even though we use qsort_r only for the dictionary builder, the macro
28
+ * _GNU_SOURCE has to be declared *before* the inclusion of any standard
29
+ * header and the script 'combine.sh' combines the whole zstd source code
30
+ * in a single file.
31
+ */
32
+ #if defined(__linux) || defined(__linux__) || defined(linux) || defined(__gnu_linux__) || \
33
+ defined(__CYGWIN__) || defined(__MSYS__)
34
+ #if !defined(_GNU_SOURCE) && !defined(__ANDROID__) /* NDK doesn't ship qsort_r(). */
35
+ #define _GNU_SOURCE
36
+ #endif
37
+ #endif
38
+
39
+ #include <limits.h>
40
+ #include <stddef.h>
41
+ #include <string.h>
42
+
43
+ #if defined(__GNUC__) && __GNUC__ >= 4
44
+ # define ZSTD_memcpy(d,s,l) __builtin_memcpy((d),(s),(l))
45
+ # define ZSTD_memmove(d,s,l) __builtin_memmove((d),(s),(l))
46
+ # define ZSTD_memset(p,v,l) __builtin_memset((p),(v),(l))
47
+ #else
48
+ # define ZSTD_memcpy(d,s,l) memcpy((d),(s),(l))
49
+ # define ZSTD_memmove(d,s,l) memmove((d),(s),(l))
50
+ # define ZSTD_memset(p,v,l) memset((p),(v),(l))
51
+ #endif
52
+
53
+ #endif /* ZSTD_DEPS_COMMON */
54
+
55
+ /* Need:
56
+ * ZSTD_malloc()
57
+ * ZSTD_free()
58
+ * ZSTD_calloc()
59
+ */
60
+ #ifdef ZSTD_DEPS_NEED_MALLOC
61
+ #ifndef ZSTD_DEPS_MALLOC
62
+ #define ZSTD_DEPS_MALLOC
63
+
64
+ #include <stdlib.h>
65
+
66
+ #define ZSTD_malloc(s) malloc(s)
67
+ #define ZSTD_calloc(n,s) calloc((n), (s))
68
+ #define ZSTD_free(p) free((p))
69
+
70
+ #endif /* ZSTD_DEPS_MALLOC */
71
+ #endif /* ZSTD_DEPS_NEED_MALLOC */
72
+
73
+ /*
74
+ * Provides 64-bit math support.
75
+ * Need:
76
+ * U64 ZSTD_div64(U64 dividend, U32 divisor)
77
+ */
78
+ #ifdef ZSTD_DEPS_NEED_MATH64
79
+ #ifndef ZSTD_DEPS_MATH64
80
+ #define ZSTD_DEPS_MATH64
81
+
82
+ #define ZSTD_div64(dividend, divisor) ((dividend) / (divisor))
83
+
84
+ #endif /* ZSTD_DEPS_MATH64 */
85
+ #endif /* ZSTD_DEPS_NEED_MATH64 */
86
+
87
+ /* Need:
88
+ * assert()
89
+ */
90
+ #ifdef ZSTD_DEPS_NEED_ASSERT
91
+ #ifndef ZSTD_DEPS_ASSERT
92
+ #define ZSTD_DEPS_ASSERT
93
+
94
+ #include <assert.h>
95
+
96
+ #endif /* ZSTD_DEPS_ASSERT */
97
+ #endif /* ZSTD_DEPS_NEED_ASSERT */
98
+
99
+ /* Need:
100
+ * ZSTD_DEBUG_PRINT()
101
+ */
102
+ #ifdef ZSTD_DEPS_NEED_IO
103
+ #ifndef ZSTD_DEPS_IO
104
+ #define ZSTD_DEPS_IO
105
+
106
+ #include <stdio.h>
107
+ #define ZSTD_DEBUG_PRINT(...) fprintf(stderr, __VA_ARGS__)
108
+
109
+ #endif /* ZSTD_DEPS_IO */
110
+ #endif /* ZSTD_DEPS_NEED_IO */
111
+
112
+ /* Only requested when <stdint.h> is known to be present.
113
+ * Need:
114
+ * intptr_t
115
+ */
116
+ #ifdef ZSTD_DEPS_NEED_STDINT
117
+ #ifndef ZSTD_DEPS_STDINT
118
+ #define ZSTD_DEPS_STDINT
119
+
120
+ #include <stdint.h>
121
+
122
+ #endif /* ZSTD_DEPS_STDINT */
123
+ #endif /* ZSTD_DEPS_NEED_STDINT */
@@ -0,0 +1,324 @@
1
+ /*
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
+ * All rights reserved.
4
+ *
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
9
+ */
10
+
11
+ #ifndef ZSTD_CCOMMON_H_MODULE
12
+ #define ZSTD_CCOMMON_H_MODULE
13
+
14
+ /* this module contains definitions which must be identical
15
+ * across compression, decompression and dictBuilder.
16
+ * It also contains a few functions useful to at least 2 of them
17
+ * and which benefit from being inlined */
18
+
19
+ /*-*************************************
20
+ * Dependencies
21
+ ***************************************/
22
+ #include "compiler.h"
23
+ #include "cpu.h"
24
+ #include "mem.h"
25
+ #include "debug.h" /* assert, DEBUGLOG, RAWLOG, g_debuglevel */
26
+ #include "error_private.h"
27
+ #define ZSTD_STATIC_LINKING_ONLY
28
+ #include "../zstd.h"
29
+ #define FSE_STATIC_LINKING_ONLY
30
+ #include "fse.h"
31
+ #include "huf.h"
32
+ #ifndef XXH_STATIC_LINKING_ONLY
33
+ # define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
34
+ #endif
35
+ #include "xxhash.h" /* XXH_reset, update, digest */
36
+ #ifndef ZSTD_NO_TRACE
37
+ # include "zstd_trace.h"
38
+ #else
39
+ # define ZSTD_TRACE 0
40
+ #endif
41
+
42
+ /* ---- static assert (debug) --- */
43
+ #define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c)
44
+ #define ZSTD_isError ERR_isError /* for inlining */
45
+ #define FSE_isError ERR_isError
46
+ #define HUF_isError ERR_isError
47
+
48
+
49
+ /*-*************************************
50
+ * shared macros
51
+ ***************************************/
52
+ #undef MIN
53
+ #undef MAX
54
+ #define MIN(a,b) ((a)<(b) ? (a) : (b))
55
+ #define MAX(a,b) ((a)>(b) ? (a) : (b))
56
+ #define BOUNDED(min,val,max) (MAX(min,MIN(val,max)))
57
+
58
+
59
+ /*-*************************************
60
+ * Common constants
61
+ ***************************************/
62
+ #define ZSTD_OPT_NUM (1<<12)
63
+
64
+ #define ZSTD_REP_NUM 3 /* number of repcodes */
65
+ static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
66
+
67
+ #define KB *(1 <<10)
68
+ #define MB *(1 <<20)
69
+ #define GB *(1U<<30)
70
+
71
+ #define BIT7 128
72
+ #define BIT6 64
73
+ #define BIT5 32
74
+ #define BIT4 16
75
+ #define BIT1 2
76
+ #define BIT0 1
77
+
78
+ #define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
79
+ static UNUSED_ATTR const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
80
+ static UNUSED_ATTR const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
81
+
82
+ #define ZSTD_FRAMEIDSIZE 4 /* magic number size */
83
+
84
+ #define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
85
+ static UNUSED_ATTR const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
86
+ typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e;
87
+
88
+ #define ZSTD_FRAMECHECKSUMSIZE 4
89
+
90
+ #define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */
91
+ #define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */) /* for a non-null block */
92
+ #define MIN_LITERALS_FOR_4_STREAMS 6
93
+
94
+ typedef enum { set_basic, set_rle, set_compressed, set_repeat } SymbolEncodingType_e;
95
+
96
+ #define LONGNBSEQ 0x7F00
97
+
98
+ #define MINMATCH 3
99
+
100
+ #define Litbits 8
101
+ #define LitHufLog 11
102
+ #define MaxLit ((1<<Litbits) - 1)
103
+ #define MaxML 52
104
+ #define MaxLL 35
105
+ #define DefaultMaxOff 28
106
+ #define MaxOff 31
107
+ #define MaxSeq MAX(MaxLL, MaxML) /* Assumption : MaxOff < MaxLL,MaxML */
108
+ #define MLFSELog 9
109
+ #define LLFSELog 9
110
+ #define OffFSELog 8
111
+ #define MaxFSELog MAX(MAX(MLFSELog, LLFSELog), OffFSELog)
112
+ #define MaxMLBits 16
113
+ #define MaxLLBits 16
114
+
115
+ #define ZSTD_MAX_HUF_HEADER_SIZE 128 /* header + <= 127 byte tree description */
116
+ /* Each table cannot take more than #symbols * FSELog bits */
117
+ #define ZSTD_MAX_FSE_HEADERS_SIZE (((MaxML + 1) * MLFSELog + (MaxLL + 1) * LLFSELog + (MaxOff + 1) * OffFSELog + 7) / 8)
118
+
119
+ static UNUSED_ATTR const U8 LL_bits[MaxLL+1] = {
120
+ 0, 0, 0, 0, 0, 0, 0, 0,
121
+ 0, 0, 0, 0, 0, 0, 0, 0,
122
+ 1, 1, 1, 1, 2, 2, 3, 3,
123
+ 4, 6, 7, 8, 9,10,11,12,
124
+ 13,14,15,16
125
+ };
126
+ static UNUSED_ATTR const S16 LL_defaultNorm[MaxLL+1] = {
127
+ 4, 3, 2, 2, 2, 2, 2, 2,
128
+ 2, 2, 2, 2, 2, 1, 1, 1,
129
+ 2, 2, 2, 2, 2, 2, 2, 2,
130
+ 2, 3, 2, 1, 1, 1, 1, 1,
131
+ -1,-1,-1,-1
132
+ };
133
+ #define LL_DEFAULTNORMLOG 6 /* for static allocation */
134
+ static UNUSED_ATTR const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG;
135
+
136
+ static UNUSED_ATTR const U8 ML_bits[MaxML+1] = {
137
+ 0, 0, 0, 0, 0, 0, 0, 0,
138
+ 0, 0, 0, 0, 0, 0, 0, 0,
139
+ 0, 0, 0, 0, 0, 0, 0, 0,
140
+ 0, 0, 0, 0, 0, 0, 0, 0,
141
+ 1, 1, 1, 1, 2, 2, 3, 3,
142
+ 4, 4, 5, 7, 8, 9,10,11,
143
+ 12,13,14,15,16
144
+ };
145
+ static UNUSED_ATTR const S16 ML_defaultNorm[MaxML+1] = {
146
+ 1, 4, 3, 2, 2, 2, 2, 2,
147
+ 2, 1, 1, 1, 1, 1, 1, 1,
148
+ 1, 1, 1, 1, 1, 1, 1, 1,
149
+ 1, 1, 1, 1, 1, 1, 1, 1,
150
+ 1, 1, 1, 1, 1, 1, 1, 1,
151
+ 1, 1, 1, 1, 1, 1,-1,-1,
152
+ -1,-1,-1,-1,-1
153
+ };
154
+ #define ML_DEFAULTNORMLOG 6 /* for static allocation */
155
+ static UNUSED_ATTR const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG;
156
+
157
+ static UNUSED_ATTR const S16 OF_defaultNorm[DefaultMaxOff+1] = {
158
+ 1, 1, 1, 1, 1, 1, 2, 2,
159
+ 2, 1, 1, 1, 1, 1, 1, 1,
160
+ 1, 1, 1, 1, 1, 1, 1, 1,
161
+ -1,-1,-1,-1,-1
162
+ };
163
+ #define OF_DEFAULTNORMLOG 5 /* for static allocation */
164
+ static UNUSED_ATTR const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG;
165
+
166
+
167
+ /*-*******************************************
168
+ * Shared functions to include for inlining
169
+ *********************************************/
170
+ static void ZSTD_copy8(void* dst, const void* src) {
171
+ #if defined(ZSTD_ARCH_ARM_NEON)
172
+ vst1_u8((uint8_t*)dst, vld1_u8((const uint8_t*)src));
173
+ #else
174
+ ZSTD_memcpy(dst, src, 8);
175
+ #endif
176
+ }
177
+ #define COPY8(d,s) do { ZSTD_copy8(d,s); d+=8; s+=8; } while (0)
178
+
179
+ /* Need to use memmove here since the literal buffer can now be located within
180
+ the dst buffer. In circumstances where the op "catches up" to where the
181
+ literal buffer is, there can be partial overlaps in this call on the final
182
+ copy if the literal is being shifted by less than 16 bytes. */
183
+ static void ZSTD_copy16(void* dst, const void* src) {
184
+ #if defined(ZSTD_ARCH_ARM_NEON)
185
+ vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src));
186
+ #elif defined(ZSTD_ARCH_X86_SSE2)
187
+ _mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src));
188
+ #elif defined(__clang__)
189
+ ZSTD_memmove(dst, src, 16);
190
+ #else
191
+ /* ZSTD_memmove is not inlined properly by gcc */
192
+ BYTE copy16_buf[16];
193
+ ZSTD_memcpy(copy16_buf, src, 16);
194
+ ZSTD_memcpy(dst, copy16_buf, 16);
195
+ #endif
196
+ }
197
+ #define COPY16(d,s) do { ZSTD_copy16(d,s); d+=16; s+=16; } while (0)
198
+
199
+ #define WILDCOPY_OVERLENGTH 32
200
+ #define WILDCOPY_VECLEN 16
201
+
202
+ typedef enum {
203
+ ZSTD_no_overlap,
204
+ ZSTD_overlap_src_before_dst
205
+ /* ZSTD_overlap_dst_before_src, */
206
+ } ZSTD_overlap_e;
207
+
208
+ /*! ZSTD_wildcopy() :
209
+ * Custom version of ZSTD_memcpy(), can over read/write up to WILDCOPY_OVERLENGTH bytes (if length==0)
210
+ * @param ovtype controls the overlap detection
211
+ * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart.
212
+ * - ZSTD_overlap_src_before_dst: The src and dst may overlap, but they MUST be at least 8 bytes apart.
213
+ * The src buffer must be before the dst buffer.
214
+ */
215
+ MEM_STATIC FORCE_INLINE_ATTR
216
+ void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e const ovtype)
217
+ {
218
+ ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src;
219
+ const BYTE* ip = (const BYTE*)src;
220
+ BYTE* op = (BYTE*)dst;
221
+ BYTE* const oend = op + length;
222
+
223
+ if (ovtype == ZSTD_overlap_src_before_dst && diff < WILDCOPY_VECLEN) {
224
+ /* Handle short offset copies. */
225
+ do {
226
+ COPY8(op, ip);
227
+ } while (op < oend);
228
+ } else {
229
+ assert(diff >= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN);
230
+ /* Separate out the first COPY16() call because the copy length is
231
+ * almost certain to be short, so the branches have different
232
+ * probabilities. Since it is almost certain to be short, only do
233
+ * one COPY16() in the first call. Then, do two calls per loop since
234
+ * at that point it is more likely to have a high trip count.
235
+ */
236
+ ZSTD_copy16(op, ip);
237
+ if (16 >= length) return;
238
+ op += 16;
239
+ ip += 16;
240
+ do {
241
+ COPY16(op, ip);
242
+ COPY16(op, ip);
243
+ }
244
+ while (op < oend);
245
+ }
246
+ }
247
+
248
+ MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
249
+ {
250
+ size_t const length = MIN(dstCapacity, srcSize);
251
+ if (length > 0) {
252
+ ZSTD_memcpy(dst, src, length);
253
+ }
254
+ return length;
255
+ }
256
+
257
+ /* define "workspace is too large" as this number of times larger than needed */
258
+ #define ZSTD_WORKSPACETOOLARGE_FACTOR 3
259
+
260
+ /* when workspace is continuously too large
261
+ * during at least this number of times,
262
+ * context's memory usage is considered wasteful,
263
+ * because it's sized to handle a worst case scenario which rarely happens.
264
+ * In which case, resize it down to free some memory */
265
+ #define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128
266
+
267
+ /* Controls whether the input/output buffer is buffered or stable. */
268
+ typedef enum {
269
+ ZSTD_bm_buffered = 0, /* Buffer the input/output */
270
+ ZSTD_bm_stable = 1 /* ZSTD_inBuffer/ZSTD_outBuffer is stable */
271
+ } ZSTD_bufferMode_e;
272
+
273
+
274
+ /*-*******************************************
275
+ * Private declarations
276
+ *********************************************/
277
+
278
+ /**
279
+ * Contains the compressed frame size and an upper-bound for the decompressed frame size.
280
+ * Note: before using `compressedSize`, check for errors using ZSTD_isError().
281
+ * similarly, before using `decompressedBound`, check for errors using:
282
+ * `decompressedBound != ZSTD_CONTENTSIZE_ERROR`
283
+ */
284
+ typedef struct {
285
+ size_t nbBlocks;
286
+ size_t compressedSize;
287
+ unsigned long long decompressedBound;
288
+ } ZSTD_frameSizeInfo; /* decompress & legacy */
289
+
290
+ /* ZSTD_invalidateRepCodes() :
291
+ * ensures next compression will not use repcodes from previous block.
292
+ * Note : only works with regular variant;
293
+ * do not use with extDict variant ! */
294
+ void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); /* zstdmt, adaptive_compression (shouldn't get this definition from here) */
295
+
296
+
297
+ typedef struct {
298
+ blockType_e blockType;
299
+ U32 lastBlock;
300
+ U32 origSize;
301
+ } blockProperties_t; /* declared here for decompress and fullbench */
302
+
303
+ /*! ZSTD_getcBlockSize() :
304
+ * Provides the size of compressed block from block header `src` */
305
+ /* Used by: decompress, fullbench */
306
+ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
307
+ blockProperties_t* bpPtr);
308
+
309
+ /*! ZSTD_decodeSeqHeaders() :
310
+ * decode sequence header from src */
311
+ /* Used by: zstd_decompress_block, fullbench */
312
+ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
313
+ const void* src, size_t srcSize);
314
+
315
+ /**
316
+ * @returns true iff the CPU supports dynamic BMI2 dispatch.
317
+ */
318
+ MEM_STATIC int ZSTD_cpuSupportsBmi2(void)
319
+ {
320
+ ZSTD_cpuid_t cpuid = ZSTD_cpuid();
321
+ return ZSTD_cpuid_bmi1(cpuid) && ZSTD_cpuid_bmi2(cpuid);
322
+ }
323
+
324
+ #endif /* ZSTD_CCOMMON_H_MODULE */
@@ -0,0 +1,156 @@
1
+ /*
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
+ * All rights reserved.
4
+ *
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
9
+ */
10
+
11
+ #ifndef ZSTD_TRACE_H
12
+ #define ZSTD_TRACE_H
13
+
14
+ #include <stddef.h>
15
+
16
+ /* weak symbol support
17
+ * For now, enable conservatively:
18
+ * - Only GNUC
19
+ * - Only ELF
20
+ * - Only x86-64, i386, aarch64 and risc-v.
21
+ * Also, explicitly disable on platforms known not to work so they aren't
22
+ * forgotten in the future.
23
+ */
24
+ #if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && \
25
+ defined(__GNUC__) && defined(__ELF__) && \
26
+ (defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || \
27
+ defined(_M_IX86) || defined(__aarch64__) || defined(__riscv)) && \
28
+ !defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \
29
+ !defined(__CYGWIN__) && !defined(_AIX)
30
+ # define ZSTD_HAVE_WEAK_SYMBOLS 1
31
+ #else
32
+ # define ZSTD_HAVE_WEAK_SYMBOLS 0
33
+ #endif
34
+ #if ZSTD_HAVE_WEAK_SYMBOLS
35
+ # define ZSTD_WEAK_ATTR __attribute__((__weak__))
36
+ #else
37
+ # define ZSTD_WEAK_ATTR
38
+ #endif
39
+
40
+ /* Only enable tracing when weak symbols are available. */
41
+ #ifndef ZSTD_TRACE
42
+ # define ZSTD_TRACE ZSTD_HAVE_WEAK_SYMBOLS
43
+ #endif
44
+
45
+ #if ZSTD_TRACE
46
+
47
+ struct ZSTD_CCtx_s;
48
+ struct ZSTD_DCtx_s;
49
+ struct ZSTD_CCtx_params_s;
50
+
51
+ typedef struct {
52
+ /**
53
+ * ZSTD_VERSION_NUMBER
54
+ *
55
+ * This is guaranteed to be the first member of ZSTD_trace.
56
+ * Otherwise, this struct is not stable between versions. If
57
+ * the version number does not match your expectation, you
58
+ * should not interpret the rest of the struct.
59
+ */
60
+ unsigned version;
61
+ /**
62
+ * Non-zero if streaming (de)compression is used.
63
+ */
64
+ int streaming;
65
+ /**
66
+ * The dictionary ID.
67
+ */
68
+ unsigned dictionaryID;
69
+ /**
70
+ * Is the dictionary cold?
71
+ * Only set on decompression.
72
+ */
73
+ int dictionaryIsCold;
74
+ /**
75
+ * The dictionary size or zero if no dictionary.
76
+ */
77
+ size_t dictionarySize;
78
+ /**
79
+ * The uncompressed size of the data.
80
+ */
81
+ size_t uncompressedSize;
82
+ /**
83
+ * The compressed size of the data.
84
+ */
85
+ size_t compressedSize;
86
+ /**
87
+ * The fully resolved CCtx parameters (NULL on decompression).
88
+ */
89
+ struct ZSTD_CCtx_params_s const* params;
90
+ /**
91
+ * The ZSTD_CCtx pointer (NULL on decompression).
92
+ */
93
+ struct ZSTD_CCtx_s const* cctx;
94
+ /**
95
+ * The ZSTD_DCtx pointer (NULL on compression).
96
+ */
97
+ struct ZSTD_DCtx_s const* dctx;
98
+ } ZSTD_Trace;
99
+
100
+ /**
101
+ * A tracing context. It must be 0 when tracing is disabled.
102
+ * Otherwise, any non-zero value returned by a tracing begin()
103
+ * function is presented to any subsequent calls to end().
104
+ *
105
+ * Any non-zero value is treated as tracing is enabled and not
106
+ * interpreted by the library.
107
+ *
108
+ * Two possible uses are:
109
+ * * A timestamp for when the begin() function was called.
110
+ * * A unique key identifying the (de)compression, like the
111
+ * address of the [dc]ctx pointer if you need to track
112
+ * more information than just a timestamp.
113
+ */
114
+ typedef unsigned long long ZSTD_TraceCtx;
115
+
116
+ /**
117
+ * Trace the beginning of a compression call.
118
+ * @param cctx The dctx pointer for the compression.
119
+ * It can be used as a key to map begin() to end().
120
+ * @returns Non-zero if tracing is enabled. The return value is
121
+ * passed to ZSTD_trace_compress_end().
122
+ */
123
+ ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_compress_begin(
124
+ struct ZSTD_CCtx_s const* cctx);
125
+
126
+ /**
127
+ * Trace the end of a compression call.
128
+ * @param ctx The return value of ZSTD_trace_compress_begin().
129
+ * @param trace The zstd tracing info.
130
+ */
131
+ ZSTD_WEAK_ATTR void ZSTD_trace_compress_end(
132
+ ZSTD_TraceCtx ctx,
133
+ ZSTD_Trace const* trace);
134
+
135
+ /**
136
+ * Trace the beginning of a decompression call.
137
+ * @param dctx The dctx pointer for the decompression.
138
+ * It can be used as a key to map begin() to end().
139
+ * @returns Non-zero if tracing is enabled. The return value is
140
+ * passed to ZSTD_trace_compress_end().
141
+ */
142
+ ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_decompress_begin(
143
+ struct ZSTD_DCtx_s const* dctx);
144
+
145
+ /**
146
+ * Trace the end of a decompression call.
147
+ * @param ctx The return value of ZSTD_trace_decompress_begin().
148
+ * @param trace The zstd tracing info.
149
+ */
150
+ ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end(
151
+ ZSTD_TraceCtx ctx,
152
+ ZSTD_Trace const* trace);
153
+
154
+ #endif /* ZSTD_TRACE */
155
+
156
+ #endif /* ZSTD_TRACE_H */