zstd-ruby 1.5.6.6 → 1.5.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/README.md +38 -1
- data/ext/zstdruby/common.h +37 -11
- data/ext/zstdruby/libzstd/common/bits.h +92 -87
- data/ext/zstdruby/libzstd/common/bitstream.h +26 -29
- data/ext/zstdruby/libzstd/common/compiler.h +36 -22
- data/ext/zstdruby/libzstd/common/cpu.h +1 -1
- data/ext/zstdruby/libzstd/common/debug.h +0 -9
- data/ext/zstdruby/libzstd/common/error_private.c +1 -0
- data/ext/zstdruby/libzstd/common/error_private.h +0 -10
- data/ext/zstdruby/libzstd/common/fse.h +2 -17
- data/ext/zstdruby/libzstd/common/fse_decompress.c +2 -0
- data/ext/zstdruby/libzstd/common/huf.h +0 -9
- data/ext/zstdruby/libzstd/common/mem.h +7 -11
- data/ext/zstdruby/libzstd/common/pool.h +0 -9
- data/ext/zstdruby/libzstd/common/portability_macros.h +22 -9
- data/ext/zstdruby/libzstd/common/threading.h +0 -8
- data/ext/zstdruby/libzstd/common/xxhash.h +93 -19
- data/ext/zstdruby/libzstd/common/zstd_deps.h +12 -0
- data/ext/zstdruby/libzstd/common/zstd_internal.h +1 -69
- data/ext/zstdruby/libzstd/common/zstd_trace.h +5 -12
- data/ext/zstdruby/libzstd/compress/hist.c +10 -0
- data/ext/zstdruby/libzstd/compress/hist.h +7 -0
- data/ext/zstdruby/libzstd/compress/zstd_compress.c +1057 -367
- data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +227 -125
- data/ext/zstdruby/libzstd/compress/zstd_compress_literals.c +1 -1
- data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.c +7 -7
- data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.h +7 -6
- data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.c +17 -17
- data/ext/zstdruby/libzstd/compress/zstd_cwksp.h +41 -24
- data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +58 -50
- data/ext/zstdruby/libzstd/compress/zstd_double_fast.h +4 -12
- data/ext/zstdruby/libzstd/compress/zstd_fast.c +91 -74
- data/ext/zstdruby/libzstd/compress/zstd_fast.h +4 -12
- data/ext/zstdruby/libzstd/compress/zstd_lazy.c +64 -64
- data/ext/zstdruby/libzstd/compress/zstd_lazy.h +30 -39
- data/ext/zstdruby/libzstd/compress/zstd_ldm.c +48 -33
- data/ext/zstdruby/libzstd/compress/zstd_ldm.h +6 -14
- data/ext/zstdruby/libzstd/compress/zstd_opt.c +55 -51
- data/ext/zstdruby/libzstd/compress/zstd_opt.h +8 -16
- data/ext/zstdruby/libzstd/compress/zstd_preSplit.c +238 -0
- data/ext/zstdruby/libzstd/compress/zstd_preSplit.h +33 -0
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +134 -93
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +4 -15
- data/ext/zstdruby/libzstd/decompress/huf_decompress_amd64.S +10 -3
- data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +14 -11
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.c +6 -12
- data/ext/zstdruby/libzstd/decompress/zstd_decompress_internal.h +5 -5
- data/ext/zstdruby/libzstd/dictBuilder/cover.c +60 -19
- data/ext/zstdruby/libzstd/dictBuilder/divsufsort.h +0 -10
- data/ext/zstdruby/libzstd/dictBuilder/zdict.c +2 -2
- data/ext/zstdruby/libzstd/zdict.h +15 -8
- data/ext/zstdruby/libzstd/zstd.h +241 -132
- data/ext/zstdruby/libzstd/zstd_errors.h +1 -8
- data/ext/zstdruby/main.c +4 -0
- data/ext/zstdruby/zstdruby.c +92 -0
- data/lib/zstd-ruby/stream_writer.rb +1 -1
- data/lib/zstd-ruby/version.rb +1 -1
- metadata +5 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 50ffda13ae5c1470be924476eb600af7695a2e76f0764666f8f4a90b07b35428
|
4
|
+
data.tar.gz: e28d50be732039fc58762c02870e3ecf227b5843d647fce6a5cebac91dbf8487
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 71433e488936220118f3b4611594f8eaac68c3544816e890c412fa76f531e450f44d62d4b03d900e95b47e25d37f35b73c41baeb8275917adc1013172763fc11
|
7
|
+
data.tar.gz: 6e1f7a19a7fb75633b790d3d3a5c2c20bc520a6f2ed03f17941f186f7c9e5e2a00dfa7398fc428cfd665a935f5ed516115d81a826439bfa3d304119ae1f5cc79
|
data/.gitignore
CHANGED
data/README.md
CHANGED
@@ -10,7 +10,7 @@ See https://github.com/facebook/zstd
|
|
10
10
|
Fork from https://github.com/jarredholman/ruby-zstd.
|
11
11
|
|
12
12
|
## Zstd version
|
13
|
-
[v1.5.
|
13
|
+
[v1.5.7](https://github.com/facebook/zstd/tree/v1.5.7)
|
14
14
|
|
15
15
|
## Installation
|
16
16
|
|
@@ -49,6 +49,22 @@ compressed_data = Zstd.compress(data, level: complession_level) # default compre
|
|
49
49
|
compressed_using_dict = Zstd.compress("", dict: File.read('dictionary_file'))
|
50
50
|
```
|
51
51
|
|
52
|
+
#### Compression with CDict
|
53
|
+
|
54
|
+
If you use the same dictionary repeatedly, you can speed up the setup by creating CDict in advance:
|
55
|
+
|
56
|
+
```ruby
|
57
|
+
cdict = Zstd::CDict.new(File.read('dictionary_file'))
|
58
|
+
compressed_using_dict = Zstd.compress("", dict: cdict)
|
59
|
+
```
|
60
|
+
|
61
|
+
The compression_level can be specified on creating CDict.
|
62
|
+
|
63
|
+
```ruby
|
64
|
+
cdict = Zstd::CDict.new(File.read('dictionary_file'), 5)
|
65
|
+
compressed_using_dict = Zstd.compress("", dict: cdict)
|
66
|
+
```
|
67
|
+
|
52
68
|
#### Streaming Compression
|
53
69
|
```ruby
|
54
70
|
stream = Zstd::StreamingCompress.new
|
@@ -86,6 +102,16 @@ stream << "ghi"
|
|
86
102
|
res << stream.finish
|
87
103
|
```
|
88
104
|
|
105
|
+
#### Streaming Compression with CDict of level 5
|
106
|
+
```ruby
|
107
|
+
cdict = Zstd::CDict.new(File.read('dictionary_file', 5)
|
108
|
+
stream = Zstd::StreamingCompress.new(dict: cdict)
|
109
|
+
stream << "abc" << "def"
|
110
|
+
res = stream.flush
|
111
|
+
stream << "ghi"
|
112
|
+
res << stream.finish
|
113
|
+
```
|
114
|
+
|
89
115
|
### Decompression
|
90
116
|
|
91
117
|
#### Simple Decompression
|
@@ -100,6 +126,15 @@ data = Zstd.decompress(compressed_data)
|
|
100
126
|
Zstd.decompress(compressed_using_dict, dict: File.read('dictionary_file'))
|
101
127
|
```
|
102
128
|
|
129
|
+
#### Decompression with DDict
|
130
|
+
|
131
|
+
If you use the same dictionary repeatedly, you can speed up the setup by creating DDict in advance:
|
132
|
+
|
133
|
+
```ruby
|
134
|
+
ddict = Zstd::Ddict.new(File.read('dictionary_file'))
|
135
|
+
data = Zstd.compress(compressed_using_dict, ddict)
|
136
|
+
```
|
137
|
+
|
103
138
|
#### Streaming Decompression
|
104
139
|
```ruby
|
105
140
|
cstr = "" # Compressed data
|
@@ -118,6 +153,8 @@ result << stream.decompress(cstr[0, 10])
|
|
118
153
|
result << stream.decompress(cstr[10..-1])
|
119
154
|
```
|
120
155
|
|
156
|
+
DDict can also be specified to `dict:`.
|
157
|
+
|
121
158
|
### Skippable frame
|
122
159
|
|
123
160
|
```ruby
|
data/ext/zstdruby/common.h
CHANGED
@@ -1,13 +1,15 @@
|
|
1
1
|
#ifndef ZSTD_RUBY_H
|
2
2
|
#define ZSTD_RUBY_H 1
|
3
3
|
|
4
|
+
#include <stdbool.h>
|
4
5
|
#include <ruby.h>
|
5
6
|
#ifdef HAVE_RUBY_THREAD_H
|
6
7
|
#include <ruby/thread.h>
|
7
8
|
#endif
|
8
|
-
#include <stdbool.h>
|
9
9
|
#include "./libzstd/zstd.h"
|
10
10
|
|
11
|
+
extern VALUE rb_cCDict, rb_cDDict;
|
12
|
+
|
11
13
|
static int convert_compression_level(VALUE compression_level_value)
|
12
14
|
{
|
13
15
|
if (NIL_P(compression_level_value)) {
|
@@ -34,12 +36,24 @@ static void set_compress_params(ZSTD_CCtx* const ctx, VALUE level_from_args, VAL
|
|
34
36
|
ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, compression_level);
|
35
37
|
|
36
38
|
if (kwargs_values[1] != Qundef && kwargs_values[1] != Qnil) {
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
39
|
+
if (CLASS_OF(kwargs_values[1]) == rb_cCDict) {
|
40
|
+
ZSTD_CDict* cdict = DATA_PTR(kwargs_values[1]);
|
41
|
+
size_t ref_dict_ret = ZSTD_CCtx_refCDict(ctx, cdict);
|
42
|
+
if (ZSTD_isError(ref_dict_ret)) {
|
43
|
+
ZSTD_freeCCtx(ctx);
|
44
|
+
rb_raise(rb_eRuntimeError, "%s", "ZSTD_CCtx_refCDict failed");
|
45
|
+
}
|
46
|
+
} else if (TYPE(kwargs_values[1]) == T_STRING) {
|
47
|
+
char* dict_buffer = RSTRING_PTR(kwargs_values[1]);
|
48
|
+
size_t dict_size = RSTRING_LEN(kwargs_values[1]);
|
49
|
+
size_t load_dict_ret = ZSTD_CCtx_loadDictionary(ctx, dict_buffer, dict_size);
|
50
|
+
if (ZSTD_isError(load_dict_ret)) {
|
51
|
+
ZSTD_freeCCtx(ctx);
|
52
|
+
rb_raise(rb_eRuntimeError, "%s", "ZSTD_CCtx_loadDictionary failed");
|
53
|
+
}
|
54
|
+
} else {
|
41
55
|
ZSTD_freeCCtx(ctx);
|
42
|
-
rb_raise(
|
56
|
+
rb_raise(rb_eArgError, "`dict:` must be a Zstd::CDict or a String");
|
43
57
|
}
|
44
58
|
}
|
45
59
|
}
|
@@ -113,12 +127,24 @@ static void set_decompress_params(ZSTD_DCtx* const dctx, VALUE kwargs)
|
|
113
127
|
rb_get_kwargs(kwargs, kwargs_keys, 0, 1, kwargs_values);
|
114
128
|
|
115
129
|
if (kwargs_values[0] != Qundef && kwargs_values[0] != Qnil) {
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
130
|
+
if (CLASS_OF(kwargs_values[0]) == rb_cDDict) {
|
131
|
+
ZSTD_DDict* ddict = DATA_PTR(kwargs_values[0]);
|
132
|
+
size_t ref_dict_ret = ZSTD_DCtx_refDDict(dctx, ddict);
|
133
|
+
if (ZSTD_isError(ref_dict_ret)) {
|
134
|
+
ZSTD_freeDCtx(dctx);
|
135
|
+
rb_raise(rb_eRuntimeError, "%s", "ZSTD_DCtx_refDDict failed");
|
136
|
+
}
|
137
|
+
} else if (TYPE(kwargs_values[0]) == T_STRING) {
|
138
|
+
char* dict_buffer = RSTRING_PTR(kwargs_values[0]);
|
139
|
+
size_t dict_size = RSTRING_LEN(kwargs_values[0]);
|
140
|
+
size_t load_dict_ret = ZSTD_DCtx_loadDictionary(dctx, dict_buffer, dict_size);
|
141
|
+
if (ZSTD_isError(load_dict_ret)) {
|
142
|
+
ZSTD_freeDCtx(dctx);
|
143
|
+
rb_raise(rb_eRuntimeError, "%s", "ZSTD_CCtx_loadDictionary failed");
|
144
|
+
}
|
145
|
+
} else {
|
120
146
|
ZSTD_freeDCtx(dctx);
|
121
|
-
rb_raise(
|
147
|
+
rb_raise(rb_eArgError, "`dict:` must be a Zstd::DDict or a String");
|
122
148
|
}
|
123
149
|
}
|
124
150
|
}
|
@@ -28,27 +28,29 @@ MEM_STATIC unsigned ZSTD_countTrailingZeros32_fallback(U32 val)
|
|
28
28
|
MEM_STATIC unsigned ZSTD_countTrailingZeros32(U32 val)
|
29
29
|
{
|
30
30
|
assert(val != 0);
|
31
|
-
#
|
32
|
-
#
|
33
|
-
|
34
|
-
#
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
#
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
31
|
+
#if defined(_MSC_VER)
|
32
|
+
# if STATIC_BMI2
|
33
|
+
return (unsigned)_tzcnt_u32(val);
|
34
|
+
# else
|
35
|
+
if (val != 0) {
|
36
|
+
unsigned long r;
|
37
|
+
_BitScanForward(&r, val);
|
38
|
+
return (unsigned)r;
|
39
|
+
} else {
|
40
|
+
__assume(0); /* Should not reach this code path */
|
41
|
+
}
|
42
|
+
# endif
|
43
|
+
#elif defined(__GNUC__) && (__GNUC__ >= 4)
|
44
|
+
return (unsigned)__builtin_ctz(val);
|
45
|
+
#elif defined(__ICCARM__)
|
46
|
+
return (unsigned)__builtin_ctz(val);
|
47
|
+
#else
|
48
|
+
return ZSTD_countTrailingZeros32_fallback(val);
|
49
|
+
#endif
|
49
50
|
}
|
50
51
|
|
51
|
-
MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val)
|
52
|
+
MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val)
|
53
|
+
{
|
52
54
|
assert(val != 0);
|
53
55
|
{
|
54
56
|
static const U32 DeBruijnClz[32] = {0, 9, 1, 10, 13, 21, 2, 29,
|
@@ -67,86 +69,89 @@ MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val) {
|
|
67
69
|
MEM_STATIC unsigned ZSTD_countLeadingZeros32(U32 val)
|
68
70
|
{
|
69
71
|
assert(val != 0);
|
70
|
-
#
|
71
|
-
#
|
72
|
-
|
73
|
-
#
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
#
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
72
|
+
#if defined(_MSC_VER)
|
73
|
+
# if STATIC_BMI2
|
74
|
+
return (unsigned)_lzcnt_u32(val);
|
75
|
+
# else
|
76
|
+
if (val != 0) {
|
77
|
+
unsigned long r;
|
78
|
+
_BitScanReverse(&r, val);
|
79
|
+
return (unsigned)(31 - r);
|
80
|
+
} else {
|
81
|
+
__assume(0); /* Should not reach this code path */
|
82
|
+
}
|
83
|
+
# endif
|
84
|
+
#elif defined(__GNUC__) && (__GNUC__ >= 4)
|
85
|
+
return (unsigned)__builtin_clz(val);
|
86
|
+
#elif defined(__ICCARM__)
|
87
|
+
return (unsigned)__builtin_clz(val);
|
88
|
+
#else
|
89
|
+
return ZSTD_countLeadingZeros32_fallback(val);
|
90
|
+
#endif
|
88
91
|
}
|
89
92
|
|
90
93
|
MEM_STATIC unsigned ZSTD_countTrailingZeros64(U64 val)
|
91
94
|
{
|
92
95
|
assert(val != 0);
|
93
|
-
#
|
94
|
-
#
|
95
|
-
|
96
|
-
#
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
#
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
96
|
+
#if defined(_MSC_VER) && defined(_WIN64)
|
97
|
+
# if STATIC_BMI2
|
98
|
+
return (unsigned)_tzcnt_u64(val);
|
99
|
+
# else
|
100
|
+
if (val != 0) {
|
101
|
+
unsigned long r;
|
102
|
+
_BitScanForward64(&r, val);
|
103
|
+
return (unsigned)r;
|
104
|
+
} else {
|
105
|
+
__assume(0); /* Should not reach this code path */
|
106
|
+
}
|
107
|
+
# endif
|
108
|
+
#elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(__LP64__)
|
109
|
+
return (unsigned)__builtin_ctzll(val);
|
110
|
+
#elif defined(__ICCARM__)
|
111
|
+
return (unsigned)__builtin_ctzll(val);
|
112
|
+
#else
|
113
|
+
{
|
114
|
+
U32 mostSignificantWord = (U32)(val >> 32);
|
115
|
+
U32 leastSignificantWord = (U32)val;
|
116
|
+
if (leastSignificantWord == 0) {
|
117
|
+
return 32 + ZSTD_countTrailingZeros32(mostSignificantWord);
|
118
|
+
} else {
|
119
|
+
return ZSTD_countTrailingZeros32(leastSignificantWord);
|
117
120
|
}
|
118
|
-
|
121
|
+
}
|
122
|
+
#endif
|
119
123
|
}
|
120
124
|
|
121
125
|
MEM_STATIC unsigned ZSTD_countLeadingZeros64(U64 val)
|
122
126
|
{
|
123
127
|
assert(val != 0);
|
124
|
-
#
|
125
|
-
#
|
126
|
-
|
127
|
-
#
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
#
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
128
|
+
#if defined(_MSC_VER) && defined(_WIN64)
|
129
|
+
# if STATIC_BMI2
|
130
|
+
return (unsigned)_lzcnt_u64(val);
|
131
|
+
# else
|
132
|
+
if (val != 0) {
|
133
|
+
unsigned long r;
|
134
|
+
_BitScanReverse64(&r, val);
|
135
|
+
return (unsigned)(63 - r);
|
136
|
+
} else {
|
137
|
+
__assume(0); /* Should not reach this code path */
|
138
|
+
}
|
139
|
+
# endif
|
140
|
+
#elif defined(__GNUC__) && (__GNUC__ >= 4)
|
141
|
+
return (unsigned)(__builtin_clzll(val));
|
142
|
+
#elif defined(__ICCARM__)
|
143
|
+
return (unsigned)(__builtin_clzll(val));
|
144
|
+
#else
|
145
|
+
{
|
146
|
+
U32 mostSignificantWord = (U32)(val >> 32);
|
147
|
+
U32 leastSignificantWord = (U32)val;
|
148
|
+
if (mostSignificantWord == 0) {
|
149
|
+
return 32 + ZSTD_countLeadingZeros32(leastSignificantWord);
|
150
|
+
} else {
|
151
|
+
return ZSTD_countLeadingZeros32(mostSignificantWord);
|
148
152
|
}
|
149
|
-
|
153
|
+
}
|
154
|
+
#endif
|
150
155
|
}
|
151
156
|
|
152
157
|
MEM_STATIC unsigned ZSTD_NbCommonBytes(size_t val)
|
@@ -14,9 +14,6 @@
|
|
14
14
|
#ifndef BITSTREAM_H_MODULE
|
15
15
|
#define BITSTREAM_H_MODULE
|
16
16
|
|
17
|
-
#if defined (__cplusplus)
|
18
|
-
extern "C" {
|
19
|
-
#endif
|
20
17
|
/*
|
21
18
|
* This API consists of small unitary functions, which must be inlined for best performance.
|
22
19
|
* Since link-time-optimization is not available for all compilers,
|
@@ -32,7 +29,6 @@ extern "C" {
|
|
32
29
|
#include "error_private.h" /* error codes and messages */
|
33
30
|
#include "bits.h" /* ZSTD_highbit32 */
|
34
31
|
|
35
|
-
|
36
32
|
/*=========================================
|
37
33
|
* Target specific
|
38
34
|
=========================================*/
|
@@ -52,12 +48,13 @@ extern "C" {
|
|
52
48
|
/*-******************************************
|
53
49
|
* bitStream encoding API (write forward)
|
54
50
|
********************************************/
|
51
|
+
typedef size_t BitContainerType;
|
55
52
|
/* bitStream can mix input from multiple sources.
|
56
53
|
* A critical property of these streams is that they encode and decode in **reverse** direction.
|
57
54
|
* So the first bit sequence you add will be the last to be read, like a LIFO stack.
|
58
55
|
*/
|
59
56
|
typedef struct {
|
60
|
-
|
57
|
+
BitContainerType bitContainer;
|
61
58
|
unsigned bitPos;
|
62
59
|
char* startPtr;
|
63
60
|
char* ptr;
|
@@ -65,7 +62,7 @@ typedef struct {
|
|
65
62
|
} BIT_CStream_t;
|
66
63
|
|
67
64
|
MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity);
|
68
|
-
MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
|
65
|
+
MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, BitContainerType value, unsigned nbBits);
|
69
66
|
MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC);
|
70
67
|
MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
|
71
68
|
|
@@ -74,7 +71,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
|
|
74
71
|
* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code.
|
75
72
|
*
|
76
73
|
* bits are first added to a local register.
|
77
|
-
* Local register is
|
74
|
+
* Local register is BitContainerType, 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
|
78
75
|
* Writing data into memory is an explicit operation, performed by the flushBits function.
|
79
76
|
* Hence keep track how many bits are potentially stored into local register to avoid register overflow.
|
80
77
|
* After a flushBits, a maximum of 7 bits might still be stored into local register.
|
@@ -90,7 +87,6 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
|
|
90
87
|
/*-********************************************
|
91
88
|
* bitStream decoding API (read backward)
|
92
89
|
**********************************************/
|
93
|
-
typedef size_t BitContainerType;
|
94
90
|
typedef struct {
|
95
91
|
BitContainerType bitContainer;
|
96
92
|
unsigned bitsConsumed;
|
@@ -106,7 +102,7 @@ typedef enum { BIT_DStream_unfinished = 0, /* fully refilled */
|
|
106
102
|
} BIT_DStream_status; /* result of BIT_reloadDStream() */
|
107
103
|
|
108
104
|
MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize);
|
109
|
-
MEM_STATIC
|
105
|
+
MEM_STATIC BitContainerType BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits);
|
110
106
|
MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
|
111
107
|
MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
|
112
108
|
|
@@ -125,7 +121,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
|
|
125
121
|
/*-****************************************
|
126
122
|
* unsafe API
|
127
123
|
******************************************/
|
128
|
-
MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC,
|
124
|
+
MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, BitContainerType value, unsigned nbBits);
|
129
125
|
/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */
|
130
126
|
|
131
127
|
MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC);
|
@@ -163,10 +159,15 @@ MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC,
|
|
163
159
|
return 0;
|
164
160
|
}
|
165
161
|
|
166
|
-
FORCE_INLINE_TEMPLATE
|
162
|
+
FORCE_INLINE_TEMPLATE BitContainerType BIT_getLowerBits(BitContainerType bitContainer, U32 const nbBits)
|
167
163
|
{
|
168
|
-
#if
|
169
|
-
|
164
|
+
#if STATIC_BMI2 && !defined(ZSTD_NO_INTRINSICS)
|
165
|
+
# if (defined(__x86_64__) || defined(_M_X64)) && !defined(__ILP32__)
|
166
|
+
return _bzhi_u64(bitContainer, nbBits);
|
167
|
+
# else
|
168
|
+
DEBUG_STATIC_ASSERT(sizeof(bitContainer) == sizeof(U32));
|
169
|
+
return _bzhi_u32(bitContainer, nbBits);
|
170
|
+
# endif
|
170
171
|
#else
|
171
172
|
assert(nbBits < BIT_MASK_SIZE);
|
172
173
|
return bitContainer & BIT_mask[nbBits];
|
@@ -177,7 +178,7 @@ FORCE_INLINE_TEMPLATE size_t BIT_getLowerBits(size_t bitContainer, U32 const nbB
|
|
177
178
|
* can add up to 31 bits into `bitC`.
|
178
179
|
* Note : does not check for register overflow ! */
|
179
180
|
MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
|
180
|
-
|
181
|
+
BitContainerType value, unsigned nbBits)
|
181
182
|
{
|
182
183
|
DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32);
|
183
184
|
assert(nbBits < BIT_MASK_SIZE);
|
@@ -190,7 +191,7 @@ MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
|
|
190
191
|
* works only if `value` is _clean_,
|
191
192
|
* meaning all high bits above nbBits are 0 */
|
192
193
|
MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC,
|
193
|
-
|
194
|
+
BitContainerType value, unsigned nbBits)
|
194
195
|
{
|
195
196
|
assert((value>>nbBits) == 0);
|
196
197
|
assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
|
@@ -237,7 +238,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
|
|
237
238
|
BIT_addBitsFast(bitC, 1, 1); /* endMark */
|
238
239
|
BIT_flushBits(bitC);
|
239
240
|
if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */
|
240
|
-
return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
|
241
|
+
return (size_t)(bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
|
241
242
|
}
|
242
243
|
|
243
244
|
|
@@ -298,12 +299,12 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
|
298
299
|
return srcSize;
|
299
300
|
}
|
300
301
|
|
301
|
-
FORCE_INLINE_TEMPLATE
|
302
|
+
FORCE_INLINE_TEMPLATE BitContainerType BIT_getUpperBits(BitContainerType bitContainer, U32 const start)
|
302
303
|
{
|
303
304
|
return bitContainer >> start;
|
304
305
|
}
|
305
306
|
|
306
|
-
FORCE_INLINE_TEMPLATE
|
307
|
+
FORCE_INLINE_TEMPLATE BitContainerType BIT_getMiddleBits(BitContainerType bitContainer, U32 const start, U32 const nbBits)
|
307
308
|
{
|
308
309
|
U32 const regMask = sizeof(bitContainer)*8 - 1;
|
309
310
|
/* if start > regMask, bitstream is corrupted, and result is undefined */
|
@@ -313,7 +314,7 @@ FORCE_INLINE_TEMPLATE size_t BIT_getMiddleBits(BitContainerType bitContainer, U3
|
|
313
314
|
* such cpus old (pre-Haswell, 2013) and their performance is not of that
|
314
315
|
* importance.
|
315
316
|
*/
|
316
|
-
#if defined(__x86_64__) || defined(
|
317
|
+
#if defined(__x86_64__) || defined(_M_X64)
|
317
318
|
return (bitContainer >> (start & regMask)) & ((((U64)1) << nbBits) - 1);
|
318
319
|
#else
|
319
320
|
return (bitContainer >> (start & regMask)) & BIT_mask[nbBits];
|
@@ -326,7 +327,7 @@ FORCE_INLINE_TEMPLATE size_t BIT_getMiddleBits(BitContainerType bitContainer, U3
|
|
326
327
|
* On 32-bits, maxNbBits==24.
|
327
328
|
* On 64-bits, maxNbBits==56.
|
328
329
|
* @return : value extracted */
|
329
|
-
FORCE_INLINE_TEMPLATE
|
330
|
+
FORCE_INLINE_TEMPLATE BitContainerType BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
|
330
331
|
{
|
331
332
|
/* arbitrate between double-shift and shift+mask */
|
332
333
|
#if 1
|
@@ -342,7 +343,7 @@ FORCE_INLINE_TEMPLATE size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits
|
|
342
343
|
|
343
344
|
/*! BIT_lookBitsFast() :
|
344
345
|
* unsafe version; only works if nbBits >= 1 */
|
345
|
-
MEM_STATIC
|
346
|
+
MEM_STATIC BitContainerType BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
|
346
347
|
{
|
347
348
|
U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;
|
348
349
|
assert(nbBits >= 1);
|
@@ -358,18 +359,18 @@ FORCE_INLINE_TEMPLATE void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
|
|
358
359
|
* Read (consume) next n bits from local register and update.
|
359
360
|
* Pay attention to not read more than nbBits contained into local register.
|
360
361
|
* @return : extracted value. */
|
361
|
-
FORCE_INLINE_TEMPLATE
|
362
|
+
FORCE_INLINE_TEMPLATE BitContainerType BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits)
|
362
363
|
{
|
363
|
-
|
364
|
+
BitContainerType const value = BIT_lookBits(bitD, nbBits);
|
364
365
|
BIT_skipBits(bitD, nbBits);
|
365
366
|
return value;
|
366
367
|
}
|
367
368
|
|
368
369
|
/*! BIT_readBitsFast() :
|
369
370
|
* unsafe version; only works if nbBits >= 1 */
|
370
|
-
MEM_STATIC
|
371
|
+
MEM_STATIC BitContainerType BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)
|
371
372
|
{
|
372
|
-
|
373
|
+
BitContainerType const value = BIT_lookBitsFast(bitD, nbBits);
|
373
374
|
assert(nbBits >= 1);
|
374
375
|
BIT_skipBits(bitD, nbBits);
|
375
376
|
return value;
|
@@ -450,8 +451,4 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
|
|
450
451
|
return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8));
|
451
452
|
}
|
452
453
|
|
453
|
-
#if defined (__cplusplus)
|
454
|
-
}
|
455
|
-
#endif
|
456
|
-
|
457
454
|
#endif /* BITSTREAM_H_MODULE */
|