large_object_store 1.5.0 → 1.6.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Readme.md +13 -0
- data/lib/large_object_store/version.rb +1 -1
- data/lib/large_object_store.rb +38 -10
- metadata +18 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9e41ad488600c979693e5e2c58958c8fd9eb3f3d51ae53c908415b920f1180a9
|
4
|
+
data.tar.gz: 22063ad5444b0f50a39c48a08e52e537bba6b21f31b8ee7751d331dfb3574412
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6e76ee6a7a7679cdf1b797aa65e3d59656dafeade0240db262de7312d640fdc15e3de1af92323aaed10a8e2d26c762265afc1ae9bb05978b39220224096b0276
|
7
|
+
data.tar.gz: '05937831063dbe4d29766cb4237b82b11fd950a3c879d6fe9461cbf557493ed235b0e181447cca254a9043166145c278b3c1171ad7f9c9f81ed92e70b5864b1a'
|
data/Readme.md
CHANGED
@@ -26,6 +26,19 @@ store.write("a" * 1000, compress: true, compress_limit: 100) # compress when gre
|
|
26
26
|
store.write("a" * 1000, raw: true) # store as string to avoid marshaling overhead
|
27
27
|
```
|
28
28
|
|
29
|
+
zstd
|
30
|
+
====
|
31
|
+
|
32
|
+
[zstd compression](https://engineering.fb.com/2016/08/31/core-data/smaller-and-faster-data-compression-with-zstandard/), a modern improvement over the venerable zlib compression algorithm, is supported by passing the `zstd` flag when writing items:
|
33
|
+
|
34
|
+
```
|
35
|
+
store.write("a" * 10_000_000, compress: true, zstd: true)
|
36
|
+
```
|
37
|
+
|
38
|
+
For backwards compatibility and to enable safe roll-out of the change in working systems, the `zstd` flag defaults to `false`.
|
39
|
+
|
40
|
+
zstd decompression is used when the zstd magic number is detected at the beginning of compressed data, so `zstd: true` does not need to be passed when reading/fetching items.
|
41
|
+
|
29
42
|
Author
|
30
43
|
======
|
31
44
|
[Ana Martinez](https://github.com/anamartinez)<br/>
|
data/lib/large_object_store.rb
CHANGED
@@ -1,11 +1,12 @@
|
|
1
1
|
require "large_object_store/version"
|
2
2
|
require "zlib"
|
3
|
+
require "zstd-ruby"
|
3
4
|
require "securerandom"
|
4
5
|
|
5
6
|
module LargeObjectStore
|
6
7
|
UUID_BYTES = 16
|
7
8
|
UUID_SIZE = UUID_BYTES * 2
|
8
|
-
CACHE_VERSION =
|
9
|
+
CACHE_VERSION = 4
|
9
10
|
MAX_OBJECT_SIZE = 1024**2
|
10
11
|
ITEM_HEADER_SIZE = 100
|
11
12
|
DEFAULT_COMPRESS_LIMIT = 16*1024
|
@@ -13,6 +14,8 @@ module LargeObjectStore
|
|
13
14
|
COMPRESSED = 1
|
14
15
|
RAW = 2
|
15
16
|
FLAG_RADIX = 32 # we can store 32 different states
|
17
|
+
ZSTD_MAGIC = "\x28\xB5\x2F\xFD".force_encoding('ASCII-8BIT')
|
18
|
+
ZSTD_COMPRESS_LEVEL = 3 # Default level recommended by zstd authors
|
16
19
|
|
17
20
|
def self.wrap(*args)
|
18
21
|
RailsWrapper.new(*args)
|
@@ -31,7 +34,7 @@ module LargeObjectStore
|
|
31
34
|
|
32
35
|
def write(key, value, **options)
|
33
36
|
options = options.dup
|
34
|
-
value = serialize(value,
|
37
|
+
value = serialize(value, options)
|
35
38
|
|
36
39
|
# calculate slice size; note that key length is a factor because
|
37
40
|
# the key is stored on the same slab page as the value
|
@@ -107,7 +110,7 @@ module LargeObjectStore
|
|
107
110
|
|
108
111
|
# convert a object to a string
|
109
112
|
# modifies options
|
110
|
-
def serialize(value,
|
113
|
+
def serialize(value, options)
|
111
114
|
flag = NORMAL
|
112
115
|
|
113
116
|
if options.delete(:raw)
|
@@ -117,28 +120,53 @@ module LargeObjectStore
|
|
117
120
|
value = @serializer.dump(value)
|
118
121
|
end
|
119
122
|
|
120
|
-
if compress?(value,
|
123
|
+
if compress?(value, options)
|
121
124
|
flag |= COMPRESSED
|
122
|
-
value =
|
125
|
+
value = compress(value, options)
|
123
126
|
end
|
124
127
|
|
125
128
|
value.prepend(flag.to_s(FLAG_RADIX))
|
126
129
|
end
|
127
130
|
|
131
|
+
def compress(value, options)
|
132
|
+
if options[:zstd]
|
133
|
+
Zstd.compress(value, ZSTD_COMPRESS_LEVEL)
|
134
|
+
else
|
135
|
+
Zlib::Deflate.deflate(value)
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
def decompress(data)
|
140
|
+
if data.start_with?(ZSTD_MAGIC)
|
141
|
+
Zstd.decompress(data)
|
142
|
+
else
|
143
|
+
Zlib::Inflate.inflate(data)
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
128
147
|
# opposite operations and order of serialize
|
129
148
|
def deserialize(raw_data)
|
130
149
|
data = raw_data.dup
|
131
150
|
flag = data.slice!(0, 1).to_i(FLAG_RADIX)
|
132
|
-
data =
|
151
|
+
data = decompress(data) if flag & COMPRESSED == COMPRESSED
|
133
152
|
data = @serializer.load(data) if flag & RAW != RAW
|
134
153
|
data
|
135
154
|
end
|
136
155
|
|
137
156
|
# Don't pass compression on to Rails, we're doing it ourselves.
|
138
|
-
def compress?(value,
|
139
|
-
return unless options
|
140
|
-
|
141
|
-
|
157
|
+
def compress?(value, options)
|
158
|
+
return unless options[:compress]
|
159
|
+
|
160
|
+
compress_limit = options[:compress_limit] || DEFAULT_COMPRESS_LIMIT
|
161
|
+
should_compress = value.bytesize > compress_limit
|
162
|
+
|
163
|
+
if should_compress
|
164
|
+
# Pass compress: false to Rails in case the default is true
|
165
|
+
options[:compress] = false
|
166
|
+
options.delete(:compress_limit)
|
167
|
+
end
|
168
|
+
|
169
|
+
should_compress
|
142
170
|
end
|
143
171
|
|
144
172
|
def key(key, i)
|
metadata
CHANGED
@@ -1,15 +1,29 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: large_object_store
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ana Martinez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-
|
12
|
-
dependencies:
|
11
|
+
date: 2023-06-22 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: zstd-ruby
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 1.5.5
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 1.5.5
|
13
27
|
description:
|
14
28
|
email: acemacu@gmail.com
|
15
29
|
executables: []
|
@@ -31,7 +45,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
31
45
|
requirements:
|
32
46
|
- - ">="
|
33
47
|
- !ruby/object:Gem::Version
|
34
|
-
version: '2.
|
48
|
+
version: '2.6'
|
35
49
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
36
50
|
requirements:
|
37
51
|
- - ">="
|