large_object_store 1.5.0 → 1.6.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 92be121f082879185822f734dd5fe7935ce04aacb05e900ce4dbfe89b23d712c
4
- data.tar.gz: 14854eba125bd1837eb444ec25d65da9ae99228455f600bb4c9cb38902d3ac8a
3
+ metadata.gz: 9e41ad488600c979693e5e2c58958c8fd9eb3f3d51ae53c908415b920f1180a9
4
+ data.tar.gz: 22063ad5444b0f50a39c48a08e52e537bba6b21f31b8ee7751d331dfb3574412
5
5
  SHA512:
6
- metadata.gz: 489ecb7a501deda4f45feb995585999717a595ef6e4539641a6866215d23cf89eb3f745aac02d3b95a4581356405acb21f40f79b010954a3a037cd053172fe5d
7
- data.tar.gz: 6c8ad1eead221be4324a5d6557926b8702469797448d68e62a19ded103ad52a0a1dc146599d2f69b1dd6ab5b237659e275d07f9a1542b9aa7a735dc5bb95acf3
6
+ metadata.gz: 6e76ee6a7a7679cdf1b797aa65e3d59656dafeade0240db262de7312d640fdc15e3de1af92323aaed10a8e2d26c762265afc1ae9bb05978b39220224096b0276
7
+ data.tar.gz: '05937831063dbe4d29766cb4237b82b11fd950a3c879d6fe9461cbf557493ed235b0e181447cca254a9043166145c278b3c1171ad7f9c9f81ed92e70b5864b1a'
data/Readme.md CHANGED
@@ -26,6 +26,19 @@ store.write("a" * 1000, compress: true, compress_limit: 100) # compress when gre
26
26
  store.write("a" * 1000, raw: true) # store as string to avoid marshaling overhead
27
27
  ```
28
28
 
29
+ zstd
30
+ ====
31
+
32
+ [zstd compression](https://engineering.fb.com/2016/08/31/core-data/smaller-and-faster-data-compression-with-zstandard/), a modern improvement over the venerable zlib compression algorithm, is supported by passing the `zstd` flag when writing items:
33
+
34
+ ```
35
+ store.write("a" * 10_000_000, compress: true, zstd: true)
36
+ ```
37
+
38
+ For backwards compatibility and to enable safe roll-out of the change in working systems, the `zstd` flag defaults to `false`.
39
+
40
+ zstd decompression is used when the zstd magic number is detected at the beginning of compressed data, so `zstd: true` does not need to be passed when reading/fetching items.
41
+
29
42
  Author
30
43
  ======
31
44
  [Ana Martinez](https://github.com/anamartinez)<br/>
@@ -1,3 +1,3 @@
1
1
  module LargeObjectStore
2
- VERSION = "1.5.0"
2
+ VERSION = "1.6.0"
3
3
  end
@@ -1,11 +1,12 @@
1
1
  require "large_object_store/version"
2
2
  require "zlib"
3
+ require "zstd-ruby"
3
4
  require "securerandom"
4
5
 
5
6
  module LargeObjectStore
6
7
  UUID_BYTES = 16
7
8
  UUID_SIZE = UUID_BYTES * 2
8
- CACHE_VERSION = 3
9
+ CACHE_VERSION = 4
9
10
  MAX_OBJECT_SIZE = 1024**2
10
11
  ITEM_HEADER_SIZE = 100
11
12
  DEFAULT_COMPRESS_LIMIT = 16*1024
@@ -13,6 +14,8 @@ module LargeObjectStore
13
14
  COMPRESSED = 1
14
15
  RAW = 2
15
16
  FLAG_RADIX = 32 # we can store 32 different states
17
+ ZSTD_MAGIC = "\x28\xB5\x2F\xFD".force_encoding('ASCII-8BIT')
18
+ ZSTD_COMPRESS_LEVEL = 3 # Default level recommended by zstd authors
16
19
 
17
20
  def self.wrap(*args)
18
21
  RailsWrapper.new(*args)
@@ -31,7 +34,7 @@ module LargeObjectStore
31
34
 
32
35
  def write(key, value, **options)
33
36
  options = options.dup
34
- value = serialize(value, **options)
37
+ value = serialize(value, options)
35
38
 
36
39
  # calculate slice size; note that key length is a factor because
37
40
  # the key is stored on the same slab page as the value
@@ -107,7 +110,7 @@ module LargeObjectStore
107
110
 
108
111
  # convert a object to a string
109
112
  # modifies options
110
- def serialize(value, **options)
113
+ def serialize(value, options)
111
114
  flag = NORMAL
112
115
 
113
116
  if options.delete(:raw)
@@ -117,28 +120,53 @@ module LargeObjectStore
117
120
  value = @serializer.dump(value)
118
121
  end
119
122
 
120
- if compress?(value, **options)
123
+ if compress?(value, options)
121
124
  flag |= COMPRESSED
122
- value = Zlib::Deflate.deflate(value)
125
+ value = compress(value, options)
123
126
  end
124
127
 
125
128
  value.prepend(flag.to_s(FLAG_RADIX))
126
129
  end
127
130
 
131
+ def compress(value, options)
132
+ if options[:zstd]
133
+ Zstd.compress(value, ZSTD_COMPRESS_LEVEL)
134
+ else
135
+ Zlib::Deflate.deflate(value)
136
+ end
137
+ end
138
+
139
+ def decompress(data)
140
+ if data.start_with?(ZSTD_MAGIC)
141
+ Zstd.decompress(data)
142
+ else
143
+ Zlib::Inflate.inflate(data)
144
+ end
145
+ end
146
+
128
147
  # opposite operations and order of serialize
129
148
  def deserialize(raw_data)
130
149
  data = raw_data.dup
131
150
  flag = data.slice!(0, 1).to_i(FLAG_RADIX)
132
- data = Zlib::Inflate.inflate(data) if flag & COMPRESSED == COMPRESSED
151
+ data = decompress(data) if flag & COMPRESSED == COMPRESSED
133
152
  data = @serializer.load(data) if flag & RAW != RAW
134
153
  data
135
154
  end
136
155
 
137
156
  # Don't pass compression on to Rails, we're doing it ourselves.
138
- def compress?(value, **options)
139
- return unless options.delete(:compress)
140
- compress_limit = options.delete(:compress_limit) || DEFAULT_COMPRESS_LIMIT
141
- value.bytesize > compress_limit
157
+ def compress?(value, options)
158
+ return unless options[:compress]
159
+
160
+ compress_limit = options[:compress_limit] || DEFAULT_COMPRESS_LIMIT
161
+ should_compress = value.bytesize > compress_limit
162
+
163
+ if should_compress
164
+ # Pass compress: false to Rails in case the default is true
165
+ options[:compress] = false
166
+ options.delete(:compress_limit)
167
+ end
168
+
169
+ should_compress
142
170
  end
143
171
 
144
172
  def key(key, i)
metadata CHANGED
@@ -1,15 +1,29 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: large_object_store
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.5.0
4
+ version: 1.6.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ana Martinez
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-03-30 00:00:00.000000000 Z
12
- dependencies: []
11
+ date: 2023-06-22 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: zstd-ruby
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: 1.5.5
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: 1.5.5
13
27
  description:
14
28
  email: acemacu@gmail.com
15
29
  executables: []
@@ -31,7 +45,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
31
45
  requirements:
32
46
  - - ">="
33
47
  - !ruby/object:Gem::Version
34
- version: '2.5'
48
+ version: '2.6'
35
49
  required_rubygems_version: !ruby/object:Gem::Requirement
36
50
  requirements:
37
51
  - - ">="