large_object_store 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/large_object_store.rb +44 -21
- data/lib/large_object_store/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4c187c263e1ccb215c60054877952d2b83d5cc6e
|
4
|
+
data.tar.gz: 81506891e872accdb8cc722d578fce1f78e169ac
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 22c8236b2d9ae23d9fc3af9f77b2931f99c4077828773aadd6420b2e40a4d1089356efde9ef9fff367ff03e33404efce6340274cce30f285891fa6bbd7769f99
|
7
|
+
data.tar.gz: d42348f49af53a3f1a45b6c1994f690f6678df1ab25eb9af061daeb3e1b936191a7d429edf135a430a5235312f823b47dc407d263b2004465c343ffb5bc75d4b
|
data/lib/large_object_store.rb
CHANGED
@@ -1,7 +1,16 @@
|
|
1
1
|
require "large_object_store/version"
|
2
2
|
require "zlib"
|
3
|
+
require "securerandom"
|
3
4
|
|
4
5
|
module LargeObjectStore
|
6
|
+
UUID_BYTES = 16
|
7
|
+
UUID_SIZE = UUID_BYTES * 2
|
8
|
+
CACHE_VERSION = 2
|
9
|
+
MAX_OBJECT_SIZE = 1024**2
|
10
|
+
ITEM_HEADER_SIZE = 100
|
11
|
+
DEFAULT_COMPRESS_LIMIT = 16*1024
|
12
|
+
COMPRESSED = 'z'
|
13
|
+
NORMAL = '0'
|
5
14
|
|
6
15
|
def self.wrap(store)
|
7
16
|
RailsWrapper.new(store)
|
@@ -10,66 +19,74 @@ module LargeObjectStore
|
|
10
19
|
class RailsWrapper
|
11
20
|
attr_reader :store
|
12
21
|
|
13
|
-
MAX_OBJECT_SIZE = 1024**2
|
14
|
-
ITEM_HEADER_SIZE = 100
|
15
|
-
|
16
22
|
def initialize(store)
|
17
23
|
@store = store
|
18
24
|
end
|
19
25
|
|
20
26
|
def write(key, value, options = {})
|
21
27
|
value = Marshal.dump(value)
|
22
|
-
|
28
|
+
|
29
|
+
options = options.dup
|
30
|
+
compressed = false
|
31
|
+
if options.delete(:compress)
|
32
|
+
# Don't pass compression on to Rails, we're doing it ourselves.
|
33
|
+
compress_limit = options.delete(:compress_limit) || DEFAULT_COMPRESS_LIMIT
|
34
|
+
if value.bytesize > compress_limit
|
35
|
+
value = Zlib::Deflate.deflate(value)
|
36
|
+
compressed = true
|
37
|
+
end
|
38
|
+
end
|
39
|
+
value.prepend(compressed ? COMPRESSED : NORMAL)
|
23
40
|
|
24
41
|
# calculate slice size; note that key length is a factor because
|
25
42
|
# the key is stored on the same slab page as the value
|
26
|
-
slice_size = MAX_OBJECT_SIZE - ITEM_HEADER_SIZE - key.bytesize
|
43
|
+
slice_size = MAX_OBJECT_SIZE - ITEM_HEADER_SIZE - UUID_SIZE - key.bytesize
|
27
44
|
|
28
45
|
# store number of pages
|
29
46
|
pages = (value.size / slice_size.to_f).ceil
|
30
47
|
|
31
48
|
if pages == 1
|
32
|
-
@store.write(
|
49
|
+
@store.write(key(key, 0), value, options)
|
33
50
|
else
|
51
|
+
# store meta
|
52
|
+
uuid = SecureRandom.hex(UUID_BYTES)
|
53
|
+
return false unless @store.write(key(key, 0), [pages, uuid], options) # invalidates the old cache
|
54
|
+
|
34
55
|
# store object
|
35
56
|
page = 1
|
36
57
|
loop do
|
37
58
|
slice = value.slice!(0, slice_size)
|
38
59
|
break if slice.size == 0
|
39
60
|
|
40
|
-
return false unless @store.write(
|
61
|
+
return false unless @store.write(key(key, page), slice.prepend(uuid), options.merge(raw: true))
|
41
62
|
page += 1
|
42
63
|
end
|
43
|
-
|
44
|
-
@store.write("#{key}_0", pages, options)
|
64
|
+
true
|
45
65
|
end
|
46
66
|
end
|
47
67
|
|
48
68
|
def read(key)
|
49
69
|
# read pages
|
50
|
-
pages = @store.read(
|
70
|
+
pages, uuid = @store.read(key(key, 0))
|
51
71
|
return if pages.nil?
|
52
72
|
|
53
73
|
data = if pages.is_a?(Fixnum)
|
54
74
|
# read sliced data
|
55
|
-
keys = Array.new(pages).each_with_index.map{|_,i|
|
75
|
+
keys = Array.new(pages).each_with_index.map{|_,i| key(key, i+1) }
|
56
76
|
slices = @store.read_multi(*keys).values
|
57
|
-
return nil if slices.compact.size
|
58
|
-
slices.
|
77
|
+
return nil if slices.compact.size != pages
|
78
|
+
slices.map! { |s| [s.slice!(0, UUID_SIZE), s] }
|
79
|
+
return nil unless slices.map(&:first).uniq == [uuid]
|
80
|
+
slices.map!(&:last).join("")
|
59
81
|
else
|
60
82
|
pages
|
61
83
|
end
|
62
84
|
|
63
|
-
if data.
|
85
|
+
if data.slice!(0, 1) == COMPRESSED
|
64
86
|
data = Zlib::Inflate.inflate(data)
|
65
87
|
end
|
66
88
|
|
67
|
-
|
68
|
-
Marshal.load(data)
|
69
|
-
rescue Exception => e
|
70
|
-
Rails.logger.error "Cannot read large_object_store key #{key} : #{e.message} #{e.backtrace.inspect}"
|
71
|
-
nil
|
72
|
-
end
|
89
|
+
Marshal.load(data)
|
73
90
|
end
|
74
91
|
|
75
92
|
def fetch(key, options={})
|
@@ -81,7 +98,13 @@ module LargeObjectStore
|
|
81
98
|
end
|
82
99
|
|
83
100
|
def delete(key)
|
84
|
-
@store.delete(
|
101
|
+
@store.delete(key(key, 0))
|
102
|
+
end
|
103
|
+
|
104
|
+
private
|
105
|
+
|
106
|
+
def key(key, i)
|
107
|
+
"#{key}_#{CACHE_VERSION}_#{i}"
|
85
108
|
end
|
86
109
|
end
|
87
110
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: large_object_store
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ana Martinez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-11-
|
11
|
+
date: 2015-11-18 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description:
|
14
14
|
email: acemacu@gmail.com
|