large_object_store 1.1.1 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: d47e0ee45bc6fbe4d657974b55acde2b1ed6052e
4
- data.tar.gz: 461403d43371d4d95d432b2cf695b0f74d0e070a
3
+ metadata.gz: 4c187c263e1ccb215c60054877952d2b83d5cc6e
4
+ data.tar.gz: 81506891e872accdb8cc722d578fce1f78e169ac
5
5
  SHA512:
6
- metadata.gz: 9ee0ca3c5bbae29874718241d39f7dfecbfb1492503928285f3fe2e1aa4dead61d0744c0c68ca4c92b00bff645cc32dbb4a6138c956a55b06ef373dd6bd8fcfc
7
- data.tar.gz: 58834c45621095c238af2d3e8f3f291a62fcbb15a83ce8ad32e17aea4664588db926fd9b535a7e3c30b84720cd7738bf8e450d18aeb37f82c397f6e7a344bcce
6
+ metadata.gz: 22c8236b2d9ae23d9fc3af9f77b2931f99c4077828773aadd6420b2e40a4d1089356efde9ef9fff367ff03e33404efce6340274cce30f285891fa6bbd7769f99
7
+ data.tar.gz: d42348f49af53a3f1a45b6c1994f690f6678df1ab25eb9af061daeb3e1b936191a7d429edf135a430a5235312f823b47dc407d263b2004465c343ffb5bc75d4b
@@ -1,7 +1,16 @@
1
1
  require "large_object_store/version"
2
2
  require "zlib"
3
+ require "securerandom"
3
4
 
4
5
  module LargeObjectStore
6
+ UUID_BYTES = 16
7
+ UUID_SIZE = UUID_BYTES * 2
8
+ CACHE_VERSION = 2
9
+ MAX_OBJECT_SIZE = 1024**2
10
+ ITEM_HEADER_SIZE = 100
11
+ DEFAULT_COMPRESS_LIMIT = 16*1024
12
+ COMPRESSED = 'z'
13
+ NORMAL = '0'
5
14
 
6
15
  def self.wrap(store)
7
16
  RailsWrapper.new(store)
@@ -10,66 +19,74 @@ module LargeObjectStore
10
19
  class RailsWrapper
11
20
  attr_reader :store
12
21
 
13
- MAX_OBJECT_SIZE = 1024**2
14
- ITEM_HEADER_SIZE = 100
15
-
16
22
  def initialize(store)
17
23
  @store = store
18
24
  end
19
25
 
20
26
  def write(key, value, options = {})
21
27
  value = Marshal.dump(value)
22
- value = Zlib::Deflate.deflate(value) if options.delete(:compress)
28
+
29
+ options = options.dup
30
+ compressed = false
31
+ if options.delete(:compress)
32
+ # Don't pass compression on to Rails, we're doing it ourselves.
33
+ compress_limit = options.delete(:compress_limit) || DEFAULT_COMPRESS_LIMIT
34
+ if value.bytesize > compress_limit
35
+ value = Zlib::Deflate.deflate(value)
36
+ compressed = true
37
+ end
38
+ end
39
+ value.prepend(compressed ? COMPRESSED : NORMAL)
23
40
 
24
41
  # calculate slice size; note that key length is a factor because
25
42
  # the key is stored on the same slab page as the value
26
- slice_size = MAX_OBJECT_SIZE - ITEM_HEADER_SIZE - key.bytesize
43
+ slice_size = MAX_OBJECT_SIZE - ITEM_HEADER_SIZE - UUID_SIZE - key.bytesize
27
44
 
28
45
  # store number of pages
29
46
  pages = (value.size / slice_size.to_f).ceil
30
47
 
31
48
  if pages == 1
32
- @store.write("#{key}_0", value, options)
49
+ @store.write(key(key, 0), value, options)
33
50
  else
51
+ # store meta
52
+ uuid = SecureRandom.hex(UUID_BYTES)
53
+ return false unless @store.write(key(key, 0), [pages, uuid], options) # invalidates the old cache
54
+
34
55
  # store object
35
56
  page = 1
36
57
  loop do
37
58
  slice = value.slice!(0, slice_size)
38
59
  break if slice.size == 0
39
60
 
40
- return false unless @store.write("#{key}_#{page}", slice, options.merge(raw: true))
61
+ return false unless @store.write(key(key, page), slice.prepend(uuid), options.merge(raw: true))
41
62
  page += 1
42
63
  end
43
-
44
- @store.write("#{key}_0", pages, options)
64
+ true
45
65
  end
46
66
  end
47
67
 
48
68
  def read(key)
49
69
  # read pages
50
- pages = @store.read("#{key}_0")
70
+ pages, uuid = @store.read(key(key, 0))
51
71
  return if pages.nil?
52
72
 
53
73
  data = if pages.is_a?(Fixnum)
54
74
  # read sliced data
55
- keys = Array.new(pages).each_with_index.map{|_,i| "#{key}_#{i+1}" }
75
+ keys = Array.new(pages).each_with_index.map{|_,i| key(key, i+1) }
56
76
  slices = @store.read_multi(*keys).values
57
- return nil if slices.compact.size < pages
58
- slices.join("")
77
+ return nil if slices.compact.size != pages
78
+ slices.map! { |s| [s.slice!(0, UUID_SIZE), s] }
79
+ return nil unless slices.map(&:first).uniq == [uuid]
80
+ slices.map!(&:last).join("")
59
81
  else
60
82
  pages
61
83
  end
62
84
 
63
- if data.getbyte(0) == 0x78 && [0x01,0x9C,0xDA].include?(data.getbyte(1))
85
+ if data.slice!(0, 1) == COMPRESSED
64
86
  data = Zlib::Inflate.inflate(data)
65
87
  end
66
88
 
67
- begin
68
- Marshal.load(data)
69
- rescue Exception => e
70
- Rails.logger.error "Cannot read large_object_store key #{key} : #{e.message} #{e.backtrace.inspect}"
71
- nil
72
- end
89
+ Marshal.load(data)
73
90
  end
74
91
 
75
92
  def fetch(key, options={})
@@ -81,7 +98,13 @@ module LargeObjectStore
81
98
  end
82
99
 
83
100
  def delete(key)
84
- @store.delete("#{key}_0")
101
+ @store.delete(key(key, 0))
102
+ end
103
+
104
+ private
105
+
106
+ def key(key, i)
107
+ "#{key}_#{CACHE_VERSION}_#{i}"
85
108
  end
86
109
  end
87
110
  end
@@ -1,3 +1,3 @@
1
1
  module LargeObjectStore
2
- VERSION = "1.1.1"
2
+ VERSION = "1.2.0"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: large_object_store
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.1
4
+ version: 1.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ana Martinez
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-11-17 00:00:00.000000000 Z
11
+ date: 2015-11-18 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description:
14
14
  email: acemacu@gmail.com