large_object_store 1.1.1 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d47e0ee45bc6fbe4d657974b55acde2b1ed6052e
-  data.tar.gz: 461403d43371d4d95d432b2cf695b0f74d0e070a
+  metadata.gz: 4c187c263e1ccb215c60054877952d2b83d5cc6e
+  data.tar.gz: 81506891e872accdb8cc722d578fce1f78e169ac
 SHA512:
-  metadata.gz: 9ee0ca3c5bbae29874718241d39f7dfecbfb1492503928285f3fe2e1aa4dead61d0744c0c68ca4c92b00bff645cc32dbb4a6138c956a55b06ef373dd6bd8fcfc
-  data.tar.gz: 58834c45621095c238af2d3e8f3f291a62fcbb15a83ce8ad32e17aea4664588db926fd9b535a7e3c30b84720cd7738bf8e450d18aeb37f82c397f6e7a344bcce
+  metadata.gz: 22c8236b2d9ae23d9fc3af9f77b2931f99c4077828773aadd6420b2e40a4d1089356efde9ef9fff367ff03e33404efce6340274cce30f285891fa6bbd7769f99
+  data.tar.gz: d42348f49af53a3f1a45b6c1994f690f6678df1ab25eb9af061daeb3e1b936191a7d429edf135a430a5235312f823b47dc407d263b2004465c343ffb5bc75d4b

data/lib/large_object_store.rb

@@ -1,7 +1,16 @@
 require "large_object_store/version"
 require "zlib"
+require "securerandom"
 
 module LargeObjectStore
+  UUID_BYTES = 16
+  UUID_SIZE = UUID_BYTES * 2
+  CACHE_VERSION = 2
+  MAX_OBJECT_SIZE = 1024**2
+  ITEM_HEADER_SIZE = 100
+  DEFAULT_COMPRESS_LIMIT = 16*1024
+  COMPRESSED = 'z'
+  NORMAL = '0'
 
   def self.wrap(store)
     RailsWrapper.new(store)
@@ -10,66 +19,74 @@ module LargeObjectStore
   class RailsWrapper
     attr_reader :store
 
-    MAX_OBJECT_SIZE = 1024**2
-    ITEM_HEADER_SIZE = 100
-
     def initialize(store)
       @store = store
     end
 
     def write(key, value, options = {})
       value = Marshal.dump(value)
-      value = Zlib::Deflate.deflate(value) if options.delete(:compress)
+
+      options = options.dup
+      compressed = false
+      if options.delete(:compress)
+        # Don't pass compression on to Rails, we're doing it ourselves.
+        compress_limit = options.delete(:compress_limit) || DEFAULT_COMPRESS_LIMIT
+        if value.bytesize > compress_limit
+          value = Zlib::Deflate.deflate(value)
+          compressed = true
+        end
+      end
+      value.prepend(compressed ? COMPRESSED : NORMAL)
 
       # calculate slice size; note that key length is a factor because
       # the key is stored on the same slab page as the value
-      slice_size = MAX_OBJECT_SIZE - ITEM_HEADER_SIZE - key.bytesize
+      slice_size = MAX_OBJECT_SIZE - ITEM_HEADER_SIZE - UUID_SIZE - key.bytesize
 
       # store number of pages
       pages = (value.size / slice_size.to_f).ceil
 
       if pages == 1
-        @store.write("#{key}_0", value, options)
+        @store.write(key(key, 0), value, options)
       else
+        # store meta
+        uuid = SecureRandom.hex(UUID_BYTES)
+        return false unless @store.write(key(key, 0), [pages, uuid], options) # invalidates the old cache
+
         # store object
         page = 1
         loop do
           slice = value.slice!(0, slice_size)
           break if slice.size == 0
 
-          return false unless @store.write("#{key}_#{page}", slice, options.merge(raw: true))
+          return false unless @store.write(key(key, page), slice.prepend(uuid), options.merge(raw: true))
           page += 1
         end
-
-        @store.write("#{key}_0", pages, options)
+        true
       end
     end
 
     def read(key)
       # read pages
-      pages = @store.read("#{key}_0")
+      pages, uuid = @store.read(key(key, 0))
       return if pages.nil?
 
       data = if pages.is_a?(Fixnum)
         # read sliced data
-        keys = Array.new(pages).each_with_index.map{|_,i| "#{key}_#{i+1}" }
+        keys = Array.new(pages).each_with_index.map{|_,i| key(key, i+1) }
         slices = @store.read_multi(*keys).values
-        return nil if slices.compact.size < pages
-        slices.join("")
+        return nil if slices.compact.size != pages
+        slices.map! { |s| [s.slice!(0, UUID_SIZE), s] }
+        return nil unless slices.map(&:first).uniq == [uuid]
+        slices.map!(&:last).join("")
       else
         pages
       end
 
-      if data.getbyte(0) == 0x78 && [0x01,0x9C,0xDA].include?(data.getbyte(1))
+      if data.slice!(0, 1) == COMPRESSED
         data = Zlib::Inflate.inflate(data)
       end
 
-      begin
-        Marshal.load(data)
-      rescue Exception => e
-        Rails.logger.error "Cannot read large_object_store key #{key} : #{e.message} #{e.backtrace.inspect}"
-        nil
-      end
+      Marshal.load(data)
     end
 
     def fetch(key, options={})
@@ -81,7 +98,13 @@ module LargeObjectStore
     end
 
     def delete(key)
-      @store.delete("#{key}_0")
+      @store.delete(key(key, 0))
+    end
+
+    private
+
+    def key(key, i)
+      "#{key}_#{CACHE_VERSION}_#{i}"
     end
   end
 end

data/lib/large_object_store/version.rb

@@ -1,3 +1,3 @@
 module LargeObjectStore
-  VERSION = "1.1.1"
+  VERSION = "1.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: large_object_store
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.2.0
 platform: ruby
 authors:
 - Ana Martinez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-17 00:00:00.000000000 Z
+date: 2015-11-18 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: acemacu@gmail.com