large_object_store 1.2.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4c187c263e1ccb215c60054877952d2b83d5cc6e
-  data.tar.gz: 81506891e872accdb8cc722d578fce1f78e169ac
+  metadata.gz: f2345afc60e415d1e98b4522adc30f493728b84f
+  data.tar.gz: b994b636a09a5ec225b212b2ed5dc962d611e858
 SHA512:
-  metadata.gz: 22c8236b2d9ae23d9fc3af9f77b2931f99c4077828773aadd6420b2e40a4d1089356efde9ef9fff367ff03e33404efce6340274cce30f285891fa6bbd7769f99
-  data.tar.gz: d42348f49af53a3f1a45b6c1994f690f6678df1ab25eb9af061daeb3e1b936191a7d429edf135a430a5235312f823b47dc407d263b2004465c343ffb5bc75d4b
+  metadata.gz: cd896a3b1f8e88589e9c37d15920cdbf26839faf3bf831b051f842d8c4fabfe728a4dbd1c23af577844c0e24a3a5338f33670aa0372b8afe0b9e1e0919c83bbb
+  data.tar.gz: 224c388fa9bdc59020a0272d0f1af8e169e9b26e335ac36955f1e96df542cbedddf173bcc8b7456eee5b9e07df898df91a858ad557d66042000b109c70937f53

data/Readme.md

@@ -1,7 +1,7 @@
 Store large objects in memcache or others by slicing them.
  - uses read_multi for fast access
  - returns nil if one slice is missing
- - only uses single read/write if data is below 1MB
+ - low performance overhead, only uses single read/write if data is below 1MB
 
 Install
 =======
@@ -21,6 +21,9 @@ store.write("a", "a"*10_000_000)  # => true -> always!
 store.read("a").size              # => 10_000_000 using multi_get
 store.read("b")                   # => nil
 store.fetch("a"){ "something" }   # => "something" executes block on miss
+store.write("a" * 10_000_000, compress: true)                # compress when greater than 16k
+store.write("a" * 1000, compress: true, compress_limit: 100) # compress when greater than 100
+store.write("a" * 1000, raw: true)                           # store as string to avoid marshaling overhead
 ```
 
 Author

data/lib/large_object_store.rb

@@ -5,12 +5,14 @@ require "securerandom"
 module LargeObjectStore
   UUID_BYTES = 16
   UUID_SIZE = UUID_BYTES * 2
-  CACHE_VERSION = 2
+  CACHE_VERSION = 3
   MAX_OBJECT_SIZE = 1024**2
   ITEM_HEADER_SIZE = 100
   DEFAULT_COMPRESS_LIMIT = 16*1024
-  COMPRESSED = 'z'
-  NORMAL = '0'
+  NORMAL = 0
+  COMPRESSED = 1
+  RAW = 2
+  RADIX = 32 # we can store 32 different states
 
   def self.wrap(store)
     RailsWrapper.new(store)
@@ -24,19 +26,8 @@ module LargeObjectStore
     end
 
     def write(key, value, options = {})
-      value = Marshal.dump(value)
-
       options = options.dup
-      compressed = false
-      if options.delete(:compress)
-        # Don't pass compression on to Rails, we're doing it ourselves.
-        compress_limit = options.delete(:compress_limit) || DEFAULT_COMPRESS_LIMIT
-        if value.bytesize > compress_limit
-          value = Zlib::Deflate.deflate(value)
-          compressed = true
-        end
-      end
-      value.prepend(compressed ? COMPRESSED : NORMAL)
+      value = serialize(value, options)
 
       # calculate slice size; note that key length is a factor because
       # the key is stored on the same slab page as the value
@@ -46,7 +37,7 @@ module LargeObjectStore
       pages = (value.size / slice_size.to_f).ceil
 
       if pages == 1
-        @store.write(key(key, 0), value, options)
+        !!@store.write(key(key, 0), value, options)
       else
         # store meta
         uuid = SecureRandom.hex(UUID_BYTES)
@@ -82,11 +73,7 @@ module LargeObjectStore
         pages
       end
 
-      if data.slice!(0, 1) == COMPRESSED
-        data = Zlib::Inflate.inflate(data)
-      end
-
-      Marshal.load(data)
+      deserialize(data)
     end
 
     def fetch(key, options={})
@@ -103,6 +90,41 @@ module LargeObjectStore
 
     private
 
+    # convert a object to a string
+    # modifies options
+    def serialize(value, options)
+      flag = NORMAL
+
+      if options.delete(:raw)
+        flag |= RAW
+        value = value.to_s
+      else
+        value = Marshal.dump(value)
+      end
+
+      if compress?(value, options)
+        flag |= COMPRESSED
+        value = Zlib::Deflate.deflate(value)
+      end
+
+      value.prepend(flag.to_s(RADIX))
+    end
+
+    # opposite operations and order of serialize
+    def deserialize(data)
+      flag = data.slice!(0, 1).to_i(RADIX)
+      data = Zlib::Inflate.inflate(data) if flag & COMPRESSED == COMPRESSED
+      data = Marshal.load(data) if flag & RAW != RAW
+      data
+    end
+
+    # Don't pass compression on to Rails, we're doing it ourselves.
+    def compress?(value, options)
+      return unless options.delete(:compress)
+      compress_limit = options.delete(:compress_limit) || DEFAULT_COMPRESS_LIMIT
+      value.bytesize > compress_limit
+    end
+
     def key(key, i)
       "#{key}_#{CACHE_VERSION}_#{i}"
     end

data/lib/large_object_store/version.rb

@@ -1,3 +1,3 @@
 module LargeObjectStore
-  VERSION = "1.2.0"
+  VERSION = "1.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: large_object_store
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Ana Martinez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-18 00:00:00.000000000 Z
+date: 2015-12-07 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: acemacu@gmail.com