large_object_store 1.1.0 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 61880698dc7a3f62cc1eb51390815aa9166e8aa9
-  data.tar.gz: a76a3eb5076124cdf839fa99a5b4afcaf6ac51b7
+  metadata.gz: d47e0ee45bc6fbe4d657974b55acde2b1ed6052e
+  data.tar.gz: 461403d43371d4d95d432b2cf695b0f74d0e070a
 SHA512:
-  metadata.gz: fc8d52947cd7313d49a3135042b1e832574d86f57b9e6bfc262eb8134a9a73c37bbfaa28dbf828bba404611ea4e734bc09dfd685183cf307ba74ec842bf8bf36
-  data.tar.gz: 59bf1b2aac7827f387f672e97da33218ed00ae35f3bf92cea3c3aaddbd9e316adb2d68f25d3cd3b06ada7deb12f49a8873603df26cd3a75137832cf7977bd6c0
+  metadata.gz: 9ee0ca3c5bbae29874718241d39f7dfecbfb1492503928285f3fe2e1aa4dead61d0744c0c68ca4c92b00bff645cc32dbb4a6138c956a55b06ef373dd6bd8fcfc
+  data.tar.gz: 58834c45621095c238af2d3e8f3f291a62fcbb15a83ce8ad32e17aea4664588db926fd9b535a7e3c30b84720cd7738bf8e450d18aeb37f82c397f6e7a344bcce

data/lib/large_object_store/version.rb

@@ -1,3 +1,3 @@
 module LargeObjectStore
-  VERSION = "1.1.0"
+  VERSION = "1.1.1"
 end

data/lib/large_object_store.rb

@@ -1,4 +1,5 @@
 require "large_object_store/version"
+require "zlib"
 
 module LargeObjectStore
 
@@ -9,7 +10,8 @@ module LargeObjectStore
   class RailsWrapper
     attr_reader :store
 
-    LIMIT = 1024**2 - 100
+    MAX_OBJECT_SIZE = 1024**2
+    ITEM_HEADER_SIZE = 100
 
     def initialize(store)
       @store = store
@@ -17,26 +19,30 @@ module LargeObjectStore
 
     def write(key, value, options = {})
       value = Marshal.dump(value)
+      value = Zlib::Deflate.deflate(value) if options.delete(:compress)
+
+      # calculate slice size; note that key length is a factor because
+      # the key is stored on the same slab page as the value
+      slice_size = MAX_OBJECT_SIZE - ITEM_HEADER_SIZE - key.bytesize
 
       # store number of pages
-      pages = (value.size / LIMIT.to_f).ceil
+      pages = (value.size / slice_size.to_f).ceil
 
       if pages == 1
         @store.write("#{key}_0", value, options)
       else
-        @store.write("#{key}_0", pages, options)
-
         # store object
         page = 1
         loop do
-          slice = value.slice!(0, LIMIT)
+          slice = value.slice!(0, slice_size)
           break if slice.size == 0
 
-          @store.write("#{key}_#{page}", slice, options)
+          return false unless @store.write("#{key}_#{page}", slice, options.merge(raw: true))
           page += 1
         end
+
+        @store.write("#{key}_0", pages, options)
       end
-      true
     end
 
     def read(key)
@@ -44,16 +50,26 @@ module LargeObjectStore
       pages = @store.read("#{key}_0")
       return if pages.nil?
 
-      data = if pages.is_a?(String)
-        pages
-      else
+      data = if pages.is_a?(Fixnum)
         # read sliced data
         keys = Array.new(pages).each_with_index.map{|_,i| "#{key}_#{i+1}" }
         slices = @store.read_multi(*keys).values
         return nil if slices.compact.size < pages
         slices.join("")
+      else
+        pages
+      end
+
+      if data.getbyte(0) == 0x78 && [0x01,0x9C,0xDA].include?(data.getbyte(1))
+        data = Zlib::Inflate.inflate(data)
+      end
+
+      begin
+        Marshal.load(data)
+      rescue Exception => e
+        Rails.logger.error "Cannot read large_object_store key #{key} : #{e.message} #{e.backtrace.inspect}"
+        nil
       end
-      Marshal.load(data)
     end
 
     def fetch(key, options={})

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: large_object_store
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.1
 platform: ruby
 authors:
 - Ana Martinez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-06 00:00:00.000000000 Z
+date: 2015-11-17 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: acemacu@gmail.com
@@ -16,18 +16,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .travis.yml
-- Gemfile
-- Gemfile.lock
-- Rakefile
 - Readme.md
-- gem-public_cert.pem
-- large_object_store.gemspec
 - lib/large_object_store.rb
 - lib/large_object_store/version.rb
-- spec/large_object_store_spec.rb
-- spec/spec_helper.rb
-homepage: http://github.com/anamartinez/large_object_store
+homepage: https://github.com/anamartinez/large_object_store
 licenses:
 - MIT
 metadata: {}
@@ -37,17 +29,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: Store large objects in memcache or others

data/.travis.yml

@@ -1,3 +0,0 @@
-rvm:
-  - 1.9.2
-  - 1.9.3

data/Gemfile

@@ -1,6 +0,0 @@
-source "https://rubygems.org"
-gemspec
-
-gem "bump"
-gem "rake"
-gem "rspec", "~>2"

data/Gemfile.lock

@@ -1,28 +0,0 @@
-PATH
-  remote: .
-  specs:
-    large_object_store (1.1.0)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    bump (0.3.9)
-    diff-lcs (1.1.3)
-    rake (10.0.3)
-    rspec (2.12.0)
-      rspec-core (~> 2.12.0)
-      rspec-expectations (~> 2.12.0)
-      rspec-mocks (~> 2.12.0)
-    rspec-core (2.12.2)
-    rspec-expectations (2.12.1)
-      diff-lcs (~> 1.1.3)
-    rspec-mocks (2.12.2)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  bump
-  large_object_store!
-  rake
-  rspec (~> 2)

data/Rakefile

@@ -1,6 +0,0 @@
-require "bundler/gem_tasks"
-require "bump/tasks"
-
-task :default do
-  sh "rspec spec/"
-end

data/gem-public_cert.pem

@@ -1,20 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDMjCCAhqgAwIBAgIBADANBgkqhkiG9w0BAQUFADA/MRAwDgYDVQQDDAdtaWNo
-YWVsMRcwFQYKCZImiZPyLGQBGRYHZ3Jvc3NlcjESMBAGCgmSJomT8ixkARkWAml0
-MB4XDTEzMDIwMzE4MTMxMVoXDTE0MDIwMzE4MTMxMVowPzEQMA4GA1UEAwwHbWlj
-aGFlbDEXMBUGCgmSJomT8ixkARkWB2dyb3NzZXIxEjAQBgoJkiaJk/IsZAEZFgJp
-dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMorXo/hgbUq97+kII9H
-MsQcLdC/7wQ1ZP2OshVHPkeP0qH8MBHGg6eYisOX2ubNagF9YTCZWnhrdKrwpLOO
-cPLaZbjUjljJ3cQR3B8Yn1veV5IhG86QseTBjymzJWsLpqJ1UZGpfB9tXcsFtuxO
-6vHvcIHdzvc/OUkICttLbH+1qb6rsHUceqh+JrH4GrsJ5H4hAfIdyS2XMK7YRKbh
-h+IBu6dFWJJByzFsYmV1PDXln3UBmgAt65cmCu4qPfThioCGDzbSJrGDGLmw/pFX
-FPpVCm1zgYSb1v6Qnf3cgXa2f2wYGm17+zAVyIDpwryFru9yF/jJxE38z/DRsd9R
-/88CAwEAAaM5MDcwCQYDVR0TBAIwADAdBgNVHQ4EFgQUsiNnXHtKeMYYcr4yJVmQ
-WONL+IwwCwYDVR0PBAQDAgSwMA0GCSqGSIb3DQEBBQUAA4IBAQAlyN7kKo/NQCQ0
-AOzZLZ3WAePvStkCFIJ53tsv5Kyo4pMAllv+BgPzzBt7qi605mFSL6zBd9uLou+W
-Co3s48p1dy7CjjAfVQdmVNHF3MwXtfC2OEyvSQPi4xKR8iba8wa3xp9LVo1PuLpw
-/6DsrChWw74HfsJN6qJOK684hJeT8lBYAUfiC3wD0owoPSg+XtyAAddisR+KV5Y1
-NmVHuLtQcNTZy+gRht3ahJRMuC6QyLmkTsf+6MaenwAMkAgHdswGsJztOnNnBa3F
-y0kCSWmK6D+x/SbfS6r7Ke07MRqziJdB9GuE1+0cIRuFh8EQ+LN6HXCKM5pon/GU
-ycwMXfl0
------END CERTIFICATE-----

data/large_object_store.gemspec

@@ -1,17 +0,0 @@
-$LOAD_PATH.unshift File.expand_path("../lib", __FILE__)
-name = "large_object_store"
-require "#{name.gsub("-","/")}/version"
-
-Gem::Specification.new name, LargeObjectStore::VERSION do |s|
-  s.summary = "Store large objects in memcache or others"
-  s.authors = ["Ana Martinez"]
-  s.email = "acemacu@gmail.com"
-  s.homepage = "http://github.com/anamartinez/#{name}"
-  s.files = `git ls-files`.split("\n")
-  s.license = "MIT"
-  cert = File.expand_path("~/.ssh/gem-private_key.pem")
-  if File.exist?(cert)
-    s.signing_key = cert
-    s.cert_chain = ["gem-public_cert.pem"]
-  end
-end

data/spec/large_object_store_spec.rb

@@ -1,131 +0,0 @@
-# encoding: utf-8
-require "spec_helper"
-
-class TestCache
-  def initialize
-    @data = {}
-  end
-
-  def write(k,v, options={})
-    v = Marshal.dump(v)
-    return false if v.bytesize > 1024**2
-    @data[k] = v
-    true
-  end
-
-  def read(k)
-    real_read(k)
-  end
-
-  def real_read(k)
-    v = @data[k]
-    v.nil? ? nil : Marshal.load(v)
-  end
-
-  def read_multi(*keys)
-    Hash[keys.map{|k| [k, real_read(k)] }]
-  end
-
-  def keys
-    @data.keys
-  end
-
-  def delete(key)
-    @data.delete(key)
-  end
-end
-
-describe LargeObjectStore do
-  let(:cache) { TestCache.new }
-  let(:store) { LargeObjectStore.wrap(cache) }
-
-  it "has a VERSION" do
-    LargeObjectStore::VERSION.should =~ /^[\.\da-z]+$/
-  end
-
-  it "wraps and returns a wrapper" do
-    store.class.should == LargeObjectStore::RailsWrapper
-  end
-
-  it "can write/read big objects" do
-    store.write("a", "a"*10_000_000).should == true
-    store.read("a").size.should == 10_000_000
-  end
-
-  it "passes options when caching small" do
-    store.store.should_receive(:write).with(anything, anything, :expires_in => 111)
-    store.write("a", "a", :expires_in => 111)
-  end
-
-  it "passes options when caching big" do
-    store.store.should_receive(:write).with(anything, anything, :expires_in => 111).exactly(3).times
-    store.write("a", "a"*1_200_000, :expires_in => 111)
-  end
-
-  it "cannot read corrupted objects" do
-    store.write("a", ["a"*10_000_000]).should == true
-    store.store.write("a_4", nil)
-    store.read("a").should == nil
-  end
-
-  it "can write/read big non-string objects" do
-    store.write("a", ["a"*10_000_000]).should == true
-    store.read("a").first.size.should == 10_000_000
-  end
-
-  it "can read/write objects with encoding" do
-    store.write("a", "ß"*10_000_000).should == true
-    store.read("a").size.should == 10_000_000
-  end
-
-  it "can write/read giant objects" do
-    store.write("a", "a"*100_000_000).should == true
-    store.read("a").size.should == 100_000_000
-  end
-
-  it "uses necessary keys" do
-    store.write("a", "a"*5_000_000)
-    store.store.keys.should == ["a_0", "a_1", "a_2", "a_3", "a_4", "a_5"]
-  end
-
-  it "uses 1 key when value is small enough" do
-    store.write("a", "a"*500_000)
-    store.store.keys.should == ["a_0"]
-  end
-
-  it "uses read_multi" do
-    store.write("a", "a"*5_000_000)
-    store.store.should_receive(:read).with("a_0").and_return 5
-    store.read("a").size.should == 5_000_000
-  end
-
-  describe "#fetch" do
-    it "executes the block on miss" do
-      store.fetch("a"){ 1 }.should == 1
-    end
-
-    it "does not execute the block on hit" do
-      store.fetch("a"){ 1 }
-      store.fetch("a"){ 2 }.should == 1
-    end
-
-    it "passes the options" do
-      store.should_receive(:write).with(anything, anything, :expires_in => 111)
-      store.fetch("a", :expires_in => 111){ 2 }
-    end
-
-    it "can fetch false" do
-      store.fetch("a"){ false }.should == false
-      store.read("a").should == false
-    end
-  end
-
-  describe "#delete" do
-    it "removes all keys" do
-      store.write("a", "a"*5_000_000)
-      store.read("a").should_not == nil
-      store.delete("a")
-      store.read("a").should == nil
-    end
-  end
-end

data/spec/spec_helper.rb

@@ -1 +0,0 @@
-require "large_object_store"