large_object_store 0.0.1

data/.travis.yml

@@ -0,0 +1,4 @@
+rvm:
+  - ree
+  - 1.9.2
+  - 1.9.3

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+gemspec
+
+gem "bump"
+gem "rake"
+gem "rspec", "~>2"

data/Gemfile.lock

@@ -0,0 +1,28 @@
+PATH
+  remote: .
+  specs:
+    large_object_store (0.0.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    bump (0.3.9)
+    diff-lcs (1.1.3)
+    rake (10.0.3)
+    rspec (2.12.0)
+      rspec-core (~> 2.12.0)
+      rspec-expectations (~> 2.12.0)
+      rspec-mocks (~> 2.12.0)
+    rspec-core (2.12.2)
+    rspec-expectations (2.12.1)
+      diff-lcs (~> 1.1.3)
+    rspec-mocks (2.12.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bump
+  large_object_store!
+  rake
+  rspec (~> 2)

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "bump/tasks"
+
+task :default do
+  sh "rspec spec/"
+end

data/Readme.md

@@ -0,0 +1,32 @@
+Store large objects in memcache or others by slicing them.
+ - uses read_multi for fast access
+ - returns nil if one slice is missing
+
+Install
+=======
+
+```Bash
+gem install large_object_store
+```
+
+Usage
+=====
+
+```Ruby
+Rails.cache.write("a", "a"*10_000_000) # => false -> oops too large
+
+store = LargeObjectStore.wrap(Rails.cache)
+store.write("a", "a"*10_000_000)  # => true -> always!
+store.read("a").size              # => 10_000_000 using multi_get
+store.read("b")                   # => nil
+store.fetch("a"){ "something" }   # => "something" executes block on miss
+```
+
+Author
+======
+[Ana Martinez](https://github.com/anamartinez)<br/>
+acemacu@gmail.com<br/>
+[Michael Grosser](https://github.com/grosser)<br/>
+michael@grosser.it<br/>
+License: MIT<br/>
+[![Build Status](https://travis-ci.org/anamartinez/large_object_store.png)](https://travis-ci.org/anamartinez/large_object_store)

data/gem-public_cert.pem

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDMjCCAhqgAwIBAgIBADANBgkqhkiG9w0BAQUFADA/MRAwDgYDVQQDDAdtaWNo
+YWVsMRcwFQYKCZImiZPyLGQBGRYHZ3Jvc3NlcjESMBAGCgmSJomT8ixkARkWAml0
+MB4XDTEzMDIwMzE4MTMxMVoXDTE0MDIwMzE4MTMxMVowPzEQMA4GA1UEAwwHbWlj
+aGFlbDEXMBUGCgmSJomT8ixkARkWB2dyb3NzZXIxEjAQBgoJkiaJk/IsZAEZFgJp
+dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMorXo/hgbUq97+kII9H
+MsQcLdC/7wQ1ZP2OshVHPkeP0qH8MBHGg6eYisOX2ubNagF9YTCZWnhrdKrwpLOO
+cPLaZbjUjljJ3cQR3B8Yn1veV5IhG86QseTBjymzJWsLpqJ1UZGpfB9tXcsFtuxO
+6vHvcIHdzvc/OUkICttLbH+1qb6rsHUceqh+JrH4GrsJ5H4hAfIdyS2XMK7YRKbh
+h+IBu6dFWJJByzFsYmV1PDXln3UBmgAt65cmCu4qPfThioCGDzbSJrGDGLmw/pFX
+FPpVCm1zgYSb1v6Qnf3cgXa2f2wYGm17+zAVyIDpwryFru9yF/jJxE38z/DRsd9R
+/88CAwEAAaM5MDcwCQYDVR0TBAIwADAdBgNVHQ4EFgQUsiNnXHtKeMYYcr4yJVmQ
+WONL+IwwCwYDVR0PBAQDAgSwMA0GCSqGSIb3DQEBBQUAA4IBAQAlyN7kKo/NQCQ0
+AOzZLZ3WAePvStkCFIJ53tsv5Kyo4pMAllv+BgPzzBt7qi605mFSL6zBd9uLou+W
+Co3s48p1dy7CjjAfVQdmVNHF3MwXtfC2OEyvSQPi4xKR8iba8wa3xp9LVo1PuLpw
+/6DsrChWw74HfsJN6qJOK684hJeT8lBYAUfiC3wD0owoPSg+XtyAAddisR+KV5Y1
+NmVHuLtQcNTZy+gRht3ahJRMuC6QyLmkTsf+6MaenwAMkAgHdswGsJztOnNnBa3F
+y0kCSWmK6D+x/SbfS6r7Ke07MRqziJdB9GuE1+0cIRuFh8EQ+LN6HXCKM5pon/GU
+ycwMXfl0
+-----END CERTIFICATE-----

data/large_object_store.gemspec

@@ -0,0 +1,17 @@
+$LOAD_PATH.unshift File.expand_path("../lib", __FILE__)
+name = "large_object_store"
+require "#{name.gsub("-","/")}/version"
+
+Gem::Specification.new name, LargeObjectStore::VERSION do |s|
+  s.summary = "Store large objects in memcache or others"
+  s.authors = ["Ana Martinez"]
+  s.email = "acemacu@gmail.com"
+  s.homepage = "http://github.com/anamartinez/#{name}"
+  s.files = `git ls-files`.split("\n")
+  s.license = "MIT"
+  cert = File.expand_path("~/.ssh/gem-private_key.pem")
+  if File.exist?(cert)
+    s.signing_key = cert
+    s.cert_chain = ["gem-public_cert.pem"]
+  end
+end

data/lib/large_object_store.rb

@@ -0,0 +1,58 @@
+require "large_object_store/version"
+
+module LargeObjectStore
+
+  def self.wrap(store)
+    RailsWrapper.new(store)
+  end
+
+  class RailsWrapper
+    attr_reader :store
+
+    LIMIT = 1024**2 - 100
+
+    def initialize(store)
+      @store = store
+    end
+
+    def write(key, value, options = {})
+      value = Marshal.dump(value)
+      
+      # store number of pages
+      pages = (value.size / LIMIT.to_f).ceil
+      @store.write("#{key}_0", pages, options)
+
+      # store object
+      page = 1
+      loop do
+        slice = value.slice!(0, LIMIT)
+        break if slice.size == 0
+
+        @store.write("#{key}_#{page}", slice, options)
+        page += 1
+      end
+
+      true
+    end
+
+    def read(key)
+      # read pages
+      pages = @store.read("#{key}_0")
+      return if pages.nil?
+
+      # read sliced data
+      keys = Array.new(pages).each_with_index.map{|_,i| "#{key}_#{i+1}" }
+      slices = @store.read_multi(*keys).values
+      return nil if slices.compact.size < pages
+      Marshal.load(slices.join(""))
+    end
+
+    def fetch(key, options={})
+      value = read(key)
+      return value unless value.nil?
+      value = yield
+      write(key, value, options)
+      value
+    end
+  end
+end

data/lib/large_object_store/version.rb

@@ -0,0 +1,3 @@
+module LargeObjectStore
+  VERSION = "0.0.1"
+end

data/spec/large_object_store_spec.rb

@@ -0,0 +1,107 @@
+# encoding: utf-8
+require "spec_helper"
+
+class TestCache
+  def initialize
+    @data = {}
+  end
+
+  def write(k,v, options={})
+    v = Marshal.dump(v)
+    return false if v.bytesize > 1024**2
+    @data[k] = v
+    true
+  end
+
+  def read(k)
+    real_read(k)
+  end
+
+  def real_read(k)
+    v = @data[k]
+    v.nil? ? nil : Marshal.load(v)
+  end
+
+  def read_multi(*keys)
+    Hash[keys.map{|k| [k, real_read(k)] }]
+  end
+
+  def keys
+    @data.keys
+  end
+end
+
+describe LargeObjectStore do
+  let(:store) { LargeObjectStore.wrap(TestCache.new) }
+
+  it "has a VERSION" do
+    LargeObjectStore::VERSION.should =~ /^[\.\da-z]+$/
+  end
+
+  it "wraps and returns a wrapper" do
+    store.class.should == LargeObjectStore::RailsWrapper
+  end
+
+  it "can write/read big objects" do
+    store.write("a", "a"*10_000_000).should == true
+    store.read("a").size.should == 10_000_000
+  end
+
+  it "passes options" do
+    store.store.should_receive(:write).with(anything, anything, :expires_in => 111).twice
+    store.write("a", "a", :expires_in => 111)
+  end
+
+  it "cannot read corrupted objects" do
+    store.write("a", ["a"*10_000_000]).should == true
+    store.store.write("a_4", nil)
+    store.read("a").should == nil
+  end
+
+  it "can write/read big non-string objects" do
+    store.write("a", ["a"*10_000_000]).should == true
+    store.read("a").first.size.should == 10_000_000
+  end
+
+  it "can read/write objects with encoding" do
+    store.write("a", "ß"*10_000_000).should == true
+    store.read("a").size.should == 10_000_000
+  end
+
+  it "can write/read giant objects" do
+    store.write("a", "a"*100_000_000).should == true
+    store.read("a").size.should == 100_000_000
+  end
+
+  it "uses necessary keys" do
+    store.write("a", "a"*5_000_000)
+    store.store.keys.should == ["a_0", "a_1", "a_2", "a_3", "a_4", "a_5"]
+  end
+
+  it "uses read_multi" do
+    store.write("a", "a"*5_000_000)
+    store.store.should_receive(:read).with("a_0").and_return 5
+    store.read("a").size.should == 5_000_000
+  end
+
+  describe "#fetch" do
+    it "executes the block on miss" do
+      store.fetch("a"){ 1 }.should == 1
+    end
+
+    it "does not execute the block on hit" do
+      store.fetch("a"){ 1 }
+      store.fetch("a"){ 2 }.should == 1
+    end
+
+    it "passes the options" do
+      store.should_receive(:write).with(anything, anything, :expires_in => 111)
+      store.fetch("a", :expires_in => 111){ 2 }
+    end
+
+    it "can fetch false" do
+      store.fetch("a"){ false }.should == false
+      store.read("a").should == false
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1 @@
+require "large_object_store"

metadata

@@ -0,0 +1,56 @@
+--- !ruby/object:Gem::Specification
+name: large_object_store
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Ana Martinez
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-05-17 00:00:00.000000000 Z
+dependencies: []
+description: 
+email: acemacu@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .travis.yml
+- Gemfile
+- Gemfile.lock
+- Rakefile
+- Readme.md
+- gem-public_cert.pem
+- large_object_store.gemspec
+- lib/large_object_store.rb
+- lib/large_object_store/version.rb
+- spec/large_object_store_spec.rb
+- spec/spec_helper.rb
+homepage: http://github.com/anamartinez/large_object_store
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Store large objects in memcache or others
+test_files: []