kvom 6.8.0.beta.200.681.7c84f4a → 6.8.0.beta.200.713.e5c3150

data/.gitignore

@@ -1,4 +1,5 @@
 *.gem
 .bundle
-Gemfile.lock
-pkg/*
+/Gemfile.lock
+/pkg/*
+/spec/spec.opts

data/kvom.gemspec

@@ -23,4 +23,6 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "couchrest"
   s.add_runtime_dependency "activemodel"
   s.add_runtime_dependency "aws-sdk"
+  s.add_runtime_dependency "multi_json"
+  s.add_runtime_dependency "rack" # rack/utils by active_support/cache/file_store
 end

data/lib/kvom/storage.rb

@@ -0,0 +1,5 @@
+module Kvom
+  module Storage
+    Kvom.setup_autoload(self, __FILE__)
+  end
+end

data/lib/kvom/storage/base.rb

@@ -0,0 +1,88 @@
+require 'securerandom'
+require 'multi_json'
+require 'active_support/core_ext/hash/keys'
+
+module Kvom
+  module Storage
+    if MultiJson.respond_to?(:dump)
+      module MultiJsonCoder
+        def encode(value)
+          MultiJson.dump(value)
+        end
+
+        def decode(value)
+          MultiJson.load(value)
+        end
+      end
+    else
+      module MultiJsonCoder
+        def encode(value)
+          MultiJson.encode(value)
+        end
+
+        def decode(value)
+          MultiJson.decode(value)
+        end
+      end
+    end
+
+    class Base
+
+      include MultiJsonCoder
+
+      attr_reader :config
+
+      def initialize(config = nil)
+        @config =
+            if config
+              config.symbolize_keys
+            else
+              {}
+            end
+      end
+
+      def put(data)
+        id = generate_id
+        json_string = encode(data)
+        normalized = decode(json_string)
+        write(id, normalized, json_string)
+        {"id" => id}
+      end
+
+      def get(spec)
+        raise "Invalid key spec (not a hash): #{spec.inspect}" unless Hash === spec
+        id =
+            case
+            when spec.key?("id")
+              spec["id"]
+            when spec.key?(:id)
+              spec[:id]
+            else
+              raise "Invalid key spec (no id): #{spec.inspect}"
+            end
+        read(id)
+      end
+
+      private
+
+      def write(id, data, json_data)
+        raise "implement me in subclass!"
+      end
+
+      def read(id)
+        raise "implement me in subclass!"
+      end
+
+      if RUBY_VERSION < "1.9.2"
+        def generate_id
+          SecureRandom.base64(15).tr("+/", "-_")
+        end
+      else
+        def generate_id
+          SecureRandom.urlsafe_base64(15)
+        end
+      end
+
+    end
+  end
+end

data/lib/kvom/storage/cache_with_prefix.rb

@@ -0,0 +1,24 @@
+class Kvom::Storage::CacheWithPrefix
+
+  def initialize(cache, prefix)
+    @prefix = prefix
+    @cache = cache
+  end
+
+  def read(id)
+    cache.read(cache_id(id))
+  end
+
+  def write(id, value)
+    cache.write(cache_id(id), value)
+  end
+
+  private
+
+  attr_reader :cache, :prefix
+
+  def cache_id(id)
+    "#{prefix}#{id}"
+  end
+
+end

data/lib/kvom/storage/file_system_storage.rb

@@ -0,0 +1,27 @@
+# require 'active_support/cache/file_store' # fails to autoload Cache::Store (no separate file)
+require 'active_support/cache'
+
+module Kvom
+  module Storage
+    class FileSystemStorage < Base
+      def initialize(options)
+        super
+        path = config[:path] or raise "No path specified"
+        @storage = ActiveSupport::Cache::FileStore.new(path, options)
+      end
+
+      private
+
+      attr_reader :storage
+
+      def read(id)
+        storage.read(id)
+      end
+
+      def write(id, value, ignored_json_string)
+        storage.write(id, value)
+      end
+
+    end
+  end
+end

data/lib/kvom/storage/not_found.rb

@@ -0,0 +1,6 @@
+module Kvom
+  module Storage
+    module NotFound
+    end
+  end
+end

data/lib/kvom/storage/s3_storage.rb

@@ -0,0 +1,73 @@
+require 'multi_json'
+require 'aws-sdk'
+
+module ::Kvom::Storage
+  class S3Storage < Base
+
+    def initialize(options)
+      super
+      @bucket = config[:bucket] or raise "No bucket specified"
+      if "" != (bucket_prefix = config[:bucket_prefix].to_s)
+        @bucket_prefix = bucket_prefix
+      end
+
+      if (cache = config[:cache])
+        cache_prefix = config[:cache_prefix].to_s
+        @cache =
+            if "" != cache_prefix
+              CacheWithPrefix.new(cache, cache_prefix)
+            else
+              cache
+            end
+        extend CacheAware
+      end
+    end
+
+    private
+
+    attr_reader :bucket, :bucket_prefix, :cache
+
+    def read(id)
+      decode(s3_read(id))
+    end
+
+    def write(id, object, json_string)
+      s3_object(id).write(json_string)
+    end
+
+    def s3_read(id)
+      s3_object(id).read
+    rescue AWS::S3::Errors::NoSuchKey => e
+      e.extend NotFound
+      raise e
+    end
+
+    def s3_object(id)
+      @s3_objects ||= bucket.objects
+      s3_id = bucket_prefix ? "#{bucket_prefix}/#{id}" : id
+      @s3_objects[s3_id]
+    end
+
+    module CacheAware
+
+      private
+
+      def read(id)
+        object = cache.read(id)
+        if nil == object
+          object = super
+          cache.write(id, object)
+        end
+        object
+      end
+
+      def write(id, object, json_string)
+        super
+        cache.write(id, object)
+      end
+
+    end
+
+  end
+
+end

data/spec/cache_with_prefix_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+describe Kvom::Storage::CacheWithPrefix do
+  let(:wrapped_cache) {mock("cache", :read => nil, :write => nil)}
+  let(:cache_with_prefix) {Kvom::Storage::CacheWithPrefix.new(wrapped_cache, "some_prefix&")}
+
+  describe "#read" do
+    it "delegates to the wrapped cache with a prefixed cache id" do
+      wrapped_cache.should_receive(:read).with("some_prefix&x_id").and_return("cached_value")
+      cache_with_prefix.read("x_id").should == "cached_value"
+    end
+  end
+
+  describe "#write" do
+    it "delegates to the wrapped cache with a prefixed cache id" do
+      wrapped_cache.should_receive(:write).with("some_prefix&x_id", "the value")
+      cache_with_prefix.write("x_id", "the value")
+    end
+  end
+end

data/spec/storage/file_system_spec.rb

@@ -0,0 +1,37 @@
+require 'spec_helper'
+require 'fileutils'
+
+describe Kvom::Storage::FileSystemStorage do
+  STORAGE_TEST_DIR = "/tmp/kvom_fs_storage"
+
+  before(:all) do
+    FileUtils.rm_rf(STORAGE_TEST_DIR)
+  end
+
+  after(:all) do
+    FileUtils.rm_rf(STORAGE_TEST_DIR)
+  end
+
+  let(:storage) {Kvom::Storage::FileSystemStorage.new(:path => "/tmp/kvom_fs_storage")}
+
+  it "persists data at the given file system path" do
+    expect {
+      storage.put({"some" => "data"})
+    }.to change {
+      Dir.glob("#{STORAGE_TEST_DIR}/**").count
+    }
+  end
+
+  it "returns persisted data" do
+    (reference = storage.put({"more" => "data"})).should have_key("id")
+    storage.get(reference).should == {"more" => "data"}
+  end
+
+  context "when instantiated without a path" do
+    it "raises an error" do
+      expect {
+        Kvom::Storage::FileSystemStorage.new(nil)
+      }.to raise_error(/\bpath\b/)
+    end
+  end
+end

data/spec/storage/s3_spec.rb

@@ -0,0 +1,178 @@
+require 'spec_helper'
+
+describe Kvom::Storage::S3Storage do
+
+  let(:bucket) { mock(:objects => s3_objects) }
+  let(:s3_objects) { mock("s3_objects", :[] => s3_object) }
+  let(:s3_object) { mock("s3_object", :read => "{}", :write => nil) }
+  let(:different_s3_object) { mock("another_s3_object", :read => "{}", :write => nil) }
+
+  let(:storage) { Kvom::Storage::S3Storage.new(:bucket => bucket) }
+  let(:value) {[{:foo => :bar}, {'baz' => 123}]}
+  let(:desymbolized_value) {[{"foo" => "bar"}, {'baz' => 123}]}
+  let(:json_value) {'[{"foo":"bar"},{"baz":123}]'}
+  let(:secure_random_method) {RUBY_VERSION < "1.9.2" ? :base64 : :urlsafe_base64}
+
+
+  before do
+    SecureRandom.stub(secure_random_method).and_return('random_id')
+  end
+
+
+  describe '#put' do
+    it 'stores the value as JSON to S3 with a new random id and returns the id' do
+      SecureRandom.should_receive(secure_random_method).and_return('random_id')
+      bucket.objects.should_receive(:[]).with('random_id').and_return(s3_object)
+      s3_object.should_receive(:write).with(json_value)
+      storage.put(value)
+    end
+
+    it 'returns the random id' do
+      SecureRandom.should_receive(secure_random_method).and_return('unique_id')
+      storage.put([{:foo => :bar}, {'baz' => 123}]).should == {"id" => 'unique_id'}
+    end
+  end
+
+  describe '#get' do
+    context "when the lookup with the id in the bucket is successful" do
+      before do
+        bucket.objects.should_receive(:[]).with('the_id').and_return(s3_object)
+        s3_object.should_receive(:read).and_return(json_value)
+      end
+
+      it "returns the value" do
+        storage.get({"id" => "the_id"}).should == desymbolized_value
+      end
+    end
+
+    context "when the id does not exist in the bucket" do
+      before do
+        bucket.objects.should_receive(:[]).with('the_id').and_return(different_s3_object)
+        different_s3_object.should_receive(:read).and_raise(AWS::S3::Errors::NoSuchKey.new(nil, nil))
+      end
+
+      it "raise a Kvom::Storage::NotFound" do
+        expect {
+          storage.get({"id" => "the_id"})
+        }.to raise_error(Kvom::Storage::NotFound)
+      end
+    end
+  end
+
+  context "when the storage has been configured with a bucket prefix" do
+    let(:storage) { Kvom::Storage::S3Storage.new(:bucket => bucket, :bucket_prefix => "bp") }
+
+    describe '#put' do
+      it "stores with a random id within the s3 folder with the given prefix" do
+        bucket.objects.should_receive(:[]).with('bp/random_id').and_return(different_s3_object)
+        different_s3_object.should_receive(:write)
+        storage.put(value)
+      end
+
+      it 'returns the random id without a prefix' do
+        SecureRandom.should_receive(secure_random_method).and_return('unique_id')
+        storage.put(value).should == {"id" => "unique_id"}
+      end
+    end
+
+    describe '#get' do
+      it "does the s3 lookup with the bucket prefix" do
+        bucket.objects.should_receive(:[]).with('bp/the_id').and_return(different_s3_object)
+        different_s3_object.should_receive(:read).and_return('{"some":"data"}')
+        storage.get({"id" => "the_id"}).should == {"some" => "data"}
+      end
+    end
+  end
+
+  context "with an additional cache" do
+    let(:cache) { mock("cache", :read => {}, :write => nil) }
+    let(:storage) { Kvom::Storage::S3Storage.new(:bucket => bucket, :cache => cache) }
+
+    describe "#put" do
+
+      it 'writes the stored value to the cache with the same random id' do
+        cache.should_receive(:write).with('random_id', anything).ordered
+        storage.put(value)
+      end
+
+      it 'writes the value desymbolized to the cache in order to have'\
+          ' a cache hit return the value identical the one returned by S3 in case of a cache miss' do
+        value.first.should have_key(:foo)
+        value.first.should_not have_key("foo")
+        desymbolized_value.first.should_not have_key(:foo)
+        desymbolized_value.first.should have_key("foo")
+
+        cache.should_receive(:write).with('random_id', desymbolized_value)
+        storage.put(value)
+      end
+
+      it 'writes the stored value to the cache after it has been stored to S3' do
+        s3_object.should_receive(:write).ordered
+        cache.should_receive(:write).with('random_id', anything).ordered
+
+        storage.put(value)
+      end
+    end
+
+    describe "#get" do
+      context "on cache hit" do
+        before do
+          cache.should_receive(:read).with('the_id').and_return("the cached value")
+        end
+
+        it "returns the cached value without ever accessing s3" do
+          s3_object.should_not_receive(:read)
+          storage.get({"id" => 'the_id'}).should == "the cached value"
+        end
+      end
+
+      context "on cache miss" do
+        before do
+          cache.should_receive(:read).and_return(nil)
+        end
+
+        it "returns the S3 value" do
+          s3_object.should_receive(:read).and_return('{"s3":"value"}')
+          storage.get({"id" => 'the_id'}).should == {"s3" => "value"}
+        end
+
+        it "remembers the returned value in the cache" do
+          s3_object.should_receive(:read).and_return('{"s3":"value"}')
+          cache.should_receive(:write).with("the_id", {"s3" => "value"})
+
+          storage.get({"id" => 'the_id'})
+        end
+      end
+    end
+  end
+
+  context "with an additional cache with a cache prefix" do
+
+    let(:cache) { mock("cache", :read => nil, :write => nil) }
+    let(:storage) { Kvom::Storage::S3Storage.new(:bucket => bucket, :cache => cache, :cache_prefix => "cp-") }
+
+    it "reads from cache with the given prefix" do
+      cache.should_receive(:read).with("cp-an id")
+      storage.get({"id" => "an id"})
+    end
+
+    it "copies to cache with the given prefix" do
+      cache.should_receive(:write).with("cp-an id", anything)
+
+      storage.get({"id" => "an id"})
+    end
+
+    it "writes to cache with the given prefix" do
+      cache.should_receive(:write).with("cp-random_id", ["the value"])
+      storage.put(["the value"])
+    end
+  end
+
+  context "when instantiated without a bucket" do
+    it "raises an error" do
+      expect {
+        Kvom::Storage::S3Storage.new(nil)
+      }.to raise_error(/\bbucket\b/)
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: kvom
 version: !ruby/object:Gem::Version 
-  hash: 194079586554
+  hash: 27133439521
   prerelease: 6
   segments: 
   - 6
@@ -9,14 +9,12 @@ version: !ruby/object:Gem::Version
   - 0
   - beta
   - 200
-  - 681
-  - 7
+  - 713
+  - e
+  - 5
   - c
-  - 84
-  - f
-  - 4
-  - a
-  version: 6.8.0.beta.200.681.7c84f4a
+  - 3150
+  version: 6.8.0.beta.200.713.e5c3150
 platform: ruby
 authors: 
 - Kristian Hanekamp, Infopark AG
@@ -24,7 +22,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-06-26 00:00:00 +02:00
+date: 2012-06-29 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -97,6 +95,34 @@ dependencies:
   name: aws-sdk
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency 
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  version_requirements: *id006
+  name: multi_json
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency 
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  version_requirements: *id007
+  name: rack
+  prerelease: false
+  type: :runtime
 description: Use it to build object models in ruby on top of a key value store.
 email: 
 - kristian.hanekamp@infopark.de
@@ -122,10 +148,19 @@ files:
 - lib/kvom/document.rb
 - lib/kvom/model_identity.rb
 - lib/kvom/not_found.rb
-- spec/base_spec.rb
-- spec/counter_spec.rb
-- spec/model_identity_spec.rb
+- lib/kvom/storage.rb
+- lib/kvom/storage/base.rb
+- lib/kvom/storage/cache_with_prefix.rb
+- lib/kvom/storage/file_system_storage.rb
+- lib/kvom/storage/not_found.rb
+- lib/kvom/storage/s3_storage.rb
+- spec/adaptor/base_spec.rb
+- spec/adaptor/counter_spec.rb
+- spec/adaptor/model_identity_spec.rb
+- spec/cache_with_prefix_spec.rb
 - spec/spec_helper.rb
+- spec/storage/file_system_spec.rb
+- spec/storage/s3_spec.rb
 has_rdoc: true
 homepage: ""
 licenses: []
@@ -163,7 +198,10 @@ signing_key:
 specification_version: 3
 summary: Key Value Object Mapper
 test_files: 
-- spec/base_spec.rb
-- spec/counter_spec.rb
-- spec/model_identity_spec.rb
+- spec/adaptor/base_spec.rb
+- spec/adaptor/counter_spec.rb
+- spec/adaptor/model_identity_spec.rb
+- spec/cache_with_prefix_spec.rb
 - spec/spec_helper.rb
+- spec/storage/file_system_spec.rb
+- spec/storage/s3_spec.rb