bucket_brigade 0.0.1 → 0.1.0

data/README.md

@@ -5,7 +5,8 @@ like Riak CS.
 Usage:
 
 ```ruby
-require 'bucket_brigade/bucket_config
+require 'bucket_brigade/bucket_config'
+require 'bucket_brigade/bucket_list'
 
 bucket_config1 = BucketBrigade::BucketConfig.new(
     'http://192.168.37.73:8080',
@@ -21,15 +22,14 @@ bucket_config2 = BucketBrigade::BucketConfig.new(
     'test-bucket'
 )
 
-
-bucket_list = BucketBrigade::BucketList.new(
+bucket_list = BucketBrigade::BucketList.build(
     bucket_config1,
     bucket_config2,
-    )
+)
 
 open('/tmp/file') do |f|
     bucket_list.put('test-key', f)
 end
 
-puts bucket_list.get('test-key')
+puts bucket_list.get('test-key').read
 ```

data/bucket_brigade.gemspec

@@ -1,8 +1,11 @@
 # encoding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'bucket_brigade/version'
 
 Gem::Specification.new do |s|
   s.name = 'bucket_brigade'
-  s.version = '0.0.1'
+  s.version = BucketBrigade::VERSION
   s.summary = 'S3 bucket cache hierarchy'
   s.description = 'S3 bucket cache hierarchy'
   s.author = 'Pivotal Labs'

data/lib/bucket_brigade/bucket.rb

@@ -1,19 +1,17 @@
 require 'bucket_brigade'
-
 require 'aws-sdk'
-
 require 'tempfile'
 
 class BucketBrigade::Bucket
   def initialize(bucket_config)
-    @s3_bucket = AWS::S3.new(
-        s3_endpoint: bucket_config.host,
-        s3_port: bucket_config.port,
-        access_key_id: bucket_config.access_key_id,
-        secret_access_key: bucket_config.secret_access_key,
-        use_ssl: bucket_config.use_ssl,
-        s3_force_path_style: true
-    ).buckets[bucket_config.bucket]
+    @bucket_config = bucket_config
+  end
+
+  def can_connect?
+    s3.buckets.first
+    true
+  rescue Timeout::Error
+    false
   end
 
   def has_key?(key)
@@ -33,7 +31,7 @@ class BucketBrigade::Bucket
   end
 
   def object(key)
-    @s3_bucket.objects[key]
+    s3_bucket.objects[key]
   end
 
   def copy(key, other)
@@ -48,4 +46,23 @@ class BucketBrigade::Bucket
 
     temp.close!
   end
+
+  private
+
+  attr_reader :bucket_config
+
+  def s3
+    @s3 ||= AWS::S3.new(
+        s3_endpoint: bucket_config.host,
+        s3_port: bucket_config.port,
+        access_key_id: bucket_config.access_key_id,
+        secret_access_key: bucket_config.secret_access_key,
+        use_ssl: bucket_config.use_ssl,
+        s3_force_path_style: true
+    )
+  end
+
+  def s3_bucket
+    @s3_bucket ||= s3.buckets[bucket_config.bucket]
+  end
 end

data/lib/bucket_brigade/bucket_list.rb

@@ -4,17 +4,32 @@ require 'bucket_brigade/bucket'
 require 'aws-sdk'
 
 class BucketBrigade::BucketList
-  def initialize(*bucket_configs)
-    @buckets = bucket_configs.map do |bucket_config|
-      BucketBrigade::Bucket.new(bucket_config)
+  def self.build(bucket_configs)
+    good_buckets = []
+
+    bucket_configs.each do |bucket_config|
+      bucket = BucketBrigade::Bucket.new(bucket_config)
+      if bucket.can_connect?
+        good_buckets << bucket
+      else
+        warn "Unable to connect to bucket at #{bucket_config.endpoint}"
+      end
     end
+
+    new(good_buckets)
+  end
+
+  attr_reader :buckets
+
+  def initialize(buckets)
+    @buckets = buckets
   end
 
   def get(key)
-    @buckets.each_with_index do |bucket, i|
+    buckets.each_with_index do |bucket, i|
       if bucket.has_key?(key)
         if i > 0
-          prev_bucket = @buckets[i - 1]
+          prev_bucket = buckets[i - 1]
           bucket.copy(key, prev_bucket)
           return prev_bucket.get(key)
         end
@@ -25,9 +40,9 @@ class BucketBrigade::BucketList
   end
 
   def put(key, io)
-    @buckets.last.put(key, io)
+    buckets.last.put(key, io)
     # invalidate all the way down
-    @buckets.first(@buckets.size - 1).each do |bucket|
+    buckets.first(buckets.size - 1).each do |bucket|
       bucket.delete(key)
     end
   end

data/lib/bucket_brigade/version.rb

@@ -0,0 +1,3 @@
+module BucketBrigade
+  VERSION = '0.1.0'
+end

data/spec/integration/put_get_spec.rb

@@ -25,11 +25,11 @@ describe 'put and get data from the cache' do
         'bucket-brigade-test'
     )
 
-    bucket_list = BucketBrigade::BucketList.new(
+    bucket_list = BucketBrigade::BucketList.build([
         bucket_config1,
         bucket_config2,
         bucket_config3
-    )
+    ])
 
     io = StringIO.new
     io.write('test 1 2 3')

data/spec/unit/bucket_brigade/bucket_list_spec.rb

@@ -2,148 +2,181 @@ require 'bucket_brigade/bucket_config'
 require 'bucket_brigade/bucket_list'
 
 describe BucketBrigade::BucketList do
+  describe '.from_configs' do
+    describe 'when one of the buckets cannot connect' do
+      let(:working_bucket) { double('working bucket', can_connect?: true) }
+      let(:bad_bucket) { double('bad bucket', can_connect?: false) }
+      let(:working_config) { double('working config') }
+      let(:bad_config) { double('bad config', endpoint: 'bad.example.com') }
+      let(:s3_object) { double('s3 object') }
 
-  let(:bucket_configs) {
-    [
+      before do
+        expect(BucketBrigade::Bucket).to receive(:new).with(working_config).and_return(working_bucket)
+        expect(BucketBrigade::Bucket).to receive(:new).with(bad_config).and_return(bad_bucket)
+      end
+
+      subject(:bucket_list) { described_class.build([bad_config, working_config]) }
+
+      it 'creates a bucket list with only the working bucket' do
+        expect(bucket_list.buckets).to eq([working_bucket])
+      end
+
+      it 'only uses the working bucket' do
+        expect(working_bucket).to receive(:has_key?).with('key').and_return(true)
+        expect(working_bucket).to receive(:get).with('key').and_return(s3_object)
+        expect(bad_bucket).not_to receive(:get)
+
+        expect(bucket_list.get('key')).to eq(s3_object)
+      end
+    end
+  end
+
+  describe 'instance methods' do
+    let(:bucket_1) do
+      BucketBrigade::Bucket.new(
         BucketBrigade::BucketConfig.new(
-            'http://test-endpoint-1:1234',
-            'test-access-key-id-1',
-            'test-secret-access-key-1',
-            'test-bucket-1'
-        ),
+          'http://test-endpoint-1:1234',
+          'test-access-key-id-1',
+          'test-secret-access-key-1',
+          'test-bucket-1'
+        )
+      )
+    end
+
+    let(:bucket_2) do
+      BucketBrigade::Bucket.new(
         BucketBrigade::BucketConfig.new(
-            'http://test-endpoint-2:1234',
-            'test-access-key-id-2',
-            'test-secret-access-key-2',
-            'test-bucket-2'
+          'http://test-endpoint-2:1234',
+          'test-access-key-id-2',
+          'test-secret-access-key-2',
+          'test-bucket-2'
         )
-    ]
-  }
+      )
+    end
 
-  let(:s3_1) { instance_double(AWS::S3) }
-  let(:s3_2) { instance_double(AWS::S3) }
+    let(:buckets) { [bucket_1, bucket_2] }
 
-  let(:bucket_collection_1) { instance_double(AWS::S3::BucketCollection) }
-  let(:bucket_collection_2) { instance_double(AWS::S3::BucketCollection) }
+    let(:s3_1) { instance_double(AWS::S3, buckets: bucket_collection_1) }
+    let(:s3_2) { instance_double(AWS::S3, buckets: bucket_collection_2) }
 
-  let(:bucket_1) { instance_double(AWS::S3::Bucket) }
-  let(:bucket_2) { instance_double(AWS::S3::Bucket) }
+    let(:bucket_collection_1) { instance_double(AWS::S3::BucketCollection, first: nil) }
+    let(:bucket_collection_2) { instance_double(AWS::S3::BucketCollection, first: nil) }
 
-  let(:object_collection_1) { instance_double(AWS::S3::ObjectCollection) }
-  let(:object_collection_2) { instance_double(AWS::S3::ObjectCollection) }
+    let(:s3_bucket_1) { instance_double(AWS::S3::Bucket, objects: object_collection_1) }
+    let(:s3_bucket_2) { instance_double(AWS::S3::Bucket, objects: object_collection_2) }
 
-  let(:object_1) { instance_double(AWS::S3::S3Object) }
-  let(:object_2) { instance_double(AWS::S3::S3Object) }
+    let(:object_collection_1) { instance_double(AWS::S3::ObjectCollection) }
+    let(:object_collection_2) { instance_double(AWS::S3::ObjectCollection) }
 
-  before do
-    allow(AWS::S3).to receive(:new).with(
-                          s3_endpoint: 'test-endpoint-1',
-                          s3_port: 1234,
-                          access_key_id: 'test-access-key-id-1',
-                          secret_access_key: 'test-secret-access-key-1',
-                          use_ssl: false,
+    let(:object_1) { instance_double(AWS::S3::S3Object) }
+    let(:object_2) { instance_double(AWS::S3::S3Object) }
+
+    before do
+      allow(AWS::S3).to receive(:new).with(
+                          s3_endpoint:         'test-endpoint-1',
+                          s3_port:             1234,
+                          access_key_id:       'test-access-key-id-1',
+                          secret_access_key:   'test-secret-access-key-1',
+                          use_ssl:             false,
                           s3_force_path_style: true
-                      ).and_return(s3_1)
-    allow(s3_1).to receive(:buckets).and_return(bucket_collection_1)
-    allow(bucket_collection_1).to receive(:[]).with('test-bucket-1').and_return(bucket_1)
-    allow(bucket_1).to receive(:objects).and_return(object_collection_1)
-    allow(object_collection_1).to receive(:[]).with('test-key').and_return(object_1)
-
-    allow(AWS::S3).to receive(:new).with(
-                          s3_endpoint: 'test-endpoint-2',
-                          s3_port: 1234,
-                          access_key_id: 'test-access-key-id-2',
-                          secret_access_key: 'test-secret-access-key-2',
-                          use_ssl: false,
+                        ).and_return(s3_1)
+      allow(bucket_collection_1).to receive(:[]).with('test-bucket-1').and_return(s3_bucket_1)
+      allow(object_collection_1).to receive(:[]).with('test-key').and_return(object_1)
+
+      allow(AWS::S3).to receive(:new).with(
+                          s3_endpoint:         'test-endpoint-2',
+                          s3_port:             1234,
+                          access_key_id:       'test-access-key-id-2',
+                          secret_access_key:   'test-secret-access-key-2',
+                          use_ssl:             false,
                           s3_force_path_style: true
-                      ).and_return(s3_2)
-    allow(s3_2).to receive(:buckets).and_return(bucket_collection_2)
-    allow(bucket_collection_2).to receive(:[]).with('test-bucket-2').and_return(bucket_2)
-    allow(bucket_2).to receive(:objects).and_return(object_collection_2)
-    allow(object_collection_2).to receive(:[]).with('test-key').and_return(object_2)
-  end
+                        ).and_return(s3_2)
+      allow(bucket_collection_2).to receive(:[]).with('test-bucket-2').and_return(s3_bucket_2)
+      allow(object_collection_2).to receive(:[]).with('test-key').and_return(object_2)
+    end
 
-  subject(:bucket_list) { described_class.new(*bucket_configs) }
+    subject(:bucket_list) { described_class.new(buckets) }
 
-  describe '#get' do
-    context 'when the object is found in the first bucket' do
-      before do
-        allow(object_1).to receive(:exists?).and_return(true)
-      end
+    describe '#get' do
+      context 'when the object is found in the first bucket' do
+        before do
+          allow(object_1).to receive(:exists?).and_return(true)
+        end
 
-      it 'returns the object from the first bucket' do
-        expect(bucket_list.get('test-key')).to eq(object_1)
+        it 'returns the object from the first bucket' do
+          expect(bucket_list.get('test-key')).to eq(object_1)
+        end
       end
-    end
 
-    context 'when the object is not found in any buckets' do
-      before do
-        allow(object_1).to receive(:exists?).and_return(false)
-        allow(object_2).to receive(:exists?).and_return(false)
-      end
+      context 'when the object is not found in any buckets' do
+        before do
+          allow(object_1).to receive(:exists?).and_return(false)
+          allow(object_2).to receive(:exists?).and_return(false)
+        end
 
-      it 'returns nil' do
-        expect(bucket_list.get('test-key')).to eq(nil)
+        it 'returns nil' do
+          expect(bucket_list.get('test-key')).to eq(nil)
+        end
       end
-    end
-
-    context 'when the object is found in the second bucket' do
-      let(:temp) { instance_double(Tempfile) }
 
-      before do
-        allow(object_1).to receive(:exists?).and_return(false)
-        allow(object_2).to receive(:exists?).and_return(true)
+      context 'when the object is found in the second bucket' do
+        let(:temp) { instance_double(Tempfile) }
 
-        allow(Tempfile).to receive(:new).with('bucket-copy').and_return(temp)
-        allow(temp).to receive(:rewind)
-        allow(temp).to receive(:close!)
-      end
+        before do
+          allow(object_1).to receive(:exists?).and_return(false)
+          allow(object_2).to receive(:exists?).and_return(true)
 
-      it 'writes the object to the first bucket' do
-        allow(object_2).to receive(:read)
-        expect(object_1).to receive(:write).with(temp)
-        bucket_list.get('test-key')
-      end
+          allow(Tempfile).to receive(:new).with('bucket-copy').and_return(temp)
+          allow(temp).to receive(:rewind)
+          allow(temp).to receive(:close!)
+        end
 
-      it 'returns the object from the first bucket' do
-        allow(object_2).to receive(:read)
+        it 'writes the object to the first bucket' do
+          allow(object_2).to receive(:read)
+          expect(object_1).to receive(:write).with(temp)
+          bucket_list.get('test-key')
+        end
 
-        allow(object_1).to receive(:write).with(temp) {
-          allow(object_1).to receive(:exists?).and_return(true)
-        }
-        expect(bucket_list.get('test-key')).to eq(object_1)
-      end
+        it 'returns the object from the first bucket' do
+          allow(object_2).to receive(:read)
 
-    end
+          allow(object_1).to receive(:write).with(temp) {
+            allow(object_1).to receive(:exists?).and_return(true)
+          }
+          expect(bucket_list.get('test-key')).to eq(object_1)
+        end
 
-    context 'when the object is found in both buckets' do
-      before do
-        allow(object_1).to receive(:exists?).and_return(true)
-        allow(object_2).to receive(:exists?).and_return(true)
       end
 
-      it 'returns the object from the first bucket' do
-        expect(bucket_list.get('test-key')).to eq(object_1)
+      context 'when the object is found in both buckets' do
+        before do
+          allow(object_1).to receive(:exists?).and_return(true)
+          allow(object_2).to receive(:exists?).and_return(true)
+        end
+
+        it 'returns the object from the first bucket' do
+          expect(bucket_list.get('test-key')).to eq(object_1)
+        end
       end
     end
-  end
 
-  describe '#put' do
-    context 'when writing an object' do
-      let(:io) { instance_double(File) }
+    describe '#put' do
+      context 'when writing an object' do
+        let(:io) { instance_double(File) }
 
-      it 'puts the object in the last bucket' do
-        allow(object_1).to receive(:delete)
+        it 'puts the object in the last bucket' do
+          allow(object_1).to receive(:delete)
 
-        expect(object_2).to receive(:write).with(io)
-        bucket_list.put('test-key', io)
-      end
+          expect(object_2).to receive(:write).with(io)
+          bucket_list.put('test-key', io)
+        end
 
-      it 'delete the object from the first bucket' do
-        allow(object_2).to receive(:write).with(io)
+        it 'delete the object from the first bucket' do
+          allow(object_2).to receive(:write).with(io)
 
-        expect(object_1).to receive(:delete)
-        bucket_list.put('test-key', io)
+          expect(object_1).to receive(:delete)
+          bucket_list.put('test-key', io)
+        end
       end
     end
   end

data/spec/unit/bucket_brigade/bucket_spec.rb

@@ -1,10 +1,8 @@
 require 'bucket_brigade/bucket'
 require 'bucket_brigade/bucket_config'
-
 require 'aws-sdk'
 
 describe BucketBrigade::Bucket do
-
   subject(:bucket) { described_class.new(bucket_config) }
 
   let(:bucket_config) {
@@ -36,6 +34,37 @@ describe BucketBrigade::Bucket do
     allow(object_collection).to receive(:[]).with('test-key').and_return(object)
   end
 
+  describe '#can_connect?' do
+    context 'when able to connect to the S3 bucket' do
+      it 'returns true' do
+        allow(bucket_collection).to receive(:first)
+        expect(bucket.can_connect?).to eq(true)
+      end
+    end
+
+    context 'when unable to connect to the S3 bucket' do
+      before do
+        bad_bucket_collection = double
+        expect(bad_bucket_collection).to receive(:first).and_raise(error_class)
+        expect(s3).to receive(:buckets).and_return(bad_bucket_collection)
+      end
+
+      context 'because of a read timeout' do
+        let(:error_class) { Net::ReadTimeout }
+        it 'returns false' do
+          expect(bucket.can_connect?).to eq(false)
+        end
+      end
+
+      context 'because of an open timeout' do
+        let(:error_class) { Net::OpenTimeout }
+        it 'returns false' do
+          expect(bucket.can_connect?).to eq(false)
+        end
+      end
+    end
+  end
+
   describe '#has_key' do
     context 'when the object does not exist' do
       before { allow(object).to receive(:exists?).and_return(false) }

metadata

@@ -1,18 +1,20 @@
 --- !ruby/object:Gem::Specification
 name: bucket_brigade
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
+  prerelease: 
 platform: ruby
 authors:
 - Pivotal Labs
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-01 00:00:00.000000000 Z
+date: 2014-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -20,6 +22,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -27,6 +30,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec-legacy_formatters
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -34,6 +38,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -41,6 +46,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -48,6 +54,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -67,6 +74,7 @@ files:
 - lib/bucket_brigade/bucket.rb
 - lib/bucket_brigade/bucket_config.rb
 - lib/bucket_brigade/bucket_list.rb
+- lib/bucket_brigade/version.rb
 - spec/integration/put_get_spec.rb
 - spec/unit/bucket_brigade/bucket_config_spec.rb
 - spec/unit/bucket_brigade/bucket_list_spec.rb
@@ -74,25 +82,26 @@ files:
 homepage: https://github.com/pivotal-cf-experimental/bucket_brigade
 licenses:
 - Apache 2.0
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 1.8.23
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: S3 bucket cache hierarchy
 test_files: []

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: 8cdb69753762ea4144d0ac29a7debeff8fcefe8d
-  data.tar.gz: 9dfc9ad55eba0aa9e4b923949e477023a13b7c49
-SHA512:
-  metadata.gz: 31a691b784f454d86ee13e308622cb51ebaff8bc3a9a3c714fa94745025e4ba71738ce82513a36f9ba6a9e2b9a7bd5252408ff1e9a09e77a02b0dfd71bc56009
-  data.tar.gz: a1f16f71752d1488930d0308fcb3865791a87f3dd59799051431ba5171795f2ed2902bca6b26ce6530e80461374f6d411442a06cc527bfddbe7884f23a1b51f6