bucket_brigade 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8cdb69753762ea4144d0ac29a7debeff8fcefe8d
+  data.tar.gz: 9dfc9ad55eba0aa9e4b923949e477023a13b7c49
+SHA512:
+  metadata.gz: 31a691b784f454d86ee13e308622cb51ebaff8bc3a9a3c714fa94745025e4ba71738ce82513a36f9ba6a9e2b9a7bd5252408ff1e9a09e77a02b0dfd71bc56009
+  data.tar.gz: a1f16f71752d1488930d0308fcb3865791a87f3dd59799051431ba5171795f2ed2902bca6b26ce6530e80461374f6d411442a06cc527bfddbe7884f23a1b51f6

data/.gitignore

@@ -0,0 +1 @@
+Gemfile.lock

data/.rspec

@@ -0,0 +1,2 @@
+--require rspec/legacy_formatters
+--color

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/README.md

@@ -0,0 +1,35 @@
+This gem makes it easy to use a cache hierarchy of S3 buckets. Slower, more durable
+layers can be on Amazon S3. Faster local layers can use another S3-compatible service
+like Riak CS.
+
+Usage:
+
+```ruby
+require 'bucket_brigade/bucket_config
+
+bucket_config1 = BucketBrigade::BucketConfig.new(
+    'http://192.168.37.73:8080',
+    'access_key_id',
+    'secret_access_key',
+    'test-bucket'
+)
+
+bucket_config2 = BucketBrigade::BucketConfig.new(
+    'https://s3.amazonaws.com'
+    'access_key_id',
+    'secret_access_key',
+    'test-bucket'
+)
+
+
+bucket_list = BucketBrigade::BucketList.new(
+    bucket_config1,
+    bucket_config2,
+    )
+
+open('/tmp/file') do |f|
+    bucket_list.put('test-key', f)
+end
+
+puts bucket_list.get('test-key')
+```

data/bucket_brigade.gemspec

@@ -0,0 +1,17 @@
+# encoding: utf-8
+
+Gem::Specification.new do |s|
+  s.name = 'bucket_brigade'
+  s.version = '0.0.1'
+  s.summary = 'S3 bucket cache hierarchy'
+  s.description = 'S3 bucket cache hierarchy'
+  s.author = 'Pivotal Labs'
+  s.homepage = 'https://github.com/pivotal-cf-experimental/bucket_brigade'
+  s.license = 'Apache 2.0'
+  s.email = 'support@cloudfoundry.com'
+  s.files = `git ls-files`.split("\n")
+
+  s.add_development_dependency 'rspec', '~> 3.0.0'
+  s.add_development_dependency 'rspec-legacy_formatters', '~> 1.0.0'
+  s.add_dependency 'aws-sdk', '~> 1.49.0'
+end

data/lib/bucket_brigade.rb

@@ -0,0 +1 @@
+module BucketBrigade; end

data/lib/bucket_brigade/bucket.rb

@@ -0,0 +1,51 @@
+require 'bucket_brigade'
+
+require 'aws-sdk'
+
+require 'tempfile'
+
+class BucketBrigade::Bucket
+  def initialize(bucket_config)
+    @s3_bucket = AWS::S3.new(
+        s3_endpoint: bucket_config.host,
+        s3_port: bucket_config.port,
+        access_key_id: bucket_config.access_key_id,
+        secret_access_key: bucket_config.secret_access_key,
+        use_ssl: bucket_config.use_ssl,
+        s3_force_path_style: true
+    ).buckets[bucket_config.bucket]
+  end
+
+  def has_key?(key)
+    object(key).exists?
+  end
+
+  def get(key)
+    has_key?(key) ? object(key) : nil
+  end
+
+  def put(key, io)
+    object(key).write(io)
+  end
+
+  def delete(key)
+    object(key).delete
+  end
+
+  def object(key)
+    @s3_bucket.objects[key]
+  end
+
+  def copy(key, other)
+    temp = Tempfile.new('bucket-copy')
+    object(key).read do |chunk|
+      temp.write(chunk)
+    end
+
+    temp.rewind
+
+    other.object(key).write(temp)
+
+    temp.close!
+  end
+end

data/lib/bucket_brigade/bucket_config.rb

@@ -0,0 +1,25 @@
+require 'bucket_brigade'
+
+require 'uri'
+
+class BucketBrigade::BucketConfig
+  attr_reader :endpoint
+  attr_reader :access_key_id
+  attr_reader :secret_access_key
+  attr_reader :bucket
+  attr_reader :host
+  attr_reader :port
+  attr_reader :use_ssl
+
+  def initialize(endpoint, access_key_id, secret_access_key, bucket)
+    @endpoint = endpoint
+    @access_key_id = access_key_id
+    @secret_access_key = secret_access_key
+    @bucket = bucket
+
+    uri = URI(@endpoint)
+    @host = uri.host
+    @port = uri.port
+    @use_ssl = uri.scheme == 'https'
+  end
+end

data/lib/bucket_brigade/bucket_list.rb

@@ -0,0 +1,34 @@
+require 'bucket_brigade'
+require 'bucket_brigade/bucket'
+
+require 'aws-sdk'
+
+class BucketBrigade::BucketList
+  def initialize(*bucket_configs)
+    @buckets = bucket_configs.map do |bucket_config|
+      BucketBrigade::Bucket.new(bucket_config)
+    end
+  end
+
+  def get(key)
+    @buckets.each_with_index do |bucket, i|
+      if bucket.has_key?(key)
+        if i > 0
+          prev_bucket = @buckets[i - 1]
+          bucket.copy(key, prev_bucket)
+          return prev_bucket.get(key)
+        end
+        return bucket.get(key)
+      end
+    end
+    nil
+  end
+
+  def put(key, io)
+    @buckets.last.put(key, io)
+    # invalidate all the way down
+    @buckets.first(@buckets.size - 1).each do |bucket|
+      bucket.delete(key)
+    end
+  end
+end

data/spec/integration/put_get_spec.rb

@@ -0,0 +1,42 @@
+require 'bucket_brigade/bucket_config'
+require 'bucket_brigade/bucket_list'
+
+describe 'put and get data from the cache' do
+
+  it 'puts and gets a file from the cache' do
+    bucket_config1 = BucketBrigade::BucketConfig.new(
+        ENV.fetch('LOCAL_S3_ENDPOINT'),
+        ENV.fetch('LOCAL_S3_ID'),
+        ENV.fetch('LOCAL_S3_SECRET'),
+        'test'
+    )
+
+    bucket_config2 = BucketBrigade::BucketConfig.new(
+        ENV.fetch('LOCAL_S3_ENDPOINT'),
+        ENV.fetch('LOCAL_S3_ID'),
+        ENV.fetch('LOCAL_S3_SECRET'),
+        'test2'
+    )
+
+    bucket_config3 = BucketBrigade::BucketConfig.new(
+        'http://s3.amazonaws.com',
+        ENV.fetch('S3_ID'),
+        ENV.fetch('S3_SECRET'),
+        'bucket-brigade-test'
+    )
+
+    bucket_list = BucketBrigade::BucketList.new(
+        bucket_config1,
+        bucket_config2,
+        bucket_config3
+    )
+
+    io = StringIO.new
+    io.write('test 1 2 3')
+    io.rewind
+
+    bucket_list.put('test', io)
+
+    expect(bucket_list.get('test').read).to eq('test 1 2 3')
+  end
+end

data/spec/unit/bucket_brigade/bucket_config_spec.rb

@@ -0,0 +1,72 @@
+require 'bucket_brigade/bucket_config'
+
+describe BucketBrigade::BucketConfig do
+  let(:endpoint) { 'http://host.com:1234' }
+  let(:access_key_id) { 'test-access-key-id' }
+  let(:secret_access_key) { 'secret-access-key' }
+  let(:bucket) { 'bucket' }
+
+  subject(:bucket_config) do
+    described_class.new(endpoint, access_key_id, secret_access_key, bucket)
+  end
+
+  describe '#endpoint' do
+    it 'returns the endpoint' do
+      expect(bucket_config.endpoint).to eq(endpoint)
+    end
+  end
+
+  describe '#access_key_id' do
+    it 'returns the access_key_id' do
+      expect(bucket_config.access_key_id).to eq(access_key_id)
+    end
+  end
+
+  describe '#secret_access_key' do
+    it 'returns the secret_access_key' do
+      expect(bucket_config.secret_access_key).to eq(secret_access_key)
+    end
+  end
+
+  describe '#bucket' do
+    it 'returns the bucket' do
+      expect(bucket_config.bucket).to eq(bucket)
+    end
+  end
+
+  describe '#port' do
+    context 'when the port is provided' do
+      it 'returns the port' do
+        expect(bucket_config.port).to eq(1234)
+      end
+    end
+
+    context 'when the port is not provided' do
+      let(:endpoint) { 'http://host.com' }
+      it 'returns port 80' do
+        expect(bucket_config.port).to eq(80)
+      end
+    end
+  end
+
+  describe '#host' do
+    it 'returns the host' do
+      expect(bucket_config.host).to eq('host.com')
+    end
+  end
+
+  describe '#use_ssl' do
+    context 'when the endpoint is https' do
+      let(:endpoint) { 'https://host.com/' }
+      it 'returns true' do
+        expect(bucket_config.use_ssl).to eq(true)
+      end
+    end
+
+    context 'when the endpoint is not https' do
+      it 'returns false' do
+        expect(bucket_config.use_ssl).to eq(false)
+      end
+    end
+  end
+end

data/spec/unit/bucket_brigade/bucket_list_spec.rb

@@ -0,0 +1,150 @@
+require 'bucket_brigade/bucket_config'
+require 'bucket_brigade/bucket_list'
+
+describe BucketBrigade::BucketList do
+
+  let(:bucket_configs) {
+    [
+        BucketBrigade::BucketConfig.new(
+            'http://test-endpoint-1:1234',
+            'test-access-key-id-1',
+            'test-secret-access-key-1',
+            'test-bucket-1'
+        ),
+        BucketBrigade::BucketConfig.new(
+            'http://test-endpoint-2:1234',
+            'test-access-key-id-2',
+            'test-secret-access-key-2',
+            'test-bucket-2'
+        )
+    ]
+  }
+
+  let(:s3_1) { instance_double(AWS::S3) }
+  let(:s3_2) { instance_double(AWS::S3) }
+
+  let(:bucket_collection_1) { instance_double(AWS::S3::BucketCollection) }
+  let(:bucket_collection_2) { instance_double(AWS::S3::BucketCollection) }
+
+  let(:bucket_1) { instance_double(AWS::S3::Bucket) }
+  let(:bucket_2) { instance_double(AWS::S3::Bucket) }
+
+  let(:object_collection_1) { instance_double(AWS::S3::ObjectCollection) }
+  let(:object_collection_2) { instance_double(AWS::S3::ObjectCollection) }
+
+  let(:object_1) { instance_double(AWS::S3::S3Object) }
+  let(:object_2) { instance_double(AWS::S3::S3Object) }
+
+  before do
+    allow(AWS::S3).to receive(:new).with(
+                          s3_endpoint: 'test-endpoint-1',
+                          s3_port: 1234,
+                          access_key_id: 'test-access-key-id-1',
+                          secret_access_key: 'test-secret-access-key-1',
+                          use_ssl: false,
+                          s3_force_path_style: true
+                      ).and_return(s3_1)
+    allow(s3_1).to receive(:buckets).and_return(bucket_collection_1)
+    allow(bucket_collection_1).to receive(:[]).with('test-bucket-1').and_return(bucket_1)
+    allow(bucket_1).to receive(:objects).and_return(object_collection_1)
+    allow(object_collection_1).to receive(:[]).with('test-key').and_return(object_1)
+
+    allow(AWS::S3).to receive(:new).with(
+                          s3_endpoint: 'test-endpoint-2',
+                          s3_port: 1234,
+                          access_key_id: 'test-access-key-id-2',
+                          secret_access_key: 'test-secret-access-key-2',
+                          use_ssl: false,
+                          s3_force_path_style: true
+                      ).and_return(s3_2)
+    allow(s3_2).to receive(:buckets).and_return(bucket_collection_2)
+    allow(bucket_collection_2).to receive(:[]).with('test-bucket-2').and_return(bucket_2)
+    allow(bucket_2).to receive(:objects).and_return(object_collection_2)
+    allow(object_collection_2).to receive(:[]).with('test-key').and_return(object_2)
+  end
+
+  subject(:bucket_list) { described_class.new(*bucket_configs) }
+
+  describe '#get' do
+    context 'when the object is found in the first bucket' do
+      before do
+        allow(object_1).to receive(:exists?).and_return(true)
+      end
+
+      it 'returns the object from the first bucket' do
+        expect(bucket_list.get('test-key')).to eq(object_1)
+      end
+    end
+
+    context 'when the object is not found in any buckets' do
+      before do
+        allow(object_1).to receive(:exists?).and_return(false)
+        allow(object_2).to receive(:exists?).and_return(false)
+      end
+
+      it 'returns nil' do
+        expect(bucket_list.get('test-key')).to eq(nil)
+      end
+    end
+
+    context 'when the object is found in the second bucket' do
+      let(:temp) { instance_double(Tempfile) }
+
+      before do
+        allow(object_1).to receive(:exists?).and_return(false)
+        allow(object_2).to receive(:exists?).and_return(true)
+
+        allow(Tempfile).to receive(:new).with('bucket-copy').and_return(temp)
+        allow(temp).to receive(:rewind)
+        allow(temp).to receive(:close!)
+      end
+
+      it 'writes the object to the first bucket' do
+        allow(object_2).to receive(:read)
+        expect(object_1).to receive(:write).with(temp)
+        bucket_list.get('test-key')
+      end
+
+      it 'returns the object from the first bucket' do
+        allow(object_2).to receive(:read)
+
+        allow(object_1).to receive(:write).with(temp) {
+          allow(object_1).to receive(:exists?).and_return(true)
+        }
+        expect(bucket_list.get('test-key')).to eq(object_1)
+      end
+
+    end
+
+    context 'when the object is found in both buckets' do
+      before do
+        allow(object_1).to receive(:exists?).and_return(true)
+        allow(object_2).to receive(:exists?).and_return(true)
+      end
+
+      it 'returns the object from the first bucket' do
+        expect(bucket_list.get('test-key')).to eq(object_1)
+      end
+    end
+  end
+
+  describe '#put' do
+    context 'when writing an object' do
+      let(:io) { instance_double(File) }
+
+      it 'puts the object in the last bucket' do
+        allow(object_1).to receive(:delete)
+
+        expect(object_2).to receive(:write).with(io)
+        bucket_list.put('test-key', io)
+      end
+
+      it 'delete the object from the first bucket' do
+        allow(object_2).to receive(:write).with(io)
+
+        expect(object_1).to receive(:delete)
+        bucket_list.put('test-key', io)
+      end
+    end
+  end
+end

data/spec/unit/bucket_brigade/bucket_spec.rb

@@ -0,0 +1,145 @@
+require 'bucket_brigade/bucket'
+require 'bucket_brigade/bucket_config'
+
+require 'aws-sdk'
+
+describe BucketBrigade::Bucket do
+
+  subject(:bucket) { described_class.new(bucket_config) }
+
+  let(:bucket_config) {
+    BucketBrigade::BucketConfig.new(
+        'http://host.com:1234',
+        'test-access-key-id',
+        'test-secret-access-key',
+        'bucket')
+  }
+
+  let(:s3) { instance_double(AWS::S3) }
+  let(:bucket_collection) { instance_double(AWS::S3::BucketCollection) }
+  let(:s3_bucket) { instance_double(AWS::S3::Bucket) }
+  let(:object_collection) { instance_double(AWS::S3::ObjectCollection) }
+  let(:object) { instance_double(AWS::S3::S3Object) }
+
+  before do
+    allow(AWS::S3).to receive(:new).with(
+                          s3_endpoint: 'host.com',
+                          s3_port: 1234,
+                          access_key_id: 'test-access-key-id',
+                          secret_access_key: 'test-secret-access-key',
+                          use_ssl: false,
+                          s3_force_path_style: true
+                      ).and_return(s3)
+    allow(s3).to receive(:buckets).and_return(bucket_collection)
+    allow(bucket_collection).to receive(:[]).with('bucket').and_return(s3_bucket)
+    allow(s3_bucket).to receive(:objects).and_return(object_collection)
+    allow(object_collection).to receive(:[]).with('test-key').and_return(object)
+  end
+
+  describe '#has_key' do
+    context 'when the object does not exist' do
+      before { allow(object).to receive(:exists?).and_return(false) }
+
+      it 'returns false' do
+        expect(bucket).to_not have_key('test-key')
+      end
+    end
+
+    context 'when the object exists' do
+      before { allow(object).to receive(:exists?).and_return(true) }
+
+      it 'returns true' do
+        expect(bucket).to have_key('test-key')
+      end
+    end
+  end
+
+  describe '#get' do
+    context 'when the object does not exist' do
+      before { allow(object).to receive(:exists?).and_return(false) }
+
+      it 'returns nil' do
+        expect(bucket.get('test-key')).to be(nil)
+      end
+    end
+
+    context 'when the object exists' do
+      before { allow(object).to receive(:exists?).and_return(true) }
+
+      it 'gets an object from the bucket' do
+        expect(bucket.get('test-key')).to eq(object)
+      end
+    end
+  end
+
+  describe '#put' do
+    it 'puts an object in the bucket' do
+      io = instance_double(File)
+      expect(object).to receive(:write).with(io)
+      bucket.put('test-key', io)
+    end
+  end
+
+  describe '#delete' do
+    it 'delete the object from the bucket' do
+      expect(object).to receive(:delete)
+      bucket.delete('test-key')
+    end
+  end
+
+  describe '#object' do
+    it 'gets the object from the bucket' do
+      expect(bucket.object('test-key')).to eq(object)
+    end
+  end
+
+  describe '#copy' do
+    let(:other_bucket_config) {
+      BucketBrigade::BucketConfig.new(
+          'http://host2.com:1234',
+          'test-access-key-id',
+          'test-secret-access-key',
+          'bucket')
+    }
+
+    let(:other_bucket) { described_class.new(other_bucket_config) }
+
+    let(:other_s3) { instance_double(AWS::S3) }
+    let(:other_bucket_collection) { instance_double(AWS::S3::BucketCollection) }
+    let(:other_s3_bucket) { instance_double(AWS::S3::Bucket) }
+    let(:other_object_collection) { instance_double(AWS::S3::ObjectCollection) }
+    let(:other_object) { instance_double(AWS::S3::S3Object) }
+
+    before do
+      allow(AWS::S3).to receive(:new).with(
+                            s3_endpoint: 'host2.com',
+                            s3_port: 1234,
+                            access_key_id: 'test-access-key-id',
+                            secret_access_key: 'test-secret-access-key',
+                            use_ssl: false,
+                            s3_force_path_style: true
+                        ).and_return(other_s3)
+
+      allow(other_s3).to receive(:buckets).and_return(other_bucket_collection)
+      allow(other_bucket_collection).to receive(:[]).with('bucket').and_return(
+                                            other_s3_bucket)
+      allow(other_s3_bucket).to receive(:objects).and_return(
+                                    other_object_collection)
+      allow(other_object_collection).to receive(:[]).with('test-key').and_return(
+                                            other_object)
+    end
+
+    it 'copies an object from one bucket to another' do
+      temp = instance_double(Tempfile)
+      allow(Tempfile).to receive(:new).with('bucket-copy').and_return(temp)
+      allow(temp).to receive(:write).with('test')
+      allow(temp).to receive(:rewind)
+      allow(object).to receive(:read).with(no_args).and_yield('test')
+
+      expect(other_object).to receive(:write).with(temp)
+      expect(temp).to receive(:close!)
+
+      bucket.copy('test-key', other_bucket)
+    end
+  end
+end

metadata

@@ -0,0 +1,98 @@
+--- !ruby/object:Gem::Specification
+name: bucket_brigade
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Pivotal Labs
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-08-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: rspec-legacy_formatters
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.49.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.49.0
+description: S3 bucket cache hierarchy
+email: support@cloudfoundry.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- README.md
+- bucket_brigade.gemspec
+- lib/bucket_brigade.rb
+- lib/bucket_brigade/bucket.rb
+- lib/bucket_brigade/bucket_config.rb
+- lib/bucket_brigade/bucket_list.rb
+- spec/integration/put_get_spec.rb
+- spec/unit/bucket_brigade/bucket_config_spec.rb
+- spec/unit/bucket_brigade/bucket_list_spec.rb
+- spec/unit/bucket_brigade/bucket_spec.rb
+homepage: https://github.com/pivotal-cf-experimental/bucket_brigade
+licenses:
+- Apache 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: S3 bucket cache hierarchy
+test_files: []