alephant-storage 1.1.1 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a67098b9f5dd6561721bfb134d95afe3466235e1
-  data.tar.gz: 26ca720ff670c24cb6af6da20c036a2f4c39fc07
+  metadata.gz: 511f0d39c816881993bc6677398496356b367a4c
+  data.tar.gz: 90a2236c5d3f49caa4c0f1f26d64b36d96d75f40
 SHA512:
-  metadata.gz: ad8005de05e06602ad9f3bba4f63484ad0fc54c5eefc39b8e800f80f8072b916aa46f39d54ba0898b31103fa666227bfea334a05b482aaca111f1507334de673
-  data.tar.gz: 78c192408a5b3c7286253ee876c31161a5e86c1764f7f2d8941b30f3b247d21880bed9fe6f13950eb7f8858fda84f98057c4ef2531620099e9a80356ff60ca06
+  metadata.gz: e8517904fc38cacfd6d2dea8e7bbe7d35cf0daa749c7676f962fa29fc55033b1142df388c0e67ec14f675d0e7dc2ecae35c373b4fff0a2df17516ec7e86136bc
+  data.tar.gz: 6ff74d1808163c5323f60b1351ab983c4fbe3cd26dbe2fc036af9522560eb15efa598311a9912ba7b902e4c5d8c687d25a0a539662770cc0f064fd604f42fa94

data/.gitignore

@@ -3,6 +3,7 @@
 Gemfile.lock
 .rspec
 *.gem
+*.log
 
 /pkg
 /tmp

data/.ruby-version

@@ -1 +1 @@
-jruby-1.7.17
+2.3

data/.travis.yml

@@ -1,9 +1,10 @@
 language: ruby
 rvm:
+  - "2.3"
   - "jruby"
 notifications:
   email:
     recipients:
-      - DigitalNewsFrameworksTeam@bbc.co.uk
+      - D&ENewsFrameworksTeam@bbc.co.uk
     on_failure: change
     on_success: never

data/{alephant-cache.gemspec → alephant-storage.gemspec}

@@ -7,7 +7,7 @@ Gem::Specification.new do |spec|
   spec.name          = "alephant-storage"
   spec.version       = Alephant::Storage::VERSION
   spec.authors       = ["BBC News"]
-  spec.email         = ["FutureMediaNewsRubyGems@bbc.co.uk"]
+  spec.email         = ["D&ENewsFrameworksTeam@bbc.co.uk"]
   spec.summary       = "Simple abstraction layer over S3 for get/put."
   spec.homepage      = "https://github.com/BBC-News/alephant-storage"
   spec.license       = "MIT"
@@ -27,6 +27,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.5"
   spec.add_development_dependency "rake"
 
-  spec.add_runtime_dependency 'aws-sdk', '~> 1.0'
+  spec.add_runtime_dependency "aws-sdk-s3"
   spec.add_runtime_dependency 'alephant-logger'
 end

data/lib/alephant/storage.rb

@@ -1,73 +1,88 @@
 require "alephant/storage/version"
 require "alephant/logger"
-require "aws-sdk"
+require "aws-sdk-s3"
 require "date"
 
 module Alephant
   class Storage
     include Logger
-    attr_reader :id, :bucket, :path
+    attr_reader :bucket, :path
 
-    def initialize(id, path)
-      @id = id
-      @path = path
-      @bucket = AWS::S3.new.buckets[id]
+    def initialize(bucket, path)
+      @bucket = bucket
+      @path   = path
 
       logger.info(
         "event"  => "StorageInitialized",
-        "id"     => id,
+        "bucket" => bucket,
         "path"   => path,
         "method" => "#{self.class}#initialize"
       )
     end
 
     def clear
-      bucket.objects.with_prefix(path).delete_all
       logger.info(
         "event"  => "StorageCleared",
+        "bucket" => bucket,
         "path"   => path,
         "method" => "#{self.class}#clear"
       )
-    end
 
-    def put(id, data, content_type = "text/plain", meta = {})
-      bucket.objects["#{path}/#{id}"].write(
-        data,
-        {
-          :content_type => content_type,
-          :metadata     => meta
+      objects = client.list_objects(
+        bucket: bucket,
+        prefix: path
+      )
+
+      client.delete_objects(
+        bucket: bucket,
+        delete: {
+          objects: objects.data.contents.map { |o| { key: o.key } }
         }
       )
+    end
 
+    def put(key, data, content_type = "text/plain", meta = {})
       logger.metric "StoragePuts"
       logger.info(
-        "event"  => "StorageObjectStored",
-        "path"   => path,
-        "id"     => id,
-        "method" => "#{self.class}#put"
+        "event"    => "StorageObjectStored",
+        "bucket"   => bucket,
+        "path"     => path,
+        "key"      => key,
+        "method"   => "#{self.class}#put"
+      )
+
+      client.put_object(
+        bucket: bucket,
+        body: data,
+        key: [path, key].join('/'),
+        content_type: content_type,
+        metadata: meta
       )
     end
 
-    def get(id)
-      object       = bucket.objects["#{path}/#{id}"]
-      content      = object.read
-      content_type = object.content_type
-      meta_data    = object.metadata.to_h.merge(add_custom_meta(object))
+    def get(key)
+      object = client.get_object(
+        bucket: bucket,
+        key: [path, key].join('/')
+      )
+
+      meta = object.metadata.merge(add_custom_meta(object))
 
       logger.metric "StorageGets"
       logger.info(
         "event"       => "StorageObjectRetrieved",
+        "bucket"      => bucket,
         "path"        => path,
-        "id"          => id,
-        "contentType" => content_type,
-        "metadata"    => meta_data,
+        "key"         => key,
+        "contentType" => object.content_type,
+        "metadata"    => meta,
         "method"      => "#{self.class}#get"
       )
 
       {
-        :content      => content,
-        :content_type => content_type,
-        :meta         => meta_data
+        :content      => object.body.read,
+        :content_type => object.content_type,
+        :meta         => Hash[meta.map { |k, v| [k.to_sym, v] }]
       }
     end
 
@@ -79,5 +94,11 @@ module Alephant
         :"head_Last-Modified" => DateTime.parse(object.last_modified.to_s).httpdate
       }
     end
+
+    def client
+      options = {}
+      options[:endpoint] = ENV['AWS_S3_ENDPOINT'] if ENV['AWS_S3_ENDPOINT']
+      @client ||= ::Aws::S3::Client.new(options)
+    end
   end
 end

data/lib/alephant/storage/version.rb

@@ -1,5 +1,5 @@
 module Alephant
   class Storage
-    VERSION = "1.1.1"
+    VERSION = "2.0.0"
   end
 end

data/spec/storage_spec.rb

@@ -2,100 +2,128 @@ require "spec_helper"
 require "date"
 
 describe Alephant::Storage do
-  let(:id)   { :id }
+  let(:bucket)   { 'my-bucket' }
+  let(:key) { 'my-key' }
   let(:path) { :path }
-  let(:data) { :data }
-  subject { Alephant::Storage }
-
-  describe "initialize(id, path)" do
-    it "sets and exposes id, path instance variables " do
-      instance = subject.new(id, path)
-      expect(instance.id).to eq(id)
-      expect(instance.path).to eq(path)
-    end
+  let(:data) { 'data' }
+  let(:fake_client) { Aws::S3::Client.new(stub_responses: true) }
+  subject { Alephant::Storage.new(bucket, path) }
 
-    it "sets bucket instance variable as S3 bucket with id" do
-      instance = subject.new(id, path)
+  before do
+    allow(subject).to receive(:client).and_return(fake_client)
+  end
 
-      expect(instance.bucket).to be_an AWS::S3::Bucket
-      expect(instance.bucket.name).to eq('id')
+  describe "#initialize" do
+    it "sets and exposes bucket, path instance variables " do
+      expect(subject.bucket).to eq(bucket)
+      expect(subject.path).to eq(path)
     end
   end
 
-  describe "clear" do
-    let(:num_object_in_path) { 100 }
-    it "deletes all objects for a path" do
-      filtered_object_collection = double()
-      expect(filtered_object_collection)
-        .to receive(:delete_all)
+  describe "#clear" do
+    before do
+      fake_client.stub_responses(:list_objects, contents: fake_objects)
+    end
 
-      s3_object_collection = double()
-      expect(s3_object_collection)
-        .to receive(:with_prefix)
-        .with(path)
-        .and_return(filtered_object_collection)
+    context 'with objects' do
+      let(:fake_objects) { [ { key: key } ] }
 
-      s3_bucket = double()
-      expect(s3_bucket).to receive(:objects).and_return(s3_object_collection)
+      it 'deletes all objects for a path' do
+        expect(subject.clear.data).to be_a(Aws::S3::Types::DeleteObjectsOutput)
+      end
+    end
 
-      expect_any_instance_of(AWS::S3).to receive(:buckets).and_return({ id => s3_bucket })
+    context 'with no objects' do
+      let(:fake_objects) { [] }
 
-      instance = subject.new(id, path)
-      instance.clear
+      it 'returns DeleteObjectsOutput response' do
+        expect(subject.clear.data).to be_a(Aws::S3::Types::DeleteObjectsOutput)
+      end
     end
   end
 
-  describe "put(id, data)" do
-    it "sets bucket path/id content data" do
-      s3_object_collection = double()
-      expect(s3_object_collection).to receive(:write).with(:data, { :content_type => 'foo/bar', :metadata=>{} })
+  describe "#put" do
+    it "sets bucket path/key content data" do
+      expect(subject.put(key, data, 'foo/bar').data).to be_a(Aws::S3::Types::PutObjectOutput)
+    end
+  end
 
-      s3_bucket = double()
-      expect(s3_bucket).to receive(:objects).and_return(
-        {
-          "path/id" => s3_object_collection
-        }
-      )
+  describe "#get" do
+    context 'with object response' do
+      before do
+        fake_client.stub_responses(:get_object, {
+          body: 'content',
+          etag: 'foo_123',
+          metadata: meta,
+          content_type: content_type,
+          last_modified: Time.parse(last_modified)
+        })
+      end
+
+      context 'with meta' do
+        let(:meta) do
+          {
+            'foo' => 'bar'
+          }
+        end
+
+        let(:content_type) { 'foo/bar' }
+        let(:last_modified) { '2016-04-11 10:39:57 +0000' }
+
+        it "gets bucket path/key content data" do
+          expected_hash = {
+            :content      => "content",
+            :content_type => "foo/bar",
+            :meta         => {
+              :foo                  => 'bar',
+              :head_ETag            => "foo_123",
+              :"head_Last-Modified" => "Mon, 11 Apr 2016 10:39:57 GMT"
+            }
+          }
+
+          expect(subject.get(key)).to eq(expected_hash)
+        end
+      end
+
+      context 'with no meta' do
+        let(:meta) { {} }
+
+        let(:content_type) { 'foo/bar' }
+        let(:last_modified) { '2016-04-11 10:39:57 +0000' }
+
+        it "gets bucket path/key content data" do
+          expected_hash = {
+            :content      => "content",
+            :content_type => "foo/bar",
+            :meta         => {
+              :head_ETag            => "foo_123",
+              :"head_Last-Modified" => "Mon, 11 Apr 2016 10:39:57 GMT"
+            }
+          }
+
+          expect(subject.get(key)).to eq(expected_hash)
+        end
+      end
+    end
 
-      expect_any_instance_of(AWS::S3).to receive(:buckets).and_return({ id => s3_bucket })
-      instance = subject.new(id, path)
+    context 'object does not exist' do
+      before do
+        fake_client.stub_responses(:get_object, 'NoSuchKey')
+      end
 
-      instance.put(id, data, "foo/bar")
+      it 'raises NoSuchKey' do
+        expect { subject.get(key) }.to raise_error(Aws::S3::Errors::NoSuchKey)
+      end
     end
-  end
 
-  describe "get(id)" do
-    it "gets bucket path/id content data" do
-      s3_object_collection = double()
-      expect(s3_object_collection).to receive(:read).and_return("content")
-      expect(s3_object_collection).to receive(:content_type).and_return("foo/bar" )
-      expect(s3_object_collection).to receive(:metadata).and_return({ :foo => :bar })
-      expect(s3_object_collection).to receive(:etag).and_return("foo_123")
-      expect(s3_object_collection).to receive(:last_modified).and_return(DateTime.parse("2016-04-11 10:39:57 +0000"))
-
-      s3_bucket = double()
-      expect(s3_bucket).to receive(:objects).and_return(
-        {
-          "path/id" => s3_object_collection
-        }
-      )
-
-      expect_any_instance_of(AWS::S3).to receive(:buckets).and_return({ id => s3_bucket })
-
-      instance = subject.new(id, path)
-      object_hash = instance.get(id)
-
-      expected_hash = {
-        :content      => "content",
-        :content_type => "foo/bar",
-        :meta         => {
-          :foo                  => :bar,
-          :head_ETag            => "foo_123",
-          :"head_Last-Modified" => "Mon, 11 Apr 2016 10:39:57 GMT"
-        }
-      }
-
-      expect(object_hash).to eq(expected_hash)
+    context 'bucket does not exist' do
+      before do
+        fake_client.stub_responses(:get_object, 'NoSuchBucket')
+      end
+
+      it 'raises NoSuchBucket' do
+        expect { subject.get(key) }.to raise_error(Aws::S3::Errors::NoSuchBucket)
+      end
     end
   end
 end

metadata

@@ -1,185 +1,185 @@
 --- !ruby/object:Gem::Specification
 name: alephant-storage
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 2.0.0
 platform: ruby
 authors:
 - BBC News
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-14 00:00:00.000000000 Z
+date: 2018-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: rspec
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: rspec-nc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: rspec-nc
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: guard
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: guard
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: guard-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: guard-rspec
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: pry
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: pry-remote
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: pry-remote
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: pry-nav
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: pry-nav
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.5'
-  name: bundler
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.5'
 - !ruby/object:Gem::Dependency
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: rake
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
-  name: aws-sdk
-  prerelease: false
+        version: '0'
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
+  name: alephant-logger
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: alephant-logger
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description:
+description: 
 email:
-- FutureMediaNewsRubyGems@bbc.co.uk
+- D&ENewsFrameworksTeam@bbc.co.uk
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .ruby-version
-- .travis.yml
+- ".gitignore"
+- ".ruby-version"
+- ".travis.yml"
 - Gemfile
 - Guardfile
 - LICENSE.txt
 - README.md
 - Rakefile
-- alephant-cache.gemspec
+- alephant-storage.gemspec
 - lib/alephant/storage.rb
 - lib/alephant/storage/version.rb
 - spec/spec_helper.rb
@@ -188,24 +188,24 @@ homepage: https://github.com/BBC-News/alephant-storage
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.1.9
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.6.12
+signing_key: 
 specification_version: 4
 summary: Simple abstraction layer over S3 for get/put.
 test_files: