saviour 0.4.6 → 0.4.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5b1d3de91b6a03bd4e483d2012346575d1fb444d
-  data.tar.gz: eca43877819318d52cabb2bcc6a02f71f131ee03
+  metadata.gz: 5936e61e689f63ba357bd92fc4b57e6d97cb48e8
+  data.tar.gz: cad8d0487163dca1e329e4b0bfd902162d5eb234
 SHA512:
-  metadata.gz: ab1eabe24fdc96f7785c9a72d6da2250c6ae9362aca1da91817c08846d70443b8919ca41c14cc34afc5652d4081dcac80a2450a3e30cbeee5ce9c2b82ac5fd29
-  data.tar.gz: 9db4871ec035aa11f7d9fcce3cd72ed7eb7e96e44567a5f21552e02df9a520a9057b455941ead2513cac13cdf14ce44d1d9c42012ec4af1e01007e855feef6ef
+  metadata.gz: ef0ef25467254b34eb00d0103567d469f43c8b72ebec94448b88543f9330999a829c0453f44aa8b1aabc47f220bc79a4e0f26577051c61bcd870d63fdc9773a4
+  data.tar.gz: 70ec47375eafbf65a8b26fdc89c728aadaceed58e134a0967751592755da1b54bfbf0356743dc8656e8c5858982dcc1e0bf382b4ee17141e527088848687f4d7

data/lib/saviour/s3_storage.rb

@@ -1,5 +1,5 @@
 begin
-  require 'fog/aws'
+  require 'aws-sdk-s3'
 rescue LoadError
 end
 
@@ -15,9 +15,10 @@ module Saviour
       @create_options = conf.delete(:create_options) { {} }
       conf.fetch(:aws_access_key_id) { raise(ArgumentError, "aws_access_key_id is required") }
       conf.fetch(:aws_secret_access_key) { raise(ArgumentError, "aws_secret_access_key is required") }
+      @region = conf[:region] || raise(ArgumentError, "region is required")
     end
 
-    def write(contents, path)
+    def write(file_or_contents, path)
       path = sanitize_leading_slash(path)
 
       # http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
@@ -25,45 +26,50 @@ module Saviour
         raise(KeyTooLarge, "The key in S3 must be at max 1024 bytes, this key is too big: #{path}")
       end
 
-      directory.files.create({
-                                 key: path,
-                                 body: contents,
-                                 public: true
-                             }.merge(@create_options)
-      )
+      client.put_object(@create_options.merge(body: file_or_contents, bucket: @bucket, key: path))
     end
 
     def write_from_file(file, path)
+      file.rewind
+
       write(file, path)
     end
 
     def read_to_file(path, dest_file)
+      dest_file.binmode
       dest_file.rewind
-      dest_file.write(read(path))
-    end
+      dest_file.truncate(0)
 
-    def read(path)
-      real_path = sanitize_leading_slash(path)
+      io = get_file_stringio(path)
+      while data = io.read(1024 * 1024)
+        dest_file.write(data)
+      end
 
-      file = directory.files.get(real_path)
-      raise FileNotPresent, "Trying to read an unexisting path: #{path}" unless file
+      dest_file.flush
+    end
 
-      file.body
+    def read(path)
+      get_file_stringio(path).read
     end
 
     def delete(path)
-      real_path = sanitize_leading_slash(path)
-
-      file = directory.files.get(real_path)
-      raise FileNotPresent, "Trying to delete an unexisting path: #{path}" unless file
+      path = sanitize_leading_slash(path)
 
-      file.destroy
+      client.delete_object(
+        bucket: @bucket,
+        key: path,
+      )
     end
 
     def exists?(path)
       path = sanitize_leading_slash(path)
 
-      !!directory.files.head(path)
+      !!client.head_object(
+        bucket: @bucket,
+        key: path
+      )
+    rescue Aws::S3::Errors::NotFound
+      false
     end
 
     def public_url(path)
@@ -77,13 +83,15 @@ module Saviour
       source_path = sanitize_leading_slash(source_path)
       destination_path = sanitize_leading_slash(destination_path)
 
-      connection.copy_object(
-        @bucket, source_path,
-        @bucket, destination_path,
-        @create_options
+      client.copy_object(
+        @create_options.merge(
+          copy_source: "/#{@bucket}/#{source_path}",
+          bucket: @bucket,
+          key: destination_path
+        )
       )
-
-      FileUtils.cp(real_path(source_path), real_path(destination_path))
+    rescue Aws::S3::Errors::NoSuchKey
+      raise FileNotPresent, "Trying to cp an unexisting path: #{source_path}"
     end
 
     def mv(source_path, destination_path)
@@ -93,6 +101,17 @@ module Saviour
 
     private
 
+    def get_file_stringio(path)
+      path = sanitize_leading_slash(path)
+
+      client.get_object(
+        bucket: @bucket,
+        key: path
+      ).body
+    rescue Aws::S3::Errors::NotFound, Aws::S3::Errors::NoSuchKey
+      raise FileNotPresent, "Trying to read an unexisting path: #{path}"
+    end
+
     def public_url_prefix
       if @public_url_prefix.respond_to?(:call)
         @public_url_prefix.call
@@ -105,12 +124,12 @@ module Saviour
       path.gsub(/\A\/*/, '')
     end
 
-    def directory
-      @directory ||= connection.directories.new(key: @bucket)
-    end
-
-    def connection
-      @connection ||= Fog::Storage.new({ provider: 'AWS' }.merge(@conf))
+    def client
+      @client ||= Aws::S3::Client.new(
+        access_key_id: @conf[:aws_access_key_id],
+        secret_access_key: @conf[:aws_secret_access_key],
+        region: @region
+      )
     end
   end
 end

data/lib/saviour/version.rb

@@ -1,3 +1,3 @@
 module Saviour
-  VERSION = "0.4.6"
+  VERSION = "0.4.7"
 end

data/saviour.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake"
   spec.add_development_dependency "sqlite3"
   spec.add_development_dependency "appraisal"
-  spec.add_development_dependency "fog-aws"
+  spec.add_development_dependency "aws-sdk-s3"
   spec.add_development_dependency "mime-types"
   spec.add_development_dependency "get_process_mem"
 end

data/spec/feature/memory_usage_spec.rb

@@ -30,6 +30,13 @@ describe "memory usage" do
     end
   end
 
+  def with_no_gc
+    GC.disable
+    yield
+  ensure
+    GC.enable
+  end
+
   describe "is kept low when using exclusively with_file processors" do
     let(:uploader) {
       Class.new(Saviour::BaseUploader) {
@@ -46,12 +53,14 @@ describe "memory usage" do
       a = base_klass.create!
 
       with_tempfile do |f|
-        base_line = GetProcessMem.new.mb
+        with_no_gc do
+          base_line = GetProcessMem.new.mb
 
-        a.update_attributes! file: f
+          a.update_attributes! file: f
 
-        # Expect memory usage to grow below 10% of the file size
-        expect(GetProcessMem.new.mb - base_line).to be < size_to_test / 10
+          # Expect memory usage to grow below 10% of the file size
+          expect(GetProcessMem.new.mb - base_line).to be < size_to_test / 10
+        end
       end
     end
   end
@@ -72,12 +81,14 @@ describe "memory usage" do
       a = base_klass.create!
 
       with_tempfile do |f|
-        base_line = GetProcessMem.new.mb
+        with_no_gc do
+          base_line = GetProcessMem.new.mb
 
-        a.update_attributes! file: f
+          a.update_attributes! file: f
 
-        # Expect memory usage to grow at least the size of the file
-        expect(GetProcessMem.new.mb - base_line).to be > size_to_test
+          # Expect memory usage to grow at least the size of the file
+          expect(GetProcessMem.new.mb - base_line).to be > size_to_test
+        end
       end
     end
   end

data/spec/feature/original_assigned_file_is_not_modified_spec.rb

@@ -0,0 +1,43 @@
+require 'spec_helper'
+
+describe "Original assigned file" do
+  before { allow(Saviour::Config).to receive(:storage).and_return(Saviour::LocalStorage.new(local_prefix: @tmpdir, public_url_prefix: "http://domain.com")) }
+
+  let(:uploader) {
+    Class.new(Saviour::BaseUploader) do
+      store_dir { "/store/dir/#{model.id}" }
+      process_with_file do |file, name|
+        ::File.delete(file.path)
+
+        f = Tempfile.new("test")
+        f.write("Hello")
+        f.flush
+
+        [f, name]
+      end
+    end
+  }
+
+  let(:klass) {
+    klass = Class.new(Test) {
+      include Saviour::Model
+    }
+    klass.attach_file :file, uploader
+    klass
+  }
+
+  it "is preserved even after deleting the incoming file from a processor" do
+    f = Tempfile.new("test")
+    f.write("original")
+    f.flush
+
+    a = klass.create! file: f
+
+    expect(a.file.read).to eq "Hello"
+
+    expect(::File.file?(f.path)).to be_truthy
+    expect(::File.read(f.path)).to eq "original"
+
+    f.close!
+  end
+end

data/spec/models/s3_storage_spec.rb

@@ -1,9 +1,23 @@
 require 'spec_helper'
 
 describe Saviour::S3Storage do
-  subject { Saviour::S3Storage.new(bucket: "fake-bucket", aws_access_key_id: "stub", aws_secret_access_key: "stub", public_url_prefix: "https://fake-bucket.s3.amazonaws.com") }
-  let!(:mocked_s3) { MockedS3Helper.new }
-  before { mocked_s3.start!(bucket_name: "fake-bucket") }
+  let(:injected_client) { Aws::S3::Client.new(stub_responses: true) }
+
+  let(:storage_options) {
+    {
+      bucket: "fake-bucket",
+      aws_access_key_id: "stub",
+      aws_secret_access_key: "stub",
+      region: "fake",
+      public_url_prefix: "https://fake-bucket.s3.amazonaws.com"
+    }
+  }
+
+  subject {
+    storage = Saviour::S3Storage.new(storage_options)
+    allow(storage).to receive(:client).and_return(injected_client)
+    storage
+  }
 
   context do
     it "fails when no keys are provided" do
@@ -18,12 +32,8 @@ describe Saviour::S3Storage do
 
     it "writting a new file" do
       with_test_file("camaloon.jpg") do |file, _|
-        expect(mocked_s3.exists?(destination_path)).to be_falsey
-
         contents = file.read
-        subject.write(contents, destination_path)
-
-        expect(mocked_s3.read(destination_path)).to eq contents
+        expect(subject.write(contents, destination_path)).to be_truthy
       end
     end
 
@@ -32,18 +42,6 @@ describe Saviour::S3Storage do
       expect { subject.write("contents", key) }.to raise_error.with_message(/The key in S3 must be at max 1024 bytes, this key is too big/)
     end
 
-    it "overwrites the existing file" do
-      mocked_s3.write("some dummy contents", destination_path)
-      expect(mocked_s3.exists?(destination_path)).to be_truthy
-
-      with_test_file("camaloon.jpg") do |file, _|
-        contents = file.read
-        subject.write(contents, destination_path)
-        expect(mocked_s3.read(destination_path)).to eq contents
-      end
-    end
-
-
     it "ignores leading slash" do
       subject.write("trash contents", "/folder/file.out")
       expect(subject.exists?("folder/file.out")).to be_truthy
@@ -52,14 +50,21 @@ describe Saviour::S3Storage do
     end
 
     describe "fog create options" do
-      subject { Saviour::S3Storage.new(bucket: "fake-bucket", aws_access_key_id: "stub", aws_secret_access_key: "stub", public_url_prefix: "https://fake-bucket.s3.amazonaws.com", create_options: { 'Cache-Control' => 'max-age=31536000' }) }
+      let(:storage_options) {
+        {
+          bucket: "fake-bucket",
+          aws_access_key_id: "stub",
+          aws_secret_access_key: "stub",
+          public_url_prefix: "https://fake-bucket.s3.amazonaws.com",
+          create_options: { cache_control: 'max-age=31536000', acl: "public-read" },
+          region: "fake"
+        }
+      }
 
       it "uses passed options to create new files in S3" do
         with_test_file("camaloon.jpg") do |file, _|
           contents = file.read
-          subject.write(contents, destination_path)
-          file_data = mocked_s3.head(destination_path)
-          expect(file_data.cache_control).to eq "max-age=31536000"
+          expect(subject.write(contents, destination_path)).to be_truthy
         end
       end
     end
@@ -69,15 +74,12 @@ describe Saviour::S3Storage do
     let(:destination_path) { "dest/file.jpeg" }
 
     it "reads an existing file" do
-      with_test_file("camaloon.jpg") do |file, _|
-        contents = file.read
-
-        mocked_s3.write(contents, destination_path)
-        expect(subject.read(destination_path)).to eq contents
-      end
+      injected_client.stub_responses(:get_object, body: "hello")
+      expect(subject.read(destination_path)).to eq "hello"
     end
 
     it "fails if the file do not exists" do
+      injected_client.stub_responses(:get_object, 'NotFound')
       expect { subject.read("nope.rar") }.to raise_error(Saviour::FileNotPresent)
     end
   end
@@ -86,18 +88,7 @@ describe Saviour::S3Storage do
     let(:destination_path) { "dest/file.jpeg" }
 
     it "deletes an existing file" do
-      with_test_file("camaloon.jpg") do |file, _|
-        contents = file.read
-        mocked_s3.write(contents, destination_path)
-
-        expect(mocked_s3.exists?(destination_path)).to be_truthy
-        subject.delete("/dest/file.jpeg")
-        expect(mocked_s3.exists?(destination_path)).to be_falsey
-      end
-    end
-
-    it "fails if the file do not exists" do
-      expect { subject.delete("nope.rar") }.to raise_error(Saviour::FileNotPresent)
+      expect(subject.delete(destination_path)).to be_truthy
     end
   end
 
@@ -105,14 +96,11 @@ describe Saviour::S3Storage do
     let(:destination_path) { "dest/file.jpeg" }
 
     it "with existing file" do
-      with_test_file("camaloon.jpg") do |file, _|
-        contents = file.read
-        mocked_s3.write(contents, destination_path)
-        expect(subject.exists?(destination_path)).to be_truthy
-      end
+      expect(subject.exists?(destination_path)).to be_truthy
     end
 
     it "with no file" do
+      injected_client.stub_responses(:head_object, 'NotFound')
       expect(subject.exists?("unexisting_file.zip")).to be_falsey
     end
   end
@@ -121,37 +109,40 @@ describe Saviour::S3Storage do
     let(:destination_path) { "dest/file.jpeg" }
 
     context do
-      subject { Saviour::S3Storage.new(bucket: "fake-bucket", aws_access_key_id: "stub", aws_secret_access_key: "stub") }
+      let(:storage_options) {
+        {
+          bucket: "fake-bucket",
+          aws_access_key_id: "stub",
+          aws_secret_access_key: "stub",
+          region: "fake"
+        }
+      }
 
       it "fails if not provided the prefix" do
-        with_test_file("camaloon.jpg") do |file, _|
-          contents = file.read
-          mocked_s3.write(contents, destination_path)
-          expect { subject.public_url(destination_path) }.to raise_error(Saviour::S3Storage::MissingPublicUrlPrefix)
-        end
+        expect { subject.public_url(destination_path) }.to raise_error(Saviour::S3Storage::MissingPublicUrlPrefix)
       end
     end
 
     context do
-      subject { Saviour::S3Storage.new(bucket: "fake-bucket", aws_access_key_id: "stub", aws_secret_access_key: "stub", public_url_prefix: -> { "https://#{Time.now.hour}.s3.amazonaws.com" }) }
+      let(:storage_options) {
+        {
+          bucket: "fake-bucket",
+          aws_access_key_id: "stub",
+          aws_secret_access_key: "stub",
+          public_url_prefix: -> { "https://#{Time.now.hour}.s3.amazonaws.com" },
+          region: "fake"
+        }
+      }
 
       it "allow to use a lambda for dynamic url prefixes" do
         allow(Time).to receive(:now).and_return(Time.new(2015, 1, 1, 13, 2, 1))
 
-        with_test_file("camaloon.jpg") do |file, _|
-          contents = file.read
-          mocked_s3.write(contents, destination_path)
-          expect(subject.public_url(destination_path)).to eq "https://13.s3.amazonaws.com/dest/file.jpeg"
-        end
+        expect(subject.public_url(destination_path)).to eq "https://13.s3.amazonaws.com/dest/file.jpeg"
       end
     end
 
     it do
-      with_test_file("camaloon.jpg") do |file, _|
-        contents = file.read
-        mocked_s3.write(contents, destination_path)
-        expect(subject.public_url(destination_path)).to eq "https://fake-bucket.s3.amazonaws.com/dest/file.jpeg"
-      end
+      expect(subject.public_url(destination_path)).to eq "https://fake-bucket.s3.amazonaws.com/dest/file.jpeg"
     end
   end
 end

data/spec/spec_helper.rb

@@ -36,8 +36,6 @@ require 'support/models'
 
 RSpec.configure do |config|
   config.around do |example|
-    Fog.mock!
-
     Dir.mktmpdir { |dir|
       @tmpdir = dir
       example.run
@@ -47,7 +45,6 @@ RSpec.configure do |config|
   config.before do
     Test.delete_all
   end
-  config.after { Fog::Mock.reset }
 end
 
 def with_tempfile(ext = ".jpg")
@@ -68,45 +65,3 @@ def with_test_file(name)
     yield(temp, File.basename(temp.path))
   end
 end
-
-class MockedS3Helper
-  attr_reader :directory
-
-  def start!(bucket_name: nil)
-    @directory = connection.directories.create(key: bucket_name)
-  end
-
-  def write(contents, path)
-    directory.files.create(
-        key: path,
-        body: contents,
-        public: true
-    )
-  end
-
-  def read(path)
-    directory.files.get(path).body
-  end
-
-  def delete(path)
-    directory.files.get(path).destroy
-  end
-
-  def head(path)
-    directory.files.head(path)
-  end
-
-  def exists?(path)
-    !!head(path)
-  end
-
-  def public_url(path)
-    directory.files.get(path).public_url
-  end
-
-  private
-
-  def connection
-    @connection ||= Fog::Storage.new(provider: 'AWS', aws_access_key_id: "stub", aws_secret_access_key: "stub")
-  end
-end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: saviour
 version: !ruby/object:Gem::Version
-  version: 0.4.6
+  version: 0.4.7
 platform: ruby
 authors:
 - Roger Campos
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-04 00:00:00.000000000 Z
+date: 2018-01-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -109,7 +109,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: fog-aws
+  name: aws-sdk-s3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -191,6 +191,7 @@ files:
 - spec/feature/follow_file_spec.rb
 - spec/feature/halt_processor_spec.rb
 - spec/feature/memory_usage_spec.rb
+- spec/feature/original_assigned_file_is_not_modified_spec.rb
 - spec/feature/persisted_path_spec.rb
 - spec/feature/processors_api_spec.rb
 - spec/feature/reload_model_spec.rb