s3_backup 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c8df1802a52812399f8b1f91abebe675329f5dd0
-  data.tar.gz: 1d3f6f9c0407707e6906ce249e59f091c0c6aafd
+  metadata.gz: f8d4e48f3d5a0125d3d100f14b18834b66560986
+  data.tar.gz: d01288f1afffeccebd2256c971366be957e45400
 SHA512:
-  metadata.gz: d931c51fca7861bb7f5fac42563fcb6b1da05299ac56c09d790bedfc2d23fcb958d3da9f646cb0996b04a7bde31ab6085fd00766776b7649158808e7ab4d7af7
-  data.tar.gz: 35fa04c64c85e1374d62e3bf89abc8834c05897e86e440eb439fe744c429a95ad611b6067d4415f7cae3a7046f804a19ea6f3c3d38f45de06658fe535fa1d3cb
+  metadata.gz: 1f361b09a6dd62d72f63dda4ed20c99817ca856656f222ca40a388a530b86722550d835dfed540d0b3798219ca8aebeef7a2ef9aeac2d192723e83ff02d2e16f
+  data.tar.gz: b4ed851c477696e54ed578a2a43fe091091e9b9b6715407564b6f1c42cc3827eb585e17733015535e273b3433aaea6a5950bbacf16ba054255f0ac0a5a7870bd

data/README.md

@@ -42,6 +42,9 @@ s3:
   aws_secret_access_key: <%= ENV['AWS_SECRET_ACCESS_KEY'] %>
   bucket: <%= ENV['S3_BUCKET'] %>
   aws_region: <%= ENV['AWS_REGION'] %>
+  aws_endpoint: <%= ENV['AWS_ENDPOINT'] %>
+  server_side_encryption: 'AES256'
+  stub_responses: false
   pg_path: rds_backup
   redis_path: redis_backup
   keep: 5

data/lib/s3_backup/config.rb

@@ -12,8 +12,11 @@ module S3Backup
       attr_accessor :redis_dump_path
       attr_accessor :aws_access_key_id
       attr_accessor :aws_secret_access_key
+      attr_accessor :aws_endpoint
+      attr_accessor :aws_server_side_encryption
       attr_accessor :bucket
       attr_accessor :aws_region
+      attr_accessor :aws_stub_responses
       attr_accessor :s3_pg_path
       attr_accessor :s3_redis_path
       attr_accessor :s3_keep
@@ -29,13 +32,16 @@ module S3Backup
 
         self.redis_dump_path = config('redis', 'dump_path')
 
-        self.aws_access_key_id     = config('s3', 'aws_access_key_id')
-        self.aws_secret_access_key = config('s3', 'aws_secret_access_key')
-        self.bucket                = config('s3', 'bucket')
-        self.aws_region            = config('s3', 'aws_region')
-        self.s3_keep               = config('s3', 'keep')
-        self.s3_pg_path            = config('s3', 'pg_path')
-        self.s3_redis_path         = config('s3', 'redis_path')
+        self.aws_access_key_id          = config('s3', 'aws_access_key_id')
+        self.aws_secret_access_key      = config('s3', 'aws_secret_access_key')
+        self.bucket                     = config('s3', 'bucket')
+        self.aws_region                 = config('s3', 'aws_region')
+        self.aws_endpoint               = config('s3', 'aws_endpoint')
+        self.aws_server_side_encryption = config('s3', 'server_side_encryption')
+        self.aws_stub_responses         = config('s3', 'stub_responses')
+        self.s3_keep                    = config('s3', 'keep')
+        self.s3_pg_path                 = config('s3', 'pg_path')
+        self.s3_redis_path              = config('s3', 'redis_path')
 
         self.tables = config('tables') || {}
 

data/lib/s3_backup/pg/backup.rb

@@ -17,11 +17,11 @@ module S3Backup
         Obfuscate.new(pg_dump_file.path, obfuscated_file.path).obfuscate_dump!
         puts 'Obfuscation done.'
         puts 'Upload to S3 ...'
-        S3Backup::S3.new.upload!(obfucated_file_name, Config.s3_pg_path, obfuscated_file.path)
+        S3Backup::Storage::S3.new.upload!(obfucated_file_name, Config.s3_pg_path, obfuscated_file.path)
         puts 'Uploaded.'
         puts 'Clean environement.'
         clean_env
-        S3Backup::S3.new.clean!(db_name, Config.s3_pg_path)
+        S3Backup::Storage::S3.new.clean!(db_name, Config.s3_pg_path)
       end
 
       private

data/lib/s3_backup/pg/import.rb

@@ -16,7 +16,7 @@ module S3Backup
         puts 'Setup local database ...'
         setup_local_database
         puts 'Downloading pg database ...'
-        S3Backup::S3.new.download!(pg_database_name, Config.s3_pg_path, pg_dump_s3_file.path)
+        S3Backup::Storage::S3.new.download!(pg_database_name, Config.s3_pg_path, pg_dump_s3_file.path)
         umcompress_file
         puts "Loading data in #{database} ..."
         load_file

data/lib/s3_backup/redis/backup.rb

@@ -9,11 +9,11 @@ module S3Backup
         compress_file
         puts 'Compressed.'
         puts 'Upload to S3 ...'
-        S3Backup::S3.new.upload!(compressed_file_name, Config.s3_redis_path, compressed_file.path)
+        S3Backup::Storage::S3.new.upload!(compressed_file_name, Config.s3_redis_path, compressed_file.path)
         puts 'Uploaded.'
         puts 'Clean environement.'
         clean_env
-        S3Backup::S3.new.clean!(base_s3_name, Config.s3_redis_path)
+        S3Backup::Storage::S3.new.clean!(base_s3_name, Config.s3_redis_path)
       end
 
       private

data/lib/s3_backup/redis/import.rb

@@ -18,7 +18,7 @@ module S3Backup
         puts 'Stop redis database ...'
         stop_redis_database
         puts 'Downloading redis database ...'
-        S3Backup::S3.new.download!(redis_s3_file_name, Config.s3_redis_path, redis_dump_s3_file.path)
+        S3Backup::Storage::S3.new.download!(redis_s3_file_name, Config.s3_redis_path, redis_dump_s3_file.path)
         umcompress_file
         copy_file
         puts 'Start redis database ...'

data/lib/s3_backup/storage/s3.rb

@@ -0,0 +1,92 @@
+require 'aws-sdk-s3'
+require 'zlib'
+require 'ruby-progressbar'
+
+module S3Backup
+  module Storage
+    class S3
+
+      attr_reader :connection
+
+      def initialize
+        @connection = Aws::S3::Client.new(
+          credentials: Aws::Credentials.new(
+            Config.aws_access_key_id,
+            Config.aws_secret_access_key
+          ),
+          region:              Config.aws_region,
+          endpoint:            Config.aws_endpoint,
+          stub_responses:      Config.aws_stub_responses
+        )
+      end
+
+      def upload!(file_name, bucket_path, file_path)
+        upload_options = {
+          bucket: Config.bucket,
+          key: File.join(bucket_path, file_name),
+          body: File.open(file_path)
+        }
+
+        upload_options[:server_side_encryption] = Config.aws_server_side_encryption if Config.aws_server_side_encryption
+        @connection.put_object(upload_options)
+        
+        true
+      end
+
+      def download!(database_name, bucket_path, file_path)
+        prefix = File.join(bucket_path, database_name)
+        s3_backup_file = @connection.list_objects(bucket: Config.bucket, prefix: prefix).contents.sort_by(&:last_modified).reverse.first
+
+        raise "#{database_name} file not found on s3" unless s3_backup_file
+
+        file = File.open(file_path, 'wb')
+        puts "File size: #{(s3_backup_file.size.to_f / 1024 / 1024).round(4)}MB, writing to #{file_path}"
+        total_bytes = s3_backup_file.size
+        remaining_bytes = s3_backup_file.size
+        progress_bar
+        
+        @connection.get_object(bucket: Config.bucket, key: s3_backup_file.key) do |chunk|
+          update_progress_bar(total_bytes, remaining_bytes)
+          file.write chunk
+          remaining_bytes -= chunk.size
+        end
+        file.close
+
+        true
+      end
+
+      def clean!(base_name, bucket_path)
+        prefix = File.join(bucket_path, base_name)
+
+        s3_files = @connection.list_objects(bucket: Config.bucket, prefix: prefix).contents.sort_by(&:last_modified).reverse
+        files_to_remove = s3_files[(Config.s3_keep || 1)..-1]
+
+        return true if files_to_remove.nil? || files_to_remove.empty?
+
+        @connection.delete_objects(
+          bucket: Config.bucket,
+          delete: {
+            objects: files_to_remove.map {|f| {key: f.key} }
+          }
+        )
+
+        true
+      end
+
+      private
+
+      def progress_bar
+        @progress_bar ||= ProgressBar.create(
+          format:         "%a %b\u{15E7}%i %p%% %t",
+          progress_mark:  ' ',
+          remainder_mark: "\u{FF65}"
+        )
+      end
+
+      def update_progress_bar(total, remaining)
+        progress_bar.progress = (((total - remaining) * 100) / total).to_i
+      end
+
+    end
+  end
+end

data/lib/s3_backup/version.rb

@@ -1,3 +1,3 @@
 module S3Backup # :nodoc:
-  VERSION = '0.0.5'.freeze
+  VERSION = '0.0.6'.freeze
 end

data/lib/s3_backup.rb

@@ -1,5 +1,5 @@
 require 's3_backup/config'
-require 's3_backup/s3'
+require 's3_backup/storage/s3'
 
 require 's3_backup/pg/obfuscate'
 require 's3_backup/pg/backup'
@@ -36,7 +36,7 @@ module S3Backup
 
       require_s3_params
       Config.requires!(:s3_pg_path)
-      file_downloaded = S3Backup::S3.new.download!(pg_database_name, Config.s3_pg_path, filename)
+      file_downloaded = S3Backup::Storage::S3.new.download!(pg_database_name, Config.s3_pg_path, filename)
       abort 'Unable to download file' unless file_downloaded
       filename
     end

data/spec/fixtures/obfuscate_default.yml

@@ -9,7 +9,10 @@ s3:
   aws_access_key_id: XXX
   aws_secret_access_key: DSLK
   bucket: my_bucket
-  aws_region: p-southeast-2
+  aws_region: ap-southeast-2
+  aws_endpoint: https://s3.amazonaws.com
+  stub_responses: true
+  server_side_encryption: AES256
   pg_path: pg_backup
 
 tables:

data/spec/s3_backup/storage/s3_spec.rb

@@ -0,0 +1,66 @@
+require 'spec_helper'
+
+describe S3Backup::Storage::S3 do
+  let(:configuration_file) { 'spec/fixtures/obfuscate_default.yml' }
+
+  before { S3Backup::Config.load!(configuration_file) }
+  subject(:s3) { S3Backup::Storage::S3.new }
+
+  describe '#upload' do
+    before do
+      client = s3.instance_variable_get('@connection')
+      client.stub_responses(:put_object)
+    end
+
+    it 'should upload a file via AWS S3' do
+      s3.upload!('test', 'test-bucket', 'spec/fixtures/obfuscate_default.yml')
+    end
+  end
+
+  describe '#download' do
+    before do
+      client = s3.instance_variable_get('@connection')
+      client.stub_responses(:list_objects,
+                            {
+                              contents:  [
+                                {
+                                  key: 'offers/1/img.jgp',
+                                  last_modified: Time.now,
+                                  size: 1234
+                                }
+                              ]
+                            })
+    end
+
+    it 'should retrieve the latest file' do
+      temp = Tempfile.new('s3-backup')
+      expect(s3.download!('test', 'test-bucket', temp.path)).to eq(true)
+    end
+  end
+
+  describe '#clean!' do
+    before do
+      client = s3.instance_variable_get('@connection')
+      client.stub_responses(:list_objects,
+                            {
+                              contents:  [
+                                {
+                                  key: 'test1-012.tgz',
+                                  last_modified: Time.now,
+                                  size: 1234
+                                },
+                                {
+                                  key: 'test1-123.tgz',
+                                  last_modified: Time.now - 3600,
+                                  size: 1234
+                                }
+                              ]
+                            })
+    end
+
+    it 'should clean the old files' do
+      s3.clean!('test', 'test-bucket')
+    end
+  end
+
+end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: s3_backup
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Tom Floc'h
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-16 00:00:00.000000000 Z
+date: 2018-02-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: fog
+  name: aws-sdk-s3
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.41'
+        version: '1.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.41'
+        version: '1.8'
 - !ruby/object:Gem::Dependency
   name: faker
   requirement: !ruby/object:Gem::Requirement
@@ -113,12 +113,13 @@ files:
 - lib/s3_backup/railtie.rb
 - lib/s3_backup/redis/backup.rb
 - lib/s3_backup/redis/import.rb
-- lib/s3_backup/s3.rb
+- lib/s3_backup/storage/s3.rb
 - lib/s3_backup/version.rb
 - lib/tasks/pg.rake
 - lib/tasks/redis.rake
 - spec/fixtures/obfuscate_default.yml
 - spec/s3_backup/pg/obfuscate_spec.rb
+- spec/s3_backup/storage/s3_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/arkes/s3_backup
 licenses:
@@ -140,7 +141,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.4.5.2
 signing_key: 
 specification_version: 4
 summary: Postgres, redis backup and restore

data/lib/s3_backup/s3.rb

@@ -1,81 +0,0 @@
-require 'fog'
-require 'zlib'
-require 'ruby-progressbar'
-
-module S3Backup
-  class S3
-
-    attr_reader :connection
-
-    def initialize
-      @connection = Fog::Storage.new(
-        provider:              'AWS',
-        aws_access_key_id:     Config.aws_access_key_id,
-        aws_secret_access_key: Config.aws_secret_access_key,
-        region:                Config.aws_region
-      )
-    end
-
-    def upload!(file_name, bucket_path, file_path)
-      directory = @connection.directories.get(Config.bucket)
-
-      directory.files.create(
-        key:    File.join(bucket_path, file_name),
-        body:   File.open(file_path),
-        public: false
-      )
-
-      true
-    end
-
-    def download!(database_name, bucket_path, file_path)
-      prefix = File.join(bucket_path, database_name)
-      directory = connection.directories.get(Config.bucket, prefix: prefix)
-
-      s3_backup_file = directory.files.sort_by(&:last_modified).reverse.first
-
-      raise "#{database_name} file not found on s3" unless s3_backup_file
-
-      file = File.open(file_path, 'wb')
-      puts "File size: #{s3_backup_file.content_length / 1024 / 1024}MB, writing to #{file_path}"
-      progress_bar
-      
-      directory.files.get(s3_backup_file.key) do |chunk, remaining_bytes, total_bytes|
-        update_progress_bar(total_bytes, remaining_bytes)
-        file.write chunk
-      end
-      file.close
-
-      true
-    end
-
-    def clean!(base_name, bucket_path)
-      prefix = File.join(bucket_path, base_name)
-      directory = connection.directories.get(Config.bucket, prefix: prefix)
-
-      s3_files = directory.files.sort_by(&:last_modified).reverse
-      files_to_remove = s3_files[Config.s3_keep..-1]
-
-      return true if files_to_remove.blank?
-
-      files_to_remove.each(&:destroy)
-
-      true
-    end
-
-    private
-
-    def progress_bar
-      @progress_bar ||= ProgressBar.create(
-        format:         "%a %b\u{15E7}%i %p%% %t",
-        progress_mark:  ' ',
-        remainder_mark: "\u{FF65}"
-      )
-    end
-
-    def update_progress_bar(total, remaining)
-      progress_bar.progress = (((total - remaining) * 100) / total).to_i
-    end
-
-  end
-end