s3_rotate 1.0.0

data/spec/s3_rotate/core/backup_uploader_spec.rb

@@ -0,0 +1,145 @@
+# standard
+require 'date'
+
+# 3rd part
+require 'fog-aws'
+
+# s3_rotate
+require File.expand_path("../../../../lib/s3_rotate/core/backup_uploader", __FILE__)
+require File.expand_path("../../../../lib/s3_rotate/utils/file_utils", __FILE__)
+
+describe S3Rotate::BackupUploader do
+
+  before :each do
+    @client   = S3Rotate::S3Client.new('key', 'secret', 'bucket', 'region')
+    @uploader = S3Rotate::BackupUploader.new(@client)
+  end
+
+  describe '#initialize' do
+
+    it 'sets the client' do
+      expect(@uploader.s3_client).not_to eq nil
+    end
+
+  end
+
+  describe '#upload' do
+
+    it 'uploads all the new files until reaching an already uploaded file' do
+      # mock
+      @client.connection.directories.get('bucket').files.create(key: '/backup_name/daily/2020-01-02.tgz', body: 'some data')
+      @client.connection.directories.get('bucket').files.create(key: '/backup_name/daily/2020-01-03.tgz', body: 'some data')
+      @client.connection.directories.get('bucket').files.create(key: '/backup_name/daily/2020-01-04.tgz', body: 'some data')
+      @client.connection.directories.get('bucket').files.create(key: '/backup_name/weekly/2020-01-05.tgz', body: 'some data')
+      @client.connection.directories.get('bucket').files.create(key: '/backup_name/monthly/2020-01-06.tgz', body: 'some data')
+      @client.connection.directories.get('bucket').files.create(key: '/other_backup_name/daily/2020-01-07.tgz', body: 'some data')
+
+      allow(S3Rotate::FileUtils).to receive(:date_from_filename).and_call_original
+      allow(S3Rotate::FileUtils).to receive(:extension_from_filename).and_call_original
+      allow(S3Rotate::FileUtils).to receive(:files_in_directory).with('/path/to/dir').and_return([
+        'some-backup-2020-01-01.tgz',
+        'some-backup-2020-01-02.tgz',
+        'some-backup-2020-01-03.tgz',
+        'some-backup-2020-01-04.tgz',
+        'some-backup-2020-01-05.tgz',
+        'some-backup-2020-01-06.tgz',
+        'some-backup-2020-01-07.tgz'
+      ])
+      allow(@client).to receive(:upload_local_backup_to_s3).and_return nil
+      allow(File).to receive(:open).and_return "raw_data"
+
+      # perform test
+      @uploader.upload('backup_name', '/path/to/dir')
+
+      # verify result
+      expect(@client).to have_received(:upload_local_backup_to_s3).exactly(3)
+      expect(@client).to have_received(:upload_local_backup_to_s3).with('backup_name', Date.new(2020, 1, 7), 'daily', '.tgz', 'raw_data')
+      expect(@client).to have_received(:upload_local_backup_to_s3).with('backup_name', Date.new(2020, 1, 6), 'daily', '.tgz', 'raw_data')
+      expect(@client).to have_received(:upload_local_backup_to_s3).with('backup_name', Date.new(2020, 1, 5), 'daily', '.tgz', 'raw_data')
+
+      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).exactly(4)
+      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-07.tgz', /\d{4}-\d{2}-\d{2}/)
+      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-06.tgz', /\d{4}-\d{2}-\d{2}/)
+      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-05.tgz', /\d{4}-\d{2}-\d{2}/)
+      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-04.tgz', /\d{4}-\d{2}-\d{2}/)
+
+      expect(S3Rotate::FileUtils).to have_received(:extension_from_filename).exactly(4)
+      expect(S3Rotate::FileUtils).to have_received(:extension_from_filename).with('some-backup-2020-01-07.tgz')
+      expect(S3Rotate::FileUtils).to have_received(:extension_from_filename).with('some-backup-2020-01-06.tgz')
+      expect(S3Rotate::FileUtils).to have_received(:extension_from_filename).with('some-backup-2020-01-05.tgz')
+      expect(S3Rotate::FileUtils).to have_received(:extension_from_filename).with('some-backup-2020-01-04.tgz')
+    end
+
+    it 'uploads files without extensions' do
+      # mock
+      allow(S3Rotate::FileUtils).to receive(:date_from_filename).and_call_original
+      allow(S3Rotate::FileUtils).to receive(:extension_from_filename).and_return nil
+      allow(S3Rotate::FileUtils).to receive(:files_in_directory).with('/path/to/dir').and_return([
+        'some-backup-2020-01-01.tgz',
+        'some-backup-2020-01-02.tgz',
+        'some-backup-2020-01-03.tgz',
+      ])
+      allow(@client).to receive(:upload_local_backup_to_s3).and_return nil
+      allow(File).to receive(:open).and_return "raw_data"
+
+      # perform test
+      @uploader.upload('backup_name', '/path/to/dir')
+
+      # verify result
+      expect(@client).to have_received(:upload_local_backup_to_s3).exactly(3)
+      expect(@client).to have_received(:upload_local_backup_to_s3).with('backup_name', Date.new(2020, 1, 3), 'daily', nil, 'raw_data')
+      expect(@client).to have_received(:upload_local_backup_to_s3).with('backup_name', Date.new(2020, 1, 2), 'daily', nil, 'raw_data')
+      expect(@client).to have_received(:upload_local_backup_to_s3).with('backup_name', Date.new(2020, 1, 1), 'daily', nil, 'raw_data')
+
+      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).exactly(3)
+      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-03.tgz', /\d{4}-\d{2}-\d{2}/)
+      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-02.tgz', /\d{4}-\d{2}-\d{2}/)
+      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-01.tgz', /\d{4}-\d{2}-\d{2}/)
+
+      expect(S3Rotate::FileUtils).to have_received(:extension_from_filename).exactly(3)
+      expect(S3Rotate::FileUtils).to have_received(:extension_from_filename).with('some-backup-2020-01-03.tgz')
+      expect(S3Rotate::FileUtils).to have_received(:extension_from_filename).with('some-backup-2020-01-02.tgz')
+      expect(S3Rotate::FileUtils).to have_received(:extension_from_filename).with('some-backup-2020-01-01.tgz')
+    end
+
+    it 'does not upload files with broken date' do
+      # mock
+      original_date_from_filename = S3Rotate::FileUtils.method(:date_from_filename)
+      allow(S3Rotate::FileUtils).to receive(:date_from_filename) do |filename, date_regex|
+        if filename == 'some-backup-2020-01-02.tgz'
+          nil
+        else
+          original_date_from_filename.call(filename, date_regex)
+        end
+      end
+      allow(S3Rotate::FileUtils).to receive(:extension_from_filename).and_call_original
+      allow(S3Rotate::FileUtils).to receive(:files_in_directory).with('/path/to/dir').and_return([
+        'some-backup-2020-01-01.tgz',
+        'some-backup-2020-01-02.tgz',
+        'some-backup-2020-01-03.tgz',
+      ])
+      allow(@client).to receive(:upload_local_backup_to_s3).and_return nil
+      allow(File).to receive(:open).and_return "raw_data"
+
+      # perform test
+      @uploader.upload('backup_name', '/path/to/dir')
+
+      # verify result
+      expect(@client).to have_received(:upload_local_backup_to_s3).exactly(2)
+      expect(@client).to have_received(:upload_local_backup_to_s3).with('backup_name', Date.new(2020, 1, 3), 'daily', '.tgz', 'raw_data')
+      expect(@client).to have_received(:upload_local_backup_to_s3).with('backup_name', Date.new(2020, 1, 1), 'daily', '.tgz', 'raw_data')
+
+      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).exactly(3)
+      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-03.tgz', /\d{4}-\d{2}-\d{2}/)
+      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-02.tgz', /\d{4}-\d{2}-\d{2}/)
+      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-01.tgz', /\d{4}-\d{2}-\d{2}/)
+
+      expect(S3Rotate::FileUtils).to have_received(:extension_from_filename).exactly(3)
+      expect(S3Rotate::FileUtils).to have_received(:extension_from_filename).with('some-backup-2020-01-03.tgz')
+      expect(S3Rotate::FileUtils).to have_received(:extension_from_filename).with('some-backup-2020-01-02.tgz')
+      expect(S3Rotate::FileUtils).to have_received(:extension_from_filename).with('some-backup-2020-01-01.tgz')
+    end
+
+  end
+
+end

data/spec/s3_rotate/utils/file_utils_spec.rb

@@ -0,0 +1,61 @@
+# standard
+require 'date'
+
+# s3_rotate
+require File.expand_path("../../../../lib/s3_rotate/utils/file_utils", __FILE__)
+
+describe S3Rotate::FileUtils do
+
+  describe '#date_from_filename' do
+
+    it 'parses Date.parse formats' do
+      expect(S3Rotate::FileUtils::date_from_filename("/path/to/file-2020-12-13-backup.tar.gz")).to eq Date.new(2020, 12, 13)
+    end
+
+    it 'parses timestamp formats' do
+      expect(S3Rotate::FileUtils::date_from_filename("/path/to/file-1580098737-backup.tar.gz", /file-(\d+)-backup.tar.gz/)).to eq Date.new(2020, 1, 27)
+    end
+
+    it 'raises if the regex matched nothing' do
+      expect{ S3Rotate::FileUtils::date_from_filename("/path/to/file-1580098737-backup.tar.gz") }.to raise_error(RuntimeError, "Invalid date_regex or filename format")
+    end
+
+    it 'raises if the regex matched nothing' do
+      expect{ S3Rotate::FileUtils::date_from_filename("/path/to/file-1580098737-backup.tar.gz", /file-\d+-backup.tar.gz/) }.to raise_error(RuntimeError, "Date format not supported")
+    end
+
+  end
+
+  describe '#extension_from_filename' do
+
+    it 'handles no extension' do
+      expect(S3Rotate::FileUtils::extension_from_filename("backup")).to eq nil
+    end
+
+    it 'handles short extension' do
+      expect(S3Rotate::FileUtils::extension_from_filename("backup.tgz")).to eq '.tgz'
+    end
+
+    it 'handles long extension' do
+      expect(S3Rotate::FileUtils::extension_from_filename("backup.tar.gz.1")).to eq '.tar.gz.1'
+    end
+
+    it 'handles absolute paths' do
+      expect(S3Rotate::FileUtils::extension_from_filename("/path/to.file/backup.tar.gz.1")).to eq '.tar.gz.1'
+    end
+
+  end
+
+  describe 'files_in_directory' do
+
+    it 'returns the directory files ordered ASC' do
+      expect(S3Rotate::FileUtils::files_in_directory("#{__dir__}/mock")).to eq [ "backup-2020-01-02.test", "backup-2020-01-03.test", "backup-2020-02-01.test", "backup-2021-01-01.test" ]
+    end
+
+    it 'raises for invalid directories' do
+      expect{ S3Rotate::FileUtils::files_in_directory("/invalid/path") }.to raise_error(RuntimeError, "Invalid directory /invalid/path")
+    end
+
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,15 @@
+RSpec.configure do |c|
+
+  c.before :each do
+    Fog.mock!
+
+    fog = Fog::Storage.new(aws_access_key_id: 'key', aws_secret_access_key: 'secret', provider: 'AWS', region: 'region')
+    fog.directories.create(key: 'bucket')
+  end
+
+  c.after :each do
+    Fog::Mock.reset
+    Fog.unmock!
+  end
+
+end

metadata

@@ -0,0 +1,76 @@
+--- !ruby/object:Gem::Specification
+name: s3_rotate
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Simon Ninon
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-01-27 00:00:00.000000000 Z
+dependencies: []
+description: AWS S3 upload with rotation mechanism
+email: simon.ninon@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- examples/example.rb
+- lib/s3_rotate.rb
+- lib/s3_rotate/aws/s3_client.rb
+- lib/s3_rotate/core/backup_manager.rb
+- lib/s3_rotate/core/backup_rotator.rb
+- lib/s3_rotate/core/backup_uploader.rb
+- lib/s3_rotate/utils/file_utils.rb
+- s3_rotate.gemspec
+- spec/s3_rotate/aws/s3_client_spec.rb
+- spec/s3_rotate/core/backup_manager_spec.rb
+- spec/s3_rotate/core/backup_rotator_spec.rb
+- spec/s3_rotate/core/backup_uploader_spec.rb
+- spec/s3_rotate/utils/file_utils_spec.rb
+- spec/s3_rotate/utils/mock/backup-2020-01-02.test
+- spec/s3_rotate/utils/mock/backup-2020-01-03.test
+- spec/s3_rotate/utils/mock/backup-2020-02-01.test
+- spec/s3_rotate/utils/mock/backup-2021-01-01.test
+- spec/spec_helper.rb
+homepage: https://github.com/Whova/s3_rotate
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.1
+signing_key: 
+specification_version: 4
+summary: AWS S3 upload with rotation mechanism
+test_files:
+- spec/s3_rotate/aws/s3_client_spec.rb
+- spec/s3_rotate/core/backup_manager_spec.rb
+- spec/s3_rotate/core/backup_rotator_spec.rb
+- spec/s3_rotate/core/backup_uploader_spec.rb
+- spec/s3_rotate/utils/file_utils_spec.rb
+- spec/s3_rotate/utils/mock/backup-2020-01-02.test
+- spec/s3_rotate/utils/mock/backup-2020-01-03.test
+- spec/s3_rotate/utils/mock/backup-2020-02-01.test
+- spec/s3_rotate/utils/mock/backup-2021-01-01.test
+- spec/spec_helper.rb