s3_rotate 1.0.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 16f04d3b048eedc89e3a4efaacdb46009bda296f842ace1d1a0c56f87bb88158
-  data.tar.gz: df1fd512ebb22a0800ea822364e76f69c0a5409d89e90b95ee7731dcd81f0bed
+  metadata.gz: bb5f480435edb48a8d0d8195774fd5280d988ab2c219351a9fa18aba4ff5f89e
+  data.tar.gz: 6da33cabbbd97c8b5256fcb4f0cbd282606745676b6fbaecb746a6b2868925a5
 SHA512:
-  metadata.gz: 89caf936d757c6d2afd3bc2eba65361e9d79ed1547b1fed1f51317925af7ddd458dce94cdd6a4402cf07d35889bb3be105c047dbdc6c690bf9f3c71fb715a26e
-  data.tar.gz: 30e6a7117c71466ff2bc99dafe37d97edfc574d945c30c6b4886d06f2644a2dae68db59f23d1f6dc11c3e7f32928927ad11182a24e69f1e0749203aee090cc62
+  metadata.gz: c11dd83c9a0aa17194542d8e398af2f992d64a9c9b1d4a1182fbda65ba5aa67a2bcf368753d2f70e55f2e4827d7de6c54ba0db7457bb50a2b027a9c24ca04574
+  data.tar.gz: 7375433ebc482f99592e8e1b241c4d479c72f2e54ef03dfde53d4eba2c078f703dfe1437dc68cf05310e06ae99e99c82faee57a0ac99e19a2a868689a324afba

data/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019 Whova, Inc.
+Copyright (c) 2019 Simon Ninon
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -147,10 +147,11 @@ Prototype:
 # @param backup_name        String containing the name of the backup to upload
 # @param local_backups_path String containing the path to the directory containing the backups
 # @param date_regex         Regex returning the date contained in the filename of each backup
+# @param date_format        Format to be used by DateTime.strptime to parse the extracted date
 #
 # @return nothing
 #
-def upload(backup_name, local_backups_path, date_regex=/\d{4}-\d{2}-\d{2}/)
+def upload(backup_name, local_backups_path, date_regex=/\d{4}-\d{2}-\d{2}/, date_format="%Y-%m-%d")
 ```
 
 Example:
@@ -165,14 +166,11 @@ backup_manager.upload("defect-dojo-backup", "/var/opt/defect-dojo/backups")
 - `backup_name`: This is how you want to name your group of backup. This will be used to create a directory on the AWS S3 bucket under which your backup will be stored. It can be anything you want
 - `local_backups_path`: This is the path to the local directory containing your backups to be uploaded
 - `date_regex`: To rotate backups from daily to weekly, and from weekly to monthly, `S3 Backup` needs to determine which date is related to each backup file. This is done by extracting the date information from the filename, using a regex specified in `date_regex`.
+- `date_format`: This complements `date_regex` and gives the format to be used to convert the string matched by `date_regex` into a `Date` object using `DateTime.strptime`.
 
-`date_regex` is the most important part here: without it, `S3 Rotate` does not know when your backup was generated and can not rotate your backups properly. Here are some examples of regex:
-- if your backups are like `1578804325_2020_01_11_12.6.2_gitlab_backup.tar`, you can use `date_regex=/\d{4}-\d{2}-\d{2}/` (this will match `2020_01_11_12`)
-- if your backups are like `1578804325_gitlab_backup.tar`, you can use `date_regex=/(\d+)_gitlab_backup.tar/` (this will match `1578804325`)
-
-As of now, `date_regex` can be:
-- any string that can be parsed by `Date.parse`
-- a timestamp
+`date_regex` & `date_format` is the most important part here: without it, `S3 Rotate` does not know when your backup was generated and can not rotate your backups properly. Here are some examples of regex:
+- if your backups are like `1578804325_2020_01_11_12.6.2_gitlab_backup.tar`, you can use `date_regex=/\d{4}-\d{2}-\d{2}/` (this will match `2020_01_11_12`) & `date_format="%Y-%m-%d"`
+- if your backups are like `1578804325_gitlab_backup.tar`, you can use `date_regex=/(\d+)_gitlab_backup.tar/` (this will match `1578804325`) % `date_format="%s"`
 
 ### S3Rotate::BackupManager.rotate
 Prototype:
@@ -228,7 +226,7 @@ $> bundle exec rspec .
 
 
 ## Author
-[Simon Ninon](https://github.com/Cylix), for [Whova](https://whova.com)
+[Simon Ninon](https://github.com/Cylix)
 
 
 

data/examples/example.rb

@@ -1,7 +1,33 @@
-#!/usr/bin/env ruby
-
 require 's3_rotate'
 
-backup_manager = S3Rotate::BackupManager.new('aws_access_key_id', 'aws_secret_access_key', 'bucket_name', 'region')
-backup_manager.upload("defect-dojo-backup", "/var/opt/defect-dojo/backups")
-backup_manager.rotate("defect-dojo-backup", "/var/opt/defect-dojo/backups")
+# Query the environment
+aws_access_key_id     = ENV["AWS_ACCESS_KEY_ID"]
+aws_access_key_secret = ENV["AWS_ACCESS_KEY_SECRET"]
+bucket_name           = ENV["BUCKET_NAME"]
+region                = ENV["REGION"]
+
+if not aws_access_key_id or not aws_access_key_secret or not bucket_name or not region
+  puts "Misconfigured Environment: Please make sure to set the following Environment Variables"
+  puts " - AWS_ACCESS_KEY_ID"
+  puts " - AWS_ACCESS_KEY_SECRET"
+  puts " - BUCKET_NAME"
+  puts " - REGION"
+  exit -1
+end
+
+# Init S3 Rotate
+backup_manager = S3Rotate::BackupManager.new(aws_access_key_id, aws_access_key_secret, bucket_name, region)
+
+# Upload backups to S3
+backup_manager.upload("backup-dojo", "/data/dojo", date_regex=/\d{4}_\d{2}_\d{2}/, date_format="%Y_%m_%d")
+backup_manager.upload("backup-gitlab", "/data/gitlab/app-backup", date_regex=/\d{4}_\d{2}_\d{2}/, date_format="%Y_%m_%d")
+backup_manager.upload("backup-splunk", "/data/splunk/backup", date_regex=/\d{4}_\d{2}_\d{2}/, date_format="%Y_%m_%d")
+backup_manager.upload("backup-taiga", "/data/taiga", date_regex=/\d{4}_\d{2}_\d{2}/, date_format="%Y_%m_%d")
+backup_manager.upload("backup-trac", "/data/trac/backup", date_regex=/\d{4}_\d{2}_\d{2}/, date_format="%Y_%m_%d")
+
+# Rotate backups
+backup_manager.rotate("backup-dojo", "/data/dojo", max_local=3, max_daily=14, max_weekly=8, max_monthly=6)
+backup_manager.rotate("backup-gitlab", "/data/gitlab/app-backup", max_local=3, max_daily=14, max_weekly=8, max_monthly=6)
+backup_manager.rotate("backup-splunk", "/data/splunk/backup", max_local=3, max_daily=14, max_weekly=8, max_monthly=6)
+backup_manager.rotate("backup-taiga", "/data/taiga", max_local=3, max_daily=14, max_weekly=8, max_monthly=6)
+backup_manager.rotate("backup-trac", "/data/trac/backup", max_local=3, max_daily=14, max_weekly=8, max_monthly=6)

data/lib/s3_rotate/aws/s3_client.rb

@@ -1,9 +1,15 @@
 require 'fog-aws'
+require 'logger'
+
+require 's3_rotate/utils/logging'
 
 module S3Rotate
 
   class S3Client
 
+    # logger
+    include Logging
+
     # attributes
     attr_accessor :access_key
     attr_accessor :access_secret
@@ -84,10 +90,28 @@ module S3Rotate
     # @return created S3 Bucket File
     #
     def upload(backup_name, backup_date, type, extension, data)
+      logger.info("uploading /#{backup_name}/#{type}/#{backup_date.to_s}#{extension}")
+
       # 104857600 bytes => 100 megabytes
       bucket.files.create(key: "/#{backup_name}/#{type}/#{backup_date.to_s}#{extension}", body: data, multipart_chunk_size: 104857600)
     end
 
+    #
+    # Copy an existing file on AWS S3
+    #
+    # @param backup_name   String containing the name of the backup being updated
+    # @param file          S3 File, file to be copied
+    # @param type          String representing the type the backup is being copied to, one of "daily", "weekly" or "monthly"
+    #
+    # @return created S3 Bucket File
+    #
+    def copy(backup_name, file, type)
+      logger.info("copying #{file.key} to /#{backup_name}/#{type}/#{file.key.split('/').last}")
+
+      # 104857600 bytes => 100 megabytes
+      file.copy(@bucket_name, "/#{backup_name}/#{type}/#{file.key.split('/').last}")
+    end
+
   end
 
 end

data/lib/s3_rotate/core/backup_manager.rb

@@ -34,11 +34,12 @@ module S3Rotate
     # @param backup_name        String containing the name of the backup to upload
     # @param local_backups_path String containing the path to the directory containing the backups
     # @param date_regex         Regex returning the date contained in the filename of each backup
+    # @param date_format        Format to be used by DateTime.strptime to parse the extracted date
     #
     # @return nothing
     #
-    def upload(backup_name, local_backups_path, date_regex=/\d{4}-\d{2}-\d{2}/)
-      @uploader.upload(backup_name, local_backups_path, date_regex)
+    def upload(backup_name, local_backups_path, date_regex=/\d{4}-\d{2}-\d{2}/, date_format="%Y-%m-%d")
+      @uploader.upload(backup_name, local_backups_path, date_regex, date_format)
     end
 
     #

data/lib/s3_rotate/core/backup_rotator.rb

@@ -1,5 +1,6 @@
 # s3_rotate
 require 's3_rotate/utils/file_utils'
+require 's3_rotate/utils/logging'
 
 module S3Rotate
 
@@ -9,6 +10,9 @@ module S3Rotate
   #
   class BackupRotator
 
+    # logger
+    include Logging
+
     # attributes
     attr_accessor :s3_client
 
@@ -69,13 +73,18 @@ module S3Rotate
       daily_backups.each do |backup|
         # promote to weekly if applicable
         if should_promote_daily_to_weekly?(backup.key, recent_weekly_file)
-          recent_weekly_file = promote(backup_name, backup.key, backup.body, "weekly").key
+          recent_weekly_file = promote(backup_name, backup, "weekly").key
         end
       end
 
       # cleanup old files
       if daily_backups.length > max_daily
-        daily_backups.each_with_index { |backup, i| backup.destroy if i < daily_backups.length - max_daily }
+        daily_backups.each_with_index do |backup, i|
+          if i < daily_backups.length - max_daily
+            logger.info("removing #{backup.key}")
+            backup.destroy
+          end
+        end
       end
     end
 
@@ -106,13 +115,18 @@ module S3Rotate
       weekly_backups.each do |backup|
         # promote to monthly if applicable
         if should_promote_weekly_to_monthly?(backup.key, recent_monthly_file)
-          recent_monthly_file = promote(backup_name, backup.key, backup.body, "monthly").key
+          recent_monthly_file = promote(backup_name, backup, "monthly").key
         end
       end
 
       # cleanup old files
       if weekly_backups.length > max_weekly
-        weekly_backups.each_with_index { |backup, i| backup.destroy if i < weekly_backups.length - max_weekly }
+        weekly_backups.each_with_index do |backup, i|
+          if i < weekly_backups.length - max_weekly
+            logger.info("removing #{backup.key}")
+            backup.destroy
+          end
+        end
       end
     end
 
@@ -132,7 +146,12 @@ module S3Rotate
 
       # cleanup old files
       if monthly_backups.length > max_monthly
-        monthly_backups.each_with_index { |backup, i| backup.destroy if i < monthly_backups.length - max_monthly }
+        monthly_backups.each_with_index do |backup, i|
+          if i < monthly_backups.length - max_monthly
+            logger.info("removing #{backup.key}")
+            backup.destroy
+          end
+        end
       end
     end
 
@@ -152,7 +171,10 @@ module S3Rotate
 
       # cleanup old files
       if local_backups.length > max_local
-        local_backups[0..(local_backups.length - max_local - 1)].each { |backup| File.delete("#{local_backups_path}/#{backup}") }
+        local_backups[0..(local_backups.length - max_local - 1)].each do |backup|
+          logger.info("removing #{local_backups_path}/#{backup}")
+          File.delete("#{local_backups_path}/#{backup}")
+        end
       end
     end
 
@@ -182,7 +204,7 @@ module S3Rotate
       end
 
       # perform date comparison
-      return (date_daily_file - date_weekly_file).abs >= 7
+      return date_daily_file - date_weekly_file >= 7
     end
 
     #
@@ -215,23 +237,17 @@ module S3Rotate
     end
 
     #
-    # Promote a daily backup into a weekly backup
+    # Promote a backup into a different type of backup backup (for example, daily into weekly)
     # This operation keeps the original daily file, and creates a new weekly backup
     #
     # @param backup_name   String containing the name of the backup being updated
-    # @param filename      String, filename of the backup you want to promote
-    # @param body          String, body of the file you want to promote
-    # @param type          String representing the type of backup being uploaded, one of "daily", "weekly" or "monthly"
+    # @param file          S3 File, file to be promoted
+    # @param type          String representing the type the backup is being promoted into, one of "daily", "weekly" or "monthly"
     #
     # @return created S3 Bucket File
     #
-    def promote(backup_name, filename, body, type)
-      # parse the date & extension
-      backup_date      = FileUtils::date_from_filename(filename)
-      backup_extension = FileUtils::extension_from_filename(filename)
-
-      # upload
-      @s3_client.upload(backup_name, backup_date, type, backup_extension, body)
+    def promote(backup_name, file, type)
+      @s3_client.copy(backup_name, file, type)
     end
 
   end

data/lib/s3_rotate/core/backup_uploader.rb

@@ -31,17 +31,18 @@ module S3Rotate
     # @param backup_name        String containing the name of the backup to upload
     # @param local_backups_path String containing the path to the directory containing the backups
     # @param date_regex         Regex returning the date contained in the filename of each backup
+    # @param date_format        Format to be used by DateTime.strptime to parse the extracted date
     #
     # @return nothing
     #
-    def upload(backup_name, local_backups_path, date_regex=/\d{4}-\d{2}-\d{2}/)
+    def upload(backup_name, local_backups_path, date_regex=/\d{4}-\d{2}-\d{2}/, date_format="%Y-%m-%d")
       # get backup files
       local_backups = FileUtils::files_in_directory(local_backups_path).reverse
 
       # upload local backups until we find one backup already uploaded
       local_backups.each do |local_backup|
         # parse the date & extension
-        backup_date      = FileUtils::date_from_filename(local_backup, date_regex)
+        backup_date      = FileUtils::date_from_filename(local_backup, date_regex, date_format)
         backup_extension = FileUtils::extension_from_filename(local_backup)
 
         # skip invalid files
@@ -51,7 +52,7 @@ module S3Rotate
         break if @s3_client.exists?(backup_name, backup_date, "daily", extension=backup_extension)
 
         # upload file
-        @s3_client.upload_local_backup_to_s3(backup_name, backup_date, "daily", backup_extension, File.open(local_backup))
+        @s3_client.upload(backup_name, backup_date, "daily", backup_extension, File.open("#{local_backups_path}/#{local_backup}"))
       end
     end
 

data/lib/s3_rotate/utils/file_utils.rb

@@ -10,10 +10,11 @@ module S3Rotate
     #
     # @param filename     String containing the filename to be parsed.
     # @param date_regex   Regex returning the date contained in the filename
+    # @param date_format  Format to be used by DateTime.strptime to parse the extracted date
     #
     # @return Date instance, representing the parsed date
     #
-    def FileUtils.date_from_filename(filename, date_regex=/\d{4}-\d{2}-\d{2}/)
+    def FileUtils.date_from_filename(filename, date_regex=/\d{4}-\d{2}-\d{2}/, date_format="%Y-%m-%d")
       # match the date in the filename
       match = filename.match(date_regex)
 
@@ -30,14 +31,9 @@ module S3Rotate
 
       # regular date
       begin
-        if date_str.include?("-")
-          Date.parse(date_str)
-        # timestamp
-        else
-          DateTime.strptime(date_str, "%s").to_date
-        end
+        DateTime.strptime(date_str, date_format).to_date
       rescue
-        raise "Date format not supported"
+        raise "Invalid date_format"
       end
     end
 

data/lib/s3_rotate/utils/logging.rb

@@ -0,0 +1,19 @@
+module S3Rotate
+
+  module Logging
+
+    def logger
+      Logging.logger
+    end
+
+    def self.logger
+      @logger ||= Logger.new(STDOUT)
+    end
+
+    def self.level(level)
+      logger.level = level
+    end
+
+  end
+
+end

data/s3_rotate.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |s|
   s.homepage    = 'https://github.com/Whova/s3_rotate'
   s.license     = 'MIT'
 
-  s.version     = '1.0.1'
+  s.version     = '1.1.0'
   s.date        = Date.today.to_s
 
   s.authors     = ["Simon Ninon"]

data/spec/s3_rotate/aws/s3_client_spec.rb

@@ -125,4 +125,25 @@ describe S3Rotate::S3Client do
 
   end
 
+  describe '#copy' do
+
+    it 'copies backup' do
+      # mock data
+      file = @client.connection.directories.get('bucket').files.create(key: '/backup_name/daily/2020-01-12.tgz', body: 'some data')
+
+      # perform test
+      @client.copy('backup_name', file, 'weekly')
+
+      # verify result
+      expect(@client.connection.directories.get('bucket', prefix: '/backup_name/weekly').files.length).to eq 1
+      expect(@client.connection.directories.get('bucket', prefix: '/backup_name/weekly').files[0].key).to eq '/backup_name/weekly/2020-01-12.tgz'
+      expect(@client.connection.directories.get('bucket', prefix: '/backup_name/weekly').files[0].body).to eq 'some data'
+
+      expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily').files.length).to eq 1
+      expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily').files[0].key).to eq '/backup_name/daily/2020-01-12.tgz'
+      expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily').files[0].body).to eq 'some data'
+    end
+
+  end
+
 end

data/spec/s3_rotate/core/backup_manager_spec.rb

@@ -33,10 +33,10 @@ describe S3Rotate::BackupManager do
 
     it 'calls uploader.upload' do
       # mock
-      allow(@manager.uploader).to receive(:upload).with('backup_name', '/path/to/dir', /\d{4}-\d{2}-\d{2}/).and_return('upload_result')
+      allow(@manager.uploader).to receive(:upload).with('backup_name', '/path/to/dir', /\d{4}-\d{2}-\d{2}/, "%Y-%m-%d").and_return('upload_result')
 
       # perform test
-      expect(@manager.upload('backup_name', '/path/to/dir', /\d{4}-\d{2}-\d{2}/)).to eq 'upload_result'
+      expect(@manager.upload('backup_name', '/path/to/dir', /\d{4}-\d{2}-\d{2}/, "%Y-%m-%d")).to eq 'upload_result'
     end
 
   end

data/spec/s3_rotate/core/backup_rotator_spec.rb

@@ -748,24 +748,21 @@ describe S3Rotate::BackupRotator do
 
   describe '#promote' do
 
-    it 'promotes backup with extension' do
-      # perform test
-      @rotator.promote('backup_name', '/gitlab/daily/2020-01-02.tgz', 'some data', 'weekly')
-
-      # verify result
-      expect(@client.connection.directories.get('bucket', prefix: '/backup_name/weekly').files.length).to eq 1
-      expect(@client.connection.directories.get('bucket', prefix: '/backup_name/weekly').files[0].key).to eq '/backup_name/weekly/2020-01-02.tgz'
-      expect(@client.connection.directories.get('bucket', prefix: '/backup_name/weekly').files[0].body).to eq 'some data'
-    end
+    it 'promotes backup' do
+      # mock data
+      file = @client.connection.directories.get('bucket').files.create(key: '/backup_name/daily/2020-01-12.tgz', body: 'some data')
 
-    it 'promotes backup without extension' do
       # perform test
-      @rotator.promote('backup_name', '/gitlab/daily/2020-01-02', 'some data', 'weekly')
+      @rotator.promote('backup_name', file, 'weekly')
 
       # verify result
       expect(@client.connection.directories.get('bucket', prefix: '/backup_name/weekly').files.length).to eq 1
-      expect(@client.connection.directories.get('bucket', prefix: '/backup_name/weekly').files[0].key).to eq '/backup_name/weekly/2020-01-02'
+      expect(@client.connection.directories.get('bucket', prefix: '/backup_name/weekly').files[0].key).to eq '/backup_name/weekly/2020-01-12.tgz'
       expect(@client.connection.directories.get('bucket', prefix: '/backup_name/weekly').files[0].body).to eq 'some data'
+
+      expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily').files.length).to eq 1
+      expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily').files[0].key).to eq '/backup_name/daily/2020-01-12.tgz'
+      expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily').files[0].body).to eq 'some data'
     end
 
   end

data/spec/s3_rotate/core/backup_uploader_spec.rb

@@ -45,23 +45,23 @@ describe S3Rotate::BackupUploader do
         'some-backup-2020-01-06.tgz',
         'some-backup-2020-01-07.tgz'
       ])
-      allow(@client).to receive(:upload_local_backup_to_s3).and_return nil
+      allow(@client).to receive(:upload).and_return nil
       allow(File).to receive(:open).and_return "raw_data"
 
       # perform test
       @uploader.upload('backup_name', '/path/to/dir')
 
       # verify result
-      expect(@client).to have_received(:upload_local_backup_to_s3).exactly(3)
-      expect(@client).to have_received(:upload_local_backup_to_s3).with('backup_name', Date.new(2020, 1, 7), 'daily', '.tgz', 'raw_data')
-      expect(@client).to have_received(:upload_local_backup_to_s3).with('backup_name', Date.new(2020, 1, 6), 'daily', '.tgz', 'raw_data')
-      expect(@client).to have_received(:upload_local_backup_to_s3).with('backup_name', Date.new(2020, 1, 5), 'daily', '.tgz', 'raw_data')
+      expect(@client).to have_received(:upload).exactly(3)
+      expect(@client).to have_received(:upload).with('backup_name', Date.new(2020, 1, 7), 'daily', '.tgz', 'raw_data')
+      expect(@client).to have_received(:upload).with('backup_name', Date.new(2020, 1, 6), 'daily', '.tgz', 'raw_data')
+      expect(@client).to have_received(:upload).with('backup_name', Date.new(2020, 1, 5), 'daily', '.tgz', 'raw_data')
 
       expect(S3Rotate::FileUtils).to have_received(:date_from_filename).exactly(4)
-      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-07.tgz', /\d{4}-\d{2}-\d{2}/)
-      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-06.tgz', /\d{4}-\d{2}-\d{2}/)
-      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-05.tgz', /\d{4}-\d{2}-\d{2}/)
-      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-04.tgz', /\d{4}-\d{2}-\d{2}/)
+      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-07.tgz', /\d{4}-\d{2}-\d{2}/, "%Y-%m-%d")
+      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-06.tgz', /\d{4}-\d{2}-\d{2}/, "%Y-%m-%d")
+      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-05.tgz', /\d{4}-\d{2}-\d{2}/, "%Y-%m-%d")
+      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-04.tgz', /\d{4}-\d{2}-\d{2}/, "%Y-%m-%d")
 
       expect(S3Rotate::FileUtils).to have_received(:extension_from_filename).exactly(4)
       expect(S3Rotate::FileUtils).to have_received(:extension_from_filename).with('some-backup-2020-01-07.tgz')
@@ -79,22 +79,22 @@ describe S3Rotate::BackupUploader do
         'some-backup-2020-01-02.tgz',
         'some-backup-2020-01-03.tgz',
       ])
-      allow(@client).to receive(:upload_local_backup_to_s3).and_return nil
+      allow(@client).to receive(:upload).and_return nil
       allow(File).to receive(:open).and_return "raw_data"
 
       # perform test
       @uploader.upload('backup_name', '/path/to/dir')
 
       # verify result
-      expect(@client).to have_received(:upload_local_backup_to_s3).exactly(3)
-      expect(@client).to have_received(:upload_local_backup_to_s3).with('backup_name', Date.new(2020, 1, 3), 'daily', nil, 'raw_data')
-      expect(@client).to have_received(:upload_local_backup_to_s3).with('backup_name', Date.new(2020, 1, 2), 'daily', nil, 'raw_data')
-      expect(@client).to have_received(:upload_local_backup_to_s3).with('backup_name', Date.new(2020, 1, 1), 'daily', nil, 'raw_data')
+      expect(@client).to have_received(:upload).exactly(3)
+      expect(@client).to have_received(:upload).with('backup_name', Date.new(2020, 1, 3), 'daily', nil, 'raw_data')
+      expect(@client).to have_received(:upload).with('backup_name', Date.new(2020, 1, 2), 'daily', nil, 'raw_data')
+      expect(@client).to have_received(:upload).with('backup_name', Date.new(2020, 1, 1), 'daily', nil, 'raw_data')
 
       expect(S3Rotate::FileUtils).to have_received(:date_from_filename).exactly(3)
-      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-03.tgz', /\d{4}-\d{2}-\d{2}/)
-      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-02.tgz', /\d{4}-\d{2}-\d{2}/)
-      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-01.tgz', /\d{4}-\d{2}-\d{2}/)
+      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-03.tgz', /\d{4}-\d{2}-\d{2}/, "%Y-%m-%d")
+      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-02.tgz', /\d{4}-\d{2}-\d{2}/, "%Y-%m-%d")
+      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-01.tgz', /\d{4}-\d{2}-\d{2}/, "%Y-%m-%d")
 
       expect(S3Rotate::FileUtils).to have_received(:extension_from_filename).exactly(3)
       expect(S3Rotate::FileUtils).to have_received(:extension_from_filename).with('some-backup-2020-01-03.tgz')
@@ -118,21 +118,21 @@ describe S3Rotate::BackupUploader do
         'some-backup-2020-01-02.tgz',
         'some-backup-2020-01-03.tgz',
       ])
-      allow(@client).to receive(:upload_local_backup_to_s3).and_return nil
+      allow(@client).to receive(:upload).and_return nil
       allow(File).to receive(:open).and_return "raw_data"
 
       # perform test
       @uploader.upload('backup_name', '/path/to/dir')
 
       # verify result
-      expect(@client).to have_received(:upload_local_backup_to_s3).exactly(2)
-      expect(@client).to have_received(:upload_local_backup_to_s3).with('backup_name', Date.new(2020, 1, 3), 'daily', '.tgz', 'raw_data')
-      expect(@client).to have_received(:upload_local_backup_to_s3).with('backup_name', Date.new(2020, 1, 1), 'daily', '.tgz', 'raw_data')
+      expect(@client).to have_received(:upload).exactly(2)
+      expect(@client).to have_received(:upload).with('backup_name', Date.new(2020, 1, 3), 'daily', '.tgz', 'raw_data')
+      expect(@client).to have_received(:upload).with('backup_name', Date.new(2020, 1, 1), 'daily', '.tgz', 'raw_data')
 
       expect(S3Rotate::FileUtils).to have_received(:date_from_filename).exactly(3)
-      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-03.tgz', /\d{4}-\d{2}-\d{2}/)
-      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-02.tgz', /\d{4}-\d{2}-\d{2}/)
-      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-01.tgz', /\d{4}-\d{2}-\d{2}/)
+      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-03.tgz', /\d{4}-\d{2}-\d{2}/, "%Y-%m-%d")
+      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-02.tgz', /\d{4}-\d{2}-\d{2}/, "%Y-%m-%d")
+      expect(S3Rotate::FileUtils).to have_received(:date_from_filename).with('some-backup-2020-01-01.tgz', /\d{4}-\d{2}-\d{2}/, "%Y-%m-%d")
 
       expect(S3Rotate::FileUtils).to have_received(:extension_from_filename).exactly(3)
       expect(S3Rotate::FileUtils).to have_received(:extension_from_filename).with('some-backup-2020-01-03.tgz')

data/spec/s3_rotate/utils/file_utils_spec.rb

@@ -13,15 +13,15 @@ describe S3Rotate::FileUtils do
     end
 
     it 'parses timestamp formats' do
-      expect(S3Rotate::FileUtils::date_from_filename("/path/to/file-1580098737-backup.tar.gz", /file-(\d+)-backup.tar.gz/)).to eq Date.new(2020, 1, 27)
+      expect(S3Rotate::FileUtils::date_from_filename("/path/to/file-1580098737-backup.tar.gz", /file-(\d+)-backup.tar.gz/, "%s")).to eq Date.new(2020, 1, 27)
     end
 
     it 'raises if the regex matched nothing' do
-      expect{ S3Rotate::FileUtils::date_from_filename("/path/to/file-1580098737-backup.tar.gz") }.to raise_error(RuntimeError, "Invalid date_regex or filename format")
+      expect{ S3Rotate::FileUtils::date_from_filename("/path/to/file-1580098737-backup.tar.gz", /\d{4}-\d{2}-\d{2}/, "%s") }.to raise_error(RuntimeError, "Invalid date_regex or filename format")
     end
 
-    it 'raises if the regex matched nothing' do
-      expect{ S3Rotate::FileUtils::date_from_filename("/path/to/file-1580098737-backup.tar.gz", /file-\d+-backup.tar.gz/) }.to raise_error(RuntimeError, "Date format not supported")
+    it 'raises if the matched string can not be parsed' do
+      expect{ S3Rotate::FileUtils::date_from_filename("/path/to/file-1580098737-backup.tar.gz", /file-\d+-backup.tar.gz/, "%s") }.to raise_error(RuntimeError, "Invalid date_format")
     end
 
   end

data/spec/spec_helper.rb

@@ -1,3 +1,8 @@
+require 'logger'
+require 'fog-aws'
+
+require File.expand_path("../../lib/s3_rotate/utils/logging", __FILE__)
+
 RSpec.configure do |c|
 
   c.before :each do
@@ -5,6 +10,8 @@ RSpec.configure do |c|
 
     fog = Fog::Storage.new(aws_access_key_id: 'key', aws_secret_access_key: 'secret', provider: 'AWS', region: 'region')
     fog.directories.create(key: 'bucket')
+
+    S3Rotate::Logging::level Logger::ERROR
   end
 
   c.after :each do

metadata

@@ -1,27 +1,27 @@
 --- !ruby/object:Gem::Specification
 name: s3_rotate
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.0
 platform: ruby
 authors:
 - Simon Ninon
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-28 00:00:00.000000000 Z
+date: 2020-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog-aws
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 3.5.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 3.5.2
 description: AWS S3 upload with rotation mechanism
@@ -30,8 +30,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .rspec
+- ".gitignore"
+- ".rspec"
 - Gemfile
 - LICENSE
 - README.md
@@ -42,6 +42,7 @@ files:
 - lib/s3_rotate/core/backup_rotator.rb
 - lib/s3_rotate/core/backup_uploader.rb
 - lib/s3_rotate/utils/file_utils.rb
+- lib/s3_rotate/utils/logging.rb
 - s3_rotate.gemspec
 - spec/s3_rotate/aws/s3_client_spec.rb
 - spec/s3_rotate/core/backup_manager_spec.rb
@@ -63,17 +64,16 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: 2.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.10
+rubygems_version: 3.0.1
 signing_key: 
 specification_version: 4
 summary: AWS S3 upload with rotation mechanism