s3_rotate 1.0.0

data/lib/s3_rotate/core/backup_manager.rb

@@ -0,0 +1,62 @@
+require 's3_rotate/core/backup_uploader'
+require 's3_rotate/core/backup_rotator'
+
+module S3Rotate
+
+  class BackupManager
+
+    # attributes
+    attr_accessor :s3_client
+    attr_accessor :uploader
+    attr_accessor :rotator
+
+    #
+    # Initialize a new BackupManager instance.
+    #
+    # @param key        String representing the AWS ACCESS KEY ID.
+    # @param secret     String representing the AWS ACCESS KEY SECRET.
+    # @param bucket     String representing the name of the bucket ot use.
+    # @param region     String representing the region to conect to.
+    #
+    # @return the newly instanciated object.
+    #
+    def initialize(key, secret, bucket, region)
+      @s3_client = S3Client.new(key, secret, bucket, region)
+      @uploader  = BackupUploader.new(@s3_client)
+      @rotator   = BackupRotator.new(@s3_client)
+    end
+
+    #
+    # Upload local backup files to AWS S3
+    # Only uploads new backups
+    # Only uploads backups as daily backups: use `rotate` to generate the weekly & monthly files
+    #
+    # @param backup_name        String containing the name of the backup to upload
+    # @param local_backups_path String containing the path to the directory containing the backups
+    # @param date_regex         Regex returning the date contained in the filename of each backup
+    #
+    # @return nothing
+    #
+    def upload(backup_name, local_backups_path, date_regex=/\d{4}-\d{2}-\d{2}/)
+      @uploader.upload(backup_name, local_backups_path, date_regex)
+    end
+
+    #
+    # Rotate files (local, daily, weekly, monthly) and apply maximum limits for each type
+    #
+    # @param backup_name        String containing the name of the backup to rotate
+    # @param local_backups_path String containing the path to the directory containing the backups
+    # @param max_local          Integer specifying the maximum number of local backups to keep
+    # @param max_daily          Integer specifying the maximum number of daily backups to keep
+    # @param max_weekly         Integer specifying the maximum number of weekly backups to keep
+    # @param max_monthly        Integer specifying the maximum number of monthly backups to keep
+    #
+    # @return nothing
+    #
+    def rotate(backup_name, local_backups_dir, max_local=3, max_daily=7, max_weekly=4, max_monthly=3)
+      @rotator.rotate(backup_name, local_backups_dir, max_local, max_daily, max_weekly, max_monthly)
+    end
+
+  end
+
+end

data/lib/s3_rotate/core/backup_rotator.rb

@@ -0,0 +1,239 @@
+# s3_rotate
+require 's3_rotate/utils/file_utils'
+
+module S3Rotate
+
+  #
+  # BackupRotator Class
+  # Handles backup rotation locally and on S3
+  #
+  class BackupRotator
+
+    # attributes
+    attr_accessor :s3_client
+
+    #
+    # Initialize a new BackupRotator instance.
+    #
+    # @param s3_client    S3Client instance
+    #
+    # @return the newly instanciated object.
+    #
+    def initialize(s3_client)
+      @s3_client = s3_client
+    end
+
+    #
+    # Rotate files (local, daily, weekly, monthly) and apply maximum limits for each type
+    #
+    # @param backup_name        String containing the name of the backup to rotate
+    # @param local_backups_path String containing the path to the directory containing the backups
+    # @param max_local          Integer specifying the maximum number of local backups to keep
+    # @param max_daily          Integer specifying the maximum number of daily backups to keep
+    # @param max_weekly         Integer specifying the maximum number of weekly backups to keep
+    # @param max_monthly        Integer specifying the maximum number of monthly backups to keep
+    #
+    # @return nothing
+    #
+    def rotate(backup_name, local_backups_dir, max_local=3, max_daily=7, max_weekly=4, max_monthly=3)
+      rotate_local(local_backups_dir, max_local)
+      rotate_daily(backup_name, max_daily)
+      rotate_weekly(backup_name, max_weekly)
+      rotate_monthly(backup_name, max_monthly)
+    end
+
+    #
+    # Rotate daily files
+    #
+    # @param backup_name   String containing the name of the backup being rotated
+    # @param max_daily     Integer specifying the maximum number of daily backups to keep
+    #   - If there are less than `max_daily` daily files: do nothing
+    #   - If there are more than `max_daily` daily files: delete the oldest files to leave `max_daily` files
+    #
+    # The rotation works as follows:
+    #   - Less than 7 days datediff between the oldest daily file and the most recent weekly file: do nothing
+    #   - More than 7 days datediff between the oldest daily file and the most recent weekly file: promote the oldest daily file to weekly file
+    #   - In both cases, apply the `max_daily`
+    #
+    # @return nothing
+    #
+    def rotate_daily(backup_name, max_daily)
+      # get backup files
+      daily_backups  = @s3_client.remote_backups(backup_name, "daily").files
+      weekly_backups = @s3_client.remote_backups(backup_name, "weekly").files
+
+      # get most recent weekly file
+      recent_weekly_file = weekly_backups.last
+
+      # look through daily backups to find which oness should be promoted
+      daily_backups.each do |backup|
+        # promote to weekly if applicable
+        if should_promote_daily_to_weekly?(backup.key, recent_weekly_file&.key)
+          recent_weekly_file = promote(backup_name, backup.key, backup.body, "weekly")
+        end
+      end
+
+      # cleanup old files
+      if daily_backups.length > max_daily
+        daily_backups.each_with_index { |backup, i| backup.destroy if i < daily_backups.length - max_daily }
+      end
+    end
+
+    #
+    # Rotate weekly files
+    #
+    # @param backup_name   String containing the name of the backup being rotated
+    # @param max_weekly    Integer specifying the maximum number of weekly backups to keep
+    #   - If there are less than `max_weekly` weekly files: do nothing
+    #   - If there are more than `max_weekly` weekly files: delete the oldest files to leave `max_weekly` files
+    #
+    # The rotation works as follows:
+    #   - Less than 1 month datediff between the oldest weekly file and the most recent monthly file: do nothing
+    #   - More than 1 month datediff between the oldest weekly file and the most recent monthly file: promote the oldest daily file to weekly file
+    #   - In both cases, apply the `max_weekly`
+    #
+    # @return nothing
+    #
+    def rotate_weekly(backup_name, max_weekly)
+      # get backup files
+      weekly_backups  = @s3_client.remote_backups(backup_name, "weekly").files
+      monthly_backups = @s3_client.remote_backups(backup_name, "monthly").files
+
+      # get most recent monthly file
+      recent_monthly_file = monthly_backups.last
+
+      # look through weekly backups to find which oness should be promoted
+      weekly_backups.each do |backup|
+        # promote to monthly if applicable
+        if should_promote_weekly_to_monthly?(backup.key, recent_monthly_file&.key)
+          recent_monthly_file = promote(backup_name, backup.key, backup.body, "monthly")
+        end
+      end
+
+      # cleanup old files
+      if weekly_backups.length > max_weekly
+        weekly_backups.each_with_index { |backup, i| backup.destroy if i < weekly_backups.length - max_weekly }
+      end
+    end
+
+    #
+    # Rotate monthly files
+    #
+    # @param backup_name   String containing the name of the backup being rotated
+    # @param max_monthly   Integer specifying the maximum number of month backups to keep
+    #   - If there are less than `max_monthly` monthly files: do nothing
+    #   - If there are more than `max_monthly` monthly files: delete the oldest files to leave `max_monthly` files
+    #
+    # @return nothing
+    #
+    def rotate_monthly(backup_name, max_monthly)
+      # get backup files
+      monthly_backups = @s3_client.remote_backups(backup_name, "monthly").files
+
+      # cleanup old files
+      if monthly_backups.length > max_monthly
+        monthly_backups.each_with_index { |backup, i| backup.destroy if i < monthly_backups.length - max_monthly }
+      end
+    end
+
+    #
+    # Rotate local files
+    #
+    # @param local_backups_path   String containing the path to the directory containing the backups
+    # @param max_local            Integer specifying the maximum number of local backups to keep
+    #   - If there are less than `max_local` local files: do nothing
+    #   - If there are more than `max_local` local files: delete the oldest files to leave `max_local` files
+    #
+    # @return nothing
+    #
+    def rotate_local(local_backups_path, max_local)
+      # get backup files
+      local_backups = FileUtils::files_in_directory(local_backups_path)
+
+      # cleanup old files
+      if local_backups.length > max_local
+        local_backups[0..(local_backups.length - max_local - 1)].each { |backup| File.delete("#{local_backups_path}/#{backup}") }
+      end
+    end
+
+    #
+    # Check whether `daily_file` should be promoted into a weekly file
+    # Only promote a daily file if the most recent weekly backup is one week old
+    #
+    # @param daily_file     String, filename of the daily backup to be checked for promotion
+    # @param weekly_file    String, filename of the most recent weekly backup
+    #
+    # @return Boolean, True or False, whether the file should be promoted
+    #
+    def should_promote_daily_to_weekly?(daily_file, weekly_file)
+      # never promote if no daily file
+      return false if not daily_file
+
+      # always promote if no weekly file
+      return true if not weekly_file
+
+      # retrieve the date of each file
+      begin
+        date_daily_file  = FileUtils::date_from_filename(daily_file)
+        date_weekly_file = FileUtils::date_from_filename(weekly_file)
+      rescue
+        print "Wrong date (Date.parse in should_promote_daily_to_weekly)."
+        return false
+      end
+
+      # perform date comparison
+      return (date_daily_file - date_weekly_file).abs >= 7
+    end
+
+    #
+    # Check whether `weekly_file` should be promoted into a monthly file
+    # Only promote a weekly file if the most recent monthly backup is one month old
+    #
+    # @param weekly_file    String, filename of the weekly backup to be checked for promotion
+    # @param monthly_file   String, filename of the most recent monthly backup
+    #
+    # @return Boolean, True or False, whether the file should be promoted
+    #
+    def should_promote_weekly_to_monthly?(weekly_file, monthly_file)
+      # never promote if no weekly file
+      return false if not weekly_file
+
+      # always promote if no monthly file
+      return true if not monthly_file
+
+      # retrieve the date of each file
+      begin
+        date_weekly_file  = FileUtils::date_from_filename(weekly_file)
+        date_monthly_file = FileUtils::date_from_filename(monthly_file)
+      rescue
+        print "Wrong date (Date.parse in should_promote_weekly_to_monthly)."
+        return false
+      end
+
+      # perform date comparison
+      return date_weekly_file.prev_month >= date_monthly_file
+    end
+
+    #
+    # Promote a daily backup into a weekly backup
+    # This operation keeps the original daily file, and creates a new weekly backup
+    #
+    # @param backup_name   String containing the name of the backup being updated
+    # @param filename      String, filename of the backup you want to promote
+    # @param body          String, body of the file you want to promote
+    # @param type          String representing the type of backup being uploaded, one of "daily", "weekly" or "monthly"
+    #
+    # @return created S3 Bucket File
+    #
+    def promote(backup_name, filename, body, type)
+      # parse the date & extension
+      backup_date      = FileUtils::date_from_filename(filename)
+      backup_extension = FileUtils::extension_from_filename(filename)
+
+      # upload
+      @s3_client.upload(backup_name, backup_date, type, backup_extension, body)
+    end
+
+  end
+
+end

data/lib/s3_rotate/core/backup_uploader.rb

@@ -0,0 +1,60 @@
+# s3_rotate
+require 's3_rotate/utils/file_utils'
+
+module S3Rotate
+
+  #
+  # BackupUploader Class
+  # Handles backup uploads with the right format
+  #
+  class BackupUploader
+
+    # attributes
+    attr_accessor :s3_client
+
+    #
+    # Initialize a new BackupUploader instance.
+    #
+    # @param s3_client    S3Client instance
+    #
+    # @return the newly instanciated object.
+    #
+    def initialize(s3_client)
+      @s3_client = s3_client
+    end
+
+    #
+    # Upload local backup files to AWS S3
+    # Only uploads new backups
+    # Only uploads backups as daily backups: use `rotate` to generate the weekly & monthly files
+    #
+    # @param backup_name        String containing the name of the backup to upload
+    # @param local_backups_path String containing the path to the directory containing the backups
+    # @param date_regex         Regex returning the date contained in the filename of each backup
+    #
+    # @return nothing
+    #
+    def upload(backup_name, local_backups_path, date_regex=/\d{4}-\d{2}-\d{2}/)
+      # get backup files
+      local_backups = FileUtils::files_in_directory(local_backups_path).reverse
+
+      # upload local backups until we find one backup already uploaded
+      local_backups.each do |local_backup|
+        # parse the date & extension
+        backup_date      = FileUtils::date_from_filename(local_backup, date_regex)
+        backup_extension = FileUtils::extension_from_filename(local_backup)
+
+        # skip invalid files
+        next if not backup_date
+
+        # stop uploading once we reach a file already uploaded
+        break if @s3_client.exists?(backup_name, backup_date, "daily", extension=backup_extension)
+
+        # upload file
+        @s3_client.upload_local_backup_to_s3(backup_name, backup_date, "daily", backup_extension, File.open(local_backup))
+      end
+    end
+
+  end
+
+end

data/lib/s3_rotate/utils/file_utils.rb

@@ -0,0 +1,65 @@
+require 'date'
+
+module S3Rotate
+
+  module FileUtils
+
+    #
+    # Parse the date in a filename
+    # Date can be any format recognize by Date.parse, or be a timestamp
+    #
+    # @param filename     String containing the filename to be parsed.
+    # @param date_regex   Regex returning the date contained in the filename
+    #
+    # @return Date instance, representing the parsed date
+    #
+    def FileUtils.date_from_filename(filename, date_regex=/\d{4}-\d{2}-\d{2}/)
+      # match the date in the filename
+      match    = filename.match(date_regex)
+      date_str = match&.captures&.first || match&.to_s
+
+      # if nothing could be match, immediately fail
+      raise "Invalid date_regex or filename format" if not date_str
+
+      # regular date
+      begin
+        if date_str.include?("-")
+          Date.parse(date_str)
+        # timestamp
+        else
+          DateTime.strptime(date_str, "%s").to_date
+        end
+      rescue
+        raise "Date format not supported"
+      end
+    end
+
+    #
+    # Parse the extension in a filename
+    #
+    # @param filename     String containing the filename to be parsed
+    #
+    # @return String containing the extension of the filename if relevant, None otherwise
+    #
+    def FileUtils.extension_from_filename(filename)
+      if filename.include?('.')
+        '.' + filename.split('/').last.split('.')[1..-1].join('.')
+      end
+    end
+
+    #
+    # Get the list of files in the specified directory
+    #
+    # @param directory   String containing the path to the directory
+    #
+    # @return array of filenames, in ascending date order
+    #
+    def FileUtils.files_in_directory(directory)
+      Dir.entries(directory).select { |f| !File.directory? f }.sort
+    rescue
+      raise "Invalid directory #{directory}"
+    end
+
+  end
+
+end

data/s3_rotate.gemspec

@@ -0,0 +1,15 @@
+require 'date'
+
+Gem::Specification.new do |s|
+  s.name        = 's3_rotate'
+  s.version     = '1.0.0'
+  s.homepage    = 'https://github.com/Whova/s3_rotate'
+  s.date        = Date.today.to_s
+  s.summary     = "AWS S3 upload with rotation mechanism"
+  s.description = s.summary
+  s.authors     = ["Simon Ninon"]
+  s.email       = 'simon.ninon@gmail.com'
+  s.files       = `git ls-files`.split("\n")
+  s.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.license     = 'MIT'
+end

data/spec/s3_rotate/aws/s3_client_spec.rb

@@ -0,0 +1,128 @@
+# standard
+require 'date'
+
+# 3rd part
+require 'fog-aws'
+
+# s3_rotate
+require File.expand_path("../../../../lib/s3_rotate/aws/s3_client", __FILE__)
+
+describe S3Rotate::S3Client do
+
+  before :each do
+    @client = S3Rotate::S3Client.new('key', 'secret', 'bucket', 'region')
+  end
+
+  describe '#initialize' do
+
+    it 'sets the access_key' do
+      expect(@client.access_key).to eq 'key'
+    end
+
+    it 'sets the access_secret' do
+      expect(@client.access_secret).to eq 'secret'
+    end
+
+    it 'sets the bucket_name' do
+      expect(@client.bucket_name).to eq 'bucket'
+    end
+
+    it 'sets the region' do
+      expect(@client.region).to eq 'region'
+    end
+
+  end
+
+  describe '#connection' do
+
+    it 'sets the connection when unset' do
+      # mock
+      @client.connection = nil
+
+      # perform test
+      expect(@client.connection).not_to eq nil
+    end
+
+    it 'does not set the connection when set' do
+      # mock
+      @client.connection = "some connection"
+
+      # perform test
+      expect(@client.connection).to eq "some connection"
+    end
+
+  end
+
+  describe '#bucket' do
+
+    it 'gets the bucket' do
+      expect(@client.bucket).not_to eq nil
+      expect(@client.bucket.key).to eq 'bucket'
+    end
+
+  end
+
+  describe '#remote_backups' do
+
+    before do
+      @client.connection.directories.get('bucket').files.create(key: '/backup_name/daily/2020-01-01.tgz', body: 'some data')
+      @client.connection.directories.get('bucket').files.create(key: '/backup_name/daily/2020-01-02.tgz', body: 'some data')
+      @client.connection.directories.get('bucket').files.create(key: '/backup_name/daily/2020-02-03.tgz', body: 'some data')
+      @client.connection.directories.get('bucket').files.create(key: '/backup_name/daily/2021-02-04.tgz', body: 'some data')
+      @client.connection.directories.get('bucket').files.create(key: '/backup_name/weekly/2020-01-03.tgz', body: 'some data')
+      @client.connection.directories.get('bucket').files.create(key: '/backup_name/monthly/2020-01-04.tgz', body: 'some data')
+      @client.connection.directories.get('bucket').files.create(key: '/other_backup_name/daily/2020-01-05.tgz', body: 'some data')
+    end
+
+    it 'gets the remote backups' do
+      expect(@client.remote_backups('backup_name', 'daily')).not_to eq nil
+      expect(@client.remote_backups('backup_name', 'daily').files).not_to eq nil
+      expect(@client.remote_backups('backup_name', 'daily').files.length).to eq 4
+      expect(@client.remote_backups('backup_name', 'daily').files[0].key).to eq "/backup_name/daily/2020-01-01.tgz"
+      expect(@client.remote_backups('backup_name', 'daily').files[1].key).to eq "/backup_name/daily/2020-01-02.tgz"
+      expect(@client.remote_backups('backup_name', 'daily').files[2].key).to eq "/backup_name/daily/2020-02-03.tgz"
+      expect(@client.remote_backups('backup_name', 'daily').files[3].key).to eq "/backup_name/daily/2021-02-04.tgz"
+    end
+
+  end
+
+  describe '#exists?' do
+
+    before do
+      @client.connection.directories.get('bucket').files.create(key: '/backup_name/daily/2020-01-01.tgz', body: 'some data')
+    end
+
+    it 'returns true for existing backups' do
+      expect(@client.exists?('backup_name', Date.new(2020, 1, 1), 'daily', '.tgz')).to eq true
+    end
+
+    it 'returns false for wrong extension' do
+      expect(@client.exists?('backup_name', Date.new(2020, 1, 1), 'daily', '.tar.gz')).to eq false
+    end
+
+    it 'returns false for wrong type' do
+      expect(@client.exists?('backup_name', Date.new(2020, 1, 1), 'weekly', '.tgz')).to eq false
+    end
+
+    it 'returns false for wrong date' do
+      expect(@client.exists?('backup_name', Date.new(2020, 1, 2), 'daily', '.tgz')).to eq false
+    end
+
+    it 'returns false for backup name' do
+      expect(@client.exists?('other_backup_name', Date.new(2020, 1, 1), 'daily', '.tgz')).to eq false
+    end
+
+  end
+
+  describe '#upload' do
+
+    it 'uploads files' do
+      @client.upload('backup_name', Date.new(2020, 1, 1), 'daily', '.tgz', 'hello world')
+      expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily/2020-01-01.tgz').files.length).to eq 1
+      expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily/2020-01-01.tgz').files.first.key).to eq '/backup_name/daily/2020-01-01.tgz'
+      expect(@client.connection.directories.get('bucket', prefix: '/backup_name/daily/2020-01-01.tgz').files.first.body).to eq 'hello world'
+    end
+
+  end
+
+end