scalingo_backups_manager 0.4.0 → 0.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0c13c0008cbcd861729fbbc5cb43a3b53ed63db136a2b24393b077c2fbb6b50e
-  data.tar.gz: d93398d512fe1d0e9fdf964e431fc5c570c3201c5475e51174d3e7f297ef02c8
+  metadata.gz: 7be1ca03d45192dc394492ef8fb5c0a8b852ca08b4f96bbc4469960fda01e2f6
+  data.tar.gz: 721b03b36837bff777bbc9e79fdc85e188fa03524fac0405c92844209ba2fa59
 SHA512:
-  metadata.gz: 26cc222b56f4199eb7a1bd06f1c9e787d53a5f3a2a979836f991fc368369252c53cfeb7cbbd457882e4bce32aa07f09dcf8813a5d73a9007de4e958bfefe94fd
-  data.tar.gz: 0577310f2c5cceef264cd75789572577b7c218ff536cdb878a2ab40e6190f9e5d6b4e429587e596e7f6c71b7f1b787f431dbe4c0d7eee6a04c94a795fcd72a3f
+  metadata.gz: 4015ae6b481e82b6a565caa4db86738df7a819875241e405cc2ca742874ef8d09d3f68e50d13459a7cb341c07b4813aed998978a950afbafb6b97f6a4c115732
+  data.tar.gz: 405ef4f1c59acd60099b9a5bd71246e906d181d86399ba998b4290f5dce976210ffe02b420aebb4f0a514975de13674aa007e54050efa60650e6e7a52a14c724

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    scalingo_backups_manager (0.3.0)
+    scalingo_backups_manager (0.6.0)
       httparty (~> 0.18)
       net-sftp (~> 3.0.0)
       scalingo (~> 3.0)

data/lib/scalingo_backups_manager/cli.rb

@@ -128,40 +128,62 @@ module ScalingoBackupsManager
     def upload_to_ftp
       invoke :download, [], application: options[:application], addon: options[:addon]
       configuration = Configuration.new
+      opts = {
+        webhooks: configuration.config[:webhooks]
+      }
       configuration.for_each_addons do |application, addon|
-        config = addon.sftp_config
+        step = 1
+        sftp_config = addon.sftp_config
         path = ("#{addon.config[:path]}" || "backups/#{addon.addon_provider[:id]}") + "/#{Time.now.strftime("%Y%m%d")}.tar.gz"
         next unless File.exists?(path)
+        puts "** Upload backup for #{application.name} **"
 
-        sftp = ScalingoBackupsManager::SftpTools.new(config[:auth])
+        sftp = ScalingoBackupsManager::SftpTools.new(sftp_config[:auth])
 
         folders = [
-          config.dig(:auth, :dir),
-          config.dig(:dir) || application.name,
+          sftp_config.dig(:auth, :dir),
+          sftp_config.dig(:dir) || application.name,
           addon.addon_provider[:id]
         ]
 
-        if config[:retention].blank?
+        if sftp_config[:retention].blank?
           remote_path = "/" + [folders].delete_if(&:blank?).join("/")
           sftp.mkdir!(remote_path)
-          sftp.upload_file(path, remote_path)
+          sftp.upload_file(path, remote_path, options: opts)
           next
         end
 
-        config[:retention].each do |k, v|
-          folders << config.dig(:retention, k, :dir)
-          remote_path = "/" + [folders].delete_if(&:blank?).join("/")
+        sftp_config[:retention].each do |k, retention_config|
+          retention_folders = folders.dup
+          retention_folders << sftp_config.dig(:retention, k, :dir)
+          remote_path = "/" + retention_folders.delete_if(&:blank?).join("/")
+          puts "#{step} - Creating remote directory at #{remote_path}"
+          step += 1
           sftp.mkdir!(remote_path)
           case k
           when "daily"
-            sftp.upload_file(path, remote_path)
+            sftp.upload_file(path, remote_path, options: opts)
+            files = sftp.list_files(remote_path)
+            puts "#{step} - Checking daily backups"
+            step += 1
+            if files.size > retention_config[:ttl]
+              files_to_remove = files.sort_by(&:name).shift(files.size - retention_config[:ttl])
+              puts "#{step} - Removing #{files_to_remove.size} backups because of ttl configuration"
+              files_to_remove.each do |file|
+                puts "Removing file #{remote_path + "/" + file.name}"
+                sftp.remove!(remote_path + "/" + file.name)
+              end
+            end
           when "monthly"
             next unless Date.today.day == 1
-            sftp.upload_file(path, remote_path)
+            sftp.upload_file(path, remote_path, options: opts)
+            puts "#{step} - Checking monthly backups"
+            step += 1
           end
         end
 
       end
+      FileUtils.rm_r 'backups/'
     end
 
   end

data/lib/scalingo_backups_manager/notification.rb

@@ -0,0 +1,30 @@
+require 'httparty'
+module ScalingoBackupsManager
+
+  class Notification
+
+    def self.send_slack_notification(hook_url, message)
+      HTTParty.post(
+        hook_url,
+        body: {
+          message: message
+        }.to_json,
+        headers: { 'Content-Type' => 'application/json' }
+      )
+    end
+
+    def self.send_discord_notification(hook_url, message)
+      payload = {
+        user: 'Scalingo backups manager',
+        content: message
+      }.to_json
+      HTTParty.post(
+        hook_url,
+        body: payload,
+        headers: { 'Content-Type': 'application/json' }
+      )
+    end
+
+  end
+
+end

data/lib/scalingo_backups_manager/restore/mysql.rb

@@ -63,7 +63,7 @@ module ScalingoBackupsManager
         puts "*** Restoring backup to mysql database ***"
         puts "Command: #{restore_cmd}"
         system(restore_cmd)
-        #FileUtils.rm_r destination_path unless opts[:skip_rm]
+        FileUtils.rm_r destination_path unless opts[:skip_rm]
       end
 
     end

data/lib/scalingo_backups_manager/restore/postgres.rb

@@ -34,21 +34,27 @@ module ScalingoBackupsManager
           user: rails_db_config["user"],
         }
 
-        restore_cmd = "/usr/bin/env psql #{config[:database]} -h #{opts[:host] || config[:host]}"
+        restore_cmd = "/usr/bin/env pg_restore"
 
+        file_path = Dir["#{destination_path}*.pgsql"]
+        if file_path.empty?
+          puts "*** No SQL file found in tar ***"
+          return
+        end
+        restore_cmd << " #{file_path.first}"
+
+        if config[:host].present?
+          restore_cmd << " -h #{opts[:host] || config[:host] || 'localhost'}"
+        end
         if config[:user].present?
-          restore_cmd << " --u #{config[:user]}"
-          if config[:password].present?
-            restore_cmd << " --password"
-            restore_cmd << " #{config[:password]}"
-          end
+          restore_cmd << " -U #{config[:user]}"
         end
 
-        if config[:port].present?
+        if opts[:port].present? || config[:port].present?
           restore_cmd << " -p #{opts[:port] || config[:port] || 5432}"
         end
 
-        restore_cmd << " < #{destination_path}"
+        restore_cmd << " -d #{config[:database]}"
 
         puts "*** Restoring backup to Postgres database ***"
         system(restore_cmd)

data/lib/scalingo_backups_manager/sftp_tools.rb

@@ -1,3 +1,4 @@
+require 'scalingo_backups_manager/notification'
 require 'net/sftp'
 
 module ScalingoBackupsManager
@@ -14,6 +15,22 @@ module ScalingoBackupsManager
       end
     end
 
+    def list_files(path)
+      files = []
+      start do |sftp|
+        sftp.dir.glob("#{path}", "*.tar.gz").each do |file|
+          files << file
+        end
+      end
+      files
+    end
+
+    def remove!(path)
+      start do |sftp|
+        sftp.remove!(path)
+      end
+    end
+
     def mkdir!(path)
       start do |sftp|
         folder_tree = []
@@ -29,10 +46,19 @@ module ScalingoBackupsManager
       end
     end
 
-    def upload_file(filepath, remote_dir)
+    def upload_file(filepath, remote_dir, options: {})
       filename = filepath.split('/').last
       start do |sftp|
-        sftp.upload!(filepath, "#{remote_dir}/#{filename}")
+        begin
+          sftp.upload!(filepath, "#{remote_dir}/#{filename}")
+        rescue
+          if options.dig(:webhooks, :slack_webhook_url)
+            ScalingoBackupsManager::Notification.send_slack_notification(options.dig(:webhooks, :slack_webhook_url), "An error has occured while uploading backup, see the logs for more information")
+          end
+          if options.dig(:webhooks, :discord_webhook_url)
+            ScalingoBackupsManager::Notification.send_discord_notification(options.dig(:webhooks, :discord_webhook_url), "An error has occured while uploading backup, see the logs for more information")
+          end
+        end
       end
     end
   end

data/lib/scalingo_backups_manager/version.rb

@@ -1,3 +1,3 @@
 module ScalingoBackupsManager
-  VERSION = "0.4.0"
+  VERSION = "0.6.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scalingo_backups_manager
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.6.3
 platform: ruby
 authors:
 - Kevin Clercin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-06-01 00:00:00.000000000 Z
+date: 2021-06-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -92,6 +92,7 @@ files:
 - lib/scalingo_backups_manager/backup.rb
 - lib/scalingo_backups_manager/cli.rb
 - lib/scalingo_backups_manager/configuration.rb
+- lib/scalingo_backups_manager/notification.rb
 - lib/scalingo_backups_manager/restore/mongodb.rb
 - lib/scalingo_backups_manager/restore/mysql.rb
 - lib/scalingo_backups_manager/restore/postgres.rb