scalingo_backups_manager 0.3.0 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0098acfcaa51bcbcd0ab7675849a739bdf6617f970968c975ea6eaf9d218b7dd'
-  data.tar.gz: fe08d492ab01f1495b733d5dcbccd1004f1d78ae930ea5db0fb700640f91a969
+  metadata.gz: bec014dcdbd0fe4aa8a6e646b131b43f7af832aa3a13d22f24900d9dcfeff857
+  data.tar.gz: 1758ce4b459fe4448ddeaeda52302e94b48083685148d398061bc4be96211574
 SHA512:
-  metadata.gz: da90a3650e78a0b00960dbdad25c6493ea475803c31ad7cdb70ea8b86638520c02a65b0d4bbe99272716918eb9f2b73de4a8f79a5dfc276ffe5af51aac1545b1
-  data.tar.gz: 7ea91ada14e91ac0e4ec23c73eac3ba929ad3d099aee8463d47d0f4feb2af3a7fae4b4b4e2ab8bbe6045d73eb53e19c610871851dbef73f24deb7cfd3b67e94d
+  metadata.gz: 7b7a1f1f853aa11ea8135effcc15df970aedf269145cccb471a430705f1130e71919116f93ed1f9b2c2062a141191f214d9c99bfe5bc5f3a806abf591453b3a0
+  data.tar.gz: d313ddc959d7757050d9276e79a516395e119dc32de69278736d350a31a0ac1581a1c41b1b490ba1b842801710ba93e4985dd469312a7a0e5c88448b2494fe98

data/Gemfile

@@ -5,3 +5,4 @@ gemspec
 
 gem "rake", "~> 12.0"
 gem "rspec", "~> 3.0"
+

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    scalingo_backups_manager (0.3.0)
+    scalingo_backups_manager (0.6.0)
       httparty (~> 0.18)
+      net-sftp (~> 3.0.0)
       scalingo (~> 3.0)
       thor (~> 1.1)
 
@@ -33,6 +34,9 @@ GEM
     multi_json (1.15.0)
     multi_xml (0.6.0)
     multipart-post (2.1.1)
+    net-sftp (3.0.0)
+      net-ssh (>= 5.0.0, < 7.0.0)
+    net-ssh (6.1.0)
     rake (12.3.3)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)

data/README.md

@@ -3,7 +3,6 @@
 
 This gem allow to download backups of multiple scalingo applications and addons in order to be restore in local database or be send to an SFTP server
 
-
 ## TODO
 
 - Mysql

data/lib/scalingo_backups_manager/addon.rb

@@ -5,19 +5,20 @@ module ScalingoBackupsManager
 
   class Addon
 
-    attr_accessor :application, :addon, :config
+    attr_accessor :application, :addon, :config, :sftp_config
 
     DEFAULT_DATABASE_API_ROOT_URL = "https://db-api.osc-fr1.scalingo.com"
 
-    def self.find(app, id, config: {})
+    def self.find(app, id, config: {}, sftp_config: {})
       addon = Configuration.client.addons.find(app.id, id).data
-      self.new(app, addon, config: config)
+      self.new(app, addon, config: config, sftp_config: sftp_config)
     end
 
-    def initialize(app, addon, config: {})
+    def initialize(app, addon, config: {}, sftp_config: {})
       raise "Application must be set" unless app && app.is_a?(ScalingoBackupsManager::Application)
       @application = app
       @addon = addon
+      @sftp_config = sftp_config
       @config = config
     end
 

data/lib/scalingo_backups_manager/cli.rb

@@ -5,6 +5,7 @@ require 'scalingo_backups_manager/addon'
 require 'scalingo_backups_manager/restore/mongodb'
 require 'scalingo_backups_manager/restore/postgres'
 require 'scalingo_backups_manager/restore/mysql'
+require 'scalingo_backups_manager/sftp_tools'
 
 module ScalingoBackupsManager
 
@@ -83,6 +84,7 @@ module ScalingoBackupsManager
         backup = backups.first
         download_link = backup.download_link
         if download_link
+          puts "Downloading #{application.name} last backup"
           path = ("#{addon.config[:path]}" || "backups/#{addon.addon_provider[:id]}") + "/#{Time.now.strftime("%Y%m%d")}.tar.gz"
           if File.exist?(path)
             puts "Backup already download, skipping..."
@@ -122,6 +124,68 @@ module ScalingoBackupsManager
       end
     end
 
+    desc "upload_to_ftp", "Upload last backup to FTP"
+    def upload_to_ftp
+      invoke :download, [], application: options[:application], addon: options[:addon]
+      configuration = Configuration.new
+      opts = {
+        webhooks: configuration.config[:webhooks]
+      }
+      configuration.for_each_addons do |application, addon|
+        step = 1
+        sftp_config = addon.sftp_config
+        path = ("#{addon.config[:path]}" || "backups/#{addon.addon_provider[:id]}") + "/#{Time.now.strftime("%Y%m%d")}.tar.gz"
+        next unless File.exists?(path)
+        puts "** Upload backup for #{application.name} **"
+
+        sftp = ScalingoBackupsManager::SftpTools.new(sftp_config[:auth])
+
+        folders = [
+          sftp_config.dig(:auth, :dir),
+          sftp_config.dig(:dir) || application.name,
+          addon.addon_provider[:id]
+        ]
+
+        if sftp_config[:retention].blank?
+          remote_path = "/" + [folders].delete_if(&:blank?).join("/")
+          sftp.mkdir!(remote_path)
+          sftp.upload_file(path, remote_path, options: opts)
+          next
+        end
+
+        sftp_config[:retention].each do |k, retention_config|
+          retention_folders = folders.dup
+          retention_folders << sftp_config.dig(:retention, k, :dir)
+          remote_path = "/" + retention_folders.delete_if(&:blank?).join("/")
+          puts "#{step} - Creating remote directory at #{remote_path}"
+          step += 1
+          sftp.mkdir!(remote_path)
+          case k
+          when "daily"
+            sftp.upload_file(path, remote_path, options: opts)
+            files = sftp.list_files(remote_path)
+            puts "#{step} - Checking daily backups"
+            step += 1
+            if files.size > retention_config[:ttl]
+              files_to_remove = files.sort_by(&:name).shift(files.size - retention_config[:ttl])
+              puts "#{step} - Removing #{files_to_remove.size} backups because of ttl configuration"
+              files_to_remove.each do |file|
+                puts "Removing file #{remote_path + "/" + file.name}"
+                sftp.remove!(remote_path + "/" + file.name)
+              end
+            end
+          when "monthly"
+            next unless Date.today.day == 1
+            sftp.upload_file(path, remote_path, options: opts)
+            puts "#{step} - Checking monthly backups"
+            step += 1
+          end
+        end
+
+      end
+      FileUtils.rm_r 'backups/'
+    end
+
   end
 
   private

data/lib/scalingo_backups_manager/configuration.rb

@@ -16,7 +16,6 @@ module ScalingoBackupsManager
     def self.create_file
       File.open(FILE_NAME, 'w+') do |f|
         f.flock(File::LOCK_EX)
-        content = f.read
         f.rewind
         f.write({ region: 'osc-fr-1', apps: []}.to_yaml)
         f.flush
@@ -34,7 +33,7 @@ module ScalingoBackupsManager
       File.open(FILE_NAME, "w+") do |f|
         f.flock(File::LOCK_EX)
         f.truncate(0)
-        f.write(config.to_yaml)
+        f.write(config.to_hash.to_yaml)
         f.flush
       end
     end
@@ -45,7 +44,7 @@ module ScalingoBackupsManager
         puts "Configuration file does not exist"
         return
       end
-      @config = YAML.load(@file.read) || { apps: [] }
+      @config = (YAML.load(@file.read) || { apps: [] }).with_indifferent_access
     end
 
     def add_addon_to_app(application, addon)
@@ -53,16 +52,21 @@ module ScalingoBackupsManager
       self.class.write_config(@config)
     end
 
-    def for_each_addons(application_uid, addon_uid)
+    def for_each_addons(application_uid = nil, addon_uid = nil)
+      client_sftp_config = @config[:sftp] || {}
+
       @config[:apps].each do |application_name, application_config|
         next if application_uid && application_uid != application_name.to_s
         next unless application_config[:id]
         application = ScalingoBackupsManager::Application.find(application_config[:id])
         next unless application_config[:addons] && application_config[:addons].size > 0
+
+        app_sftp_config = (application_config[:sftp] || {}).reverse_merge(client_sftp_config)
+
         application_config[:addons].each do |addon_name, addon_config|
           next if addon_uid && addon_uid != addon_name.to_s
           next unless addon_config[:id]
-          addon = ScalingoBackupsManager::Addon.find(application, addon_config[:id], config: addon_config)
+          addon = ScalingoBackupsManager::Addon.find(application, addon_config[:id], config: addon_config, sftp_config: (addon_config[:sftp] || {}).reverse_merge(app_sftp_config))
           yield(application, addon) if block_given?
         end
       end

data/lib/scalingo_backups_manager/notification.rb

@@ -0,0 +1,30 @@
+require 'httparty'
+module ScalingoBackupsManager
+
+  class Notification
+
+    def self.send_slack_notification(hook_url, message)
+      HTTParty.post(
+        hook_url,
+        body: {
+          message: message
+        }.to_json,
+        headers: { 'Content-Type' => 'application/json' }
+      )
+    end
+
+    def self.send_discord_notification(hook_url, message)
+      payload = {
+        user: 'Scalingo backups manager',
+        content: message
+      }.to_json
+      HTTParty.post(
+        hook_url,
+        body: payload,
+        headers: { 'Content-Type': 'application/json' }
+      )
+    end
+
+  end
+
+end

data/lib/scalingo_backups_manager/restore/postgres.rb

@@ -44,7 +44,7 @@ module ScalingoBackupsManager
           end
         end
 
-        if config[:port].present?
+        if opts[:port].present? && config[:port].present?
           restore_cmd << " -p #{opts[:port] || config[:port] || 5432}"
         end
 

data/lib/scalingo_backups_manager/sftp_tools.rb

@@ -0,0 +1,66 @@
+require 'scalingo_backups_manager/notification'
+require 'net/sftp'
+
+module ScalingoBackupsManager
+  class SftpTools
+    attr_accessor :ftp_host
+
+    def initialize(ftp_host)
+      @ftp_host = ftp_host
+    end
+
+    def start
+      Net::SFTP.start(@ftp_host[:host], @ftp_host[:user], password: @ftp_host[:password], port: @ftp_host[:port]) do |sftp|
+        yield(sftp) if block_given?
+      end
+    end
+
+    def list_files(path)
+      files = []
+      start do |sftp|
+        sftp.dir.glob("#{path}", "*.tar.gz").each do |file|
+          files << file
+        end
+      end
+      files
+    end
+
+    def remove!(path)
+      start do |sftp|
+        sftp.remove!(path)
+      end
+    end
+
+    def mkdir!(path)
+      start do |sftp|
+        folder_tree = []
+        path.split("/").each do |folder_name|
+          next if folder_name.blank?
+
+          folder_tree << folder_name
+          begin
+            sftp.mkdir!(folder_tree.join("/"))
+          rescue
+          end
+        end
+      end
+    end
+
+    def upload_file(filepath, remote_dir, options: {})
+      filename = filepath.split('/').last
+      start do |sftp|
+        begin
+          sftp.upload!(filepath, "#{remote_dir}/#{filename}")
+        rescue
+          if options.dig(:webhooks, :slack_webhook_url)
+            ScalingoBackupsManager::Notification.send_slack_notification(options.dig(:webhooks, :slack_webhook_url), "An error has occured while uploading backup, see the logs for more information")
+          end
+          if options.dig(:webhooks, :discord_webhook_url)
+            ScalingoBackupsManager::Notification.send_discord_notification(options.dig(:webhooks, :discord_webhook_url), "An error has occured while uploading backup, see the logs for more information")
+          end
+        end
+      end
+    end
+  end
+
+end

data/lib/scalingo_backups_manager/version.rb

@@ -1,3 +1,3 @@
 module ScalingoBackupsManager
-  VERSION = "0.3.0"
+  VERSION = "0.6.2"
 end

data/scalingo_backups_manager.gemspec

@@ -18,6 +18,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "thor", '~> 1.1'
   spec.add_dependency 'httparty', "~> 0.18"
   spec.add_dependency 'scalingo', '~> 3.0'
+  spec.add_dependency 'net-sftp', '~> 3.0.0'
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scalingo_backups_manager
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.6.2
 platform: ruby
 authors:
 - Kevin Clercin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-05-31 00:00:00.000000000 Z
+date: 2021-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: net-sftp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
 description: 
 email:
 - kevin@9troisquarts.com
@@ -78,9 +92,11 @@ files:
 - lib/scalingo_backups_manager/backup.rb
 - lib/scalingo_backups_manager/cli.rb
 - lib/scalingo_backups_manager/configuration.rb
+- lib/scalingo_backups_manager/notification.rb
 - lib/scalingo_backups_manager/restore/mongodb.rb
 - lib/scalingo_backups_manager/restore/mysql.rb
 - lib/scalingo_backups_manager/restore/postgres.rb
+- lib/scalingo_backups_manager/sftp_tools.rb
 - lib/scalingo_backups_manager/version.rb
 - scalingo_backups_manager.gemspec
 homepage: https://github.com/9troisquarts/scalingo_backups_manager