scalingo_backups_manager 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0098acfcaa51bcbcd0ab7675849a739bdf6617f970968c975ea6eaf9d218b7dd'
-  data.tar.gz: fe08d492ab01f1495b733d5dcbccd1004f1d78ae930ea5db0fb700640f91a969
+  metadata.gz: 0c13c0008cbcd861729fbbc5cb43a3b53ed63db136a2b24393b077c2fbb6b50e
+  data.tar.gz: d93398d512fe1d0e9fdf964e431fc5c570c3201c5475e51174d3e7f297ef02c8
 SHA512:
-  metadata.gz: da90a3650e78a0b00960dbdad25c6493ea475803c31ad7cdb70ea8b86638520c02a65b0d4bbe99272716918eb9f2b73de4a8f79a5dfc276ffe5af51aac1545b1
-  data.tar.gz: 7ea91ada14e91ac0e4ec23c73eac3ba929ad3d099aee8463d47d0f4feb2af3a7fae4b4b4e2ab8bbe6045d73eb53e19c610871851dbef73f24deb7cfd3b67e94d
+  metadata.gz: 26cc222b56f4199eb7a1bd06f1c9e787d53a5f3a2a979836f991fc368369252c53cfeb7cbbd457882e4bce32aa07f09dcf8813a5d73a9007de4e958bfefe94fd
+  data.tar.gz: 0577310f2c5cceef264cd75789572577b7c218ff536cdb878a2ab40e6190f9e5d6b4e429587e596e7f6c71b7f1b787f431dbe4c0d7eee6a04c94a795fcd72a3f

data/Gemfile

@@ -5,3 +5,4 @@ gemspec
 
 gem "rake", "~> 12.0"
 gem "rspec", "~> 3.0"
+

data/Gemfile.lock

@@ -3,6 +3,7 @@ PATH
   specs:
     scalingo_backups_manager (0.3.0)
       httparty (~> 0.18)
+      net-sftp (~> 3.0.0)
       scalingo (~> 3.0)
       thor (~> 1.1)
 
@@ -33,6 +34,9 @@ GEM
     multi_json (1.15.0)
     multi_xml (0.6.0)
     multipart-post (2.1.1)
+    net-sftp (3.0.0)
+      net-ssh (>= 5.0.0, < 7.0.0)
+    net-ssh (6.1.0)
     rake (12.3.3)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)

data/README.md

@@ -3,7 +3,6 @@
 
 This gem allow to download backups of multiple scalingo applications and addons in order to be restore in local database or be send to an SFTP server
 
-
 ## TODO
 
 - Mysql

data/lib/scalingo_backups_manager/addon.rb

@@ -5,19 +5,20 @@ module ScalingoBackupsManager
 
   class Addon
 
-    attr_accessor :application, :addon, :config
+    attr_accessor :application, :addon, :config, :sftp_config
 
     DEFAULT_DATABASE_API_ROOT_URL = "https://db-api.osc-fr1.scalingo.com"
 
-    def self.find(app, id, config: {})
+    def self.find(app, id, config: {}, sftp_config: {})
       addon = Configuration.client.addons.find(app.id, id).data
-      self.new(app, addon, config: config)
+      self.new(app, addon, config: config, sftp_config: sftp_config)
     end
 
-    def initialize(app, addon, config: {})
+    def initialize(app, addon, config: {}, sftp_config: {})
       raise "Application must be set" unless app && app.is_a?(ScalingoBackupsManager::Application)
       @application = app
       @addon = addon
+      @sftp_config = sftp_config
       @config = config
     end
 

data/lib/scalingo_backups_manager/cli.rb

@@ -5,6 +5,7 @@ require 'scalingo_backups_manager/addon'
 require 'scalingo_backups_manager/restore/mongodb'
 require 'scalingo_backups_manager/restore/postgres'
 require 'scalingo_backups_manager/restore/mysql'
+require 'scalingo_backups_manager/sftp_tools'
 
 module ScalingoBackupsManager
 
@@ -83,6 +84,7 @@ module ScalingoBackupsManager
         backup = backups.first
         download_link = backup.download_link
         if download_link
+          puts "Downloading #{application.name} last backup"
           path = ("#{addon.config[:path]}" || "backups/#{addon.addon_provider[:id]}") + "/#{Time.now.strftime("%Y%m%d")}.tar.gz"
           if File.exist?(path)
             puts "Backup already download, skipping..."
@@ -122,6 +124,46 @@ module ScalingoBackupsManager
       end
     end
 
+    desc "upload_to_ftp", "Upload last backup to FTP"
+    def upload_to_ftp
+      invoke :download, [], application: options[:application], addon: options[:addon]
+      configuration = Configuration.new
+      configuration.for_each_addons do |application, addon|
+        config = addon.sftp_config
+        path = ("#{addon.config[:path]}" || "backups/#{addon.addon_provider[:id]}") + "/#{Time.now.strftime("%Y%m%d")}.tar.gz"
+        next unless File.exists?(path)
+
+        sftp = ScalingoBackupsManager::SftpTools.new(config[:auth])
+
+        folders = [
+          config.dig(:auth, :dir),
+          config.dig(:dir) || application.name,
+          addon.addon_provider[:id]
+        ]
+
+        if config[:retention].blank?
+          remote_path = "/" + [folders].delete_if(&:blank?).join("/")
+          sftp.mkdir!(remote_path)
+          sftp.upload_file(path, remote_path)
+          next
+        end
+
+        config[:retention].each do |k, v|
+          folders << config.dig(:retention, k, :dir)
+          remote_path = "/" + [folders].delete_if(&:blank?).join("/")
+          sftp.mkdir!(remote_path)
+          case k
+          when "daily"
+            sftp.upload_file(path, remote_path)
+          when "monthly"
+            next unless Date.today.day == 1
+            sftp.upload_file(path, remote_path)
+          end
+        end
+
+      end
+    end
+
   end
 
   private

data/lib/scalingo_backups_manager/configuration.rb

@@ -16,7 +16,6 @@ module ScalingoBackupsManager
     def self.create_file
       File.open(FILE_NAME, 'w+') do |f|
         f.flock(File::LOCK_EX)
-        content = f.read
         f.rewind
         f.write({ region: 'osc-fr-1', apps: []}.to_yaml)
         f.flush
@@ -34,7 +33,7 @@ module ScalingoBackupsManager
       File.open(FILE_NAME, "w+") do |f|
         f.flock(File::LOCK_EX)
         f.truncate(0)
-        f.write(config.to_yaml)
+        f.write(config.to_hash.to_yaml)
         f.flush
       end
     end
@@ -45,7 +44,7 @@ module ScalingoBackupsManager
         puts "Configuration file does not exist"
         return
       end
-      @config = YAML.load(@file.read) || { apps: [] }
+      @config = (YAML.load(@file.read) || { apps: [] }).with_indifferent_access
     end
 
     def add_addon_to_app(application, addon)
@@ -53,16 +52,21 @@ module ScalingoBackupsManager
       self.class.write_config(@config)
     end
 
-    def for_each_addons(application_uid, addon_uid)
+    def for_each_addons(application_uid = nil, addon_uid = nil)
+      client_sftp_config = @config[:sftp] || {}
+
       @config[:apps].each do |application_name, application_config|
         next if application_uid && application_uid != application_name.to_s
         next unless application_config[:id]
         application = ScalingoBackupsManager::Application.find(application_config[:id])
         next unless application_config[:addons] && application_config[:addons].size > 0
+
+        app_sftp_config = (application_config[:sftp] || {}).reverse_merge(client_sftp_config)
+
         application_config[:addons].each do |addon_name, addon_config|
           next if addon_uid && addon_uid != addon_name.to_s
           next unless addon_config[:id]
-          addon = ScalingoBackupsManager::Addon.find(application, addon_config[:id], config: addon_config)
+          addon = ScalingoBackupsManager::Addon.find(application, addon_config[:id], config: addon_config, sftp_config: (addon_config[:sftp] || {}).reverse_merge(app_sftp_config))
           yield(application, addon) if block_given?
         end
       end

data/lib/scalingo_backups_manager/sftp_tools.rb

@@ -0,0 +1,40 @@
+require 'net/sftp'
+
+module ScalingoBackupsManager
+  class SftpTools
+    attr_accessor :ftp_host
+
+    def initialize(ftp_host)
+      @ftp_host = ftp_host
+    end
+
+    def start
+      Net::SFTP.start(@ftp_host[:host], @ftp_host[:user], password: @ftp_host[:password], port: @ftp_host[:port]) do |sftp|
+        yield(sftp) if block_given?
+      end
+    end
+
+    def mkdir!(path)
+      start do |sftp|
+        folder_tree = []
+        path.split("/").each do |folder_name|
+          next if folder_name.blank?
+
+          folder_tree << folder_name
+          begin
+            sftp.mkdir!(folder_tree.join("/"))
+          rescue
+          end
+        end
+      end
+    end
+
+    def upload_file(filepath, remote_dir)
+      filename = filepath.split('/').last
+      start do |sftp|
+        sftp.upload!(filepath, "#{remote_dir}/#{filename}")
+      end
+    end
+  end
+
+end

data/lib/scalingo_backups_manager/version.rb

@@ -1,3 +1,3 @@
 module ScalingoBackupsManager
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/scalingo_backups_manager.gemspec

@@ -18,6 +18,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "thor", '~> 1.1'
   spec.add_dependency 'httparty', "~> 0.18"
   spec.add_dependency 'scalingo', '~> 3.0'
+  spec.add_dependency 'net-sftp', '~> 3.0.0'
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scalingo_backups_manager
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Kevin Clercin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-05-31 00:00:00.000000000 Z
+date: 2021-06-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: net-sftp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
 description: 
 email:
 - kevin@9troisquarts.com
@@ -81,6 +95,7 @@ files:
 - lib/scalingo_backups_manager/restore/mongodb.rb
 - lib/scalingo_backups_manager/restore/mysql.rb
 - lib/scalingo_backups_manager/restore/postgres.rb
+- lib/scalingo_backups_manager/sftp_tools.rb
 - lib/scalingo_backups_manager/version.rb
 - scalingo_backups_manager.gemspec
 homepage: https://github.com/9troisquarts/scalingo_backups_manager