capistrano-ops 0.2.3 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 48c373221e575d786df6abaf731ae417311c95584c1ae9db507b2f509b83a9f4
-  data.tar.gz: 5aa7c63a62cc0d3f9dd8ac8af6f0eab4b16b8d0ac43f121961b5a1510757ddc5
+  metadata.gz: 6897d38c35cb8b0dfb713d16d0eb0c390c431636ea5c6565e1dc457b96a29c6b
+  data.tar.gz: 9789f0e37f0767403324c1ca68c97759a8ebdbdb5a3e4ac6d362b76dfc5064ad
 SHA512:
-  metadata.gz: 11e060e8ed7888ffb2459a16b0482bae06d9ac7818d965a976345da0e06ab608948a5c46ba31e39c3989e240f02b5fd03ee305ef7e1c2ffc0ab4c15c0d453ca4
-  data.tar.gz: afc62c529fe831ef17d8f32bb0f89a20687c031472e49b433baf1be7a9a993ce6bbac468a9ec5b0b37e982ee266709faac3d74e5d61985d3ef33e43ebc003b7f
+  metadata.gz: 4607175b27284ed6d19ca2fe3032dccde17f54cb62dfbfc43c614af031abf9990b873b89b1d905c74d2acb48dc4e94a3eec6b50bbff18dee0079b2a5264d04dd
+  data.tar.gz: 1ce1060464559ce566320e9ef7a1700d5d94ee688e6f8aea85f7dbd2548e78bfc5e6cb1f5e751b38e9b566e7411e89587d4207eb6c0349592d0cacab854e6ffc

data/.rubocop.yml

@@ -0,0 +1,21 @@
+AllCops:
+  NewCops: disable
+  TargetRubyVersion: 2.7
+Layout/LineLength:
+  Max: 160
+Metrics/MethodLength:
+  Max: 50
+Metrics/ClassLength:
+  Max: 750
+Metrics/BlockLength:
+  Max: 50
+Metrics/ParameterLists:
+  Max: 6
+Metrics/AbcSize:
+  Max: 60
+Metrics/CyclomaticComplexity:
+  Max: 25
+Metrics/PerceivedComplexity:
+  Max: 25
+Style/Documentation:
+  Enabled: false

data/README.md

@@ -46,8 +46,12 @@ require 'capistrano/ops'
 
 | Script                                           | Description                                                       |
 | ------------------------------------------------ | ----------------------------------------------------------------- |
-| `cap <environment> backup:create`                | creates backup of postgres database on the server                 |
-| `cap <environment> backup:pull`                  | download latest postgres backup from server                       |
+| `cap <environment> backup:create`                | creates backup of postgres database on the server (deprecated)    |
+| `cap <environment> backup:pull`                  | download latest postgres backup from server (deprecated)          |
+| `cap <environment> backup:database:create`       | creates backup of postgres database on the server                 |
+| `cap <environment> backup:database:pull`         | download latest postgres backup from server                       |
+| `cap <environment> backup:storage:create`        | creates backup of storage on the server                           |
+| `cap <environment> backup:storage:pull`          | download latest storage backup from server                        |
 | `cap <environment> figaro_yml:compare`           | compare local application.yml with server application.yml         |
 | `cap <environment> figaro_yml:get`               | shows env vars from server application.yml configured thru figaro |
 | `cap <environment> logs:rails`                   | display server log live                                           |
@@ -55,6 +59,8 @@ require 'capistrano/ops'
 | `cap <environment> invoke:rake TASK=<your:task>` | invoke rake task on server                                        |
 | `rake pg:dump`                                   | creates postgres database backup                                  |
 | `rake pg:remove_old_dumps`                       | remove old postgres backups                                       |
+| `rake storage:backup`                            | creates backup of storage                                         |
+| `rake storage:remove_old_backups`                | remove old storage backups                                        |
 
 ## Usage
 
@@ -72,23 +78,26 @@ production:
 
 ### Optional Settings for backup task
 
-| env                | description                                                               |                            type/options                            |
-| ------------------ | ------------------------------------------------------------------------- | :----------------------------------------------------------------: |
-| NUMBER_OF_BACKUPS  | number of backups to keep (default: 1)                                    |                              `number`                              |
-| BACKUPS_ENABLED    | enable/disable backup task (default: Rails.env == 'production')           |                             `boolean`                              |
-| DEFAULT_URL        | notification message title (default: "#{database} Backup")                |                              `string`                              |
-| NOTIFICATION_TYPE  | for notification (default: nil)                                           |                    `string` (`webhook`/`slack`)                    |
-| NOTIFICATION_LEVEL | for notification (default: nil)                                           |                     `string` (`info`/`error`)                      |
-| SLACK_SECRET       | for slack integration                                                     | `string` (e.g. `xoxb-1234567890-1234567890-1234567890-1234567890`) |
-| SLACK_CHANNEL      | for slack integration                                                     |                    `string` (e.g. `C234567890`)                    |
-| WEBHOOK_URL        | Webhook server to send message                                            |                      e.g `http://example.com`                      |
-| WEBHOOK_SECRET     | Secret to send with uses md5-hmac hexdigest in header`x-hub-signature`    |                                ---                                 |
-| BACKUP_PROVIDER    | Backup provider (default: nil)                                            |                          `string` (`s3`)                           |
-| S3_BACKUP_BUCKET   | S3 bucket name for backups                                                |                              `string`                              |
-| S3_BACKUP_REGION   | S3 region for backups                                                     |                              `string`                              |
-| S3_BACKUP_KEY      | S3 access key for backups                                                 |                              `string`                              |
-| S3_BACKUP_SECRET   | S3 secret key for backups                                                 |                              `string`                              |
-| S3_BACKUP_ENDPOINT | S3 endpoint for backups (optional, used for other S3 compatible services) |                              `string`                              |
+| env                        | description                                                                                         |                            type/options                            |
+| -------------------------- | --------------------------------------------------------------------------------------------------- | :----------------------------------------------------------------: |
+| NUMBER_OF_BACKUPS          | number of backups to keep (default: 7)                                                              |                              `number`                              |
+| NUMBER_OF_LOCAL_BACKUPS    | number of backups to keep locally (default: nil)                                                    |                              `number`                              |
+| NUMBER_OF_EXTERNAL_BACKUPS | number of backups to keep externally (default: nil)                                                 |                              `number`                              |
+| BACKUPS_ENABLED            | enable/disable backup task (default: Rails.env == 'production')                                     |                             `boolean`                              |
+| EXTERNAL_BACKUP_ENABLED    | enable/disable external backup (default: false) (only if 'BACKUPS_ENABLED', needs additional setup) |                             `boolean`                              |
+| DEFAULT_URL                | notification message title (default: "#{database} Backup")                                          |                              `string`                              |
+| NOTIFICATION_TYPE          | for notification (default: nil)                                                                     |                    `string` (`webhook`/`slack`)                    |
+| NOTIFICATION_LEVEL         | for notification (default: nil)                                                                     |                     `string` (`info`/`error`)                      |
+| SLACK_SECRET               | for slack integration                                                                               | `string` (e.g. `xoxb-1234567890-1234567890-1234567890-1234567890`) |
+| SLACK_CHANNEL              | for slack integration                                                                               |                    `string` (e.g. `C234567890`)                    |
+| WEBHOOK_URL                | Webhook server to send message                                                                      |                      e.g `http://example.com`                      |
+| WEBHOOK_SECRET             | Secret to send with uses md5-hmac hexdigest in header`x-hub-signature`                              |                                ---                                 |
+| BACKUP_PROVIDER            | Backup provider (default: nil)                                                                      |                          `string` (`s3`)                           |
+| S3_BACKUP_BUCKET           | S3 bucket name for backups                                                                          |                              `string`                              |
+| S3_BACKUP_REGION           | S3 region for backups                                                                               |                              `string`                              |
+| S3_BACKUP_KEY              | S3 access key for backups                                                                           |                              `string`                              |
+| S3_BACKUP_SECRET           | S3 secret key for backups                                                                           |                              `string`                              |
+| S3_BACKUP_ENDPOINT         | S3 endpoint for backups (optional, used for other S3 compatible services)                           |                              `string`                              |
 
 ### use with whenever/capistrano
 
@@ -102,10 +111,12 @@ set :output, -> { '2>&1 | logger -t whenever_cron' }
 
 every :day, at: '2:00 am' do
   rake 'pg:dump'
+  rake 'storage:backup'
 end
 
 every :day, at: '3:00 am' do
   rake 'pg:remove_old_dumps'
+  rake 'storage:remove_old_backups'
 end
 ```
 
@@ -153,6 +164,34 @@ if you want to use notification level you have to add this to your `application.
 NOTIFICATION_LEVEL: 'info' # default is 'error'
 ```
 
+## Backups
+
+if you want to configure the number of backups you have to add this to your `application.yml`
+
+```ruby
+NUMBER_OF_BACKUPS: 7 # default is 7 (local + external)
+```
+
+to fine tune the number of local and external backups you can use this:
+
+```ruby
+NUMBER_OF_LOCAL_BACKUPS: 7 # default is nil (local)
+NUMBER_OF_EXTERNAL_BACKUPS: 7 # default is nil (local)
+```
+
+### Backup provider
+
+if you want to use an external backup provider you have to add this to your `application.yml`
+
+```ruby
+BACKUP_PROVIDER: 's3'
+S3_BACKUP_BUCKET: '<your-s3-bucket>'
+S3_BACKUP_REGION: '<your-s3-region>'
+S3_BACKUP_KEY: '<your-s3-key>'
+S3_BACKUP_SECRET: '<your-s3-secret>'
+S3_BACKUP_ENDPOINT: '<your-s3-endpoint>' # optional, used for other S3 compatible services
+```
+
 ## Contributing
 
 1. Fork it ( https://github.com/zauberware/capistrano-ops/fork )

data/capistrano-ops.gemspec

@@ -18,12 +18,12 @@ Gem::Specification.new do |s|
   s.test_files    = `git ls-files -- test/{functional,unit}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
   s.require_paths = ['lib']
-  
-  s.required_ruby_version = '>= 2.5.0', '< 3.0.0'
-  
-  s.add_dependency 'rails'
-  s.add_dependency 'nokogiri'
+
+  s.required_ruby_version = '~> 2.7.0'
   s.add_dependency 'aws-sdk-s3', '~> 1.128'
+  s.add_dependency 'faraday'
+  s.add_dependency 'nokogiri'
+  s.add_dependency 'rails'
   s.add_development_dependency 'bundler', '~> 2.4.12'
   s.add_development_dependency 'rake', '~> 10.0'
 end

data/lib/capistrano/ops/backup/api.rb

@@ -1,38 +1,40 @@
+# frozen_string_literal: true
+
 module Backup
-    class Api
-        attr_accessor :backup_provider, :provider_config
+  class Api
+    attr_accessor :backup_provider, :provider_config
 
-        def initialize(provider: ENV['BACKUP_PROVIDER'], provider_config: {})
-            self.backup_provider = provider
-            self.provider_config = provider_config
-        end
+    def initialize(provider: ENV['BACKUP_PROVIDER'], provider_config: {})
+      self.backup_provider = provider
+      self.provider_config = provider_config
+    end
 
-        def upload(backup_file, filename)
-            case backup_provider
-            when 's3'
-                s3 = Backup::S3.new(**provider_config)
-                s3.upload(backup_file, filename)
-            when 'scp'
-                p "SCP backup not implemented yet"
-            when 'rsync'
-                p "Rsync backup not implemented yet"
-            else
-                raise Backup::Error, "Backup provider not configured"
-            end
-        end
+    def upload(backup_file, filename)
+      case backup_provider
+      when 's3'
+        s3 = Backup::S3.new(**provider_config)
+        s3.upload(backup_file, filename)
+      when 'scp'
+        p 'SCP backup not implemented yet'
+      when 'rsync'
+        p 'Rsync backup not implemented yet'
+      else
+        raise Backup::Error, 'Backup provider not configured'
+      end
+    end
 
-        def remove_old_backups(basename,keep)
-            case backup_provider
-            when 's3'
-                s3 = Backup::S3.new(**provider_config)
-                s3.remove_old_backups(basename,keep: keep)
-            when 'scp'
-                p "SCP backup not implemented yet"
-            when 'rsync'
-                p "Rsync backup not implemented yet"
-            else
-                raise Backup::Error, "Backup provider not configured"
-            end
-        end 
+    def remove_old_backups(basename, keep)
+      case backup_provider
+      when 's3'
+        s3 = Backup::S3.new(**provider_config)
+        s3.remove_old_backups(basename, keep: keep)
+      when 'scp'
+        p 'SCP backup not implemented yet'
+      when 'rsync'
+        p 'Rsync backup not implemented yet'
+      else
+        raise Backup::Error, 'Backup provider not configured'
+      end
     end
-end
+  end
+end

data/lib/capistrano/ops/backup/s3.rb

@@ -1,49 +1,52 @@
+# frozen_string_literal: true
+
 module Backup
-    require 'aws-sdk-s3'
-    class S3
-        attr_accessor :endpoint, :region, :access_key_id, :secret_access_key, :s3_client
+  require 'aws-sdk-s3'
+  class S3
+    attr_accessor :endpoint, :region, :access_key_id, :secret_access_key, :s3_client
 
-        def initialize( endpoint: ENV['S3_BACKUP_ENDPOINT'], region: ENV['S3_BACKUP_REGION'], access_key_id: ENV['S3_BACKUP_KEY'], secret_access_key: ENV['S3_BACKUP_SECRET'])
-            self.endpoint = endpoint
-            self.region = region
-            self.access_key_id = access_key_id
-            self.secret_access_key = secret_access_key
-            config = {
-                region: region,
-                access_key_id: access_key_id,
-                secret_access_key: secret_access_key
-            }
-            config[:endpoint] = endpoint unless endpoint.nil?
-            self.s3_client = Aws::S3::Client.new(config)
-        end
+    def initialize(endpoint: ENV['S3_BACKUP_ENDPOINT'], region: ENV['S3_BACKUP_REGION'], access_key_id: ENV['S3_BACKUP_KEY'],
+                   secret_access_key: ENV['S3_BACKUP_SECRET'])
+      self.endpoint = endpoint
+      self.region = region
+      self.access_key_id = access_key_id
+      self.secret_access_key = secret_access_key
+      config = {
+        region: region,
+        access_key_id: access_key_id,
+        secret_access_key: secret_access_key
+      }
+      config[:endpoint] = endpoint unless endpoint.nil?
+      self.s3_client = Aws::S3::Client.new(config)
+    end
 
-        def upload(backup_file, key)
-            begin
-                s3_client.put_object(
-                    bucket: ENV['S3_BACKUP_BUCKET'],
-                    key: key,
-                    body: File.open(backup_file)
-                )
-            rescue => e
-                puts "Error uploading backup to S3: #{e.message}"
-                raise e
-            end
-            "File uploaded to S3"
-        end
+    def upload(backup_file, key)
+      begin
+        s3_client.put_object(
+          bucket: ENV['S3_BACKUP_BUCKET'],
+          key: key,
+          body: File.open(backup_file)
+        )
+      rescue StandardError => e
+        puts "Error uploading backup to S3: #{e.message}"
+        raise e
+      end
+      'File uploaded to S3'
+    end
 
-        def remove_old_backups(basename, keep: 5)
-            bucket = ENV['S3_BACKUP_BUCKET']
-            all_items = s3_client.list_objects_v2(bucket:bucket, prefix: basename).contents
-            count = all_items.count
-            if count <= keep
-                p 'Nothing to remove'
-                return
-            end
-            items = all_items.sort_by(&:last_modified).reverse.slice(keep..-1).map(&:key)
-            items.each do |item|
-                p "Removing #{item} from S3"
-                s3_client.delete_object(bucket: bucket, key: item)
-            end
-        end
+    def remove_old_backups(basename, keep: 5)
+      bucket = ENV['S3_BACKUP_BUCKET']
+      all_items = s3_client.list_objects_v2(bucket: bucket, prefix: basename).contents
+      count = all_items.count
+      if count <= keep
+        p 'Nothing to remove'
+        return
+      end
+      items = all_items.sort_by(&:last_modified).reverse.slice(keep..-1).map(&:key)
+      items.each do |item|
+        p "Removing #{item} from S3"
+        s3_client.delete_object(bucket: bucket, key: item)
+      end
     end
-end
+  end
+end

data/lib/capistrano/ops/backup.rb

@@ -1,7 +1,8 @@
+# frozen_string_literal: true
+
 require 'capistrano/ops/backup/api'
 require 'capistrano/ops/backup/s3'
 
-
 module Backup
-    class Error < StandardError; end
-end
+  class Error < StandardError; end
+end

data/lib/capistrano/ops/capistrano/v3/tasks/backup/backup_helper.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module BackupHelper
+  def backup_file_name(type)
+    regex = type == 'storage' ? "'.{0,}\.tar.gz'" : "'.{0,}\.dump'"
+    @backup_file_name ||= capture "cd #{shared_path}/backups && ls -lt | grep -E -i #{regex} | head -n 1 | awk '{print $9}'"
+  end
+
+  def backup_file_size
+    @backup_file_size ||= capture "cd #{shared_path}/backups && wc -c #{@backup_file_name} | awk '{print $1}'"
+  end
+
+  def download_backup(backup_file, type)
+    puts "Downloading #{type} backup"
+    download! "#{shared_path}/backups/#{backup_file}"
+    puts "Download finished\nDeleting temporary backup..."
+    cleanup_backup(backup_file, "Download finished\nDeleting temporary backup...")
+  end
+
+  def cleanup_backup(backup_file, message)
+    puts message
+    execute "cd #{shared_path}/backups && rm #{backup_file}"
+    puts 'Temporary backup deleted'
+  end
+
+  def question(question, default = 'n', &block)
+    print "#{question} #{default.downcase == 'n' ? '(y/N)' : '(Y/n)'}: "
+    input = $stdin.gets.strip.downcase
+    answer = (input.empty? ? default : input).downcase.to_s
+
+    if %w[y n].include?(answer)
+      block.call(answer == 'y')
+    else
+      question(question, default, &block)
+    end
+  end
+
+  def prepare_env
+    @env = "RAILS_ENV=#{fetch(:stage)}"
+    @path_cmd = "PATH=$HOME/.rbenv/versions/#{RUBY_VERSION}/bin:$PATH"
+    @test_command = "cd #{release_path} && #{@path_cmd} && #{@env}"
+  end
+
+  def size_str(size)
+    units = %w[B KB MB GB TB]
+    e = (Math.log(size) / Math.log(1024)).floor
+    s = format('%.2f', size.to_f / 1024**e)
+    s.sub(/\.?0*$/, units[e])
+  end
+end

data/lib/capistrano/ops/capistrano/v3/tasks/backup/database/create.rake

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require_relative '../backup_helper'
+namespace :backup do
+  namespace :database do
+    include BackupHelper
+    # Default to :app role
+    rake_roles = fetch(:rake_roles, :app)
+    
+    desc 'create a backup of the server database'
+    task :create do
+      on roles(rake_roles) do
+        puts 'Creating backup...'
+        execute "#{prepare_env} BACKUPS_ENABLED=true bundle exec rake pg:dump"
+      end
+    end
+  end
+end

data/lib/capistrano/ops/capistrano/v3/tasks/backup/database/pull.rake

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require_relative '../backup_helper'
+namespace :backup do
+  namespace :database do
+    include BackupHelper
+    # Default to :app role
+    rake_roles = fetch(:rake_roles, :app)
+
+    desc 'pull latest database dump from server to local'
+    task :pull do
+      on roles(rake_roles) do
+        puts 'Creating temporary backup...'
+        execute "#{prepare_env} BACKUPS_ENABLED=true EXTERNAL_BACKUP_ENABLED=false bundle exec rake pg:dump"
+        puts "Backup created\nPrepare download..."
+        backup_file = backup_file_name('database')
+        backup_size = backup_file_size
+        question("Backup size: #{size_str(backup_size.to_i)}. Continue?", 'n') do |answer|
+          if answer
+            download_backup(backup_file, 'database')
+          else
+            cleanup_backup(backup_file, "Aborting...\nDeleting temporary backup...")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/capistrano/ops/capistrano/v3/tasks/backup/storage/create.rake

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require_relative '../backup_helper'
+namespace :backup do
+  namespace :storage do
+    include BackupHelper
+    # Default to :app role
+    rake_roles = fetch(:rake_roles, :app)
+    
+    desc 'create storage dump on server'
+    task :create do
+      on roles(rake_roles) do
+        puts 'Creating backup...'
+        execute "#{prepare_env} BACKUPS_ENABLED=true EXTERNAL_BACKUP_ENABLED=false bundle exec rake storage:backup"
+      end
+    end
+  end
+end

data/lib/capistrano/ops/capistrano/v3/tasks/backup/storage/pull.rake

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative '../backup_helper'
+namespace :backup do
+  namespace :storage do
+    include BackupHelper
+    # Default to :app role
+    rake_roles = fetch(:rake_roles, :app)
+    desc 'pull latest storage dump from server to local'
+    task :pull do
+      on roles(rake_roles) do
+        puts 'Creating temporary backup...'
+        execute "#{prepare_env} BACKUPS_ENABLED=true EXTERNAL_BACKUP_ENABLED=false bundle exec rake storage:backup"
+        puts 'Backup created'
+        backup_file = backup_file_name('storage')
+        backup_size = backup_file_size
+
+        puts 'Prepare download...'
+
+        question("Backup size: #{size_str(backup_size.to_i)}. Continue?", 'n') do |answer|
+          if answer
+            download_backup(backup_file, 'storage')
+          else
+            cleanup_backup(backup_file, "Aborting...\nDeleting temporary backup...")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/capistrano/ops/capistrano/v3/tasks/backup.rake

@@ -1,29 +1,25 @@
 # frozen_string_literal: true
 
+require_relative 'backup/backup_helper'
+
 namespace :backup do
+  include BackupHelper
   # Default to :app role
   rake_roles = fetch(:rake_roles, :app)
 
-  desc 'create a backup of the server database'
+  desc 'create a backup of the server database (deprecated, use backup:database:create instead)'
   task :create do
     on roles(rake_roles) do
-      env = "RAILS_ENV=#{fetch(:stage)}"
-      # rubocop:disable Layout/LineLength
-      path_cmd = "PATH=$HOME/.rbenv/versions/#{RUBY_VERSION}/bin:$PATH"
-      # rubocop:enable Layout/LineLength
-      execute "cd #{release_path} && #{path_cmd} && #{env} BACKUPS_ENABLED=true bundle exec rake pg:dump"
+      warn "deprecated: use 'backup:database:create' instead, in future versions this task will be removed"
+      execute "#{prepare_env} BACKUPS_ENABLED=true EXTERNAL_BACKUP_ENABLED=false bundle exec rake pg:dump"
     end
   end
-  desc 'pull latest database backups from server to local'
+  desc 'pull latest database backups from server to local (deprecated, use backup:database:pull instead)'
   task :pull do
     on roles(rake_roles) do
-      # rubocop:disable Layout/LineLength
-      execute "cd #{shared_path}/backups && tar -czf #{shared_path}/backups.tar.gz $(ls -lt | grep -E -i '.{0,}\.dump' | head -n 1 | awk '{print $9}')"
-      # rubocop:enable Layout/LineLength
-      download! "#{shared_path}/backups.tar.gz", 'backups.tar.gz'
-      execute "rm #{shared_path}/backups.tar.gz"
-      system 'tar -xzf backups.tar.gz'
-      system 'rm backups.tar.gz'
+      warn "deprecated: use 'backup:database:pull' instead, in future versions this task will be removed"
+      backup_file = backup_file_name('database')
+      download! "#{shared_path}/backups/#{backup_file}"
     end
   end
 end