backupper 0.5.2 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 427e5d2913d10307c2a02b49cf8948aec658561267d602553b1c8a2f02af113a
-  data.tar.gz: cf4a1a6fe666551223bea18ed1a53d8a6aa170d23e1a5bcc52f92c6282f5fea6
+  metadata.gz: a381517df098479aaa0237669fe34e93eb27616af160d6b2d60ea055d70c1835
+  data.tar.gz: 45d9d097401c396d046646003876f0a4078f25ec77cde66d84450e4f02eb3e96
 SHA512:
-  metadata.gz: 72cca9939837dd3d45ac6853ba2985349ac53659a8e34195b61b9133b08ffb35846346e3b656e933b587badbd27ff585124a9071c805fc7e80ca1f64ea19262e
-  data.tar.gz: 8a743ab45a61dfbf13c6f31511d4e8339a2f905c25c5602049073153ea09c0d1795daea96148d81c83fdd0ae72bf4296b26a32a0844c5c6e0a882f4c1fa67025
+  metadata.gz: 4c4ebcb074e0f30928d4952b0c80f0617d3bbe7bfe0277d1ef0cb26dbad0f6bc1c14bd012dff0aec880bcf35db2c09267e5087de56432e94221dfc8adc3aa7fb
+  data.tar.gz: 3bb3d1486bf4cc093175aa5fd28cb6afe5f6f7691d72d116dc730e333904b0a61b7637abd687d8e90609e5a9d76824f938f837949e122c790dec61ad33528504

data/Gemfile.lock

@@ -1,38 +1,47 @@
 PATH
   remote: .
   specs:
-    backupper (0.5.1)
-      mail (~> 2.7)
-      sshkit (~> 1.15)
+    backupper (0.6.0)
+      mail (~> 2.9)
+      sshkit (~> 1.25)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    byebug (11.0.1)
-    date (3.3.3)
-    mail (2.8.0)
+    base64 (0.3.0)
+    date (3.5.1)
+    logger (1.7.0)
+    mail (2.9.1)
+      logger
       mini_mime (>= 0.1.1)
       net-imap
       net-pop
       net-smtp
-    mini_mime (1.1.2)
-    net-imap (0.3.4)
+    mini_mime (1.1.5)
+    net-imap (0.6.4.1)
       date
       net-protocol
     net-pop (0.1.2)
       net-protocol
-    net-protocol (0.2.1)
+    net-protocol (0.2.2)
       timeout
-    net-scp (4.0.0)
+    net-scp (4.1.0)
       net-ssh (>= 2.6.5, < 8.0.0)
-    net-smtp (0.3.3)
+    net-sftp (4.0.0)
+      net-ssh (>= 5.0.0, < 8.0.0)
+    net-smtp (0.5.1)
       net-protocol
-    net-ssh (7.0.1)
-    rake (10.5.0)
-    sshkit (1.21.3)
+    net-ssh (7.3.3)
+    ostruct (0.6.3)
+    rake (13.4.2)
+    sshkit (1.25.0)
+      base64
+      logger
       net-scp (>= 1.1.2)
+      net-sftp (>= 2.1.2)
       net-ssh (>= 2.8.0)
-    timeout (0.3.1)
+      ostruct
+    timeout (0.6.1)
 
 PLATFORMS
   ruby
@@ -40,9 +49,8 @@ PLATFORMS
 
 DEPENDENCIES
   backupper!
-  bundler (~> 2)
-  byebug (~> 11)
-  rake (~> 10.0)
+  bundler
+  rake
 
 BUNDLED WITH
-   2.3.8
+  4.0.15

data/README.md

@@ -20,6 +20,10 @@ mailer:
   to: pioz@email.com      # Send report email to this address
   password: Pa$$w0rD      # Gmail account password
 
+default:                  # options here are merged into all database configurations below
+  port: 22
+  db_username: backup
+
 db1:
   disabled: false                              # true to disable this backup
   dump: '/home/backup/db1'                     # path where to save the dump of the database
@@ -27,12 +31,13 @@ db1:
   username: user                               # server ssh username
   host: '1.2.3.4'                              # server ssh ip
   port: 22                                     # server ssh port
-  password: Pa$$w0rD                           # server ssh password
-  adapter: mysql                               # database to backup (supported are mysql or postgresql)
+  password: Pa$$w0rD                           # server ssh password (also used as sudo password when sudo is true)
+  sudo: false                                  # true to run the dump command with sudo
+  adapter: mysql                               # adapter to use to perform the backup (mysql, postgresql or mydumper)
   database: db_name                            # database name
   db_username: db_user                         # database username
   db_password: db_Pa$$w0rD                     # database password
-  dump_options: '--single-transaction --quick' # dump command extra options
+  dump_options: '--single-transaction --quick' # dump command extra options (shell-style quoted or YAML array)
 
 db2:
   disabled: false
@@ -55,12 +60,25 @@ After done all backups a report email is sent to you using gmail smtp service (r
 To backup only specific databases you can pass the option `-o db1,db3`. In this way only databases under the key `db1` and `db3` will be backupped,
 ignoring the others configuration keys.
 
-## Resources
-* [Blog post about Backupper](https://tech.uqido.com/2018/07/16/backupper-tool-backup-database/) (italian only).
+## Custom adapters
+
+An adapter is just a module method of `DumpCommand` that returns the shell command used to dump the database:
+
+```ruby
+def self.myadapter(database:, username: 'root', password: nil, dump_options: nil, working_dir: nil, outfile:)
+  return "mydump #{database} -u #{username} > #{outfile}"
+end
+```
+
+All the values arrive already shell-escaped: interpolate them as they are, without wrapping them in quotes. `dump_options` is parsed as shell-style arguments, then each argument is escaped before it reaches the adapter. The command must write the dump to `outfile`; `working_dir` is an empty temporary directory you can use as scratch space (both are removed from the server after the download). If your adapter produces a file format different from `.sql.bz2`, add its extension to `DumpCommand::EXTENSIONS`.
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/uqido/backupper.
+Bug reports and pull requests are welcome on GitHub at https://github.com/pioz/backupper.
+
+## Resources
+* [Blog post about Backupper](https://tech.uqido.com/2018/07/16/backupper-tool-backup-database/) (italian only).
+
 
 ## License
 

data/Rakefile

@@ -1,2 +1 @@
 require 'bundler/gem_tasks'
-task default: :spec

data/backupper.gemspec

@@ -6,11 +6,11 @@ Gem::Specification.new do |spec|
   spec.name          = 'backupper'
   spec.version       = Backupper::VERSION
   spec.authors       = ['pioz']
-  spec.email         = ['enrico.pilotto@uqido.com']
+  spec.email         = ['epilotto@gmx.com']
 
   spec.summary       = 'Tool to backup databases'
   spec.description   = 'Backupper is a tool to backup all your databases spread all over the world'
-  spec.homepage      = 'https://github.com/uqido/backupper'
+  spec.homepage      = 'https://github.com/pioz/backupper'
   spec.license       = 'MIT'
 
   # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
@@ -28,10 +28,10 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(/^bin\//) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_runtime_dependency 'mail', '~> 2.7'
-  spec.add_runtime_dependency 'sshkit', '~> 1.15'
+  spec.required_ruby_version = '>= 2.5'
+  spec.add_runtime_dependency 'mail', '~> 2.9'
+  spec.add_runtime_dependency 'sshkit', '~> 1.25'
 
-  spec.add_development_dependency 'bundler', '~> 2'
-  spec.add_development_dependency 'byebug', '~> 11'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'rake'
 end

data/lib/backupper/backupper.rb

@@ -1,10 +1,12 @@
 require 'fileutils'
+require 'shellwords'
 require 'sshkit'
 require 'sshkit/dsl'
 require 'yaml'
 
 require_relative 'dump_command'
 require_relative 'mailer'
+require_relative 'sshkit'
 
 class Backupper
   include SSHKit::DSL
@@ -28,7 +30,14 @@ class Backupper
 
   def backup!(only_these_keys = nil)
     filtered_conf = @conf
-    filtered_conf = @conf.slice(*only_these_keys) if only_these_keys
+    if only_these_keys
+      filtered_conf = @conf.slice(*only_these_keys)
+      (only_these_keys - @conf.keys).each do |k|
+        err = "Configuration key '#{k}' not found or disabled!"
+        error(k, err)
+        @logger.error err
+      end
+    end
     filtered_conf.each do |k, options|
       @logger.info "[#{Time.now}] ⬇️  backing up #{k}..."
       o, err = setup_options(options)
@@ -38,7 +47,7 @@ class Backupper
         next
       end
       begin
-        download_dump(
+        do_and_download_dump(
           key:          k,
           adapter:      o['adapter'],
           url:          o['url'],
@@ -47,6 +56,7 @@ class Backupper
           db_username:  o['db_username'],
           db_password:  o['db_password'],
           dump_options: o['dump_options'],
+          sudo:         o['sudo'],
           outdir:       o['outdir'],
           extra_copy:   o['extra_copy']
         )
@@ -58,31 +68,45 @@ class Backupper
     send_report_email!
   end
 
-  def download_dump(key:, adapter: 'mysql', url:, password: nil, database:, db_username: 'root', db_password: nil, dump_options: nil, outdir:, extra_copy: nil)
-    t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-    path = nil
-    filename = "#{key}__#{database}.sql.bz2"
-    tempfile = File.join('/tmp', filename)
-    dumpname = "#{Time.now.strftime('%Y-%m-%d_%H-%M-%S')}__#{filename}"
-    path = File.join(outdir, dumpname)
-    on(url) do |client|
-      client.password = password
-      execute 'set -o pipefail; ' + DumpCommand.send(adapter, database: database, username: db_username, password: db_password, dump_options: dump_options, outfile: tempfile)
-      download! tempfile, path
-      execute :rm, tempfile
-    end
-    extra_copy = check_dir(extra_copy)
-    FileUtils.cp(path, extra_copy) if extra_copy
-    @report[key] = {
-      path: File.absolute_path(path),
-      size: (File.size(path).to_f / 2**20).round(2),
-      time: (Process.clock_gettime(Process::CLOCK_MONOTONIC) - t).round(2)
-    }
-    @report[key].merge!({ extra_copy: File.absolute_path(File.join(extra_copy, dumpname)) }) if extra_copy
-  end
-
   private
 
+    def do_and_download_dump(key:, adapter: 'mysql', url:, password: nil, database:, db_username: 'root', db_password: nil, dump_options: nil, sudo: false, outdir:, extra_copy: nil)
+      t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      filename = "#{key}__#{database}#{DumpCommand::EXTENSIONS[adapter]}"
+      temp_file = File.join('/tmp', filename)
+      working_dirname = "#{key}__#{database}"
+      temp_working_dir = File.join('/tmp', working_dirname)
+      dump_name = "#{Time.now.strftime('%Y-%m-%d_%H-%M-%S')}__#{filename}"
+      path = File.join(outdir, dump_name)
+      outfile = Shellwords.escape(temp_file)
+      working_dir = Shellwords.escape(temp_working_dir)
+      on(url) do |host|
+        host.password = password
+        sudo_execute :mkdir, '-p', working_dir, password: password, sudo: sudo
+        command = DumpCommand.public_send(
+          adapter,
+          database: Shellwords.escape(database),
+          username: Shellwords.escape(db_username),
+          password: db_password && Shellwords.escape(db_password),
+          dump_options: dump_options,
+          working_dir: working_dir,
+          outfile: outfile
+        )
+        sudo_execute 'set -o pipefail; ' + command, password: password, sudo: sudo
+        download! temp_file, path
+        sudo_execute :rm, outfile, password: password, sudo: sudo
+        sudo_execute :rm, '-r', working_dir, password: password, sudo: sudo
+      end
+      extra_copy = check_dir(extra_copy)
+      FileUtils.cp(path, extra_copy) if extra_copy
+      @report[key] = {
+        path: File.absolute_path(path),
+        size: (File.size(path).to_f / 2**20).round(2),
+        time: (Process.clock_gettime(Process::CLOCK_MONOTONIC) - t).round(2)
+      }
+      @report[key][:extra_copy] = File.absolute_path(File.join(extra_copy, dump_name)) if extra_copy
+    end
+
     def setup_options(options)
       o = @default.merge(options)
       o['outdir'] = check_dir(o['dump'].to_s)
@@ -94,17 +118,38 @@ class Backupper
       o['url'] = "#{o['username']}@#{o['url']}" if o['username']
       o['url'] = "#{o['url']}:#{o['port']}" if o['port']
       o['adapter'] ||= 'mysql'
-      unless DumpCommand.respond_to?(o['adapter'])
-        return nil, "Cannot handle adapter '#{o['adapter']}'"
+      unless DumpCommand.singleton_methods(false).map(&:to_s).include?(o['adapter'])
+        return nil, "Cannot handle adapter '#{o['adapter']}' (supported: #{DumpCommand.singleton_methods(false).join(', ')})"
+      end
+      begin
+        shellescape_dump_options(o['dump_options'])
+      rescue ArgumentError => e
+        return nil, e.message
       end
 
       return o, nil
     end
 
+    def shellescape_dump_options(dump_options)
+      return nil if dump_options.nil?
+
+      options =
+        if dump_options.is_a?(Array)
+          dump_options.map(&:to_s)
+        else
+          Shellwords.split(dump_options.to_s)
+        end
+      return nil if options.empty?
+
+      return options.map { |option| Shellwords.escape(option) }.join(' ')
+    rescue ArgumentError => e
+      raise ArgumentError, "Invalid dump_options: #{e.message}"
+    end
+
     def send_report_email!
       if @report.any? && @mailer['from'] && @mailer['to'] && @mailer['password']
         begin
-          Mailer.send(@report, from: @mailer['from'], to: @mailer['to'], password: @mailer['password'], address: @mailer['address'] || 'smtp.gmail.com', port: @mailer['port'] || 587, authentication: @mailer['authentication'] || 'plain')
+          Mailer.deliver(@report, from: @mailer['from'], to: @mailer['to'], password: @mailer['password'], address: @mailer['address'] || 'smtp.gmail.com', port: @mailer['port'] || 587, authentication: @mailer['authentication'] || 'plain')
         rescue Net::SMTPAuthenticationError => e
           @logger.error e
         end

data/lib/backupper/dump_command.rb

@@ -1,16 +1,27 @@
 module DumpCommand
-  def self.mysql(database:, username: 'root', password: nil, dump_options: nil, outfile:)
+  # Extension of the file produced by each adapter ('.sql.bz2' if not listed here)
+  EXTENSIONS = Hash.new('.sql.bz2').update('mydumper' => '.tar.bz2').freeze
+
+  def self.mysql(database:, username: 'root', password: nil, dump_options: nil, working_dir: nil, outfile:)
+    params = []
+    params << "-u#{username}"
+    params << "-p#{password}" if password
+    params << dump_options if dump_options
+    return "mysqldump #{database} #{params.join(' ')} | bzip2 > #{outfile}"
+  end
+
+  def self.postgresql(database:, username: 'root', password: nil, dump_options: nil, working_dir: nil, outfile:)
     params = []
-    params << "-u'#{username}'"
-    params << "-p'#{password}'" if password
+    params << "-U #{username}"
     params << dump_options if dump_options
-    return "mysqldump '#{database}' #{params.join(' ')} | bzip2 > '#{outfile}'"
+    return "PGPASSWORD=#{password} pg_dump #{database} #{params.join(' ')} | bzip2 > #{outfile}"
   end
 
-  def self.postgresql(database:, username: 'root', password: nil, dump_options: nil, outfile:)
+  def self.mydumper(database:, username: 'root', password: nil, dump_options: nil, working_dir: nil, outfile:)
     params = []
-    params << "-U '#{username}'"
+    params << "--user #{username}"
+    params << "--password #{password}" if password
     params << dump_options if dump_options
-    return "PGPASSWORD='#{password}' pg_dump '#{database}' #{params.join(' ')} | bzip2 > '#{outfile}'"
+    return "mydumper --database #{database} #{params.join(' ')} --outputdir #{working_dir} && tar -cjf #{outfile} -C #{File.dirname(working_dir)} #{File.basename(working_dir)}"
   end
 end

data/lib/backupper/mailer.rb

@@ -1,7 +1,7 @@
 require 'mail'
 
 class Mailer
-  def self.send(report, from:, to:, password:, address:, port:, authentication:)
+  def self.deliver(report, from:, to:, password:, address:, port:, authentication:)
     Mail.defaults do
       delivery_method :smtp, {
         address:              address,
@@ -12,7 +12,7 @@ class Mailer
         enable_starttls_auto: true
       }
     end
-    Mail.deliver(to: to, from: from, subject: generate_subject(report), body: generate_body(report))
+    Mail.deliver(to: to, from: from, subject: generate_subject(report), body: generate_body(report), charset: 'UTF-8')
   end
 
   class << self

data/lib/backupper/sshkit.rb

@@ -0,0 +1,23 @@
+require 'shellwords'
+require 'sshkit'
+
+module SSHKitSudoExecute
+  def sudo_execute(*command, password: nil, sudo: false, **options)
+    return execute(*command, options) unless sudo
+
+    prompt = 'SSHKitSudoPassword'
+    execute(
+      :sudo,
+      '-S',
+      '-p', prompt,
+      '--',
+      'bash', '-c', command.map(&:to_s).join(' ').shellescape,
+      options.merge(
+        interaction_handler: {
+          /#{Regexp.escape(prompt)}/ => "#{password}\n"
+        }
+      )
+    )
+  end
+end
+SSHKit::Backend::Abstract.include(SSHKitSudoExecute)

data/lib/backupper/version.rb

@@ -1,3 +1,3 @@
 class Backupper
-  VERSION = '0.5.2'.freeze
+  VERSION = '0.6.0'.freeze
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: backupper
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.6.0
 platform: ruby
 authors:
 - pioz
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-02-02 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mail
@@ -16,74 +15,60 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '2.9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '2.9'
 - !ruby/object:Gem::Dependency
   name: sshkit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.15'
+        version: '1.25'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.15'
+        version: '1.25'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2'
-- !ruby/object:Gem::Dependency
-  name: byebug
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '11'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '11'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
 description: Backupper is a tool to backup all your databases spread all over the
   world
 email:
-- enrico.pilotto@uqido.com
+- epilotto@gmx.com
 executables:
 - backupper
 extensions: []
@@ -101,13 +86,13 @@ files:
 - lib/backupper/backupper.rb
 - lib/backupper/dump_command.rb
 - lib/backupper/mailer.rb
+- lib/backupper/sshkit.rb
 - lib/backupper/version.rb
-homepage: https://github.com/uqido/backupper
+homepage: https://github.com/pioz/backupper
 licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -115,15 +100,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.3
-signing_key:
+rubygems_version: 4.0.15
 specification_version: 4
 summary: Tool to backup databases
 test_files: []