outback 0.0.14 → 1.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 4503e45304f0be857fab9999cddf991b4a3823668951f0ee476eb94c80abd420
+  data.tar.gz: 49451b535df2adcf44c66089101de73cd38fd9d4494cda5503db130bca36674e
+SHA512:
+  metadata.gz: b88bc0e63f3145bf2d84e032c725e01de590f3d8560de0cb4d60b064895c81aa63827614f729b95d86c89282b106dbcb76afe6f7a9281fbaaef6449ecb937a37
+  data.tar.gz: 1dd915c8b1bf994bd7f78de43f834f4b3b1424da88636c788de8708fb30ca8b9ad05827dbd236e7070013e00484cd61d18ef0301186f870ffa0529f991ed9fed

data/CHANGELOG

@@ -1,3 +1,14 @@
+*1.1.0 (July 8, 2024)*
+
+* Support Ruby 3
+* Do not fail on store or purge
+
+*1.0.2 (September 20, 2016)*
+
+* Add SCP target
+* Add encryption with OpenSSL
+* Drop activesupport dependency
+
 *0.0.12 (August 16, 2012)*
 
 * Fix warning caused by methodphitamine.rb

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 onrooby
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -28,7 +28,7 @@ instantiated during a single run.
 
 ```` ruby
 Outback::Configuration.new 'name' do
-  source :directory, '/ver/www' do
+  source :directory, '/var/www' do
     exclude '/var/www/foo'
     exclude '/var/www/icons/*.png'
   end
@@ -45,6 +45,13 @@ Outback::Configuration.new 'name' do
     # database 'specific_database'  
   end
 
+  processor :encryption do
+    password 'very secure encryption password'
+    
+    # The default cipher is aes-256-cbc
+    #cipher 'openssl supported cipher'
+  end
+
   # Amazon S3 storage
   target :s3 do
     access_key  'S3 access key'
@@ -56,6 +63,19 @@ Outback::Configuration.new 'name' do
     # Just omit the definition to keep archives forever.
     ttl         1.month
   end
+  
+  # SFTP storage
+  target :sftp, 'example.com' do
+    user 'backupuser'
+    # Usually, SSH keys will be used to connect to the server.
+    # Alternatively, a password can be given here.
+    #password 'your password'
+
+    # Default port is 22
+    # port 1234
+    ttl 1.day
+    path '/backups'
+  end
 
   # Store on a local filesystem path
   target :directory, '/media/backups/daily' do
@@ -79,7 +99,7 @@ Default configurations and commandline options
 If you place your backup configurations in the file `/etc/outback.conf` they
 will be read automatically when the outback executable is invoked. Make
 sure to have correct permissions on the configuration files, as they might
-include database passwords.
+include database and encryption passwords.
 
 Alternatively, you can pass in the configuration file to read as a
 commandline argument. The default configuration file in /etc will then be
@@ -131,4 +151,10 @@ Other commandline options are:
 * `-t` `--test`    test configuration, then exit
 * `-l` `--list`    list available configurations, then exit
 * `-h` `--help`    display help
-* `--version`      display version
+* `--version`      display version
+
+Known Issues
+------------
+
+The s3 gem currently doesn't support V4 signatures, which are required for
+s3 regions introduced after January 2014.

data/lib/outback/archive.rb

@@ -1,30 +1,19 @@
 module Outback
   class Archive
-    NAME_PATTERN = /([A-Za-z0-9.\-]+)_(\d{14})_(\w+)/
+    NAME_PATTERN = /\A([A-Za-z0-9.\-]+)_(\d{14})_(\w+)/
     
-    attr_reader :filename, :backup_name, :timestamp, :source_name, :parent
+    attr_reader :filename, :backup_name, :timestamp, :source_name, :size
     
-    def initialize(filename, parent)
-      @filename, @parent = Pathname.new(filename), parent
+    def initialize(filename)
+      @filename = Pathname.new(filename)
       unless match_data = @filename.basename.to_s.match(NAME_PATTERN)
         raise ArgumentError, 'invalid name'
       end
       @backup_name, @timestamp, @source_name = match_data.captures[0..2]
     end
     
-    def size
-      filename.size
+    def to_s
+      "#{filename}"
     end
-    
-    def open
-      filename.open
-    end
-    
-    def outdated?
-      if timestamp && parent && parent.ttl
-        Time.now - Time.parse(timestamp) > parent.ttl
-      end
-    end
-    
   end
 end

data/lib/outback/backup.rb

@@ -1,44 +1,62 @@
 module Outback
   class Backup
-    attr_reader :configuration, :name, :timestamp, :archives, :tmpdir
-    delegate :sources, :targets, :to => :configuration
+    include Logging
     
+    attr_reader :configuration, :name, :timestamp, :tmpdir
+        
     def initialize(configuration)
       raise ArgumentError, "configuration required" unless configuration.is_a?(Outback::Configuration)
       @configuration = configuration
       @name = configuration.name
-      @timestamp = Time.now.to_formatted_s(:number)
+      @timestamp = Time.now.strftime(Outback::TIME_FORMAT)
     end
     
     def run!
-      @archives  = []
-      begin
-        Outback.info "Using working directory #{configuration.tmpdir}" if configuration.tmpdir
-        @tmpdir = Dir.mktmpdir([name, timestamp], configuration.tmpdir)
-        @archives = create_archives
-        Outback.info "Created #{@archives.size} archives"
-        store_archives
-      ensure
-        FileUtils.remove_entry_secure(tmpdir)
-      end
+      logger.info "Using working directory #{configuration.tmpdir}" if configuration.tmpdir
+      @tmpdir = Dir.mktmpdir([name, timestamp], configuration.tmpdir)
+      archives = create_archives
+      logger.info "Created #{archives.size} archives"
+      archives = process_archives(archives)
+      logger.info "Processed #{archives.size} archives"
+      store_archives(archives)
       purge_targets
+    ensure
+      FileUtils.remove_entry_secure(tmpdir)
+    end
+    
+    [:sources, :processors, :targets].each do |collection|
+      define_method(collection) { configuration.public_send(collection) }
     end
     
     private
     
     def create_archives
-      archives = sources.collect do |source|
-        source.create_archives(name, timestamp, tmpdir)
-      end
-      archives.flatten.compact
+      sources.map { |source| source.create_archives(timestamp, tmpdir) }.flatten.compact
     end
     
-    def store_archives
-      targets.each { |target| target.put(archives) }
+    def process_archives(archives)
+      processors.inject(archives) { |archives, processor| processor.process!(archives) }.flatten.compact
+    end
+    
+    def store_archives(archives)
+      targets.each do |target|
+        begin
+          target.put(archives)
+        rescue => e
+          logger.info "Error while storing to #{target}: #{e.inspect}"
+          0
+        end
+      end
     end
 
     def purge_targets
-      targets.each { |target| target.purge!(name) }
+      targets.each do |target|
+        begin
+          target.purge!
+        rescue => e
+          logger.info "Error while purging #{target}: #{e.inspect}"
+        end
+      end
     end
     
   end

data/lib/outback/cli.rb

@@ -35,7 +35,11 @@ module Outback
         end
         option_parser.parse!
         Outback::Configuration.reset
-        config_file = ARGV.first || DEFAULT_CONFIGURATION_FILE
+        config_file = if ARGV.first
+          ARGV.first.start_with?('/') ? ARGV.first : File.join(Dir.pwd, ARGV.first)
+        else
+          DEFAULT_CONFIGURATION_FILE
+        end
         begin
           load config_file
         rescue ConfigurationError => conf_error
@@ -83,4 +87,4 @@ module Outback
     end
     
   end  
-end
+end

data/lib/outback/configuration.rb

@@ -1,10 +1,11 @@
 module Outback
   class Configuration
+    
     @loaded = []
 
     class << self
       def add(configuration)
-        raise ConfigurationError.new("duplicate configuration #{configuration.name}") if loaded.any?(&its.name == configuration.name)
+        raise ConfigurationError, "duplicate configuration #{configuration.name}" if loaded.any?(&its.name == configuration.name)
         loaded << configuration
       end
       
@@ -21,13 +22,13 @@ module Outback
       end
     end
 
-    attr_reader :name, :sources, :targets, :errors
+    attr_reader :name, :sources, :targets, :processors, :errors
 
     def initialize(name, &block)
-      raise(ConfigurationError.new("configuration name can't be blank")) if name.blank?
-      @name = name.to_s
-      raise(ConfigurationError.new('configuration name may not contain underscores')) if @name.include?('_')
-      @sources, @targets, @errors = [], [], []
+      name = name.to_s
+      raise ConfigurationError, "Illegal configuration name #{name.inspect}" unless name.match(/\A[a-z][a-z0-9_\-.]+\z/)
+      @name = name
+      @sources, @processors, @targets, @errors = [], [], [], []
       if block_given?
         if block.arity == 1 then yield(self) else instance_eval(&block) end
       end
@@ -48,21 +49,25 @@ module Outback
         @tmpdir
       elsif args.size == 1
         dir = Pathname.new(args.first).realpath
-        raise(ConfigurationError.new("tmpdir #{dir} is no directory")) unless dir.directory?
+        raise ConfigurationError, "tmpdir #{dir} is not a directory" unless dir.directory?
         @tmpdir = dir
       else
-        raise ConfigurationError.new("tmpdir: wrong number of arguments(#{args.size} for 1)")
+        raise ConfigurationError, "tmpdir: wrong number of arguments(#{args.size} for 1)"
       end
     end
 
     protected
 
     def source(type, *args, &block)
-      "Outback::#{type.to_s.classify}Source".constantize.configure(*args, &block).tap { |instance| sources << instance }
+      "Outback::#{type.to_s.classify}Source".constantize.configure(name, *args, &block).tap { |instance| sources << instance }
+    end
+    
+    def processor(type, *args, &block)
+      "Outback::#{type.to_s.classify}Processor".constantize.configure(name, *args, &block).tap { |instance| processors << instance }
     end
     
     def target(type, *args, &block)
-      "Outback::#{type.to_s.classify}Target".constantize.configure(*args, &block).tap { |instance| targets << instance }
+      "Outback::#{type.to_s.classify}Target".constantize.configure(name, *args, &block).tap { |instance| targets << instance }
     end
 
     def error(message)

data/lib/outback/directory_source.rb

@@ -2,7 +2,8 @@ module Outback
   class DirectorySource < Source
     attr_reader :path
     
-    def initialize(path)
+    def initialize(backup_name, path)
+      super(backup_name)
       @path = path
     end
     
@@ -18,18 +19,18 @@ module Outback
       excludes.concat(paths.map(&:to_s)).uniq!
     end
     
-    def create_archives(backup_name, timestamp, tmpdir)
+    def create_archives(timestamp, tmpdir)
       source_dir = Pathname.new(path).realpath
       archive_name = Pathname.new(tmpdir).join("#{backup_name}_#{timestamp}_#{source_name}.tar.gz")
       exclude_list = Pathname.new(tmpdir).join('exclude_list.txt')
       File.open(exclude_list, 'w') { |f| f << excludes.map { |e| e.to_s.sub(/\A\//, '') }.join("\n") }
       verbose_switch = Outback.verbose? ? 'v' : ''
       commandline = "tar -cz#{verbose_switch}pf #{archive_name} --exclude-from=#{exclude_list} --directory=/ #{source_dir.to_s.sub(/\A\//, '')}"
-      Outback.debug "executing command: #{commandline}"
+      logger.debug "executing command: #{commandline}"
       result = `#{commandline}`
-      Outback.debug "result: #{result}"
-      Outback.info "Archived directory #{path}"
-      [TempArchive.new(archive_name, self)]
+      logger.debug "result: #{result}"
+      logger.info "Archived directory #{path}"
+      [SourceArchive.new(archive_name)]
     end
   end
-end
+end

data/lib/outback/directory_target.rb

@@ -3,7 +3,8 @@ module Outback
     attr_reader :path
     attr_setter :user, :group, :directory_permissions, :archive_permissions, :ttl, :move
     
-    def initialize(path)
+    def initialize(backup_name, path)
+      super(backup_name)
       @path = Pathname.new(path)
     end
     
@@ -11,39 +12,45 @@ module Outback
       (user and group) or (not user and not group)
     end
     
-    def display_name
+    def to_s
       "directory:#{path}"
     end
     
     def put(archives)
-      Dir.mkdir(path) unless path.directory?
+      FileUtils.mkdir_p(path) unless path.directory?
       FileUtils.chmod directory_permissions || 0700, path
       size = 0
       archives.each do |archive|
         basename = Pathname.new(archive.filename).basename
         if move
-          Outback.debug "moving #{archive.filename} to #{path}"
+          logger.debug "moving #{archive.filename} to #{path}"
           FileUtils.mv archive.filename, path
         else
-          Outback.debug "copying #{archive.filename} to #{path}"
+          logger.debug "copying #{archive.filename} to #{path}"
           FileUtils.cp_r archive.filename, path
         end
         archived_file = path.join(basename)
-        Outback.debug "setting permissions for #{archived_file}"
+        logger.debug "setting permissions for #{archived_file}"
         FileUtils.chmod archive_permissions || 0600, archived_file
         if user && group
-          Outback.debug "setting owner #{user}, group #{group} for #{archived_file}"
+          logger.debug "setting owner #{user}, group #{group} for #{archived_file}"
           FileUtils.chown user, group, archived_file
         end
         size += archived_file.size
       end
-      Outback.info "#{move ? 'Moved' : 'Copied'} #{archives.size} archives (#{size} bytes) to #{display_name}"
+      logger.info "#{move ? 'Moved' : 'Copied'} #{archives.size} archives (#{size} bytes) to #{self}"
       archives.size
     end
+    
+    private
 
-    def list_archives(name)
-      path.files(Archive::NAME_PATTERN).map { |f| DirectoryArchive.new(f, self) }.select(&its.backup_name == name)
+    def list_all_archives
+      path.files(Archive::NAME_PATTERN).map { |f| build_archive(f.to_s, f.size) }
+    end
+    
+    def unlink_archive!(archive)
+      archive.filename.unlink
     end
     
   end
-end
+end

data/lib/outback/encryption_processor.rb

@@ -0,0 +1,34 @@
+require 'openssl'
+require 'open3'
+
+module Outback
+  class EncryptionProcessor < Processor
+
+    attr_setter :password, :cipher
+
+    def cipher
+      @cipher ||= 'aes-256-cbc'
+    end
+    
+    def to_s
+      "encryption:#{cipher}"
+    end
+
+    private
+    
+    def process_archive!(archive)
+      result = nil
+      outfile = Pathname.new("#{archive.filename}.enc")
+      logger.debug "Encrypting #{archive} with #{self}"
+      Open3.popen3("openssl enc -#{cipher} -pass stdin -in #{archive.filename} -out #{outfile}") do |stdin, stdout, stderr, wait_thr|
+        stdin << password
+        stdin.close
+        result = wait_thr.value
+      end
+      raise ProcessingError, "error processing archive #{archive} in #{self}" unless result.success?
+      raise ProcessingError, "outfile #{outfile} not found" unless outfile.file?
+      archive.unlink
+      SourceArchive.new(outfile)
+    end
+  end
+end

data/lib/outback/errors.rb

@@ -0,0 +1,7 @@
+module Outback
+  class Error < StandardError; end
+  
+  class ConfigurationError < Error; end
+  class BackupError < Error; end
+  class ProcessingError < Error; end
+end

data/lib/outback/logging.rb

@@ -0,0 +1,7 @@
+module Outback
+  module Logging
+    def logger
+      Outback
+    end
+  end
+end

data/lib/outback/mysql_source.rb

@@ -7,7 +7,7 @@ module Outback
     end
     
     def excludes
-      @excludes ||= []
+      @excludes ||= ['mysql', 'performance_schema', 'information_schema']
     end
     
     def database(*names)
@@ -22,13 +22,13 @@ module Outback
       user && password
     end
     
-    def create_archives(backup_name, timestamp, tmpdir)
+    def create_archives(timestamp, tmpdir)
       mysql_host = host || 'localhost'
       mysql_port = (port || 3306) unless socket
       if databases.empty?
-        #        (host=nil, user=nil, passwd=nil, db=nil, port=nil, socket=nil, flag=nil)
+        # (host=nil, user=nil, passwd=nil, db=nil, port=nil, socket=nil, flag=nil)
         mysql = Mysql.connect(mysql_host, user, password, nil, mysql_port, socket)
-        @databases = mysql.databases - excludes
+        @databases = mysql.list_dbs - excludes
         mysql.close
       end
       
@@ -37,13 +37,13 @@ module Outback
         mysql_conf_file = Pathname.new(tmpdir).join('outback_my.cnf')
         File.open(mysql_conf_file, 'w') { |f| f << "[client]\npassword=#{password}\n" }
         FileUtils.chmod 0600, mysql_conf_file
-        Outback.debug "MysqlSource: dumping database '#{database}'"
+        logger.debug "MysqlSource: dumping database '#{database}'"
         commandline = "mysqldump --defaults-extra-file=#{mysql_conf_file} --opt --user=#{user} --host=#{mysql_host} --port=#{mysql_port} #{database} | gzip > #{archive_name}"
         result = `#{commandline}`.strip
-        Outback.debug(result) unless result.blank?
-        Outback.info "Archived database #{database}"
-        TempArchive.new(archive_name, self).tap { |archive| Outback.debug "dumped #{archive.filename.basename} with #{archive.size} bytes" }
+        logger.debug(result) unless result.blank?
+        logger.info "Archived database #{database}"
+        SourceArchive.new(archive_name).tap { |archive| logger.debug "dumped #{archive.filename.basename} with #{archive.size} bytes" }
       end
     end
   end
-end
+end

data/lib/outback/processor.rb

@@ -0,0 +1,17 @@
+module Outback
+  class Processor
+    include Configurable
+    include Logging
+    
+    attr_reader :backup_name
+    
+    def initialize(backup_name)
+      @backup_name = backup_name
+    end
+        
+    def process!(archives)
+      archives.map { |archive| process_archive!(archive) }.flatten.compact
+    end
+    
+  end
+end

data/lib/outback/s3_target.rb

@@ -2,7 +2,7 @@ module Outback
   class S3Target < Target
     attr_setter :bucket_name, :access_key, :secret_key, :ttl, :prefix
     
-    def display_name
+    def to_s
       "s3:#{bucket_name}/#{prefix}"
     end
     
@@ -12,7 +12,7 @@ module Outback
     
     def service(force_reconnect = false)
       @service = nil if force_reconnect
-      @service ||= S3::Service.new(:access_key_id => access_key, :secret_access_key => secret_key)
+      @service ||= S3::Service.new(access_key_id: access_key, secret_access_key: secret_key, use_ssl: true)
     end
     
     def bucket
@@ -23,7 +23,7 @@ module Outback
       size = count = 0
       archives.each do |archive|
         object_name = [prefix.to_s, archive.filename.basename.to_s].join('/')
-        Outback.debug "S3Target: storing #{archive.filename} in s3://#{bucket_name}/#{object_name}"
+        logger.debug "S3Target: storing #{archive.filename} in s3://#{bucket_name}/#{object_name}"
         object = bucket.objects.build(object_name)
         object.content = archive.open
         object.acl = :private
@@ -32,17 +32,26 @@ module Outback
           size += archive.size
           count += 1
         else
-          Outback.error "S3 archive upload failed: #{object_name}"
+          logger.error "S3 archive upload failed: #{object_name}"
         end
       end
-      Outback.info "Uploaded #{count} archives (#{size} bytes) to #{display_name}"
+      logger.info "Uploaded #{count} archives (#{size} bytes) to #{self}"
       count
     end
     
-    def list_archives(name)
+    private
+
+    def list_all_archives
       entries = bucket.objects.select { |e| e.key.start_with?(prefix.to_s) && e.key[prefix.to_s.size..-1].match(Archive::NAME_PATTERN) }
-      entries.map { |e| S3Archive.new(e.key, self) }.select(&its.backup_name == name)
+      entries.map { |e| build_archive(e.key, e.size) }
     end
+    
+    def unlink_archive!(archive)
+      logger.debug "purging S3Archive: #{archive.filename}"
+      object = bucket.objects.find(archive.filename.to_s)
+      return true if object && object.destroy
+      raise BackupError, "could not find object #{archive.filename} for purging"
+    end
+
   end
-  
-end
+end

data/lib/outback/sftp_target.rb

@@ -0,0 +1,70 @@
+require 'net/sftp'
+
+module Outback
+  class SftpTarget < Target
+    
+    attr_reader :host
+    attr_setter :user, :password, :port, :path, :ttl, :ssh_opts
+    
+    def initialize(backup_name, host)
+      super(backup_name)
+      @host = host
+    end
+    
+    def valid?
+      host.present?
+    end
+    
+    def to_s
+      "sftp:#{user}@#{host}#{':' + port.to_s if port}:#{path}"
+    end
+    
+    def put(archives)
+      size = count = 0
+      connect do
+        archives.each do |archive|
+          basename = archive.filename.basename.to_s
+          upload_filename = path ? File.join(path, basename) : basename
+          logger.debug "SftpTarget: storing #{archive.filename} in sftp://#{user}@#{host}#{':' + port.to_s if port}:#{upload_filename}"
+          connection.upload!(archive.filename.to_s, upload_filename)
+          size += archive.size
+          count += 1
+        end
+      end
+      logger.info "Uploaded #{count} archives (#{size} bytes) to #{self}"
+      count
+    end
+
+    private
+  
+    def connect
+      result = nil
+      Net::SFTP.start(host, user, ssh_options) do |sftp|
+        @connection = sftp
+        result = yield
+      end
+      result
+    ensure
+      @connection = nil
+    end
+  
+    def ssh_options
+      options = ssh_opts || {}
+      if password
+        options[:password] = password
+        options[:auth_methods] ||= %w[password]
+      end
+      options[:port] = port if port
+      options
+    end
+    
+    def list_all_archives
+      connection.dir.entries(path).select(&:file?).map { |entry| build_archive(File.join(path, entry.name), entry.attributes.size.to_i) }
+    end
+    
+    def unlink_archive!(archive)
+      connection.remove!(archive.filename)
+    end
+
+  end
+end