simple_backup 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cd3e8ec26688c63d6e75643e4b962556b330c9ef
-  data.tar.gz: d721a3e4ee4cea8064440289e527f9633c553a17
+  metadata.gz: db18d1aeca0f0b1e397e8599de6ed70474fd49e1
+  data.tar.gz: 9e70279a4648074f77da59a0f070a869ce8c555d
 SHA512:
-  metadata.gz: ba862bc3835154a09912e16c85eab02c7b8defc932dfb755f5d5d6fcaf4f473e1ee87109194520f0a57bbac34188458000ad37aeab9b4d7549845840f8cda6cc
-  data.tar.gz: f4eb040ff8315e5b35690ceb309237e8341090185376837e677afa50c92e7b457bbeda0b5f7b87a84142946974c4384f745b32241e5fff18aa1593b7d77aa13e
+  metadata.gz: d94e122a1bfab65a376e29087564568917c939149cec9d52c37d5905dd683687b033b1132fb28bcf6cf373da9f0fc3cb6492fd8cc30fe1c09bb2d38ef3412421
+  data.tar.gz: 00bb1b44ad48000765a9aa84817cc451772886beb03dc41e07a7503cbcf3d2e4d1293844de049caa6987d5d11a0221f57534453cafa9904ff4356d38eae1c54b

data/.gitignore

@@ -1,2 +1,3 @@
 /pkg/
+/Dockerfile
 /Gemfile.lock

data/README.md

@@ -9,7 +9,8 @@ development and its API should be treat as unstable.
 ## TODO
 
 - [ ] Refactorization
-- [ ] Few backend for backup store (file, s3, ftp)
+- [ ] Few backends for backup store (file, s3, ftp)
+- [ ] Filters mechanism (e.g. PGP encryption)
 - [ ] Tests
 - [ ] Docummentation and examples
 

data/backup_example.rb

@@ -1,31 +1,41 @@
 #!/usr/bin/env ruby
 
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
 require 'simple_backup'
 
 SimpleBackup.run do
-#  log_level           :debug
-  backup_dir          '/backup'
+  log_level  :debug
+
   high_usage_treshold 0.9
   check_disk_path     '/'
   check_disk_path     '/backup'
   check_disk_path     '/home/app'
 
-  apps do
-    keep_last 9
+  default_keep_last 9
+
+  sources do
+    dir 'app-1', '/home/app/app-1', type: :capistrano, backends: 'backup'
+    dir 'app-2', '/home/app/app-2', backends: :none
+    dir 'none',  '/none'
+
+    file 'hosts', '/etc/hosts'
 
-    app '/home/app/app-1', type: :capistrano
-    app '/home/app/app-2', type: :bare
+    mysql 'test1', 'test1'
+    mysql 'test2', 'test2'
+    mysql 'test3', 'test3', exclude_tables: ['t_test1']
   end
 
-  mysql do
-    keep_last 9
+  backends do
+    local 'backup', path: '/srv/backup'
+  end
 
+  mysql do
     host 'localhost'
     port 3306
-    user 'backup'
-    pass 'backup'
-    db   'test1'
-    db   'test2', exclude_tables: ['t_test1']
+    user 'root'
+    pass 'root'
   end
 
   mailer do
@@ -33,7 +43,7 @@ SimpleBackup.run do
 
     from 'backup@localhost'
     to   'root@localhost'
-    cc   'root@localhost'
+    cc   'rb@localhost'
     bcc  'root@localhost'
   end
 end

data/lib/simple_backup/backend/abstract.rb

@@ -0,0 +1,35 @@
+module SimpleBackup
+  module Backend
+    class Abstract
+      @@logger = Utils::Logger.instance
+
+      def configure(*args)
+        raise NotImplementedError
+      end
+
+      def name=(value)
+        @name = value.gsub(/[^a-zA-Z0-9\-\_\. ]*/, '').gsub(/\s+/, '_').downcase
+      end
+
+      def name
+        @name
+      end
+
+      def type
+        self.class.name.split('::').last.gsub(/[^a-zA-Z0-9\-\_\. ]*/, '').gsub(/\s+/, '_').downcase
+      end
+
+      def desc
+        '%5s :: %s' % [type, @name]
+      end
+
+      def store(source)
+        raise NotImplementedError
+      end
+
+      def cleanup(source)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/simple_backup/backend/local.rb

@@ -0,0 +1,45 @@
+module SimpleBackup
+  module Backend
+    class Local < Abstract
+      def configure(options = {})
+        raise "Must provide :path option" unless options[:path]
+
+        @path = options[:path]
+
+        raise "#{@path} does not exists" unless ::File.exist?(@path)
+        raise "#{@path} is not a directory" unless ::File.directory?(@path)
+        raise "#{@path} is not writable" unless ::File.writable?(@path)
+      end
+
+      def store(source)
+        storage_path = get_storage_path(source)
+        FileUtils.cp source.backup_file, storage_path
+      end
+
+      def cleanup(source)
+        storage_path = get_storage_path(source)
+
+        files = ::Dir.glob(::File.join(storage_path, '*.tar.gz')).sort
+
+        to_persist = files
+        to_persist = files.slice(source.keep_last * -1, source.keep_last) if files.length > source.keep_last
+        to_remove = files - to_persist
+
+        @@logger.scope_start
+        to_remove.each do |file|
+          FileUtils.rm(file)
+          @@logger.debug "Old backup '#{file}' for source '#{source.desc.strip}' cleaned up from '#{desc.strip}'"
+        end
+        @@logger.scope_end
+      end
+
+      private
+      def get_storage_path(source)
+        path = ::File.join(@path, source.type, source.name)
+        FileUtils.mkpath path unless ::File.exist?(path)
+
+        path
+      end
+    end
+  end
+end

data/lib/simple_backup/backends.rb

@@ -0,0 +1,60 @@
+require 'singleton'
+require 'simple_backup/backend/abstract'
+
+module SimpleBackup
+  class Backends
+    include Singleton
+
+    @@logger = Utils::Logger.instance
+    @@sources = Sources.instance
+
+    def initialize
+      @backends = {}
+    end
+
+    def each(&block)
+      @backends.each(&block)
+    end
+
+    def save_and_cleanup
+      each do |name, backend|
+        @@sources.each do |name, source|
+          next unless source.backup_file and source.supports(backend)
+
+          backend.store(source)
+          @@logger.info "Source '#{source.desc.strip}' stored in backend '#{backend.desc.strip}'"
+
+          backend.cleanup(source)
+          @@logger.info "Source '#{source.desc.strip}' cleaned up in backend '#{backend.desc.strip}'"
+        end
+      end
+    end
+
+    def method_missing(method, *args)
+      backend = create_backend(method)
+
+      return nil if backend.nil?
+
+      name = args.shift
+      options = args.shift
+      options ||= {}
+
+      raise "Name '#{name}' for backend already used" if @backends.has_key?(name.to_sym)
+
+      backend.name = name
+      backend.configure(options)
+
+      @@logger.info "Created backend for: #{backend.desc}"
+      @backends[name.to_sym] = backend
+    end
+
+    private
+    def create_backend(name)
+      file = "simple_backup/backend/#{name}"
+
+      require file
+      backend_name = Object.const_get("SimpleBackup::Backend::#{name.capitalize}")
+      backend_name.new
+    end
+  end
+end

data/lib/simple_backup/dsl.rb

@@ -1,120 +1,53 @@
 module SimpleBackup
-  TIMESTAMP = Time.new.strftime('%Y%m%d%H%M%S')
-  @@status = :failed
-
-  def self.status
-    @@status
-  end
-
-  def self.run(&block)
-    Logger::info "Backup #{TIMESTAMP} started"
-
-    dsl = DSL.new
-
-    Logger::scope_start :info, "Configuration"
-    dsl.instance_eval(&block)
-    dsl.prepare
-    Logger::scope_end
-
-    dsl.run
-    dsl.cleanup
-    @@status = :succeed
-
-    Logger::info "Backup #{TIMESTAMP} finished"
-  rescue StandardError => e
-    self.handle_exception(e)
-  ensure
-    dsl.notify
-    Logger::info "Notifications for backup #{TIMESTAMP} finished"
-  end
-
-  def self.handle_exception(e)
-    Logger::error "#{e.class} => #{e.message}"
-    Logger::error "Backup #{TIMESTAMP} failed"
-    STDERR.puts "Error @ #{Time.new.strftime('%Y-%m-%dT%H:%M:%S')}"
-    STDERR.puts "#{e.inspect}"
-    STDERR.puts e.backtrace
-  end
-
   class DSL
-    def initialize
-      @storage = Storage.new
-    end
-
-    def prepare
-      if @apps_block
-        @apps = Engine::Apps.new
-        @apps.storage = @storage
-        @apps.instance_eval(&@apps_block)
-      end
-
-      if @mysql_block
-        @mysql = Engine::MySQL.new
-        @mysql.storage = @storage
-        @mysql.instance_eval(&@mysql_block)
-      end
-    end
-
-    def run
-      usage = Utils::Disk::usage
-      Logger::error "Disk high usage treshold exceeded #{usage[:high_usage]}" if usage[:high_usage_exceeded]
+    @@logger = Utils::Logger.instance
 
-      Logger::scope_start :info, "Backup job"
-      @apps.backup if @apps
-      @mysql.backup if @mysql
-      Logger::scope_end
+    def initialize(engine)
+      @engine = engine
     end
 
-    def cleanup
-      Logger::scope_start :info, "Cleanup job"
-      @apps.cleanup if @apps
-      @mysql.cleanup if @mysql
-      Logger::scope_end
-    end
-
-    def notify
-      @mailer.send if @mailer
-    rescue StandardError => e
-      SimpleBackup.handle_exception(e)
+    def log_level(level)
+      @@logger.level = level
     end
 
-    def sources
-      sources = {}
-      sources[:apps] = @apps.sources if @apps
-      sources[:mysql] = @mysql.sources if @mysql
+    def high_usage_treshold(value)
+      @@logger.info "Setting high_usage_treshold to #{value}"
 
-      sources
+      Utils::Disk.high_usage_treshold = value
     end
 
-    def log_level(level)
-      Logger::level = level
-    end
+    def check_disk_path(path)
+      @@logger.info "Adding disk path '#{path}' to usage check"
 
-    def backup_dir(dir)
-      @storage.dir = dir
+      Utils::Disk.add_path(path)
     end
 
-    def high_usage_treshold(value)
-      Logger::info "Setting high_usage_treshold to #{value}"
-      Utils::Disk.high_usage_treshold = value
+    def default_keep_last(value)
+      Sources.instance.default_keep_last = value
     end
 
-    def check_disk_path(path)
-      Logger::info "Adding disk path '#{path}' to usage check"
-      Utils::Disk.add_path(path)
+    def sources(&block)
+      sources = Sources.instance
+      sources.instance_eval(&block)
     end
 
-    def apps(&block)
-      @apps_block = block
+    def backends(&block)
+      backends = Backends.instance
+      backends.instance_eval(&block)
     end
 
     def mysql(&block)
-      @mysql_block = block
+      @@logger.info "Configuring MySQL Util"
+
+      Utils::MySQL.instance.instance_eval(&block)
     end
 
     def mailer(&block)
-      @mailer = Mailer.new(self, @storage)
+      @@logger.info "Configuring Mailer Util"
+
+      @mailer = Utils::Mailer.new
       @mailer.instance_eval(&block)
+      @engine.mailer = @mailer
     end
   end
 end

data/lib/simple_backup/engine.rb

@@ -1,3 +1,41 @@
-require 'simple_backup/engine/abstract'
-require 'simple_backup/engine/apps'
-require 'simple_backup/engine/mysql'
+module SimpleBackup
+  module Engine
+    class Engine
+
+      @@backends = Backends.instance
+      @@sources = Sources.instance
+      @@logger = Utils::Logger.instance
+      @@mysql = Utils::MySQL.instance
+
+      def mailer=(mailer)
+        @mailer = mailer
+      end
+
+      def run
+        usage = Utils::Disk::usage
+
+        @@logger.error "Disk high usage treshold exceeded #{usage[:high_usage]}" if usage[:high_usage_exceeded]
+        @@logger.scope_start :info, "Backup"
+
+        @@sources.backup
+        @@backends.save_and_cleanup
+        @@sources.cleanup
+
+        @@logger.scope_end
+      ensure
+        @@mysql.close
+      end
+
+      def notify
+        return unless @mailer
+        @@logger.scope_start :info, "Sending e-mail notification"
+
+        @mailer.send
+
+        @@logger.scope_end :info, "Notifications for backup #{TIMESTAMP} finished"
+      rescue StandardError => e
+        SimpleBackup.handle_exception(e)
+      end
+    end
+  end
+end

data/lib/simple_backup/source/abstract.rb

@@ -0,0 +1,130 @@
+require 'rubygems/package'
+require 'tmpdir'
+require 'zlib'
+
+module SimpleBackup
+  module Source
+    class Abstract
+      @@logger = Utils::Logger.instance
+
+      def configure(*args)
+        raise NotImplementedError
+      end
+
+      def keep_last=(value)
+        @keep_last = value
+      end
+
+      def keep_last
+        @keep_last
+      end
+
+      def name=(value)
+        @name = value.gsub(/[^a-zA-Z0-9\-\_\. ]*/, '').gsub(/\s+/, '_').downcase
+      end
+
+      def name
+        @name
+      end
+
+      def type
+        self.class.name.split('::').last.gsub(/[^a-zA-Z0-9\-\_\. ]*/, '').gsub(/\s+/, '_').downcase
+      end
+
+      def desc
+        '%5s :: %s' % [type, @name]
+      end
+
+      def get
+        return @backup_file if @backup_file
+
+        @@logger.scope_start :info, "Getting archive for: #{desc}"
+
+        @tmp_dir = ::Dir.mktmpdir('simple_backup-')
+        @@logger.debug "Created tmp directory #{@tmp_dir}"
+
+        data_exists = prepare_data
+
+        @@logger.warning "No data for: #{desc}" unless data_exists
+        archive_data if data_exists
+
+        FileUtils.rm_rf(@tmp_dir)
+        @@logger.debug "Removed tmp directory #{@tmp_dir}"
+
+        @backup_file
+      ensure
+        @@logger.scope_end
+      end
+
+      def cleanup
+        return nil unless @backup_file
+
+        FileUtils.rm (@backup_file)
+        @@logger.debug "Temporary backup file #{@backup_file} was removed"
+      end
+
+      def backup_file
+        @backup_file
+      end
+
+      def backends=(value)
+        @backends = []
+        @backends = @backends + value if value.kind_of?(Array)
+        @backends << value unless value.kind_of?(Array)
+      end
+
+      def supports(backend)
+        return TRUE unless @backends
+        return FALSE unless @backends.include?(backend.name)
+
+        TRUE
+      end
+
+      private
+      def prepare_data
+        raise NotImplementedError
+      end
+
+      def archive_data
+        filename = "#{type}-#{name}.#{SimpleBackup::TIMESTAMP}.tar.gz"
+        @backup_file = ::File.join(::Dir.tmpdir, filename)
+
+        ::File.open(backup_file, 'w') do |f|
+          f.write targz.string
+        end
+
+        @@logger.debug "Backup saved to temporary file #{backup_file}"
+      end
+
+      def targz
+        path = @tmp_dir
+
+        content = StringIO.new('');
+        Gem::Package::TarWriter.new(content) do |tar|
+          ::Dir[::File.join(path, '**/*')].each do |file|
+            mode = ::File.stat(file).mode
+            relative_file = file.sub(/^#{Regexp::escape path}\/?/, '')
+
+            if ::File.directory?(file)
+              tar.mkdir(relative_file, mode)
+            else
+              tar.add_file relative_file, mode do |tf|
+                ::File.open(file, 'rb') do |f|
+                  tf.write f.read
+                end
+              end
+            end
+          end
+        end
+        content.rewind
+
+        gz = StringIO.new('')
+        zip = Zlib::GzipWriter.new(gz)
+        zip.write content.string
+        zip.close
+
+        gz
+      end
+    end
+  end
+end

data/lib/simple_backup/source/dir.rb

@@ -0,0 +1,39 @@
+module SimpleBackup
+  module Source
+    class Dir < Abstract
+      def initialize
+        @strategy = :bare
+      end
+
+      def configure(path, options = {})
+        @path = path
+
+        raise "#{path} is a file - use File source instead of Dir" unless !::File.exist?(path) or ::File.directory?(path)
+        @strategy = options[:strategy] if options[:strategy]
+      end
+
+      private
+      def prepare_data
+        return false unless ::File.exist?(@path)
+
+        path_entries = get_path_entries
+        FileUtils.cp_r path_entries, @tmp_dir if path_entries
+
+        true
+      end
+
+      def get_path_entries
+        file = "simple_backup/source/dir_strategy/#{@strategy.to_s}"
+
+        require file
+        strategy_name = Object.const_get("SimpleBackup::Source::DirStrategy::#{@strategy.to_s.capitalize}")
+        strategy = strategy_name.new
+
+        strategy.get_entries(@path)
+      rescue Errno::ENOENT
+        @@logger.warning "Path '#{@path}' does not exists"
+        nil
+      end
+    end
+  end
+end

data/lib/simple_backup/source/dir_strategy/bare.rb

@@ -0,0 +1,17 @@
+module SimpleBackup
+  module Source
+    module DirStrategy
+      class Bare
+        def get_entries(path)
+          ::Dir.entries(path).map do |p|
+            if p.match(/^\.\.?$/)
+              nil
+            else
+              ::File.join(path, p)
+            end
+          end.compact
+        end
+      end
+    end
+  end
+end

data/lib/simple_backup/source/dir_strategy/capistrano.rb

@@ -0,0 +1,41 @@
+module SimpleBackup
+  module Source
+    module DirStrategy
+      class Capistrano
+        @@logger = Utils::Logger.instance
+
+        def get_entries(path)
+          shared = shared_path(path)
+          current = current_path(path)
+          paths = [current, shared].compact
+
+          if paths.empty?
+            @@logger.warning "No capistrano paths for application"
+            return nil
+          end
+
+          paths
+        end
+
+        private
+        def current_path(path)
+          current = ::Dir.new(::File.join(path, 'current') + '/')
+          @@logger.debug "Capistrano current path: #{current.path}"
+          current.path
+        rescue Errno::ENOENT
+          @@logger.warning "No capistrano current path for application"
+          nil
+        end
+
+        def shared_path(path)
+          shared = ::Dir.new(::File.join(path, 'shared'))
+          @@logger.debug "Capistrano shared path: #{shared.path}"
+          shared.path
+        rescue Errno::ENOENT
+          @@logger.warning "No capistrano shared path for application"
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/simple_backup/source/file.rb

@@ -0,0 +1,20 @@
+module SimpleBackup
+  module Source
+    class File < Abstract
+      def configure(path, options = {})
+        @path = path
+
+        raise "#{path} is a directory - use Dir source instead of File" unless !::File.exist?(path) or ::File.file?(path)
+      end
+
+      private
+      def prepare_data
+        return false unless ::File.exist?(@path)
+
+        FileUtils.cp @path, @tmp_dir
+
+        true
+      end
+    end
+  end
+end