bostonlogic-safe 0.3.0

data/examples/unit/svndump_example.rb

@@ -0,0 +1,39 @@
+require File.expand_path(File.dirname(__FILE__) + '/../example_helper')
+
+describe Astrails::Safe::Svndump do
+  def def_config
+    {
+      :options => "OPTS",
+      :repo_path => "bar/baz"
+    }
+  end
+
+  def svndump(id = :foo, config = def_config)
+    Astrails::Safe::Svndump.new(id, Astrails::Safe::Config::Node.new(nil, config))
+  end
+
+  before(:each) do
+    stub(Time).now.stub!.strftime {"NOW"}
+  end
+
+  after(:each) { Astrails::Safe::TmpFile.cleanup }
+
+  describe :backup do
+    before(:each) do
+      @svn = svndump
+    end
+
+    {
+      :id => "foo",
+      :kind => "svndump",
+      :extension => ".svn",
+      :filename => "svndump-foo.NOW",
+      :command => "svnadmin dump OPTS bar/baz",
+    }.each do |k, v|
+      it "should set #{k} to #{v}" do
+        @svn.backup.send(k).should == v
+      end
+    end
+
+  end
+end

data/lib/astrails/safe.rb

@@ -0,0 +1,71 @@
+require "aws/s3"
+require 'cloudfiles'
+require 'net/sftp'
+require 'fileutils'
+require 'benchmark'
+require 'net/smtp'
+
+require 'yaml'
+
+require 'tempfile'
+require File.dirname(__FILE__) + '/../extensions/mktmpdir'
+
+require File.dirname(__FILE__) + '/safe/tmp_file'
+
+require File.dirname(__FILE__) + '/safe/config/node'
+require File.dirname(__FILE__) + '/safe/config/builder'
+
+require File.dirname(__FILE__) + '/safe/stream'
+
+require File.dirname(__FILE__) + '/safe/backup'
+
+require File.dirname(__FILE__) + '/safe/backup'
+
+require File.dirname(__FILE__) + '/safe/source'
+require File.dirname(__FILE__) + '/safe/mysqldump'
+require File.dirname(__FILE__) + '/safe/pgdump'
+require File.dirname(__FILE__) + '/safe/archive'
+require File.dirname(__FILE__) + '/safe/svndump'
+
+require File.dirname(__FILE__) + '/safe/pipe'
+require File.dirname(__FILE__) + '/safe/gpg'
+require File.dirname(__FILE__) + '/safe/gzip'
+
+require File.dirname(__FILE__) + '/safe/sink'
+require File.dirname(__FILE__) + '/safe/local'
+require File.dirname(__FILE__) + '/safe/s3'
+require File.dirname(__FILE__) + '/safe/sftp'
+
+require File.dirname(__FILE__) + '/safe/rcloud'
+require File.dirname(__FILE__) + '/safe/notification'
+
+module Astrails
+  module Safe
+    ROOT = File.join(File.dirname(__FILE__), "..", "..")
+
+    def safe(&block)
+      config = Config::Node.new(&block)
+      #config.dump
+      
+      begin
+        [[Mysqldump, [:mysqldump, :databases]],
+         [Pgdump,    [:pgdump,    :databases]],
+         [Archive,   [:tar,       :archives]],
+         [Svndump,   [:svndump,   :repos]]
+        ].each do |klass, path|
+          if collection = config[*path]
+            collection.each do |name, config|
+              klass.new(name, config).backup.run(config, :gpg, :gzip, :local, :s3, :sftp, :rcloud)
+            end
+          end
+        end
+
+        Astrails::Safe::TmpFile.cleanup
+      rescue Exception => e
+        puts e.to_yaml if $DRY_RUN || $_VERBOSE
+        Notification.new(config, e).send_failure
+      end
+    end
+    module_function :safe
+  end
+end

data/lib/astrails/safe/archive.rb

@@ -0,0 +1,24 @@
+module Astrails
+  module Safe
+    class Archive < Source
+
+      def command
+        "tar -cf - #{@config[:options]} #{tar_exclude_files} #{tar_files}"
+      end
+
+      def extension; '.tar'; end
+
+      protected
+
+      def tar_exclude_files
+        [*@config[:exclude]].compact.map{|x| "--exclude=#{x}"} * " "
+      end
+
+      def tar_files
+        raise RuntimeError, "missing files for tar" unless @config[:files]
+        [*@config[:files]].map {|s| s.strip} * " "
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/backup.rb

@@ -0,0 +1,20 @@
+module Astrails
+  module Safe
+    class Backup
+      attr_accessor :id, :kind, :filename, :extension, :command, :compressed, :timestamp, :path
+      def initialize(opts = {})
+        opts.each do |k, v|
+          self.send("#{k}=", v)
+        end
+      end
+
+      def run(config, *mods)
+        mods.each do |mod|
+          mod = mod.to_s
+          mod[0] = mod[0..0].upcase
+          Astrails::Safe.const_get(mod).new(config, self).process
+        end
+      end
+    end
+  end
+end

data/lib/astrails/safe/config/builder.rb

@@ -0,0 +1,62 @@
+module Astrails
+  module Safe
+    module Config
+      class Builder
+        COLLECTIONS = %w/database archive repo/
+        ITEMS = %w/s3 key secret bucket path gpg password keep local mysqldump pgdump options
+        user host port socket skip_tables tar files exclude filename svndump repo_path sftp
+        rcloud username api_key container
+        notification subject host domain username password authentication port from recipients/
+        NAMES = COLLECTIONS + ITEMS
+        def initialize(node)
+          @node = node
+        end
+
+        # supported args:
+        #   args = [value]
+        #   args = [id, data]
+        #   args = [data]
+        # id/value - simple values, data - hash
+        def method_missing(sym, *args, &block)
+          return super unless NAMES.include?(sym.to_s)
+
+          # do we have id or value?
+          unless args.first.is_a?(Hash)
+            id_or_value = args.shift # nil for args == []
+          end
+
+          id_or_value = id_or_value.map {|v| v.to_s} if id_or_value.is_a?(Array)
+
+          # do we have data hash?
+          if data = args.shift
+            raise "#{sym}: hash expected: #{data.inspect}" unless data.is_a?(Hash)
+          end
+
+          #puts "#{sym}: args=#{args.inspect}, id_or_value=#{id_or_value}, data=#{data.inspect}, block=#{block.inspect}"
+
+          raise "#{sym}: unexpected: #{args.inspect}" unless args.empty?
+          raise "#{sym}: missing arguments" unless id_or_value || data || block
+
+          if COLLECTIONS.include?(sym.to_s) && id_or_value
+            data ||= {}
+          end
+
+          if !data && !block
+            # simple value assignment
+            @node[sym] = id_or_value
+
+          elsif id_or_value
+            # collection element with id => create collection node and a subnode in it
+            key = sym.to_s + "s"
+            collection = @node[key] || @node.set(key, {})
+            collection.set(id_or_value, data || {}, &block)
+
+          else
+            # simple subnode
+            @node.set(sym, data || {}, &block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/astrails/safe/config/node.rb

@@ -0,0 +1,66 @@
+require 'astrails/safe/config/builder'
+module Astrails
+  module Safe
+    module Config
+      class Node
+        attr_reader :parent
+        def initialize(parent = nil, data = {}, &block)
+          @parent, @data = parent, {}
+          data.each { |k, v| self[k] = v }
+          Builder.new(self).instance_eval(&block) if block
+        end
+
+        # looks for the path from this node DOWN. will not delegate to parent
+        def get(*path)
+          key = path.shift
+          value = @data[key.to_s]
+          return value if value && path.empty?
+
+          value && value.get(*path)
+        end
+
+        # recursive find
+        # starts at the node and continues to the parent
+        def find(*path)
+          get(*path) || @parent && @parent.find(*path)
+        end
+        alias :[] :find
+
+        def set(key, value, &block)
+          @data[key.to_s] =
+            if value.is_a?(Hash)
+              Node.new(self, value, &block)
+            else
+              raise(ArgumentError, "#{key}: no block supported for simple values") if block
+              value
+            end
+        end
+        alias :[]= :set
+
+        def each(&block)
+          @data.each(&block)
+        end
+        include Enumerable
+
+        def to_hash
+          @data.keys.inject({}) do |res, key|
+            value = @data[key]
+            res[key] = value.is_a?(Node) ? value.to_hash : value
+            res
+          end
+        end
+
+        def dump(indent = "")
+          @data.each do |key, value|
+            if value.is_a?(Node)
+              puts "#{indent}#{key}:"
+              value.dump(indent + "    ")
+            else
+              puts "#{indent}#{key}: #{value.inspect}"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/astrails/safe/gpg.rb

@@ -0,0 +1,45 @@
+module Astrails
+  module Safe
+    class Gpg < Pipe
+
+      protected
+
+      def post_process
+        @backup.compressed = true
+      end
+
+      def pipe
+        if key
+          "|gpg #{@config[:gpg, :options]} -e -r #{key}"
+        elsif password
+          "|gpg #{@config[:gpg,:options]} -c --passphrase-file #{gpg_password_file(password)}"
+        end
+      end
+
+      def extension
+        ".gpg"
+      end
+
+      def active?
+        raise RuntimeError, "can't use both gpg password and pubkey" if key && password
+
+        !!(password || key)
+      end
+
+      private
+
+      def password
+        @password ||= @config[:gpg, :password]
+      end
+
+      def key
+        @key ||= @config[:gpg, :key]
+      end
+
+      def gpg_password_file(pass)
+        return "TEMP_GENERATED_FILENAME" if $DRY_RUN
+        Astrails::Safe::TmpFile.create("gpg-pass") { |file| file.write(pass) }
+      end
+    end
+  end
+end

data/lib/astrails/safe/gzip.rb

@@ -0,0 +1,25 @@
+module Astrails
+  module Safe
+    class Gzip < Pipe
+
+      protected
+
+      def post_process
+        @backup.compressed = true
+      end
+
+      def pipe
+        "|gzip"
+      end
+
+      def extension
+        ".gz"
+      end
+
+      def active?
+        !@backup.compressed
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/local.rb

@@ -0,0 +1,48 @@
+module Astrails
+  module Safe
+    class Local < Sink
+
+      protected
+
+      def active?
+        # S3 can't upload from pipe. it needs to know file size, so we must pass through :local
+        # will change once we add SSH/FTP sink
+        true
+      end
+
+      def path
+        @path ||= File.expand_path(expand(@config[:local, :path] || raise(RuntimeError, "missing :local/:path")))
+      end
+
+      def save
+        puts "command: #{@backup.command}" if $_VERBOSE
+
+        @backup.path = full_path # need to do it outside DRY_RUN so that it will be avialable for S3 DRY_RUN
+
+        unless $DRY_RUN
+          FileUtils.mkdir_p(path) unless File.directory?(path)
+          benchmark = Benchmark.realtime do
+            system "#{@backup.command}>#{@backup.path}"
+          end
+          puts("command took " + sprintf("%.2f", benchmark) + " second(s).") if $_VERBOSE
+        end
+
+      end
+
+      def cleanup
+        return unless keep = @config[:keep, :local]
+
+        puts "listing files #{base}" if $_VERBOSE
+
+        files = Dir["#{base}*"] .
+          select{|f| File.file?(f) && File.size(f) > 0} .
+          sort
+
+        cleanup_with_limit(files, keep) do |f|
+          puts "removing local file #{f}" if $DRY_RUN || $_VERBOSE
+          File.unlink(f) unless $DRY_RUN
+        end
+      end
+    end
+  end
+end

data/lib/astrails/safe/mysqldump.rb

@@ -0,0 +1,31 @@
+module Astrails
+  module Safe
+    class Mysqldump < Source
+
+      def command
+        "mysqldump --defaults-extra-file=#{mysql_password_file} #{@config[:options]} #{mysql_skip_tables} #{@id}"
+      end
+
+      def extension; '.sql'; end
+
+      protected
+
+      def mysql_password_file
+        Astrails::Safe::TmpFile.create("mysqldump") do |file|
+          file.puts "[mysqldump]"
+          %w/user password socket host port/.each do |k|
+            v = @config[k]
+            file.puts "#{k} = #{v}" if v
+          end
+        end
+      end
+
+      def mysql_skip_tables
+        if skip_tables = @config[:skip_tables]
+          [*skip_tables].map { |t| "--ignore-table=#{@id}.#{t}" } * " "
+        end
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/notification.rb

@@ -0,0 +1,66 @@
+module Astrails
+  module Safe
+    class Notification
+
+      attr_accessor :config, :error
+      def initialize(config, error)
+        @config, @error = config, error
+      end
+      
+      def send_failure
+        if valid?
+          msg = "Subject: #{subject}\n\n"
+          msg += "#{subject} at #{Time.now.strftime('%A, %B %d, %Y %I:%M:%S %p')}\n\n"
+          msg += "Exception: #{error.to_s}\n\n"
+          msg += "Stack Trace: #{error.backtrace.join("\n")}\n"
+          Net::SMTP.start(host, port, domain, username, password, authentication) do |smtp|
+            smtp.send_message(msg, from, recipients)
+          end
+        end
+      end
+      
+      private
+      
+      def valid?
+        subject && host && domain && username && password && authentication && port && from && recipients
+      end
+       
+      def subject
+        @config[:notification, :subject]
+      end
+      
+      def host
+        @config[:notification, :host]
+      end
+      
+      def domain
+        @config[:notification, :domain]
+      end
+
+      def username
+        @config[:notification, :username]
+      end
+      
+      def password
+        @config[:notification, :password]
+      end
+      
+      def authentication
+        @config[:notification, :authentication]
+      end
+      
+      def port
+        @config[:notification, :port]
+      end
+      
+      def from
+        @config[:notification, :from]
+      end
+
+      def recipients
+        @config[:notification, :recipients]
+      end
+      
+    end
+  end
+end