webbynode-safe 0.2.5

data/lib/astrails/safe.rb

@@ -0,0 +1,62 @@
+require "aws/s3"
+require 'net/sftp'
+require 'fileutils'
+require 'benchmark'
+
+require 'tempfile'
+require 'extensions/mktmpdir'
+
+require 'astrails/safe/tmp_file'
+
+require 'astrails/safe/config/node'
+require 'astrails/safe/config/builder'
+
+require 'astrails/safe/stream'
+
+require 'astrails/safe/backup'
+
+require 'astrails/safe/backup'
+
+require 'astrails/safe/source'
+require 'astrails/safe/mongodbdump'
+require 'astrails/safe/mysqldump'
+require 'astrails/safe/pgdump'
+require 'astrails/safe/archive'
+require 'astrails/safe/svndump'
+
+require 'astrails/safe/pipe'
+require 'astrails/safe/gpg'
+require 'astrails/safe/gzip'
+
+require 'astrails/safe/sink'
+require 'astrails/safe/local'
+require 'astrails/safe/multi'
+require 'astrails/safe/s3'
+require 'astrails/safe/sftp'
+
+module Astrails
+  module Safe
+    ROOT = File.join(File.dirname(__FILE__), "..", "..")
+
+    def safe(&block)
+      config = Config::Node.new(&block)
+      #config.dump
+      
+      [[Mysqldump,   [:mysqldump,   :databases]],
+       [Mongodbdump, [:mongodbdump, :databases]],
+       [Pgdump,      [:pgdump,      :databases]],
+       [Archive,     [:tar,         :archives]],
+       [Svndump,     [:svndump,     :repos]]
+      ].each do |klass, path|
+        if collection = config[*path]
+          collection.each do |name, config|
+            klass.new(name, config).backup.run(config, :multi, :gpg, :gzip, :local, :s3, :sftp)
+          end
+        end
+      end
+
+      Astrails::Safe::TmpFile.cleanup
+    end
+    module_function :safe
+  end
+end

data/lib/astrails/safe/archive.rb

@@ -0,0 +1,24 @@
+module Astrails
+  module Safe
+    class Archive < Source
+
+      def command
+        "tar -cf - #{@config[:options]} #{tar_exclude_files} #{tar_files}"
+      end
+
+      def extension; '.tar'; end
+
+      protected
+
+      def tar_exclude_files
+        [*@config[:exclude]].compact.map{|x| "--exclude=#{x}"} * " "
+      end
+
+      def tar_files
+        raise RuntimeError, "missing files for tar" unless @config[:files]
+        [*@config[:files]].map {|s| s.strip} * " "
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/backup.rb

@@ -0,0 +1,20 @@
+module Astrails
+  module Safe
+    class Backup
+      attr_accessor :id, :kind, :filename, :extension, :command, :compressed, :timestamp, :path, :processed, :multi
+      def initialize(opts = {})
+        opts.each do |k, v|
+          self.send("#{k}=", v)
+        end
+      end
+
+      def run(config, *mods)
+        mods.each do |mod|
+          mod = mod.to_s
+          mod[0] = mod[0..0].upcase
+          Astrails::Safe.const_get(mod).new(config, self).process
+        end
+      end
+    end
+  end
+end

data/lib/astrails/safe/config/builder.rb

@@ -0,0 +1,60 @@
+module Astrails
+  module Safe
+    module Config
+      class Builder
+        COLLECTIONS = %w/database archive repo/
+        ITEMS = %w/s3 key secret bucket path gpg password keep local mysqldump pgdump mongodbdump
+        options user host port socket skip_tables tar files exclude filename svndump repo_path sftp/
+        NAMES = COLLECTIONS + ITEMS
+        def initialize(node)
+          @node = node
+        end
+
+        # supported args:
+        #   args = [value]
+        #   args = [id, data]
+        #   args = [data]
+        # id/value - simple values, data - hash
+        def method_missing(sym, *args, &block)
+          return super unless NAMES.include?(sym.to_s)
+
+          # do we have id or value?
+          unless args.first.is_a?(Hash)
+            id_or_value = args.shift # nil for args == []
+          end
+
+          id_or_value = id_or_value.map {|v| v.to_s} if id_or_value.is_a?(Array)
+
+          # do we have data hash?
+          if data = args.shift
+            raise "#{sym}: hash expected: #{data.inspect}" unless data.is_a?(Hash)
+          end
+
+          #puts "#{sym}: args=#{args.inspect}, id_or_value=#{id_or_value}, data=#{data.inspect}, block=#{block.inspect}"
+
+          raise "#{sym}: unexpected: #{args.inspect}" unless args.empty?
+          raise "#{sym}: missing arguments" unless id_or_value || data || block
+
+          if COLLECTIONS.include?(sym.to_s) && id_or_value
+            data ||= {}
+          end
+
+          if !data && !block
+            # simple value assignment
+            @node[sym] = id_or_value
+
+          elsif id_or_value
+            # collection element with id => create collection node and a subnode in it
+            key = sym.to_s + "s"
+            collection = @node[key] || @node.set(key, {})
+            collection.set(id_or_value, data || {}, &block)
+
+          else
+            # simple subnode
+            @node.set(sym, data || {}, &block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/astrails/safe/config/node.rb

@@ -0,0 +1,66 @@
+require 'astrails/safe/config/builder'
+module Astrails
+  module Safe
+    module Config
+      class Node
+        attr_reader :parent
+        def initialize(parent = nil, data = {}, &block)
+          @parent, @data = parent, {}
+          data.each { |k, v| self[k] = v }
+          Builder.new(self).instance_eval(&block) if block
+        end
+
+        # looks for the path from this node DOWN. will not delegate to parent
+        def get(*path)
+          key = path.shift
+          value = @data[key.to_s]
+          return value if value && path.empty?
+
+          value && value.get(*path)
+        end
+
+        # recursive find
+        # starts at the node and continues to the parent
+        def find(*path)
+          get(*path) || @parent && @parent.find(*path)
+        end
+        alias :[] :find
+
+        def set(key, value, &block)
+          @data[key.to_s] =
+            if value.is_a?(Hash)
+              Node.new(self, value, &block)
+            else
+              raise(ArgumentError, "#{key}: no block supported for simple values") if block
+              value
+            end
+        end
+        alias :[]= :set
+
+        def each(&block)
+          @data.each(&block)
+        end
+        include Enumerable
+
+        def to_hash
+          @data.keys.inject({}) do |res, key|
+            value = @data[key]
+            res[key] = value.is_a?(Node) ? value.to_hash : value
+            res
+          end
+        end
+
+        def dump(indent = "")
+          @data.each do |key, value|
+            if value.is_a?(Node)
+              puts "#{indent}#{key}:"
+              value.dump(indent + "    ")
+            else
+              puts "#{indent}#{key}: #{value.inspect}"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/astrails/safe/gpg.rb

@@ -0,0 +1,45 @@
+module Astrails
+  module Safe
+    class Gpg < Pipe
+
+      protected
+
+      def post_process
+        @backup.compressed = true
+      end
+
+      def pipe
+        if key
+          "|gpg #{@config[:gpg, :options]} -e -r #{key}"
+        elsif password
+          "|gpg #{@config[:gpg,:options]} -c --passphrase-file #{gpg_password_file(password)}"
+        end
+      end
+
+      def extension
+        ".gpg"
+      end
+
+      def active?
+        raise RuntimeError, "can't use both gpg password and pubkey" if key && password
+
+        !!(password || key)
+      end
+
+      private
+
+      def password
+        @password ||= @config[:gpg, :password]
+      end
+
+      def key
+        @key ||= @config[:gpg, :key]
+      end
+
+      def gpg_password_file(pass)
+        return "TEMP_GENERATED_FILENAME" if $DRY_RUN
+        Astrails::Safe::TmpFile.create("gpg-pass") { |file| file.write(pass) }
+      end
+    end
+  end
+end

data/lib/astrails/safe/gzip.rb

@@ -0,0 +1,25 @@
+module Astrails
+  module Safe
+    class Gzip < Pipe
+
+      protected
+
+      def post_process
+        @backup.compressed = true
+      end
+
+      def pipe
+        "|gzip"
+      end
+
+      def extension
+        ".gz"
+      end
+
+      def active?
+        !@backup.compressed
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/local.rb

@@ -0,0 +1,46 @@
+module Astrails
+  module Safe
+    class Local < Sink
+
+      protected
+
+      def active?
+        !backup.processed
+      end
+
+      def path
+        @path ||= File.expand_path(expand(@config[:local, :path] || raise(RuntimeError, "missing :local/:path")))
+      end
+
+      def save
+        puts "command: #{@backup.command}" if $_VERBOSE
+
+        @backup.path = full_path # need to do it outside DRY_RUN so that it will be avialable for S3 DRY_RUN
+
+        unless $DRY_RUN
+          FileUtils.mkdir_p(path) unless File.directory?(path)
+          benchmark = Benchmark.realtime do
+            system "#{@backup.command}>#{@backup.path}"
+          end
+          puts("command took " + sprintf("%.2f", benchmark) + " second(s).") if $_VERBOSE
+        end
+
+      end
+
+      def cleanup
+        return unless keep = @config[:keep, :local]
+
+        puts "listing files #{base}" if $_VERBOSE
+
+        files = Dir["#{base}*"] .
+          select{|f| File.file?(f) && File.size(f) > 0} .
+          sort
+
+        cleanup_with_limit(files, keep) do |f|
+          puts "removing local file #{f}" if $DRY_RUN || $_VERBOSE
+          File.unlink(f) unless $DRY_RUN
+        end
+      end
+    end
+  end
+end

data/lib/astrails/safe/mongodbdump.rb

@@ -0,0 +1,25 @@
+module Astrails
+  module Safe
+    class Mongodbdump < Source
+      def command
+        command = ["mongodump"]
+        command << @config[:options] if @config[:options]
+        command << "--db #{@id}"
+      
+        %w[host port out collection].each do |opt|
+          command << "--#{opt} #{@config[opt]}" if @config[opt]
+        end
+
+        command.join(" ")
+      end
+      
+      def backup
+        bkp = super
+        bkp.multi = true
+        bkp
+      end
+
+      def extension; '.mongodb.dump.tgz'; end
+    end
+  end
+end

data/lib/astrails/safe/multi.rb

@@ -0,0 +1,134 @@
+require 'tmpdir'
+
+module Astrails
+  module Safe
+    class Multi < Sink
+
+      protected
+
+      def active?
+        @backup.multi
+      end
+
+      def path
+        @path ||= File.expand_path(expand(@config[:local, :path] || raise(RuntimeError, "missing :local/:path")))
+      end
+      
+      def save
+        v "=> Backup up multiple files..."
+        v "  backup command: #{@backup.command}"
+
+        @backup.path = full_path # need to do it outside DRY_RUN so that it will be avialable for S3 DRY_RUN
+        # @backup.path << ".tgz"
+
+        compression = "tar cf#{$_VERBOSE ? "v" : ""}z #{@backup.path} dump/"
+        v "  compression command: #{compression}"
+
+        
+        unless $DRY_RUN
+          Dir.tmpdir do
+            v "  executing..."
+            v "----"
+            v ""
+
+            out = ">> /dev/null" unless $_VERBOSE
+            benchmark = Benchmark.realtime do
+              system "#{@backup.command}#{out}"
+              system "#{compression}#{out}"
+            end
+            
+            v ""
+            v "----"
+            v "  command took " + sprintf("%.2f", benchmark) + " second(s)."
+          end
+        end
+        
+        @backup.processed = true
+        @backup.compressed = true
+        
+        v "=> done!"
+        v ""
+      end
+
+      def cleanup
+        return unless keep = @config[:keep, :local]
+
+        puts "listing files #{base}" if $_VERBOSE
+
+        files = Dir["#{base}*"] .
+          select{|f| File.file?(f) && File.size(f) > 0} .
+          sort
+
+        cleanup_with_limit(files, keep) do |f|
+          puts "removing local file #{f}" if $DRY_RUN || $_VERBOSE
+          File.unlink(f) unless $DRY_RUN
+        end
+      end
+    end
+  end
+end
+
+class Dir
+   module Tmpdir
+     require 'tmpdir'
+     require 'socket'
+     require 'fileutils'
+
+     unless defined?(Super)
+       Super = Dir.send(:method, :tmpdir)
+       class << Dir
+         remove_method :tmpdir
+       end
+     end
+
+     class Error < ::StandardError; end
+
+     Hostname = Socket.gethostname || 'localhost'
+     Pid = Process.pid
+     Ppid = Process.ppid
+
+     def tmpdir *args, &block
+       options = Hash === args.last ? args.pop : {}
+
+       dirname = Super.call(*args)
+
+       return dirname unless block
+
+       turd = options['turd'] || options[:turd]
+
+       basename = [
+         Hostname,
+         Ppid,
+         Pid,
+         Thread.current.object_id.abs,
+         rand
+       ].join('-')
+
+       pathname = File.join dirname, basename
+
+       made = false
+
+       42.times do
+         begin
+           FileUtils.mkdir_p pathname
+           break(made = true)
+         rescue Object
+           sleep rand
+           :retry
+         end
+       end
+
+       raise Error, "failed to make tmpdir in #{ dirname.inspect }" unless made
+
+       begin
+         return Dir.chdir(pathname, &block)
+       ensure
+         unless turd
+           FileUtils.rm_rf(pathname) if made
+         end
+       end
+     end
+   end
+
+   extend Tmpdir
+end