akupchanko-astrails-safe 0.3.1

data/lib/astrails/safe.rb

@@ -0,0 +1,68 @@
+require "astrails/safe/version"
+
+require "aws/s3"
+require "cloudfiles"
+require 'net/sftp'
+require 'net/ftp'
+require 'fileutils'
+require 'benchmark'
+
+require 'tempfile'
+require 'extensions/mktmpdir'
+
+require 'astrails/safe/tmp_file'
+
+require 'astrails/safe/config/node'
+require 'astrails/safe/config/builder'
+
+require 'astrails/safe/stream'
+
+require 'astrails/safe/backup'
+
+require 'astrails/safe/source'
+require 'astrails/safe/mysqldump'
+require 'astrails/safe/pgdump'
+require 'astrails/safe/archive'
+require 'astrails/safe/svndump'
+require 'astrails/safe/mongodump'
+
+require 'astrails/safe/pipe'
+require 'astrails/safe/gpg'
+require 'astrails/safe/gzip'
+
+require 'astrails/safe/sink'
+require 'astrails/safe/local'
+require 'astrails/safe/s3'
+require 'astrails/safe/cloudfiles'
+require 'astrails/safe/sftp'
+require 'astrails/safe/ftp'
+
+module Astrails
+  module Safe
+    ROOT = File.join(File.dirname(__FILE__), "..", "..")
+
+    def safe(&block)
+      Config::Node.new(&block)
+    end
+
+    def process(config)
+
+      [[Mysqldump, [:mysqldump, :databases]],
+       [Pgdump,    [:pgdump,    :databases]],
+       [Mongodump, [:mongodump, :databases]],
+       [Archive,   [:tar,       :archives]],
+       [Svndump,   [:svndump,   :repos]]
+      ].each do |klass, path|
+        if collection = config[*path]
+          collection.each do |name, c|
+            klass.new(name, c).backup.run(c, :gpg, :gzip, :local, :s3, :cloudfiles, :sftp, :ftp)
+          end
+        end
+      end
+
+      Astrails::Safe::TmpFile.cleanup
+    end
+    module_function :safe
+    module_function :process
+  end
+end

data/lib/astrails/safe/archive.rb

@@ -0,0 +1,24 @@
+module Astrails
+  module Safe
+    class Archive < Source
+
+      def command
+        "tar -cf - #{config[:options]} #{tar_exclude_files} #{tar_files}"
+      end
+
+      def extension; '.tar'; end
+
+      protected
+
+      def tar_exclude_files
+        [*config[:exclude]].compact.map{|x| "--exclude=#{x}"}.join(" ")
+      end
+
+      def tar_files
+        raise RuntimeError, "missing files for tar" unless config[:files]
+        [*config[:files]].map{|s| s.strip}.join(" ")
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/backup.rb

@@ -0,0 +1,20 @@
+module Astrails
+  module Safe
+    class Backup
+      attr_accessor :id, :kind, :filename, :extension, :command, :compressed, :timestamp, :path
+      def initialize(opts = {})
+        opts.each do |k, v|
+          self.send("#{k}=", v)
+        end
+      end
+
+      def run(config, *mods)
+        mods.each do |mod|
+          mod = mod.to_s
+          mod[0] = mod[0..0].upcase
+          Astrails::Safe.const_get(mod).new(config, self).process
+        end
+      end
+    end
+  end
+end

data/lib/astrails/safe/cloudfiles.rb

@@ -0,0 +1,77 @@
+module Astrails
+  module Safe
+    class Cloudfiles < Sink
+      MAX_CLOUDFILES_FILE_SIZE = 5368709120
+
+      protected
+
+      def active?
+        container && user && api_key
+      end
+
+      def path
+        @path ||= expand(config[:cloudfiles, :path] || config[:local, :path] || ":kind/:id")
+      end
+
+      # UGLY: we need this function for the reason that
+      # we can't double mock on ruby 1.9.2, duh!
+      # so we created this func to mock it all together
+      def get_file_size(path)
+        File.stat(path).size
+      end
+
+      def save
+        raise RuntimeError, "pipe-streaming not supported for S3." unless @backup.path
+
+        # needed in cleanup even on dry run
+        cf = CloudFiles::Connection.new(user, api_key, true, service_net) unless local_only?
+        puts "Uploading #{container}:#{full_path} from #{@backup.path}" if verbose? || dry_run?
+        unless dry_run? || local_only?
+          if get_file_size(@backup.path) > MAX_CLOUDFILES_FILE_SIZE
+            STDERR.puts "ERROR: File size exceeds maximum allowed for upload to Cloud Files (#{MAX_CLOUDFILES_FILE_SIZE}): #{@backup.path}"
+            return
+          end
+          benchmark = Benchmark.realtime do
+            cf_container = cf.create_container(container)
+            o = cf_container.create_object(full_path,true)
+            o.write(File.open(@backup.path))
+          end
+          puts "...done" if verbose?
+          puts("Upload took " + sprintf("%.2f", benchmark) + " second(s).") if verbose?
+        end
+      end
+
+      def cleanup
+        return if local_only?
+
+        return unless keep = config[:keep, :cloudfiles]
+
+        puts "listing files: #{container}:#{base}*" if verbose?
+        cf = CloudFiles::Connection.new(user, api_key, true, service_net) unless local_only?
+        cf_container = cf.container(container)
+        files = cf_container.objects(:prefix => base).sort
+
+        cleanup_with_limit(files, keep) do |f|
+          puts "removing Cloud File #{container}:#{f}" if dry_run? || verbose?
+          cf_container.delete_object(f) unless dry_run? || local_only?
+        end
+      end
+
+      def container
+        config[:cloudfiles, :container]
+      end
+
+      def user
+        config[:cloudfiles, :user]
+      end
+
+      def api_key
+        config[:cloudfiles, :api_key]
+      end
+
+      def service_net
+        config[:cloudfiles, :service_net] || false
+      end
+    end
+  end
+end

data/lib/astrails/safe/config/builder.rb

@@ -0,0 +1,90 @@
+module Astrails
+  module Safe
+    module Config
+      class Builder
+
+        def initialize(node, data = {})
+          @node = node
+          data.each { |k, v| self.send k, v }
+        end
+
+
+        class << self
+          def simple_value(*names)
+            names.each do |m|
+              define_method(m) do |value|
+                ensure_uniq(m)
+                @node.set m, value
+              end
+            end
+          end
+
+          def multi_value(*names)
+            names.each do |m|
+              define_method(m) do |value|
+                value = value.map(&:to_s) if value.is_a?(Array)
+                @node.set_multi m, value
+              end
+            end
+          end
+
+          def hash_value(*names)
+            names.each do |m|
+              define_method(m) do |data = {}, &block|
+                ensure_uniq(m)
+                ensure_hash(m, data)
+                @node.set m, Node.new(@node, data || {}, &block)
+              end
+            end
+          end
+
+          def mixed_value(*names)
+            names.each do |m|
+              define_method(m) do |data={}, &block|
+                ensure_uniq(m)
+                if data.is_a?(Hash) || block
+                  ensure_hash(m, data) if block
+                  @node.set m, Node.new(@node, data, &block)
+                else
+                  @node.set m, data
+                end
+              end
+            end
+          end
+
+          def collection(*names)
+            names.each do |m|
+              define_method(m) do |id, data={}, &block|
+                raise "bad collection id: #{id.inspect}" unless id
+                ensure_hash(m, data)
+
+                name = "#{m}s"
+                collection = @node.get(name) || @node.set(name, Node.new(@node, {}))
+                collection.set id, Node.new(collection, data, &block)
+              end
+            end
+          end
+        end
+
+        simple_value :verbose, :dry_run, :local_only, :path, :command,
+          :options, :user, :host, :port, :password, :key, :secret, :bucket,
+          :api_key, :container, :socket, :service_net, :repo_path
+        multi_value :skip_tables, :exclude, :files
+        hash_value :mysqldump, :tar, :gpg, :keep, :pgdump, :tar, :svndump,
+          :sftp, :ftp, :mongodump
+        mixed_value :s3, :local, :cloudfiles
+        collection :database, :archive, :repo
+
+        private
+
+        def ensure_uniq(m)
+          raise(ArgumentError, "duplicate value for '#{m}'") if @node.get(m)
+        end
+
+        def ensure_hash(k, v)
+          raise "#{k}: hash expected: #{v.inspect}" unless v.is_a?(Hash)
+        end
+      end
+    end
+  end
+end

data/lib/astrails/safe/config/node.rb

@@ -0,0 +1,72 @@
+require 'astrails/safe/config/builder'
+module Astrails
+  module Safe
+    module Config
+      class Node
+        attr_reader :parent, :data
+
+        def initialize(parent = nil, data = {}, &block)
+          @parent = parent
+          @data = {}
+          merge data, &block
+        end
+
+        def merge data = {}, &block
+          builder = Builder.new(self, data)
+          builder.instance_eval(&block) if block
+          self
+        end
+
+        # looks for the path from this node DOWN. will not delegate to parent
+        def get(*path)
+          key = path.shift
+          value = @data[key.to_s]
+          return value if (nil != value) && path.empty?
+
+          value && value.get(*path)
+        end
+
+        # recursive find
+        # starts at the node and continues to the parent
+        def find(*path)
+          get(*path) || @parent && @parent.find(*path)
+        end
+        alias :[] :find
+
+        def set_multi(key, value)
+          @data[key.to_s] ||= []
+          @data[key.to_s].concat [*value]
+        end
+
+        def set(key, value)
+          @data[key.to_s] = value
+        end
+        alias :[]= :set
+
+        def each(&block)
+          @data.each(&block)
+        end
+        include Enumerable
+
+        def to_hash
+          @data.keys.inject({}) do |res, key|
+            value = @data[key]
+            res[key] = value.is_a?(Node) ? value.to_hash : value
+            res
+          end
+        end
+
+        def dump(indent = "")
+          @data.each do |key, value|
+            if value.is_a?(Node)
+              puts "#{indent}#{key}:"
+              value.dump(indent + "    ")
+            else
+              puts "#{indent}#{key}: #{value.inspect}"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/astrails/safe/ftp.rb

@@ -0,0 +1,104 @@
+module Astrails
+  module Safe
+    class Ftp < Sink
+
+      protected
+
+      def active?
+        host && user
+      end
+
+      def path
+        @path ||= expand(config[:ftp, :path] || config[:local, :path] || ":kind/:id")
+      end
+
+      def save
+        raise RuntimeError, "pipe-streaming not supported for FTP." unless @backup.path
+
+        puts "Uploading #{host}:#{full_path} via FTP" if verbose? || dry_run?
+
+        unless dry_run? || local_only?
+          if !port
+            port = 21
+          end
+          Net::FTP.open(host) do |ftp|
+            ftp.connect(host, port)
+            ftp.login(user, password)
+
+            dir = File.dirname(full_path)
+            parts = dir.split("/")
+            growing_path = ""
+            for part in parts
+              next if part == ""
+              if growing_path == ""
+                growing_path = part
+              else
+                growing_path = File.join(growing_path, part)
+              end
+              puts "Trying to create remote directory (#{growing_path})" if verbose?
+              begin
+                ftp.mkdir(growing_path)
+              rescue Net::FTPPermError
+                puts "Remote directory (#{growing_path}) exists, or no enough permissions" if verbose?
+              end
+            end
+
+            puts "Sending #{@backup.path} to #{full_path}" if verbose?
+            begin
+              ftp.put(@backup.path, full_path)
+            rescue Net::FTPPermError
+              puts "Ensuring remote path (#{path}) exists" if verbose?
+            end
+          end
+          puts "...done" if verbose?
+        end
+      end
+
+      def cleanup
+        return if local_only? || dry_run?
+
+        return unless keep = config[:keep, :ftp]
+
+        puts "listing files: #{host}:#{base}*" if verbose?
+        if !port
+           port = 21
+        end
+        Net::FTP.open(host) do |ftp|
+          ftp.connect(host, port)
+          ftp.login(user, password)
+          files = ftp.nlst(path)
+          pattern = File.basename("#{base}")
+          files = files.reject{ |x| !x.start_with?(pattern)}
+          puts files.collect {|x| x} if verbose?
+
+          files = files.
+            collect {|x| x }.
+            sort
+
+          cleanup_with_limit(files, keep) do |f|
+            file = File.path(f)
+            puts "removing ftp file #{host}:#{file}" if dry_run? || verbose?
+            ftp.delete(file) unless dry_run? || local_only?
+          end
+        end
+      end
+
+      def host
+        config[:ftp, :host]
+      end
+
+      def user
+        config[:ftp, :user]
+      end
+
+      def password
+        config[:ftp, :password]
+      end
+
+      def port
+        config[:ftp, :port]
+      end
+
+    end
+  end
+end