darkofabijan-astrails-safe 0.2.8

data/lib/astrails/safe/archive.rb

@@ -0,0 +1,24 @@
+module Astrails
+  module Safe
+    class Archive < Source
+
+      def command
+        "tar -cf - #{@config[:options]} #{tar_exclude_files} #{tar_files}"
+      end
+
+      def extension; '.tar'; end
+
+      protected
+
+      def tar_exclude_files
+        [*@config[:exclude]].compact.map{|x| "--exclude=#{x}"}.join(" ")
+      end
+
+      def tar_files
+        raise RuntimeError, "missing files for tar" unless @config[:files]
+        [*@config[:files]].map{|s| s.strip}.join(" ")
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/backup.rb

@@ -0,0 +1,20 @@
+module Astrails
+  module Safe
+    class Backup
+      attr_accessor :id, :kind, :filename, :extension, :command, :compressed, :timestamp, :path
+      def initialize(opts = {})
+        opts.each do |k, v|
+          self.send("#{k}=", v)
+        end
+      end
+
+      def run(config, *mods)
+        mods.each do |mod|
+          mod = mod.to_s
+          mod[0] = mod[0..0].upcase
+          Astrails::Safe.const_get(mod).new(config, self).process
+        end
+      end
+    end
+  end
+end

data/lib/astrails/safe/cloudfiles.rb

@@ -0,0 +1,70 @@
+module Astrails
+  module Safe
+    class Cloudfiles < Sink
+      MAX_CLOUDFILES_FILE_SIZE = 5368709120
+
+      protected
+
+      def active?
+        container && user && api_key
+      end
+
+      def path
+        @path ||= expand(config[:cloudfiles, :path] || config[:local, :path] || ":kind/:id")
+      end
+
+      def save
+        raise RuntimeError, "pipe-streaming not supported for Cloudfiles." unless @backup.path
+
+        # needed in cleanup even on dry run
+        cf = CloudFiles::Connection.new(user, api_key, true, service_net) unless $LOCAL
+        puts "Uploading #{container}:#{full_path} from #{@backup.path}" if $_VERBOSE || $DRY_RUN
+        unless $DRY_RUN || $LOCAL
+          if File.stat(@backup.path).size > MAX_CLOUDFILES_FILE_SIZE
+            STDERR.puts "ERROR: File size exceeds maximum allowed for upload to Cloud Files (#{MAX_CLOUDFILES_FILE_SIZE}): #{@backup.path}"
+            return
+          end
+          benchmark = Benchmark.realtime do
+            cf_container = cf.create_container(container)
+            o = cf_container.create_object(full_path,true)
+            o.write(File.open(@backup.path))
+          end
+          puts "...done" if $_VERBOSE
+          puts("Upload took " + sprintf("%.2f", benchmark) + " second(s).") if $_VERBOSE
+        end
+      end
+
+      def cleanup
+        return if $LOCAL
+
+        return unless keep = @config[:keep, :cloudfiles]
+
+        puts "listing files: #{container}:#{base}*" if $_VERBOSE
+        cf = CloudFiles::Connection.new(user, api_key, true, service_net) unless $LOCAL
+        cf_container = cf.container(container)
+        files = cf_container.objects(:prefix => base).sort
+
+        cleanup_with_limit(files, keep) do |f|
+          puts "removing Cloud File #{container}:#{f}" if $DRY_RUN || $_VERBOSE
+          cf_container.delete_object(f) unless $DRY_RUN || $LOCAL
+        end
+      end
+
+      def container
+        @config[:cloudfiles, :container]
+      end
+
+      def user
+        @config[:cloudfiles, :user]
+      end
+
+      def api_key
+        @config[:cloudfiles, :api_key]
+      end
+
+      def service_net
+        @config[:cloudfiles, :service_net] || false
+      end
+    end
+  end
+end

data/lib/astrails/safe/config/builder.rb

@@ -0,0 +1,60 @@
+module Astrails
+  module Safe
+    module Config
+      class Builder
+        COLLECTIONS = %w/database archive repo/
+        ITEMS = %w/s3 cloudfiles key secret bucket api_key container service_net path gpg password keep local mysqldump pgdump command options
+        user host port socket skip_tables tar files exclude filename svndump repo_path sftp page page_size/
+        NAMES = COLLECTIONS + ITEMS
+        def initialize(node)
+          @node = node
+        end
+
+        # supported args:
+        #   args = [value]
+        #   args = [id, data]
+        #   args = [data]
+        # id/value - simple values, data - hash
+        def method_missing(sym, *args, &block)
+          return super unless NAMES.include?(sym.to_s)
+
+          # do we have id or value?
+          unless args.first.is_a?(Hash)
+            id_or_value = args.shift # nil for args == []
+          end
+
+          id_or_value = id_or_value.map {|v| v.to_s} if id_or_value.is_a?(Array)
+
+          # do we have data hash?
+          if data = args.shift
+            raise "#{sym}: hash expected: #{data.inspect}" unless data.is_a?(Hash)
+          end
+
+          #puts "#{sym}: args=#{args.inspect}, id_or_value=#{id_or_value}, data=#{data.inspect}, block=#{block.inspect}"
+
+          raise "#{sym}: unexpected: #{args.inspect}" unless args.empty?
+          raise "#{sym}: missing arguments" unless id_or_value || data || block
+
+          if COLLECTIONS.include?(sym.to_s) && id_or_value
+            data ||= {}
+          end
+
+          if !data && !block
+            # simple value assignment
+            @node[sym] = id_or_value
+
+          elsif id_or_value
+            # collection element with id => create collection node and a subnode in it
+            key = sym.to_s + "s"
+            collection = @node[key] || @node.set(key, {})
+            collection.set(id_or_value, data || {}, &block)
+
+          else
+            # simple subnode
+            @node.set(sym, data || {}, &block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/astrails/safe/config/node.rb

@@ -0,0 +1,76 @@
+require 'astrails/safe/config/builder'
+module Astrails
+  module Safe
+    module Config
+      class Node
+        attr_reader :parent
+        attr_reader :data
+        def initialize(parent = nil, data = {}, &block)
+          @parent, @data = parent, {}
+          data.each { |k, v| self[k] = v }
+          Builder.new(self).instance_eval(&block) if block
+        end
+
+        # looks for the path from this node DOWN. will not delegate to parent
+        def get(*path)
+          key = path.shift
+          value = @data[key.to_s]
+          return value if value && path.empty?
+
+          value && value.get(*path)
+        end
+
+        # recursive find
+        # starts at the node and continues to the parent
+        def find(*path)
+          get(*path) || @parent && @parent.find(*path)
+        end
+        alias :[] :find
+
+        MULTIVALUES = %w/skip_tables exclude files/
+        def set(key, value, &block)
+          if @data[key.to_s]
+            raise(ArgumentError, "duplicate value for '#{key}'") if value.is_a?(Hash) || !MULTIVALUES.include?(key.to_s)
+          end
+
+          if value.is_a?(Hash)
+            @data[key.to_s] = Node.new(self, value, &block)
+          else
+            raise(ArgumentError, "#{key}: no block supported for simple values") if block
+            if @data[key.to_s]
+              @data[key.to_s] = @data[key.to_s].to_a + value.to_a
+            else
+              @data[key.to_s] = value
+            end
+            value
+          end
+        end
+        alias :[]= :set
+
+        def each(&block)
+          @data.each(&block)
+        end
+        include Enumerable
+
+        def to_hash
+          @data.keys.inject({}) do |res, key|
+            value = @data[key]
+            res[key] = value.is_a?(Node) ? value.to_hash : value
+            res
+          end
+        end
+
+        def dump(indent = "")
+          @data.each do |key, value|
+            if value.is_a?(Node)
+              puts "#{indent}#{key}:"
+              value.dump(indent + "    ")
+            else
+              puts "#{indent}#{key}: #{value.inspect}"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/astrails/safe/gpg.rb

@@ -0,0 +1,46 @@
+module Astrails
+  module Safe
+    class Gpg < Pipe
+
+      protected
+
+      def post_process
+        @backup.compressed = true
+      end
+
+      def pipe
+        command = @config[:gpg, :command] || 'gpg'
+        if key
+          "|#{command} #{@config[:gpg, :options]} -e -r #{key}"
+        elsif password
+          "|#{command} #{@config[:gpg, :options]} -c --passphrase-file #{gpg_password_file(password)}"
+        end
+      end
+
+      def extension
+        ".gpg"
+      end
+
+      def active?
+        raise RuntimeError, "can't use both gpg password and pubkey" if key && password
+
+        !!(password || key)
+      end
+
+      private
+
+      def password
+        @password ||= @config[:gpg, :password]
+      end
+
+      def key
+        @key ||= @config[:gpg, :key]
+      end
+
+      def gpg_password_file(pass)
+        return "TEMP_GENERATED_FILENAME" if $DRY_RUN
+        Astrails::Safe::TmpFile.create("gpg-pass") { |file| file.write(pass) }
+      end
+    end
+  end
+end

data/lib/astrails/safe/gzip.rb

@@ -0,0 +1,25 @@
+module Astrails
+  module Safe
+    class Gzip < Pipe
+
+      protected
+
+      def post_process
+        @backup.compressed = true
+      end
+
+      def pipe
+        "|gzip"
+      end
+
+      def extension
+        ".gz"
+      end
+
+      def active?
+        !@backup.compressed
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/local.rb

@@ -0,0 +1,70 @@
+module Astrails
+  module Safe
+    class Local < Sink
+
+      protected
+
+      def active?
+        # S3 can't upload from pipe. it needs to know file size, so we must pass through :local
+        # will change once we add SSH/FTP sink
+        true
+      end
+
+      def path
+        @path ||= File.expand_path(expand(@config[:local, :path] || raise(RuntimeError, "missing :local/:path")))
+      end
+
+      def save
+        puts "command: #{@backup.command}" if $_VERBOSE
+
+        # FIXME: probably need to change this to smth like @backup.finalize!
+        @backup.path = full_path # need to do it outside DRY_RUN so that it will be avialable for S3 DRY_RUN
+        
+        unless $DRY_RUN
+          FileUtils.mkdir_p(path) unless File.directory?(path)
+          benchmark = Benchmark.realtime do
+            if paging_enabled?
+              system "cd #{self.path}; #{@backup.command}>/dev/null"
+            else
+              system "#{@backup.command}>#{@backup.path}"
+            end
+          end
+          puts("command took " + sprintf("%.2f", benchmark) + " second(s).") if $_VERBOSE
+          if paging_enabled?        
+            target_dir = File.dirname(@backup.path)
+            Dir.chdir(target_dir)
+            files = Dir.glob("#{File.basename(@backup.path)}*")
+            files.collect! { |file| File.join(target_dir, file) }
+            @backup.path = files
+            puts("Paged files: #{@backup.path.inspect}") if $_VERBOSE
+          end
+        end
+
+      end
+
+      def cleanup
+        return unless keep = @config[:keep, :local]
+
+        puts "listing files #{base}" if $_VERBOSE
+
+        # TODO: cleanup ALL zero-length files
+
+        files = Dir["#{base}*"] .
+          select{|f| File.file?(f) && File.size(f) > 0} .
+          sort
+
+        cleanup_with_limit(files, keep) do |f|
+          puts "removing local file #{f}" if $DRY_RUN || $_VERBOSE
+          File.unlink(f) unless $DRY_RUN
+        end
+      end
+
+      private  
+      
+      def paging_enabled?
+        @backup.command.include?("split")
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/mysqldump.rb

@@ -0,0 +1,32 @@
+module Astrails
+  module Safe
+    class Mysqldump < Source
+
+      def command
+        "mysqldump --defaults-extra-file=#{mysql_password_file} #{@config[:options]} #{mysql_skip_tables} #{@id}"
+      end
+
+      def extension; '.sql'; end
+
+      protected
+
+      def mysql_password_file
+        Astrails::Safe::TmpFile.create("mysqldump") do |file|
+          file.puts "[mysqldump]"
+          %w/user password socket host port/.each do |k|
+            v = @config[k]
+            # values are quoted if needed
+            file.puts "#{k} = #{v.inspect}" if v
+          end
+        end
+      end
+
+      def mysql_skip_tables
+        if skip_tables = @config[:skip_tables]
+          [*skip_tables].map{ |t| "--ignore-table=#{@id}.#{t}" }.join(" ")
+        end
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/pgdump.rb

@@ -0,0 +1,36 @@
+module Astrails
+  module Safe
+    class Pgdump < Source
+
+      def command
+        if @config["password"]
+          ENV['PGPASSWORD'] = @config["password"]
+        else
+          ENV['PGPASSWORD'] = nil
+        end
+        "pg_dump #{postgres_options} #{postgres_username} #{postgres_host} #{postgres_port} #{@id}"
+      end
+
+      def extension; '.sql'; end
+
+      protected
+
+      def postgres_options
+        @config[:options]
+      end
+
+      def postgres_host
+        @config["host"] && "--host='#{@config["host"]}'"
+      end
+
+      def postgres_port
+        @config["port"] && "--port='#{@config["port"]}'"
+      end
+
+      def postgres_username
+        @config["user"] && "--username='#{@config["user"]}'"
+      end
+
+    end
+  end
+end