netguru-safe 0.2.9

data/lib/astrails/safe/gpg.rb

@@ -0,0 +1,46 @@
+module Astrails
+  module Safe
+    class Gpg < Pipe
+
+      protected
+
+      def post_process
+        @backup.compressed = true
+      end
+
+      def pipe
+        command = @config[:gpg, :command] || 'gpg'
+        if key
+          "|#{command} #{@config[:gpg, :options]} -e -r #{key}"
+        elsif password
+          "|#{command} #{@config[:gpg, :options]} -c --passphrase-file #{gpg_password_file(password)}"
+        end
+      end
+
+      def extension
+        ".gpg"
+      end
+
+      def active?
+        raise RuntimeError, "can't use both gpg password and pubkey" if key && password
+
+        !!(password || key)
+      end
+
+      private
+
+      def password
+        @password ||= @config[:gpg, :password]
+      end
+
+      def key
+        @key ||= @config[:gpg, :key]
+      end
+
+      def gpg_password_file(pass)
+        return "TEMP_GENERATED_FILENAME" if $DRY_RUN
+        Astrails::Safe::TmpFile.create("gpg-pass") { |file| file.write(pass) }
+      end
+    end
+  end
+end

data/lib/astrails/safe/gzip.rb

@@ -0,0 +1,25 @@
+module Astrails
+  module Safe
+    class Gzip < Pipe
+
+      protected
+
+      def post_process
+        @backup.compressed = true
+      end
+
+      def pipe
+        "|gzip"
+      end
+
+      def extension
+        ".gz"
+      end
+
+      def active?
+        !@backup.compressed
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/local.rb

@@ -0,0 +1,51 @@
+module Astrails
+  module Safe
+    class Local < Sink
+
+      protected
+
+      def active?
+        # S3 can't upload from pipe. it needs to know file size, so we must pass through :local
+        # will change once we add SSH/FTP sink
+        true
+      end
+
+      def path
+        @path ||= File.expand_path(expand(@config[:local, :path] || raise(RuntimeError, "missing :local/:path")))
+      end
+
+      def save
+        puts "command: #{@backup.command}" if $_VERBOSE
+
+        # FIXME: probably need to change this to smth like @backup.finalize!
+        @backup.path = full_path # need to do it outside DRY_RUN so that it will be avialable for S3 DRY_RUN
+
+        unless $DRY_RUN
+          FileUtils.mkdir_p(path) unless File.directory?(path)
+          benchmark = Benchmark.realtime do
+            system "#{@backup.command}>#{@backup.path}"
+          end
+          puts("command took " + sprintf("%.2f", benchmark) + " second(s).") if $_VERBOSE
+        end
+
+      end
+
+      def cleanup
+        return unless keep = @config[:keep, :local]
+
+        puts "listing files #{base}" if $_VERBOSE
+
+        # TODO: cleanup ALL zero-length files
+
+        files = Dir["#{base}*"] .
+          select{|f| File.file?(f) && File.size(f) > 0} .
+          sort
+
+        cleanup_with_limit(files, keep) do |f|
+          puts "removing local file #{f}" if $DRY_RUN || $_VERBOSE
+          File.unlink(f) unless $DRY_RUN
+        end
+      end
+    end
+  end
+end

data/lib/astrails/safe/mongodump.rb

@@ -0,0 +1,23 @@
+module Astrails
+  module Safe
+    class Mongodump < Source
+      
+      def command
+        opts = []
+        opts << "--host #{@config[:host]}" if @config[:host]
+        opts << "-u #{@config[:user]}" if @config[:user]
+        opts << "-p #{@config[:password]}" if @config[:password]
+        opts << "--out #{output_directory}"
+      
+        "mongodump -q \"{xxxx : { \\$ne : 0 } }\" --db #{@id} #{opts.join(" ")} && cd #{output_directory} && tar cf - ."
+      end
+    
+      def extension; '.tar'; end
+      
+      protected
+      def output_directory
+        File.join(TmpFile.tmproot, "mongodump")
+      end
+    end
+  end
+end

data/lib/astrails/safe/mysqldump.rb

@@ -0,0 +1,32 @@
+module Astrails
+  module Safe
+    class Mysqldump < Source
+
+      def command
+        "mysqldump --defaults-extra-file=#{mysql_password_file} #{@config[:options]} #{mysql_skip_tables} #{@id}"
+      end
+
+      def extension; '.sql'; end
+
+      protected
+
+      def mysql_password_file
+        Astrails::Safe::TmpFile.create("mysqldump") do |file|
+          file.puts "[mysqldump]"
+          %w/user password socket host port/.each do |k|
+            v = @config[k]
+            # values are quoted if needed
+            file.puts "#{k} = #{v.inspect}" if v
+          end
+        end
+      end
+
+      def mysql_skip_tables
+        if skip_tables = @config[:skip_tables]
+          [*skip_tables].map{ |t| "--ignore-table=#{@id}.#{t}" }.join(" ")
+        end
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/pgdump.rb

@@ -0,0 +1,36 @@
+module Astrails
+  module Safe
+    class Pgdump < Source
+
+      def command
+        if @config["password"]
+          ENV['PGPASSWORD'] = @config["password"]
+        else
+          ENV['PGPASSWORD'] = nil
+        end
+        "pg_dump #{postgres_options} #{postgres_username} #{postgres_host} #{postgres_port} #{@id}"
+      end
+
+      def extension; '.sql'; end
+
+      protected
+
+      def postgres_options
+        @config[:options]
+      end
+
+      def postgres_host
+        @config["host"] && "--host='#{@config["host"]}'"
+      end
+
+      def postgres_port
+        @config["port"] && "--port='#{@config["port"]}'"
+      end
+
+      def postgres_username
+        @config["user"] && "--username='#{@config["user"]}'"
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/pipe.rb

@@ -0,0 +1,17 @@
+module Astrails
+  module Safe
+    class Pipe < Stream
+      # process adds required commands to the current
+      # shell command string
+      # :active?, :pipe, :extension and :post_process are
+      # defined in inheriting pipe classes
+      def process
+        return unless active?
+
+        @backup.command << pipe
+        @backup.extension << extension
+        post_process
+      end
+    end
+  end
+end

data/lib/astrails/safe/s3.rb

@@ -0,0 +1,75 @@
+module Astrails
+  module Safe
+    class S3 < Sink
+      MAX_S3_FILE_SIZE = 5368709120
+
+      protected
+
+      def active?
+        bucket && key && secret
+      end
+
+      def path
+        @path ||= expand(config[:s3, :path] || config[:local, :path] || ":kind/:id")
+      end
+
+      def save
+        # FIXME: user friendly error here :)
+        raise RuntimeError, "pipe-streaming not supported for S3." unless @backup.path
+
+        puts "Uploading #{bucket}:#{full_path}" if $_VERBOSE || $DRY_RUN
+        unless $DRY_RUN || $LOCAL
+          if File.stat(@backup.path).size > MAX_S3_FILE_SIZE
+            STDERR.puts "ERROR: File size exceeds maximum allowed for upload to S3 (#{MAX_S3_FILE_SIZE}): #{@backup.path}"
+            return
+          end
+          benchmark = Benchmark.realtime do
+            File.open(@backup.path) do |file|
+              remote_bucket.objects.create(full_path, :data => file)
+            end
+          end
+          puts "...done" if $_VERBOSE
+          puts("Upload took " + sprintf("%.2f", benchmark) + " second(s).") if $_VERBOSE
+        end
+      end
+
+      def cleanup
+        return if $LOCAL
+
+        return unless keep = @config[:keep, :s3]
+
+        puts "listing files: #{bucket}:#{base}*" if $_VERBOSE
+        files = remote_bucket.objects.with_prefix(:prefix => base)
+        puts files.collect {|x| x.key} if $_VERBOSE
+
+        files = files.sort { |x,y| x.key <=> y.key }
+
+        cleanup_with_limit(files, keep) do |f|
+          puts "removing s3 file #{bucket}:#{f}" if $DRY_RUN || $_VERBOSE
+          f.delete unless $DRY_RUN || $LOCAL
+        end
+      end
+
+      def remote_bucket
+        unless @remote_bucket
+          s3 = AWS::S3.new(:access_key_id => key, :secret_access_key => secret)
+          @remote_bucket = s3.buckets.create(bucket)
+        end
+        @remote_bucket
+      end
+
+      def bucket
+        @config[:s3, :bucket]
+      end
+
+      def key
+        @config[:s3, :key]
+      end
+
+      def secret
+        @config[:s3, :secret]
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/sftp.rb

@@ -0,0 +1,88 @@
+module Astrails
+  module Safe
+    class Sftp < Sink
+
+      protected
+
+      def active?
+        host && user
+      end
+
+      def path
+        @path ||= expand(config[:sftp, :path] || config[:local, :path] || ":kind/:id")
+      end
+
+      def save
+        raise RuntimeError, "pipe-streaming not supported for SFTP." unless @backup.path
+
+        puts "Uploading #{host}:#{full_path} via SFTP" if $_VERBOSE || $DRY_RUN
+
+        unless $DRY_RUN || $LOCAL
+          opts = {}
+          opts[:password] = password if password
+          opts[:port] = port if port
+          Net::SFTP.start(host, user, opts) do |sftp|
+            puts "Sending #{@backup.path} to #{full_path}" if $_VERBOSE
+            begin
+              sftp.upload! @backup.path, full_path
+            rescue Net::SFTP::StatusException
+              puts "Ensuring remote path (#{path}) exists" if $_VERBOSE
+              # mkdir -p
+              folders = path.split('/')
+              folders.each_index do |i|
+                folder = folders[0..i].join('/')
+                puts "Creating #{folder} on remote" if $_VERBOSE
+                sftp.mkdir!(folder) rescue Net::SFTP::StatusException
+              end
+              retry
+            end
+          end
+          puts "...done" if $_VERBOSE
+        end
+      end
+
+      def cleanup
+        return if $LOCAL || $DRY_RUN
+
+        return unless keep = @config[:keep, :sftp]
+
+        puts "listing files: #{host}:#{base}*" if $_VERBOSE
+        opts = {}
+        opts[:password] = password if password
+        opts[:port] = port if port
+        Net::SFTP.start(host, user, opts) do |sftp|
+          files = sftp.dir.glob(path, File.basename("#{base}*"))
+
+          puts files.collect {|x| x.name } if $_VERBOSE
+
+          files = files.
+            collect {|x| x.name }.
+            sort
+
+          cleanup_with_limit(files, keep) do |f|
+            file = File.join(path, f)
+            puts "removing sftp file #{host}:#{file}" if $DRY_RUN || $_VERBOSE
+            sftp.remove!(file) unless $DRY_RUN || $LOCAL
+          end
+        end
+      end
+
+      def host
+        @config[:sftp, :host]
+      end
+
+      def user
+        @config[:sftp, :user]
+      end
+
+      def password
+        @config[:sftp, :password]
+      end
+
+      def port
+        @config[:sftp, :port]
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/sink.rb

@@ -0,0 +1,35 @@
+module Astrails
+  module Safe
+    class Sink < Stream
+
+      def process
+        return unless active?
+
+        save
+        cleanup
+      end
+
+      protected
+
+      # path is defined in subclass
+      # base is used in 'cleanup' to find all files that begin with base. the '.'
+      # at the end is essential to distinguish b/w foo.* and foobar.* archives for example
+      def base
+        @base ||= File.join(path, File.basename(@backup.filename).split(".").first + '.')
+      end
+
+      def full_path
+        @full_path ||= File.join(path, @backup.filename) + @backup.extension
+      end
+
+      # call block on files to be removed (all except for the LAST 'limit' files
+      def cleanup_with_limit(files, limit, &block)
+        return unless files.size > limit
+
+        to_remove = files[0..(files.size - limit - 1)]
+        # TODO: validate here
+        to_remove.each(&block)
+      end
+    end
+  end
+end

data/lib/astrails/safe/source.rb

@@ -0,0 +1,47 @@
+module Astrails
+  module Safe
+    class Source < Stream
+
+      attr_accessor :id
+      def initialize(id, config)
+        @id, @config = id.to_s, config
+      end
+
+      def timestamp
+        Time.now.strftime("%y%m%d-%H%M")
+      end
+
+      def kind
+        self.class.human_name
+      end
+
+      def filename
+        @filename ||= expand(":kind-:id.:timestamp")
+      end
+
+      def backup
+        return @backup if @backup
+        @backup = Backup.new(
+          :id        => @id,
+          :kind      => kind,
+          :extension => extension,
+          :command   => command,
+          :timestamp => timestamp
+        )
+        # can't do this in the initializer hash above since
+        # filename() calls expand() which requires @backup
+        # FIXME: move expansion to the backup (last step in ctor) assign :tags here
+        @backup.filename = filename
+        @backup
+      end
+
+      protected
+
+      def self.human_name
+        name.split('::').last.downcase
+      end
+
+    end
+  end
+end
+