akupchanko-astrails-safe 0.3.1

data/lib/astrails/safe/gpg.rb

@@ -0,0 +1,46 @@
+module Astrails
+  module Safe
+    class Gpg < Pipe
+
+      protected
+
+      def post_process
+        @backup.compressed = true
+      end
+
+      def pipe
+        command = config[:gpg, :command] || 'gpg'
+        if key
+          "|#{command} #{config[:gpg, :options]} -e -r #{key}"
+        elsif password
+          "|#{command} #{config[:gpg, :options]} -c --passphrase-file #{gpg_password_file(password)}"
+        end
+      end
+
+      def extension
+        ".gpg"
+      end
+
+      def active?
+        raise RuntimeError, "can't use both gpg password and pubkey" if key && password
+
+        !!(password || key)
+      end
+
+      private
+
+      def password
+        @password ||= config[:gpg, :password]
+      end
+
+      def key
+        @key ||= config[:gpg, :key]
+      end
+
+      def gpg_password_file(pass)
+        return "TEMP_GENERATED_FILENAME" if dry_run?
+        Astrails::Safe::TmpFile.create("gpg-pass") { |file| file.write(pass) }
+      end
+    end
+  end
+end

data/lib/astrails/safe/gzip.rb

@@ -0,0 +1,25 @@
+module Astrails
+  module Safe
+    class Gzip < Pipe
+
+      protected
+
+      def post_process
+        @backup.compressed = true
+      end
+
+      def pipe
+        "|gzip"
+      end
+
+      def extension
+        ".gz"
+      end
+
+      def active?
+        !@backup.compressed
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/local.rb

@@ -0,0 +1,51 @@
+module Astrails
+  module Safe
+    class Local < Sink
+
+      protected
+
+      def active?
+        # S3 can't upload from pipe. it needs to know file size, so we must pass through :local
+        # will change once we add SSH/FTP sink
+        true
+      end
+
+      def path
+        @path ||= File.expand_path(expand(config[:local, :path] || raise(RuntimeError, "missing :local/:path")))
+      end
+
+      def save
+        puts "command: #{@backup.command}" if verbose?
+
+        # FIXME: probably need to change this to smth like @backup.finalize!
+        @backup.path = full_path # need to do it outside DRY_RUN so that it will be avialable for S3 DRY_RUN
+
+        unless dry_run?
+          FileUtils.mkdir_p(path) unless File.directory?(path)
+          benchmark = Benchmark.realtime do
+            system "#{@backup.command}>#{@backup.path}"
+          end
+          puts("command took " + sprintf("%.2f", benchmark) + " second(s).") if verbose?
+        end
+
+      end
+
+      def cleanup
+        return unless keep = config[:keep, :local]
+
+        puts "listing files #{base}" if verbose?
+
+        # TODO: cleanup ALL zero-length files
+
+        files = Dir["#{base}*"] .
+          select{|f| File.file?(f) && File.size(f) > 0} .
+          sort
+
+        cleanup_with_limit(files, keep) do |f|
+          puts "removing local file #{f}" if dry_run? || verbose?
+          File.unlink(f) unless dry_run?
+        end
+      end
+    end
+  end
+end

data/lib/astrails/safe/mongodump.rb

@@ -0,0 +1,23 @@
+module Astrails
+  module Safe
+    class Mongodump < Source
+      
+      def command
+        opts = []
+        opts << "--host #{config[:host]}" if config[:host]
+        opts << "-u #{config[:user]}" if config[:user]
+        opts << "-p #{config[:password]}" if config[:password]
+        opts << "--out #{output_directory}"
+      
+        "mongodump -q \"{xxxx : { \\$ne : 0 } }\" --db #{@id} #{opts.join(" ")} && cd #{output_directory} && tar cf - ."
+      end
+    
+      def extension; '.tar'; end
+      
+      protected
+      def output_directory
+        File.join(TmpFile.tmproot, "mongodump")
+      end
+    end
+  end
+end

data/lib/astrails/safe/mysqldump.rb

@@ -0,0 +1,32 @@
+module Astrails
+  module Safe
+    class Mysqldump < Source
+
+      def command
+        "mysqldump --defaults-extra-file=#{mysql_password_file} #{config[:options]} #{mysql_skip_tables} #{@id}"
+      end
+
+      def extension; '.sql'; end
+
+      protected
+
+      def mysql_password_file
+        Astrails::Safe::TmpFile.create("mysqldump") do |file|
+          file.puts "[mysqldump]"
+          %w/user password socket host port/.each do |k|
+            v = config[k]
+            # values are quoted if needed
+            file.puts "#{k} = #{v.inspect}" if v
+          end
+        end
+      end
+
+      def mysql_skip_tables
+        if skip_tables = config[:skip_tables]
+          [*skip_tables].map{ |t| "--ignore-table=#{@id}.#{t}" }.join(" ")
+        end
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/pgdump.rb

@@ -0,0 +1,36 @@
+module Astrails
+  module Safe
+    class Pgdump < Source
+
+      def command
+        if config["password"]
+          ENV['PGPASSWORD'] = config["password"]
+        else
+          ENV['PGPASSWORD'] = nil
+        end
+        "pg_dump #{postgres_options} #{postgres_username} #{postgres_host} #{postgres_port} #{@id}"
+      end
+
+      def extension; '.sql'; end
+
+      protected
+
+      def postgres_options
+        config[:options]
+      end
+
+      def postgres_host
+        config["host"] && "--host='#{config["host"]}'"
+      end
+
+      def postgres_port
+        config["port"] && "--port='#{config["port"]}'"
+      end
+
+      def postgres_username
+        config["user"] && "--username='#{config["user"]}'"
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/pipe.rb

@@ -0,0 +1,17 @@
+module Astrails
+  module Safe
+    class Pipe < Stream
+      # process adds required commands to the current
+      # shell command string
+      # :active?, :pipe, :extension and :post_process are
+      # defined in inheriting pipe classes
+      def process
+        return unless active?
+
+        @backup.command << pipe
+        @backup.extension << extension
+        post_process
+      end
+    end
+  end
+end

data/lib/astrails/safe/s3.rb

@@ -0,0 +1,80 @@
+module Astrails
+  module Safe
+    class S3 < Sink
+      MAX_S3_FILE_SIZE = 5368709120
+
+      protected
+
+      def active?
+        bucket && key && secret
+      end
+
+      def path
+        @path ||= expand(config[:s3, :path] || config[:local, :path] || ":kind/:id")
+      end
+
+      def save
+        # FIXME: user friendly error here :)
+        raise RuntimeError, "pipe-streaming not supported for S3." unless @backup.path
+
+        # needed in cleanup even on dry run
+        AWS::S3::Base.establish_connection!(:access_key_id => key, :secret_access_key => secret, :use_ssl => true) unless local_only?
+
+        puts "Uploading #{bucket}:#{full_path}" if verbose? || dry_run?
+        unless dry_run? || local_only?
+          if File.stat(@backup.path).size > MAX_S3_FILE_SIZE
+            STDERR.puts "ERROR: File size exceeds maximum allowed for upload to S3 (#{MAX_S3_FILE_SIZE}): #{@backup.path}"
+            return
+          end
+          benchmark = Benchmark.realtime do
+            AWS::S3::Bucket.create(bucket) unless bucket_exists?(bucket)
+            File.open(@backup.path) do |file|
+              AWS::S3::S3Object.store(full_path, file, bucket)
+            end
+          end
+          puts "...done" if verbose?
+          puts("Upload took " + sprintf("%.2f", benchmark) + " second(s).") if verbose?
+        end
+      end
+
+      def cleanup
+        return if local_only?
+
+        return unless keep = config[:keep, :s3]
+
+        puts "listing files: #{bucket}:#{base}*" if verbose?
+        files = AWS::S3::Bucket.objects(bucket, :prefix => base, :max_keys => keep * 2)
+        puts files.collect {|x| x.key} if verbose?
+
+        files = files.
+          collect {|x| x.key}.
+          sort
+
+        cleanup_with_limit(files, keep) do |f|
+          puts "removing s3 file #{bucket}:#{f}" if dry_run? || verbose?
+          AWS::S3::Bucket.objects(bucket, :prefix => f)[0].delete unless dry_run? || local_only?
+        end
+      end
+
+      def bucket
+        config[:s3, :bucket]
+      end
+
+      def key
+        config[:s3, :key]
+      end
+
+      def secret
+        config[:s3, :secret]
+      end
+
+      private
+      
+      def bucket_exists?(bucket)
+        true if AWS::S3::Bucket.find(bucket)
+      rescue AWS::S3::NoSuchBucket
+        false
+      end
+    end
+  end
+end

data/lib/astrails/safe/sftp.rb

@@ -0,0 +1,88 @@
+module Astrails
+  module Safe
+    class Sftp < Sink
+
+      protected
+
+      def active?
+        host && user
+      end
+
+      def path
+        @path ||= expand(config[:sftp, :path] || config[:local, :path] || ":kind/:id")
+      end
+
+      def save
+        raise RuntimeError, "pipe-streaming not supported for SFTP." unless @backup.path
+
+        puts "Uploading #{host}:#{full_path} via SFTP" if verbose? || dry_run?
+
+        unless dry_run? || local_only?
+          opts = {}
+          opts[:password] = password if password
+          opts[:port] = port if port
+          Net::SFTP.start(host, user, opts) do |sftp|
+            puts "Sending #{@backup.path} to #{full_path}" if verbose?
+            begin
+              sftp.upload! @backup.path, full_path
+            rescue Net::SFTP::StatusException
+              puts "Ensuring remote path (#{path}) exists" if verbose?
+              # mkdir -p
+              folders = path.split('/')
+              folders.each_index do |i|
+                folder = folders[0..i].join('/')
+                puts "Creating #{folder} on remote" if verbose?
+                sftp.mkdir!(folder) rescue Net::SFTP::StatusException
+              end
+              retry
+            end
+          end
+          puts "...done" if verbose?
+        end
+      end
+
+      def cleanup
+        return if local_only? || dry_run?
+
+        return unless keep = config[:keep, :sftp]
+
+        puts "listing files: #{host}:#{base}*" if verbose?
+        opts = {}
+        opts[:password] = password if password
+        opts[:port] = port if port
+        Net::SFTP.start(host, user, opts) do |sftp|
+          files = sftp.dir.glob(path, File.basename("#{base}*"))
+
+          puts files.collect {|x| x.name } if verbose?
+
+          files = files.
+            collect {|x| x.name }.
+            sort
+
+          cleanup_with_limit(files, keep) do |f|
+            file = File.join(path, f)
+            puts "removing sftp file #{host}:#{file}" if dry_run? || verbose?
+            sftp.remove!(file) unless dry_run? || local_only?
+          end
+        end
+      end
+
+      def host
+        config[:sftp, :host]
+      end
+
+      def user
+        config[:sftp, :user]
+      end
+
+      def password
+        config[:sftp, :password]
+      end
+
+      def port
+        config[:sftp, :port]
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/sink.rb

@@ -0,0 +1,35 @@
+module Astrails
+  module Safe
+    class Sink < Stream
+
+      def process
+        return unless active?
+
+        save
+        cleanup
+      end
+
+      protected
+
+      # path is defined in subclass
+      # base is used in 'cleanup' to find all files that begin with base. the '.'
+      # at the end is essential to distinguish b/w foo.* and foobar.* archives for example
+      def base
+        @base ||= File.join(path, File.basename(@backup.filename).split(".").first + '.')
+      end
+
+      def full_path
+        @full_path ||= File.join(path, @backup.filename) + @backup.extension
+      end
+
+      # call block on files to be removed (all except for the LAST 'limit' files
+      def cleanup_with_limit(files, limit, &block)
+        return unless files.size > limit
+
+        to_remove = files[0..(files.size - limit - 1)]
+        # TODO: validate here
+        to_remove.each(&block)
+      end
+    end
+  end
+end

data/lib/astrails/safe/source.rb

@@ -0,0 +1,47 @@
+module Astrails
+  module Safe
+    class Source < Stream
+
+      attr_accessor :id
+      def initialize(id, config)
+        @id, @config = id.to_s, config
+      end
+
+      def timestamp
+        Time.now.strftime("%y%m%d-%H%M")
+      end
+
+      def kind
+        self.class.human_name
+      end
+
+      def filename
+        @filename ||= expand(":kind-:id.:timestamp")
+      end
+
+      def backup
+        return @backup if @backup
+        @backup = Backup.new(
+          :id        => @id,
+          :kind      => kind,
+          :extension => extension,
+          :command   => command,
+          :timestamp => timestamp
+        )
+        # can't do this in the initializer hash above since
+        # filename() calls expand() which requires @backup
+        # FIXME: move expansion to the backup (last step in ctor) assign :tags here
+        @backup.filename = filename
+        @backup
+      end
+
+      protected
+
+      def self.human_name
+        name.split('::').last.downcase
+      end
+
+    end
+  end
+end
+