bostonlogic-safe 0.3.0

data/lib/astrails/safe/pgdump.rb

@@ -0,0 +1,36 @@
+module Astrails
+  module Safe
+    class Pgdump < Source
+
+      def command
+        if @config["password"]
+          ENV['PGPASSWORD'] = @config["password"]
+        else
+          ENV['PGPASSWORD'] = nil
+        end
+        "pg_dump #{postgres_options} #{postgres_username} #{postgres_host} #{postgres_port} #{@id}"
+      end
+
+      def extension; '.sql'; end
+
+      protected
+
+      def postgres_options
+        @config[:options]
+      end
+
+      def postgres_host
+        @config["host"] && "--host='#{@config["host"]}'"
+      end
+
+      def postgres_port
+        @config["port"] && "--port='#{@config["port"]}'"
+      end
+
+      def postgres_username
+        @config["user"] && "--username='#{@config["user"]}'"
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/pipe.rb

@@ -0,0 +1,13 @@
+module Astrails
+  module Safe
+    class Pipe < Stream
+      def process
+        return unless active?
+
+        @backup.command << pipe
+        @backup.extension << extension
+        post_process
+      end
+    end
+  end
+end

data/lib/astrails/safe/rcloud.rb

@@ -0,0 +1,73 @@
+module Astrails
+  module Safe
+    class Rcloud < Sink
+
+      protected
+
+      def active?
+        container && username && api_key
+      end
+
+      def path
+        @path ||= expand(config[:rcloud, :path] || config[:local, :path] || ":kind/:id")
+      end
+
+      def save
+        raise RuntimeError, "pipe-streaming not supported for Rackspace Cloudfiles." unless @backup.path
+
+        # needed in cleanup even on dry run
+        rcf = CloudFiles::Connection.new(username, api_key)
+
+        puts "Uploading #{container}:#{full_path}" if $_VERBOSE || $DRY_RUN
+        unless $DRY_RUN || $LOCAL
+          benchmark = Benchmark.realtime do
+            if rcf.container_exists?(container)
+              rcont = rcf.container(container)
+            else
+              rcont = rcf.create_container(container)
+            end
+            rfile = rcont.create_object(full_path)
+            rfile.load_from_filename(@backup.path)
+          end
+          puts "...done" if $_VERBOSE
+          puts("Upload took " + sprintf("%.2f", benchmark) + " second(s).") if $_VERBOSE
+        end
+      end
+
+      def cleanup
+        return if $LOCAL
+
+        return unless keep = @config[:keep, :rcloud]
+        
+        rcf = CloudFiles::Connection.new(username, api_key)
+        rcont = rcf.container(container)
+
+        puts "listing files: #{container}:#{base}*" if $_VERBOSE
+        rfiles = rcont.objects_detail(:prefix => base)
+        puts rfiles.collect {|x| x[0]} if $_VERBOSE
+
+        files = rfiles.
+          collect {|x| x[0]}.
+          sort
+
+        cleanup_with_limit(files, keep) do |f|
+          puts "removing Rackspace Cloudfile #{container}:#{f}" if $DRY_RUN || $_VERBOSE
+          rcont.delete_object(f) unless $DRY_RUN || $LOCAL
+        end
+      end
+
+      def container
+        @config[:rcloud, :container]
+      end
+
+      def username
+        @config[:rcloud, :username]
+      end
+
+      def api_key
+        @config[:rcloud, :api_key]
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/s3.rb

@@ -0,0 +1,68 @@
+module Astrails
+  module Safe
+    class S3 < Sink
+
+      protected
+
+      def active?
+        bucket && key && secret
+      end
+
+      def path
+        @path ||= expand(config[:s3, :path] || config[:local, :path] || ":kind/:id")
+      end
+
+      def save
+        raise RuntimeError, "pipe-streaming not supported for S3." unless @backup.path
+
+        # needed in cleanup even on dry run
+        AWS::S3::Base.establish_connection!(:access_key_id => key, :secret_access_key => secret, :use_ssl => true) unless $LOCAL
+
+        puts "Uploading #{bucket}:#{full_path}" if $_VERBOSE || $DRY_RUN
+        unless $DRY_RUN || $LOCAL
+          benchmark = Benchmark.realtime do
+            AWS::S3::Bucket.create(bucket)
+            File.open(@backup.path) do |file|
+              AWS::S3::S3Object.store(full_path, file, bucket)
+            end
+          end
+          puts "...done" if $_VERBOSE
+          puts("Upload took " + sprintf("%.2f", benchmark) + " second(s).") if $_VERBOSE
+        end
+      end
+
+      def cleanup
+        return if $LOCAL
+
+        return unless keep = @config[:keep, :s3]
+
+
+        puts "listing files: #{bucket}:#{base}*" if $_VERBOSE
+        files = AWS::S3::Bucket.objects(bucket, :prefix => base, :max_keys => keep * 2)
+        puts files.collect {|x| x.key} if $_VERBOSE
+
+        files = files.
+          collect {|x| x.key}.
+          sort
+
+        cleanup_with_limit(files, keep) do |f|
+          puts "removing s3 file #{bucket}:#{f}" if $DRY_RUN || $_VERBOSE
+          AWS::S3::Bucket.find(bucket)[f].delete unless $DRY_RUN || $LOCAL
+        end
+      end
+
+      def bucket
+        @config[:s3, :bucket]
+      end
+
+      def key
+        @config[:s3, :key]
+      end
+
+      def secret
+        @config[:s3, :secret]
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/sftp.rb

@@ -0,0 +1,79 @@
+module Astrails
+  module Safe
+    class Sftp < Sink
+
+      protected
+
+      def active?
+        host && user
+      end
+
+      def path
+        @path ||= expand(config[:sftp, :path] || config[:local, :path] || ":kind/:id")
+      end
+
+      def save
+        raise RuntimeError, "pipe-streaming not supported for SFTP." unless @backup.path
+
+        puts "Uploading #{host}:#{full_path} via SFTP" if $_VERBOSE || $DRY_RUN
+
+        unless $DRY_RUN || $LOCAL
+          opts = {}
+          opts[:password] = password if password
+          Net::SFTP.start(host, user, opts) do |sftp|
+            puts "Sending #{@backup.path} to #{full_path}" if $_VERBOSE
+            begin
+              sftp.upload! @backup.path, full_path
+            rescue Net::SFTP::StatusException
+              puts "Ensuring remote path (#{path}) exists" if $_VERBOSE
+              # mkdir -p
+              folders = path.split('/')
+              folders.each_index do |i|
+                folder = folders[0..i].join('/')
+                puts "Creating #{folder} on remote" if $_VERBOSE
+                sftp.mkdir!(folder) rescue Net::SFTP::StatusException
+              end
+              retry
+            end
+          end
+          puts "...done" if $_VERBOSE
+        end
+      end
+
+      def cleanup
+        return if $LOCAL || $DRY_RUN
+
+        return unless keep = @config[:keep, :sftp]
+
+        puts "listing files: #{host}:#{base}*" if $_VERBOSE
+        Net::SFTP.start(host, user, :password => password) do |sftp|
+          files = sftp.dir.glob(path, File.basename("#{base}*"))
+
+          puts files.collect {|x| x.name } if $_VERBOSE
+
+          files = files.
+            collect {|x| x.name }.
+            sort
+
+          cleanup_with_limit(files, keep) do |f|
+            file = File.join(path, f)
+            puts "removing sftp file #{host}:#{file}" if $DRY_RUN || $_VERBOSE
+            sftp.remove!(file) unless $DRY_RUN || $LOCAL
+          end
+        end
+      end
+
+      def host
+        @config[:sftp, :host]
+      end
+
+      def user
+        @config[:sftp, :user]
+      end
+
+      def password
+        @config[:sftp, :password]
+      end
+    end
+  end
+end

data/lib/astrails/safe/sink.rb

@@ -0,0 +1,33 @@
+module Astrails
+  module Safe
+    class Sink < Stream
+
+      def process
+        return unless active?
+
+        save
+        cleanup
+      end
+
+      protected
+
+      # path is defined in subclass
+      def base
+        @base ||= File.join(path, File.basename(@backup.filename).split(".").first + '.')
+      end
+
+      def full_path
+        @full_path ||= File.join(path, @backup.filename) + @backup.extension
+      end
+
+      # call block on files to be removed (all except for the LAST 'limit' files
+      def cleanup_with_limit(files, limit, &block)
+        return unless files.size > limit
+
+        to_remove = files[0..(files.size - limit - 1)]
+        # TODO: validate here
+        to_remove.each(&block)
+      end
+    end
+  end
+end

data/lib/astrails/safe/source.rb

@@ -0,0 +1,46 @@
+module Astrails
+  module Safe
+    class Source < Stream
+
+      attr_accessor :id
+      def initialize(id, config)
+        @id, @config = id.to_s, config
+      end
+
+      def timestamp
+        Time.now.strftime("%y%m%d-%H%M")
+      end
+
+      def kind
+        self.class.human_name
+      end
+
+      def filename
+        @filename ||= expand(":kind-:id.:timestamp")
+      end
+
+      def backup
+        return @backup if @backup
+        @backup = Backup.new(
+          :id        => @id,
+          :kind      => kind,
+          :extension => extension,
+          :command   => command,
+          :timestamp => timestamp
+        )
+        # can't do this in the initializer hash above since
+        # filename() calls expand() which requires @backup
+        @backup.filename = filename
+        @backup
+      end
+
+      protected
+
+      def self.human_name
+        name.split('::').last.downcase
+      end
+
+    end
+  end
+end
+

data/lib/astrails/safe/stream.rb

@@ -0,0 +1,19 @@
+module Astrails
+  module Safe
+    class Stream
+
+      attr_accessor :config, :backup
+      def initialize(config, backup)
+        @config, @backup = config, backup
+      end
+
+      def expand(path)
+        path .
+        gsub(/:kind\b/, @backup.kind.to_s) .
+        gsub(/:id\b/, @backup.id.to_s) .
+        gsub(/:timestamp\b/, @backup.timestamp)
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/svndump.rb

@@ -0,0 +1,13 @@
+module Astrails
+  module Safe
+    class Svndump < Source
+
+      def command
+        "svnadmin dump #{@config[:options]} #{@config[:repo_path]}"
+      end
+
+      def extension; '.svn'; end
+
+    end
+  end
+end

data/lib/astrails/safe/tmp_file.rb

@@ -0,0 +1,48 @@
+require 'tmpdir'
+module Astrails
+  module Safe
+    module TmpFile
+      @keep_files = []
+
+      def self.tmproot
+        @tmproot ||= Dir.mktmpdir
+      end
+
+      def self.cleanup
+        begin
+          FileUtils.remove_entry_secure tmproot
+        rescue ArgumentError => e
+          if e.message =~ /parent directory is world writable/
+            puts <<-ERR
+
+
+********************************************************************************
+It looks like you have wrong permissions on your TEMP directory.  The usual
+case is when you have world writable TEMP directory withOUT the sticky bit.
+
+Try "chmod +t" on it.
+
+********************************************************************************
+
+ERR
+          else
+            raise
+          end
+        end
+        @tmproot = nil
+      end
+
+      def self.create(name)
+        # create temp directory
+
+        file = Tempfile.new(name, tmproot)
+
+        yield file
+
+        file.close
+        @keep_files << file # so that it will not get gcollected and removed from filesystem until the end
+        file.path
+      end
+    end
+  end
+end

data/lib/extensions/mktmpdir.rb

@@ -0,0 +1,45 @@
+require 'tmpdir'
+
+unless Dir.respond_to?(:mktmpdir)
+  # backward compat for 1.8.6
+  class Dir
+    def Dir.mktmpdir(prefix_suffix=nil, tmpdir=nil)
+      case prefix_suffix
+      when nil
+        prefix = "d"
+        suffix = ""
+      when String
+        prefix = prefix_suffix
+        suffix = ""
+      when Array
+        prefix = prefix_suffix[0]
+        suffix = prefix_suffix[1]
+      else
+        raise ArgumentError, "unexpected prefix_suffix: #{prefix_suffix.inspect}"
+      end
+      tmpdir ||= Dir.tmpdir
+      t = Time.now.strftime("%Y%m%d")
+      n = nil
+      begin
+        path = "#{tmpdir}/#{prefix}#{t}-#{$$}-#{rand(0x100000000).to_s(36)}"
+        path << "-#{n}" if n
+        path << suffix
+        Dir.mkdir(path, 0700)
+      rescue Errno::EEXIST
+        n ||= 0
+        n += 1
+        retry
+      end
+
+      if block_given?
+        begin
+          yield path
+        ensure
+          FileUtils.remove_entry_secure path
+        end
+      else
+        path
+      end
+    end
+  end
+end