darkofabijan-astrails-safe 0.2.8

data/lib/astrails/safe/pipe.rb

@@ -0,0 +1,17 @@
+module Astrails
+  module Safe
+    class Pipe < Stream
+      # process adds required commands to the current
+      # shell command string
+      # :active?, :pipe, :extension and :post_process are
+      # defined in inheriting pipe classes
+      def process
+        return unless active?
+
+        @backup.command << pipe
+        @backup.extension << extension
+        post_process
+      end
+    end
+  end
+end

data/lib/astrails/safe/s3.rb

@@ -0,0 +1,86 @@
+module Astrails
+  module Safe
+    class S3 < Sink
+      MAX_S3_FILE_SIZE = 5368709120
+
+      protected
+
+      def active?
+        bucket && key && secret
+      end
+
+      def path
+        @path ||= expand(config[:s3, :path] || config[:local, :path] || ":kind/:id")
+      end
+
+      def save
+        # FIXME: user friendly error here :)
+        raise RuntimeError, "pipe-streaming not supported for S3." unless @backup.path
+
+        # needed in cleanup even on dry run
+        AWS::S3::Base.establish_connection!(:access_key_id => key, :secret_access_key => secret, :use_ssl => true) unless $LOCAL
+
+        unless $DRY_RUN || $LOCAL
+          benchmark = Benchmark.realtime do
+            AWS::S3::Bucket.create(bucket)
+            store_files_to_bucket(@backup.path)
+          end
+          puts("Upload took " + sprintf("%.2f", benchmark) + " second(s).") if $_VERBOSE
+        end
+      end
+
+      def cleanup
+        return if $LOCAL
+
+        return unless keep = @config[:keep, :s3]
+
+        puts "listing files: #{bucket}:#{base}*" if $_VERBOSE
+        files = AWS::S3::Bucket.objects(bucket, :prefix => base, :max_keys => keep * 2)
+        puts files.collect {|x| x.key} if $_VERBOSE
+
+        files = files.collect {|x| x.key}.sort
+
+        cleanup_with_limit(files, keep) do |f|
+          puts "removing s3 file #{bucket}:#{f}" if $DRY_RUN || $_VERBOSE
+          AWS::S3::Bucket.find(bucket)[f].delete unless $DRY_RUN || $LOCAL
+        end
+      end
+
+      def bucket
+        @config[:s3, :bucket]
+      end
+
+      def key
+        @config[:s3, :key]
+      end
+
+      def secret
+        @config[:s3, :secret]
+      end
+      
+      def store_files_to_bucket(files)
+        if files.class == String
+          store_file(files)
+        elsif files.class == Array
+          files.each do |file|
+            store_file(file)
+          end
+        end
+      end
+
+      def store_file(filepath)
+        if File.stat(filepath).size > MAX_S3_FILE_SIZE
+          STDERR.puts "ERROR: File size exceeds maximum allowed for upload to S3 (#{MAX_S3_FILE_SIZE}): #{@backup.path}"
+          return
+        end
+        File.open(filepath) do |file|
+          destination_path = full_path(File.basename(filepath), '')
+          puts "Uploading #{bucket}:#{destination_path}" if $_VERBOSE
+          AWS::S3::S3Object.store(destination_path, file, bucket)
+          puts "...done" if $_VERBOSE
+        end
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/sftp.rb

@@ -0,0 +1,88 @@
+module Astrails
+  module Safe
+    class Sftp < Sink
+
+      protected
+
+      def active?
+        host && user
+      end
+
+      def path
+        @path ||= expand(config[:sftp, :path] || config[:local, :path] || ":kind/:id")
+      end
+
+      def save
+        raise RuntimeError, "pipe-streaming not supported for SFTP." unless @backup.path
+
+        puts "Uploading #{host}:#{full_path} via SFTP" if $_VERBOSE || $DRY_RUN
+
+        unless $DRY_RUN || $LOCAL
+          opts = {}
+          opts[:password] = password if password
+          opts[:port] = port if port
+          Net::SFTP.start(host, user, opts) do |sftp|
+            puts "Sending #{@backup.path} to #{full_path}" if $_VERBOSE
+            begin
+              sftp.upload! @backup.path, full_path
+            rescue Net::SFTP::StatusException
+              puts "Ensuring remote path (#{path}) exists" if $_VERBOSE
+              # mkdir -p
+              folders = path.split('/')
+              folders.each_index do |i|
+                folder = folders[0..i].join('/')
+                puts "Creating #{folder} on remote" if $_VERBOSE
+                sftp.mkdir!(folder) rescue Net::SFTP::StatusException
+              end
+              retry
+            end
+          end
+          puts "...done" if $_VERBOSE
+        end
+      end
+
+      def cleanup
+        return if $LOCAL || $DRY_RUN
+
+        return unless keep = @config[:keep, :sftp]
+
+        puts "listing files: #{host}:#{base}*" if $_VERBOSE
+        opts = {}
+        opts[:password] = password if password
+        opts[:port] = port if port
+        Net::SFTP.start(host, user, opts) do |sftp|
+          files = sftp.dir.glob(path, File.basename("#{base}*"))
+
+          puts files.collect {|x| x.name } if $_VERBOSE
+
+          files = files.
+            collect {|x| x.name }.
+            sort
+
+          cleanup_with_limit(files, keep) do |f|
+            file = File.join(path, f)
+            puts "removing sftp file #{host}:#{file}" if $DRY_RUN || $_VERBOSE
+            sftp.remove!(file) unless $DRY_RUN || $LOCAL
+          end
+        end
+      end
+
+      def host
+        @config[:sftp, :host]
+      end
+
+      def user
+        @config[:sftp, :user]
+      end
+
+      def password
+        @config[:sftp, :password]
+      end
+
+      def port
+        @config[:sftp, :port]
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/sink.rb

@@ -0,0 +1,35 @@
+module Astrails
+  module Safe
+    class Sink < Stream
+
+      def process
+        return unless active?
+
+        save
+        cleanup
+      end
+
+      protected
+
+      # path is defined in subclass
+      # base is used in 'cleanup' to find all files that begin with base. the '.'
+      # at the end is essential to distinguish b/w foo.* and foobar.* archives for example
+      def base
+        @base ||= File.join(path, File.basename(@backup.filename).split(".").first + '.')
+      end
+
+      def full_path(filename = @backup.filename, extension = @backup.extension)
+        File.join(path, filename) + extension
+      end
+
+      # call block on files to be removed (all except for the LAST 'limit' files
+      def cleanup_with_limit(files, limit, &block)
+        return unless files.size > limit
+
+        to_remove = files[0..(files.size - limit - 1)]
+        # TODO: validate here
+        to_remove.each(&block)
+      end
+    end
+  end
+end

data/lib/astrails/safe/source.rb

@@ -0,0 +1,47 @@
+module Astrails
+  module Safe
+    class Source < Stream
+
+      attr_accessor :id
+      def initialize(id, config)
+        @id, @config = id.to_s, config
+      end
+
+      def timestamp
+        Time.now.strftime("%y%m%d-%H%M")
+      end
+
+      def kind
+        self.class.human_name
+      end
+
+      def filename
+        @filename ||= expand(":kind-:id.:timestamp")
+      end
+
+      def backup
+        return @backup if @backup
+        @backup = Backup.new(
+          :id        => @id,
+          :kind      => kind,
+          :extension => extension,
+          :command   => command,
+          :timestamp => timestamp
+        )
+        # can't do this in the initializer hash above since
+        # filename() calls expand() which requires @backup
+        # FIXME: move expansion to the backup (last step in ctor) assign :tags here
+        @backup.filename = filename
+        @backup
+      end
+
+      protected
+
+      def self.human_name
+        name.split('::').last.downcase
+      end
+
+    end
+  end
+end
+

data/lib/astrails/safe/stream.rb

@@ -0,0 +1,20 @@
+module Astrails
+  module Safe
+    class Stream
+
+      attr_accessor :config, :backup
+      def initialize(config, backup)
+        @config, @backup = config, backup
+      end
+
+      # FIXME: move to Backup
+      def expand(path)
+        path .
+        gsub(/:kind\b/, @backup.kind.to_s) .
+        gsub(/:id\b/, @backup.id.to_s) .
+        gsub(/:timestamp\b/, @backup.timestamp)
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/svndump.rb

@@ -0,0 +1,13 @@
+module Astrails
+  module Safe
+    class Svndump < Source
+
+      def command
+        "svnadmin dump #{@config[:options]} #{@config[:repo_path]}"
+      end
+
+      def extension; '.svn'; end
+
+    end
+  end
+end

data/lib/astrails/safe/tmp_file.rb

@@ -0,0 +1,48 @@
+require 'tmpdir'
+module Astrails
+  module Safe
+    module TmpFile
+      @keep_files = []
+
+      def self.tmproot
+        @tmproot ||= Dir.mktmpdir
+      end
+
+      def self.cleanup
+        begin
+          FileUtils.remove_entry_secure tmproot
+        rescue ArgumentError => e
+          if e.message =~ /parent directory is world writable/
+            puts <<-ERR
+
+
+********************************************************************************
+It looks like you have wrong permissions on your TEMP directory.  The usual
+case is when you have world writable TEMP directory withOUT the sticky bit.
+
+Try "chmod +t" on it.
+
+********************************************************************************
+
+ERR
+          else
+            raise
+          end
+        end
+        @tmproot = nil
+      end
+
+      def self.create(name)
+        # create temp directory
+
+        file = Tempfile.new(name, tmproot)
+
+        yield file
+
+        file.close
+        @keep_files << file # so that it will not get gcollected and removed from filesystem until the end
+        file.path
+      end
+    end
+  end
+end

data/lib/extensions/mktmpdir.rb

@@ -0,0 +1,45 @@
+require 'tmpdir'
+
+unless Dir.respond_to?(:mktmpdir)
+  # backward compat for 1.8.6
+  class Dir
+    def Dir.mktmpdir(prefix_suffix=nil, tmpdir=nil)
+      case prefix_suffix
+      when nil
+        prefix = "d"
+        suffix = ""
+      when String
+        prefix = prefix_suffix
+        suffix = ""
+      when Array
+        prefix = prefix_suffix[0]
+        suffix = prefix_suffix[1]
+      else
+        raise ArgumentError, "unexpected prefix_suffix: #{prefix_suffix.inspect}"
+      end
+      tmpdir ||= Dir.tmpdir
+      t = Time.now.strftime("%Y%m%d")
+      n = nil
+      begin
+        path = "#{tmpdir}/#{prefix}#{t}-#{$$}-#{rand(0x100000000).to_s(36)}"
+        path << "-#{n}" if n
+        path << suffix
+        Dir.mkdir(path, 0700)
+      rescue Errno::EEXIST
+        n ||= 0
+        n += 1
+        retry
+      end
+
+      if block_given?
+        begin
+          yield path
+        ensure
+          FileUtils.remove_entry_secure path
+        end
+      else
+        path
+      end
+    end
+  end
+end

data/spec/integration/archive_integration_spec.rb

@@ -0,0 +1,88 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+require "fileutils"
+include FileUtils
+
+describe "tar backup" do
+  before(:all) do
+    # need both local and instance vars
+    # instance variables are used in tests
+    # local variables are used in the backup definition (instance vars can't be seen)
+    @root = root = "tmp/archive_backup_example"
+
+    # clean state
+    rm_rf @root
+    mkdir_p @root
+
+    # create source tree
+    @src = src = "#{@root}/src"
+    mkdir_p "#{@src}/q/w/e"
+    mkdir_p "#{@src}/a/s/d"
+
+    File.open("#{@src}/qwe1", "w") {|f| f.write("qwe") }
+    File.open("#{@src}/q/qwe2", "w") {|f| f.write("qwe"*2) }
+    File.open("#{@src}/q/w/qwe3", "w") {|f| f.write("qwe"*3) }
+    File.open("#{@src}/q/w/e/qwe4", "w") {|f| f.write("qwe"*4) }
+
+    File.open("#{@src}/asd1", "w") {|f| f.write("asd") }
+    File.open("#{@src}/a/asd2", "w") {|f| f.write("asd" * 2) }
+    File.open("#{@src}/a/s/asd3", "w") {|f| f.write("asd" * 3) }
+
+    @dst = dst = "#{@root}/backup"
+    mkdir_p @dst
+
+    @now = Time.now
+    @timestamp = @now.strftime("%y%m%d-%H%M")
+
+    stub(Time).now {@now} # Freeze
+
+    Astrails::Safe.safe do
+      local :path => "#{dst}/:kind"
+      tar do
+        archive :test1 do
+          files src
+          exclude "#{src}/q/w"
+          exclude "#{src}/q/w/e"
+        end
+      end
+    end
+
+    @backup = "#{dst}/archive/archive-test1.#{@timestamp}.tar.gz"
+  end
+
+  it "should create backup file" do
+    File.exists?(@backup).should be_true
+  end
+
+  describe "after extracting" do
+    before(:all) do
+      # prepare target dir
+      @target = "#{@root}/test"
+      mkdir_p @target
+      system "tar -zxvf #{@backup} -C #{@target}"
+
+      @test = "#{@target}/#{@root}/src"
+      puts @test
+    end
+
+    it "should include asd1/2/3" do
+      File.exists?("#{@test}/asd1").should be_true
+      File.exists?("#{@test}/a/asd2").should be_true
+      File.exists?("#{@test}/a/s/asd3").should be_true
+    end
+
+    it "should only include qwe 1 and 2 (no 3)" do
+      File.exists?("#{@test}/qwe1").should be_true
+      File.exists?("#{@test}/q/qwe2").should be_true
+      File.exists?("#{@test}/q/w/qwe3").should be_false
+      File.exists?("#{@test}/q/w/e/qwe4").should be_false
+    end
+
+    it "should preserve file content" do
+      File.read("#{@test}/qwe1").should == "qwe"
+      File.read("#{@test}/q/qwe2").should == "qweqwe"
+      File.read("#{@test}/a/s/asd3").should == "asdasdasd"
+    end
+  end
+
+end