astrails-safe 0.2.7 → 0.3.0

data/astrails-safe.gemspec

@@ -0,0 +1,35 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'astrails/safe/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "astrails-safe"
+  spec.version       = Astrails::Safe::VERSION
+  spec.authors       = ["Vitaly Kushner"]
+  spec.email         = ["we@astrails.com"]
+  spec.description = <<-DESC
+Astrails-Safe is a simple tool to backup databases (MySQL and PostgreSQL), Subversion repositories (with svndump) and just files.
+Backups can be stored locally or remotely and can be enctypted.
+Remote storage is supported on Amazon S3, Rackspace Cloud Files, or just plain SFTP.
+DESC
+  spec.summary       = %Q{Backup filesystem and databases (MySQL and PostgreSQL) locally or to a remote server/service (with encryption)}
+  spec.homepage      = "http://astrails.com/astrails-safe"
+  spec.license       = "MIT"
+
+  spec.default_executable = %q{astrails-safe}
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "aws-s3"
+  spec.add_dependency "cloudfiles"
+  spec.add_dependency "net-sftp"
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rr", "~> 1.0.4"
+end

data/lib/astrails/safe.rb

@@ -1,3 +1,5 @@
+require "astrails/safe/version"
+
 require "aws/s3"
 require "cloudfiles"
 require 'net/sftp'
@@ -16,8 +18,6 @@ require 'astrails/safe/stream'
 
 require 'astrails/safe/backup'
 
-require 'astrails/safe/backup'
-
 require 'astrails/safe/source'
 require 'astrails/safe/mysqldump'
 require 'astrails/safe/pgdump'
@@ -49,8 +49,8 @@ module Astrails
        [Svndump,   [:svndump,   :repos]]
       ].each do |klass, path|
         if collection = config[*path]
-          collection.each do |name, config|
-            klass.new(name, config).backup.run(config, :gpg, :gzip, :local, :s3, :cloudfiles, :sftp)
+          collection.each do |name, c|
+            klass.new(name, c).backup.run(c, :gpg, :gzip, :local, :s3, :cloudfiles, :sftp)
           end
         end
       end

data/lib/astrails/safe/archive.rb

@@ -11,12 +11,12 @@ module Astrails
       protected
 
       def tar_exclude_files
-        [*@config[:exclude]].compact.map{|x| "--exclude=#{x}"} * " "
+        [*@config[:exclude]].compact.map{|x| "--exclude=#{x}"}.join(" ")
       end
 
       def tar_files
         raise RuntimeError, "missing files for tar" unless @config[:files]
-        [*@config[:files]].map {|s| s.strip} * " "
+        [*@config[:files]].map{|s| s.strip}.join(" ")
       end
 
     end

data/lib/astrails/safe/cloudfiles.rb

@@ -1,6 +1,7 @@
 module Astrails
   module Safe
     class Cloudfiles < Sink
+      MAX_CLOUDFILES_FILE_SIZE = 5368709120
 
       protected
 
@@ -12,6 +13,13 @@ module Astrails
         @path ||= expand(config[:cloudfiles, :path] || config[:local, :path] || ":kind/:id")
       end
 
+      # UGLY: we need this function for the reason that
+      # we can't double mock on ruby 1.9.2, duh!
+      # so we created this func to mock it all together
+      def get_file_size(path)
+        File.stat(path).size
+      end
+
       def save
         raise RuntimeError, "pipe-streaming not supported for S3." unless @backup.path
 
@@ -19,10 +27,14 @@ module Astrails
         cf = CloudFiles::Connection.new(user, api_key, true, service_net) unless $LOCAL
         puts "Uploading #{container}:#{full_path} from #{@backup.path}" if $_VERBOSE || $DRY_RUN
         unless $DRY_RUN || $LOCAL
+          if get_file_size(@backup.path) > MAX_CLOUDFILES_FILE_SIZE
+            STDERR.puts "ERROR: File size exceeds maximum allowed for upload to Cloud Files (#{MAX_CLOUDFILES_FILE_SIZE}): #{@backup.path}"
+            return
+          end
           benchmark = Benchmark.realtime do
             cf_container = cf.create_container(container)
             o = cf_container.create_object(full_path,true)
-            o.write(open(@backup.path))
+            o.write(File.open(@backup.path))
           end
           puts "...done" if $_VERBOSE
           puts("Upload took " + sprintf("%.2f", benchmark) + " second(s).") if $_VERBOSE
@@ -36,11 +48,12 @@ module Astrails
 
         puts "listing files: #{container}:#{base}*" if $_VERBOSE
         cf = CloudFiles::Connection.new(user, api_key, true, service_net) unless $LOCAL
-        files = cf.container(container).objects(:prefix => base)
+        cf_container = cf.container(container)
+        files = cf_container.objects(:prefix => base).sort
 
         cleanup_with_limit(files, keep) do |f|
           puts "removing Cloud File #{container}:#{f}" if $DRY_RUN || $_VERBOSE
-          cf.container(container).delete_object(f) unless $DRY_RUN || $LOCAL
+          cf_container.delete_object(f) unless $DRY_RUN || $LOCAL
         end
       end
 

data/lib/astrails/safe/config/node.rb

@@ -38,7 +38,7 @@ module Astrails
           else
             raise(ArgumentError, "#{key}: no block supported for simple values") if block
             if @data[key.to_s]
-              @data[key.to_s] = @data[key.to_s].to_a + value.to_a
+              @data[key.to_s] = [*@data[key.to_s]] + [value]
             else
               @data[key.to_s] = value
             end

data/lib/astrails/safe/local.rb

@@ -17,6 +17,7 @@ module Astrails
       def save
         puts "command: #{@backup.command}" if $_VERBOSE
 
+        # FIXME: probably need to change this to smth like @backup.finalize!
         @backup.path = full_path # need to do it outside DRY_RUN so that it will be avialable for S3 DRY_RUN
 
         unless $DRY_RUN
@@ -34,6 +35,8 @@ module Astrails
 
         puts "listing files #{base}" if $_VERBOSE
 
+        # TODO: cleanup ALL zero-length files
+
         files = Dir["#{base}*"] .
           select{|f| File.file?(f) && File.size(f) > 0} .
           sort

data/lib/astrails/safe/mysqldump.rb

@@ -23,7 +23,7 @@ module Astrails
 
       def mysql_skip_tables
         if skip_tables = @config[:skip_tables]
-          [*skip_tables].map { |t| "--ignore-table=#{@id}.#{t}" } * " "
+          [*skip_tables].map{ |t| "--ignore-table=#{@id}.#{t}" }.join(" ")
         end
       end
 

data/lib/astrails/safe/pipe.rb

@@ -1,6 +1,10 @@
 module Astrails
   module Safe
     class Pipe < Stream
+      # process adds required commands to the current
+      # shell command string
+      # :active?, :pipe, :extension and :post_process are
+      # defined in inheriting pipe classes
       def process
         return unless active?
 

data/lib/astrails/safe/s3.rb

@@ -14,6 +14,7 @@ module Astrails
       end
 
       def save
+        # FIXME: user friendly error here :)
         raise RuntimeError, "pipe-streaming not supported for S3." unless @backup.path
 
         # needed in cleanup even on dry run
@@ -26,7 +27,7 @@ module Astrails
             return
           end
           benchmark = Benchmark.realtime do
-            AWS::S3::Bucket.create(bucket)
+            AWS::S3::Bucket.create(bucket) unless bucket_exists?(bucket)
             File.open(@backup.path) do |file|
               AWS::S3::S3Object.store(full_path, file, bucket)
             end
@@ -51,7 +52,7 @@ module Astrails
 
         cleanup_with_limit(files, keep) do |f|
           puts "removing s3 file #{bucket}:#{f}" if $DRY_RUN || $_VERBOSE
-          AWS::S3::Bucket.find(bucket)[f].delete unless $DRY_RUN || $LOCAL
+          AWS::S3::Bucket.objects(bucket, :prefix => f)[0].delete unless $DRY_RUN || $LOCAL
         end
       end
 
@@ -67,6 +68,13 @@ module Astrails
         @config[:s3, :secret]
       end
 
+      private
+      
+      def bucket_exists?(bucket)
+        true if AWS::S3::Bucket.find(bucket)
+      rescue AWS::S3::NoSuchBucket
+        false
+      end
     end
   end
 end

data/lib/astrails/safe/sink.rb

@@ -12,6 +12,8 @@ module Astrails
       protected
 
       # path is defined in subclass
+      # base is used in 'cleanup' to find all files that begin with base. the '.'
+      # at the end is essential to distinguish b/w foo.* and foobar.* archives for example
       def base
         @base ||= File.join(path, File.basename(@backup.filename).split(".").first + '.')
       end

data/lib/astrails/safe/source.rb

@@ -30,6 +30,7 @@ module Astrails
         )
         # can't do this in the initializer hash above since
         # filename() calls expand() which requires @backup
+        # FIXME: move expansion to the backup (last step in ctor) assign :tags here
         @backup.filename = filename
         @backup
       end

data/lib/astrails/safe/stream.rb

@@ -7,6 +7,7 @@ module Astrails
         @config, @backup = config, backup
       end
 
+      # FIXME: move to Backup
       def expand(path)
         path .
         gsub(/:kind\b/, @backup.kind.to_s) .

data/lib/astrails/safe/version.rb

@@ -0,0 +1,5 @@
+module Astrails
+  module Safe
+    VERSION = "0.3.0"
+  end
+end

data/{examples/integration/archive_integration_example.rb → spec/integration/archive_integration_spec.rb}

@@ -1,4 +1,4 @@
-require File.expand_path(File.dirname(__FILE__) + '/../example_helper')
+require 'spec_helper'
 
 require "fileutils"
 include FileUtils
@@ -51,7 +51,6 @@ describe "tar backup" do
   end
 
   it "should create backup file" do
-    puts "Expecting: #{@backup}"
     File.exists?(@backup).should be_true
   end
 

data/{examples/integration/cleanup_example.rb → spec/integration/cleanup_spec.rb}

@@ -1,4 +1,4 @@
-require File.expand_path(File.dirname(__FILE__) + '/../example_helper')
+require 'spec_helper'
 
 require "fileutils"
 include FileUtils
@@ -47,7 +47,6 @@ describe "tar backup" do
   end
 
   it "should create backup file" do
-    puts "Expecting: #{@backup}"
     File.exists?(@backup).should be_true
   end
 
@@ -59,4 +58,4 @@ describe "tar backup" do
     Dir["#{@dst}/archive/archive-foobar.*"].should == ["#{@dst}/archive/archive-foobar.000001.tar.gz", "#{@dst}/archive/archive-foobar.000002.tar.gz"]
   end
 
-end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,7 @@
+require 'rubygems'
+require 'bundler/setup'
+require 'astrails/safe'
+
+RSpec.configure do |config|
+  config.mock_with :rr
+end

data/{examples/unit/archive_example.rb → spec/unit/archive_spec.rb}

@@ -1,4 +1,4 @@
-require File.expand_path(File.dirname(__FILE__) + '/../example_helper')
+require 'spec_helper'
 
 describe Astrails::Safe::Archive do
 

data/spec/unit/cloudfiles_spec.rb

@@ -0,0 +1,177 @@
+require 'spec_helper'
+
+describe Astrails::Safe::Cloudfiles do
+
+  def def_config
+    {
+      :cloudfiles => {
+        :container => "_container",
+        :user    => "_user",
+        :api_key => "_api_key",
+      },
+      :keep => {
+        :cloudfiles => 2
+      }
+    }
+  end
+
+  def def_backup(extra = {})
+    {
+      :kind      => "_kind",
+      :filename  => "/backup/somewhere/_kind-_id.NOW.bar",
+      :extension => ".bar",
+      :id        => "_id",
+      :timestamp => "NOW"
+    }.merge(extra)
+  end
+
+  def cloudfiles(config = def_config, backup = def_backup)
+    Astrails::Safe::Cloudfiles.new(
+      Astrails::Safe::Config::Node.new(nil, config),
+      Astrails::Safe::Backup.new(backup)
+    )
+  end
+
+  describe :cleanup do
+
+    before(:each) do
+      @cloudfiles = cloudfiles
+
+      @files = [4,1,3,2].map { |i| "aaaaa#{i}" }
+
+      @container = "container"
+
+      stub(@container).objects(:prefix => "_kind/_id/_kind-_id.") { @files }
+      stub(@container).delete_object(anything)
+
+      stub(CloudFiles::Connection).
+        new('_user', '_api_key', true, false).stub!.
+        container('_container') {@container}
+    end
+
+    it "should check [:keep, :cloudfiles]" do
+      @cloudfiles.config[:keep].data["cloudfiles"] = nil
+      dont_allow(@cloudfiles.backup).filename
+      @cloudfiles.send :cleanup
+    end
+
+    it "should delete extra files" do
+      mock(@container).delete_object('aaaaa1')
+      mock(@container).delete_object('aaaaa2')
+      @cloudfiles.send :cleanup
+    end
+
+  end
+
+  describe :active do
+    before(:each) do
+      @cloudfiles = cloudfiles
+    end
+
+    it "should be true when all params are set" do
+      @cloudfiles.should be_active
+    end
+
+    it "should be false if container is missing" do
+      @cloudfiles.config[:cloudfiles].data["container"] = nil
+      @cloudfiles.should_not be_active
+    end
+
+    it "should be false if user is missing" do
+      @cloudfiles.config[:cloudfiles].data["user"] = nil
+      @cloudfiles.should_not be_active
+    end
+
+    it "should be false if api_key is missing" do
+      @cloudfiles.config[:cloudfiles].data["api_key"] = nil
+      @cloudfiles.should_not be_active
+    end
+  end
+
+  describe :path do
+    before(:each) do
+      @cloudfiles = cloudfiles
+    end
+    it "should use cloudfiles/path 1st" do
+      @cloudfiles.config[:cloudfiles].data["path"] = "cloudfiles_path"
+      @cloudfiles.config[:local] = {:path => "local_path"}
+      @cloudfiles.send(:path).should == "cloudfiles_path"
+    end
+
+    it "should use local/path 2nd" do
+      @cloudfiles.config[:local] = {:path => "local_path"}
+      @cloudfiles.send(:path).should == "local_path"
+    end
+
+    it "should use constant 3rd" do
+      @cloudfiles.send(:path).should == "_kind/_id"
+    end
+
+  end
+
+  describe :save do
+    def add_stubs(*stubs)
+      stubs.each do |s|
+        case s
+        when :connection
+          @connection = "connection"
+          stub(CloudFiles::Authentication).new
+          stub(CloudFiles::Connection).
+            new('_user', '_api_key', true, false) {@connection}
+        when :file_size
+          stub(@cloudfiles).get_file_size("foo") {123}
+        when :create_container
+          @container = "container"
+          stub(@container).create_object("_kind/_id/backup/somewhere/_kind-_id.NOW.bar.bar", true) {@object}
+          stub(@connection).create_container {@container}
+        when :file_open
+          stub(File).open("foo")
+        when :cloudfiles_store
+          @object = "object"
+          stub(@object).write(nil) {true}
+        end
+      end
+    end
+
+    before(:each) do
+      @cloudfiles = cloudfiles(def_config, def_backup(:path => "foo"))
+      @full_path = "_kind/_id/backup/somewhere/_kind-_id.NOW.bar.bar"
+    end
+
+    it "should fail if no backup.file is set" do
+      @cloudfiles.backup.path = nil
+      proc {@cloudfiles.send(:save)}.should raise_error(RuntimeError)
+    end
+
+    it "should establish Cloud Files connection" do
+      add_stubs(:connection, :file_size, :create_container, :file_open, :cloudfiles_store)
+      @cloudfiles.send(:save)
+    end
+
+    it "should open local file" do
+      add_stubs(:connection, :file_size, :create_container, :cloudfiles_store)
+      mock(File).open("foo")
+      @cloudfiles.send(:save)
+    end
+
+    it "should call write on the cloudfile object with files' descriptor" do
+      add_stubs(:connection, :file_size, :create_container, :cloudfiles_store)
+      stub(File).open("foo") {"qqq"}
+      mock(@object).write("qqq") {true}
+      @cloudfiles.send(:save)
+    end
+
+    it "should upload file" do
+      add_stubs(:connection, :file_size, :create_container, :file_open, :cloudfiles_store)
+      @cloudfiles.send(:save)
+    end
+
+    it "should fail on files bigger then 5G" do
+      add_stubs(:connection)
+      mock(File).stat("foo").stub!.size {5*1024*1024*1024+1}
+      mock(STDERR).puts(anything)
+      dont_allow(Benchmark).realtime
+      @cloudfiles.send(:save)
+    end
+  end
+end