webtranslateit-safe 0.4.0

data/lib/webtranslateit/safe/stream.rb

@@ -0,0 +1,40 @@
+module WebTranslateIt
+
+  module Safe
+
+    class Stream
+
+      attr_accessor :config, :backup
+
+      def initialize(config, backup)
+        @config = config
+        @backup = backup
+      end
+
+      # FIXME: move to Backup
+      def expand(path)
+        path
+          .gsub(/:kind\b/, @backup.kind.to_s)
+          .gsub(/:id\b/, @backup.id.to_s)
+          .gsub(/:timestamp\b/, @backup.timestamp)
+      end
+
+      private
+
+      def verbose?
+        config[:verbose]
+      end
+
+      def local_only?
+        config[:local_only]
+      end
+
+      def dry_run?
+        config[:dry_run]
+      end
+
+    end
+
+  end
+
+end

data/lib/webtranslateit/safe/svndump.rb

@@ -0,0 +1,17 @@
+module WebTranslateIt
+
+  module Safe
+
+    class Svndump < Source
+
+      def command
+        "svnadmin dump #{config[:options]} #{config[:repo_path]}"
+      end
+
+      def extension = '.svn'
+
+    end
+
+  end
+
+end

data/lib/webtranslateit/safe/tmp_file.rb

@@ -0,0 +1,53 @@
+require 'tmpdir'
+module WebTranslateIt
+
+  module Safe
+
+    module TmpFile
+
+      @keep_files = []
+
+      def self.tmproot
+        @tmproot ||= Dir.mktmpdir
+      end
+
+      def self.cleanup
+        begin
+          FileUtils.remove_entry_secure tmproot
+        rescue ArgumentError => e
+          raise unless e.message =~ /parent directory is world writable/
+
+          puts <<~ERR
+
+
+            ********************************************************************************
+            It looks like you have wrong permissions on your TEMP directory.  The usual
+            case is when you have world writable TEMP directory withOUT the sticky bit.
+
+            Try "chmod +t" on it.
+
+            ********************************************************************************
+
+          ERR
+          raise
+        end
+        @tmproot = nil
+      end
+
+      def self.create(name)
+        # create temp directory
+
+        file = Tempfile.new(name, tmproot)
+
+        yield file
+
+        file.close
+        @keep_files << file # so that it will not get gcollected and removed from filesystem until the end
+        file.path
+      end
+
+    end
+
+  end
+
+end

data/lib/webtranslateit/safe/version.rb

@@ -0,0 +1,9 @@
+module WebTranslateIt
+
+  module Safe
+
+    VERSION = '0.4.0'
+
+  end
+
+end

data/lib/webtranslateit/safe.rb

@@ -0,0 +1,70 @@
+require 'webtranslateit/safe/version'
+
+require 'aws/s3'
+require 'cloudfiles'
+require 'net/sftp'
+# require 'net/ftp'
+require 'fileutils'
+require 'benchmark'
+
+require 'tempfile'
+require 'extensions/mktmpdir'
+
+require 'webtranslateit/safe/tmp_file'
+
+require 'webtranslateit/safe/config/node'
+require 'webtranslateit/safe/config/builder'
+
+require 'webtranslateit/safe/stream'
+
+require 'webtranslateit/safe/backup'
+
+require 'webtranslateit/safe/source'
+require 'webtranslateit/safe/mysqldump'
+require 'webtranslateit/safe/pgdump'
+require 'webtranslateit/safe/archive'
+require 'webtranslateit/safe/svndump'
+require 'webtranslateit/safe/mongodump'
+
+require 'webtranslateit/safe/pipe'
+require 'webtranslateit/safe/gpg'
+require 'webtranslateit/safe/gzip'
+
+require 'webtranslateit/safe/sink'
+require 'webtranslateit/safe/local'
+require 'webtranslateit/safe/s3'
+require 'webtranslateit/safe/cloudfiles'
+require 'webtranslateit/safe/sftp'
+require 'webtranslateit/safe/ftp'
+
+module WebTranslateIt
+
+  module Safe
+
+    ROOT = File.join(File.dirname(__FILE__), '..', '..')
+
+    def safe(&)
+      Config::Node.new(&)
+    end
+
+    def process(config)
+      [[Mysqldump, [:mysqldump, :databases]],
+       [Pgdump,    [:pgdump,    :databases]],
+       [Mongodump, [:mongodump, :databases]],
+       [Archive,   [:tar,       :archives]],
+       [Svndump,   [:svndump,   :repos]]].each do |klass, path|
+        next unless (collection = config[*path])
+
+        collection.each do |name, c|
+          klass.new(name, c).backup.run(c, :gpg, :gzip, :local, :s3, :cloudfiles, :sftp, :ftp)
+        end
+      end
+
+      WebTranslateIt::Safe::TmpFile.cleanup
+    end
+    module_function :safe
+    module_function :process
+
+  end
+
+end

data/spec/integration/archive_integration_spec.rb

@@ -0,0 +1,89 @@
+require 'spec_helper'
+
+require 'fileutils'
+include FileUtils
+
+describe 'tar backup' do
+  before(:all) do
+    # need both local and instance vars
+    # instance variables are used in tests
+    # local variables are used in the backup definition (instance vars can't be seen)
+    @root = root = 'tmp/archive_backup_example'
+
+    # clean state
+    rm_rf @root
+    mkdir_p @root
+
+    # create source tree
+    @src = src = "#{@root}/src"
+    mkdir_p "#{@src}/q/w/e"
+    mkdir_p "#{@src}/a/s/d"
+
+    File.open("#{@src}/qwe1", 'w') {|f| f.write('qwe') }
+    File.open("#{@src}/q/qwe2", 'w') {|f| f.write('qwe'*2) }
+    File.open("#{@src}/q/w/qwe3", 'w') {|f| f.write('qwe'*3) }
+    File.open("#{@src}/q/w/e/qwe4", 'w') {|f| f.write('qwe'*4) }
+
+    File.open("#{@src}/asd1", 'w') {|f| f.write('asd') }
+    File.open("#{@src}/a/asd2", 'w') {|f| f.write('asd' * 2) }
+    File.open("#{@src}/a/s/asd3", 'w') {|f| f.write('asd' * 3) }
+
+    @dst = dst = "#{@root}/backup"
+    mkdir_p @dst
+
+    @now = Time.now
+    @timestamp = @now.strftime('%y%m%d-%H%M')
+
+    stub(Time).now {@now} # Freeze
+
+    config = WebTranslateIt::Safe.safe do
+      local :path => "#{dst}/:kind"
+      tar do
+        archive :test1 do
+          files src
+          exclude "#{src}/q/w"
+          exclude "#{src}/q/w/e"
+        end
+      end
+    end
+    WebTranslateIt::Safe.process config
+
+    @backup = "#{dst}/archive/archive-test1.#{@timestamp}.tar.gz"
+  end
+
+  it 'should create backup file' do
+    File.exist?(@backup).should be true
+  end
+
+  describe 'after extracting' do
+    before(:all) do
+      # prepare target dir
+      @target = "#{@root}/test"
+      mkdir_p @target
+      system "tar -zxvf #{@backup} -C #{@target}"
+
+      @test = "#{@target}/#{@root}/src"
+      puts @test
+    end
+
+    it 'should include asd1/2/3' do
+      File.exist?("#{@test}/asd1").should be true
+      File.exist?("#{@test}/a/asd2").should be true
+      File.exist?("#{@test}/a/s/asd3").should be true
+    end
+
+    it 'should only include qwe 1 and 2 (no 3)' do
+      File.exist?("#{@test}/qwe1").should be true
+      File.exist?("#{@test}/q/qwe2").should be true
+      File.exist?("#{@test}/q/w/qwe3").should be false
+      File.exist?("#{@test}/q/w/e/qwe4").should be false
+    end
+
+    it 'should preserve file content' do
+      File.read("#{@test}/qwe1").should == 'qwe'
+      File.read("#{@test}/q/qwe2").should == 'qweqwe'
+      File.read("#{@test}/a/s/asd3").should == 'asdasdasd'
+    end
+  end
+
+end

data/spec/integration/cleanup_spec.rb

@@ -0,0 +1,62 @@
+require 'spec_helper'
+
+require 'fileutils'
+include FileUtils
+
+describe 'tar backup' do
+  before(:all) do
+    # need both local and instance vars
+    # instance variables are used in tests
+    # local variables are used in the backup definition (instance vars can't be seen)
+    @root = root = 'tmp/cleanup_example'
+
+    # clean state
+    rm_rf @root
+    mkdir_p @root
+
+    # create source tree
+    @src = src = "#{@root}/src"
+    mkdir_p src
+
+    File.open(qwe = "#{@src}/qwe", 'w') {|f| f.write('qwe') }
+
+    @dst = dst = "#{@root}/backup"
+    mkdir_p "#{@dst}/archive"
+
+    @now = Time.now
+    @timestamp = @now.strftime('%y%m%d-%H%M')
+
+    stub(Time).now {@now} # Freeze
+
+    cp qwe, "#{dst}/archive/archive-foo.000001.tar.gz"
+    cp qwe, "#{dst}/archive/archive-foo.000002.tar.gz"
+    cp qwe, "#{dst}/archive/archive-foobar.000001.tar.gz"
+    cp qwe, "#{dst}/archive/archive-foobar.000002.tar.gz"
+
+    config = WebTranslateIt::Safe.safe do
+      local :path => "#{dst}/:kind"
+      tar do
+        keep :local => 1 # only leave the latest
+        archive :foo do
+          files src
+        end
+      end
+    end
+    WebTranslateIt::Safe.process config
+
+    @backup = "#{dst}/archive/archive-foo.#{@timestamp}.tar.gz"
+  end
+
+  it 'should create backup file' do
+    File.exist?(@backup).should be true
+  end
+
+  it 'should remove old backups' do
+    Dir["#{@dst}/archive/archive-foo.*"].should == [@backup]
+  end
+
+  it 'should NOT remove backups with base having same prefix' do
+    Dir["#{@dst}/archive/archive-foobar.*"].sort.should == ["#{@dst}/archive/archive-foobar.000001.tar.gz", "#{@dst}/archive/archive-foobar.000002.tar.gz"]
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,7 @@
+require 'rubygems'
+require 'bundler/setup'
+require 'webtranslateit/safe'
+
+RSpec.configure do |config|
+  config.mock_with :rr
+end

data/spec/webtranslateit/safe/archive_spec.rb

@@ -0,0 +1,67 @@
+require 'spec_helper'
+
+describe WebTranslateIt::Safe::Archive do
+
+  def def_config
+    {
+      options: 'OPTS',
+      files: 'apples',
+      exclude: 'oranges'
+    }
+  end
+
+  def archive(id = :foo, config = def_config)
+    WebTranslateIt::Safe::Archive.new(id, WebTranslateIt::Safe::Config::Node.new(nil, config))
+  end
+
+  after(:each) { WebTranslateIt::Safe::TmpFile.cleanup }
+
+  describe :backup do
+    before(:each) do
+      @archive = archive
+      stub(@archive).timestamp { 'NOW' }
+    end
+
+    {
+      id: 'foo',
+      kind: 'archive',
+      extension: '.tar',
+      filename: 'archive-foo.NOW',
+      command: 'tar -cf - OPTS --exclude=oranges apples'
+    }.each do |k, v|
+      it "sets #{k} to #{v}" do
+        @archive.backup.send(k).should == v
+      end
+    end
+  end
+
+  describe :tar_exclude_files do
+    it "returns '' when no excludes" do
+      archive(:foo, {}).send(:tar_exclude_files).should == ''
+    end
+
+    it 'accepts single exclude as string' do
+      archive(:foo, {exclude: 'bar'}).send(:tar_exclude_files).should == '--exclude=bar'
+    end
+
+    it 'accepts multiple exclude as array' do
+      archive(:foo, {exclude: ['foo', 'bar']}).send(:tar_exclude_files).should == '--exclude=foo --exclude=bar'
+    end
+  end
+
+  describe :tar_files do
+    it 'raises RuntimeError when no files' do
+      lambda {
+        archive(:foo, {}).send(:tar_files)
+      }.should raise_error(RuntimeError, 'missing files for tar')
+    end
+
+    it 'accepts single file as string' do
+      archive(:foo, {files: 'foo'}).send(:tar_files).should == 'foo'
+    end
+
+    it 'accepts multiple files as array' do
+      archive(:foo, {files: ['foo', 'bar']}).send(:tar_files).should == 'foo bar'
+    end
+  end
+end

data/spec/webtranslateit/safe/cloudfiles_spec.rb

@@ -0,0 +1,175 @@
+require 'spec_helper'
+
+describe WebTranslateIt::Safe::Cloudfiles do
+
+  def def_config
+    {
+      cloudfiles: {
+        container: '_container',
+        user:      '_user',
+        api_key:   '_api_key',
+      },
+      keep: { cloudfiles: 2 }
+    }
+  end
+
+  def def_backup(extra = {})
+    {
+      kind:      '_kind',
+      filename:  '/backup/somewhere/_kind-_id.NOW.bar',
+      extension: '.bar',
+      id:        '_id',
+      timestamp: 'NOW'
+    }.merge(extra)
+  end
+
+  def cloudfiles(config = def_config, backup = def_backup)
+    WebTranslateIt::Safe::Cloudfiles.new(
+      WebTranslateIt::Safe::Config::Node.new.merge(config),
+      WebTranslateIt::Safe::Backup.new(backup)
+    )
+  end
+
+  describe :cleanup do
+
+    before(:each) do
+      @cloudfiles = cloudfiles
+
+      @files = [4,1,3,2].map { |i| "aaaaa#{i}" }
+
+      @container = 'container'
+
+      stub(@container).objects(prefix: '_kind/_id/_kind-_id.') { @files }
+      stub(@container).delete_object(anything)
+
+      stub(CloudFiles::Connection).
+        new('_user', '_api_key', true, false).stub!.
+        container('_container') {@container}
+    end
+
+    it 'should check [:keep, :cloudfiles]' do
+      @cloudfiles.config[:keep].data['cloudfiles'] = nil
+      dont_allow(@cloudfiles.backup).filename
+      @cloudfiles.send :cleanup
+    end
+
+    it 'should delete extra files' do
+      mock(@container).delete_object('aaaaa1')
+      mock(@container).delete_object('aaaaa2')
+      @cloudfiles.send :cleanup
+    end
+
+  end
+
+  describe :active do
+    before(:each) do
+      @cloudfiles = cloudfiles
+    end
+
+    it 'should be true when all params are set' do
+      expect(@cloudfiles.active?).to be_truthy
+    end
+
+    it 'should be false if container is missing' do
+      @cloudfiles.config[:cloudfiles].data['container'] = nil
+      expect(@cloudfiles.active?).to be_falsy
+    end
+
+    it 'should be false if user is missing' do
+      @cloudfiles.config[:cloudfiles].data['user'] = nil
+      expect(@cloudfiles.active?).to be_falsy
+    end
+
+    it 'should be false if api_key is missing' do
+      @cloudfiles.config[:cloudfiles].data['api_key'] = nil
+      expect(@cloudfiles.active?).to be_falsy
+    end
+  end
+
+  describe :path do
+    before(:each) do
+      @cloudfiles = cloudfiles
+    end
+    it 'should use cloudfiles/path 1st' do
+      @cloudfiles.config[:cloudfiles].data['path'] = 'cloudfiles_path'
+      @cloudfiles.config[:local] = {path: 'local_path'}
+      @cloudfiles.send(:path).should == 'cloudfiles_path'
+    end
+
+    it 'should use local/path 2nd' do
+      @cloudfiles.config.merge local: {path: 'local_path'}
+      @cloudfiles.send(:path).should == 'local_path'
+    end
+
+    it 'should use constant 3rd' do
+      @cloudfiles.send(:path).should == '_kind/_id'
+    end
+
+  end
+
+  describe :save do
+    def add_stubs(*stubs)
+      stubs.each do |s|
+        case s
+        when :connection
+          @connection = 'connection'
+          stub(CloudFiles::Authentication).new
+          stub(CloudFiles::Connection).
+            new('_user', '_api_key', true, false) {@connection}
+        when :file_size
+          stub(@cloudfiles).get_file_size('foo') {123}
+        when :create_container
+          @container = 'container'
+          stub(@container).create_object('_kind/_id/backup/somewhere/_kind-_id.NOW.bar.bar', true) {@object}
+          stub(@connection).create_container {@container}
+        when :file_open
+          stub(File).open('foo')
+        when :cloudfiles_store
+          @object = 'object'
+          stub(@object).write(nil) {true}
+        end
+      end
+    end
+
+    before(:each) do
+      @cloudfiles = cloudfiles(def_config, def_backup(path: 'foo'))
+      @full_path = '_kind/_id/backup/somewhere/_kind-_id.NOW.bar.bar'
+    end
+
+    it 'should fail if no backup.file is set' do
+      @cloudfiles.backup.path = nil
+      proc {@cloudfiles.send(:save)}.should raise_error(RuntimeError)
+    end
+
+    it 'should establish Cloud Files connection' do
+      add_stubs(:connection, :file_size, :create_container, :file_open, :cloudfiles_store)
+      @cloudfiles.send(:save)
+    end
+
+    it 'should open local file' do
+      add_stubs(:connection, :file_size, :create_container, :cloudfiles_store)
+      mock(File).open('foo')
+      @cloudfiles.send(:save)
+    end
+
+    it "should call write on the cloudfile object with files' descriptor" do
+      add_stubs(:connection, :file_size, :create_container, :cloudfiles_store)
+      stub(File).open('foo') {'qqq'}
+      mock(@object).write('qqq') {true}
+      @cloudfiles.send(:save)
+    end
+
+    it 'should upload file' do
+      add_stubs(:connection, :file_size, :create_container, :file_open, :cloudfiles_store)
+      @cloudfiles.send(:save)
+    end
+
+    it 'should fail on files bigger then 5G' do
+      add_stubs(:connection)
+      mock(File).stat('foo').stub!.size {5*1024*1024*1024+1}
+      mock(STDERR).puts(anything)
+      dont_allow(Benchmark).realtime
+      @cloudfiles.send(:save)
+    end
+  end
+end