ey_cloud_server 1.4.5 → 1.4.26

data/lib/ey_backup/processors/splitter.rb

@@ -1,39 +1,35 @@
 module EY
   module Backup
-    module Splitter
-      extend Spawner
-      extend Logging
-      extend self
+    class Splitter
+      include Spawner
+      include Logging
 
       CHUNK_SIZE = 1024 * 64
       MAX_FILE_SIZE = (4.5*1024*1024*1024).to_i #4.5GB
 
-      def load(backup)
-        if backup.files.size > 1
-          backup.files = join(backup.files)
-        end
+      def self.dump(file, split_size)
+        split_size ||= MAX_FILE_SIZE
+        new.dump(file, split_size)
+      end
 
-        backup
+      def self.load(files)
+        new.load(files)
       end
 
-      def dump(command, files)
-        if files.size != 1
-          fatal "Can't split multiple files."
+      def load(files)
+        if files.size > 1
+          join(files)
+        else
+          files.first
         end
-
       end
 
-      def dump(backup)
-        output_files = backup.files.map do |file|
-          if File.size(file) >= split_size
-            split(file)
-          else
-            file
-          end
-        end.flatten
-
-        backup.files = output_files
-        backup
+      def dump(file, split_size)
+        if File.size(file) >= split_size
+          split(file, split_size)
+        else
+          [file]
+        end
       end
 
       def join(input_files)
@@ -46,12 +42,10 @@ module EY
                 output << chunk
               end
             end
-
-            FileUtils.rm(input)
           end
         end
 
-        [filename]
+        filename
       end
 
       def sort(input_files)
@@ -62,7 +56,7 @@ module EY
         file[/\.part(\d+)/, 1].to_i
       end
 
-      def split(file)
+      def split(file, split_size)
         total_size, part = 0, 0
         files = []
         File.open(file, 'r') do |i|
@@ -83,14 +77,8 @@ module EY
           end
         end
 
-        FileUtils.rm(file)
-
         files
       end
-
-      def split_size
-        EY::Backup.config.split_size || MAX_FILE_SIZE
-      end
     end
   end
 end

data/lib/ey_backup/spawner.rb

@@ -19,21 +19,15 @@ module EY
       end
 
       def run(command)
-        pid, *_ = Open4.popen4(command)
-        Process::waitpid2(pid)
+        unless runs?(command)
+          raise "Failed to run #{command.inspect}"
+        end
       end
 
-      def popen(command, stdout, stdin = nil, stderr = logger.stderr)
-        ioify(stdout, stdin, stderr) do |o, i, e|
-          Open4.popen4(command) do |pid, stdin, stdout, stderr|
-            while chunk = i.read(CHUNK_SIZE)
-              stdin << chunk
-            end
-
-            stdin.flush
-            stdin.close
-          end
-        end
+      def runs?(command)
+        pid, *_ = Open4.popen4(command)
+        pid, status = Process::waitpid2(pid)
+        status.success?
       end
 
       def ioify(stdout, stdin, stderr, &block)

data/lib/ey_cloud_server.rb

@@ -5,5 +5,5 @@ module EY
   end
 end
 
-require 'ey-flex/version'
+require 'ey_cloud_server/version'
 require 'ey_cloud_server/mysql_start'

data/lib/{ey-flex → ey_cloud_server}/version.rb

@@ -1,3 +1,3 @@
 module EY::CloudServer
-  VERSION = '1.4.5'
+  VERSION = '1.4.26'
 end

data/spec/big-brother_spec.rb

@@ -0,0 +1,12 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe "BigBrother" do
+  it "works" do
+    setup_dna({})
+
+    response = EY::BigBrother.check(tmp_dir + "/dna.json")
+    response.should == "{}"
+
+    teardown_dna({})
+  end
+end

data/spec/bucket_minder_spec.rb

@@ -0,0 +1,113 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe "BucketMinder" do
+  def setup_s3(secret_id, secret_key, region)
+    @raw_s3 = Fog::Storage.new(:provider => 'AWS', :region => region, :aws_access_key_id => secret_id, :aws_secret_access_key => secret_key)
+  end
+
+  def setup_bucket(secret_id, secret_key, bucket_name, region = nil)
+    @bucket_minder = EY::BucketMinder.new(secret_id, secret_key, bucket_name, region)
+  end
+
+  describe "in us-east-1" do
+    before do
+      @config        = spec_config
+
+      @bucket_name   = "ey-backup-#{Digest::SHA1.hexdigest(@config['aws_secret_id'])[0..11]}"
+      @region        = 'us-east-1'
+
+      setup_s3(@config['aws_secret_id'], @config['aws_secret_key'], @region)
+    end
+
+    it "sets up the bucket" do
+      setup_bucket(@config['aws_secret_id'], @config['aws_secret_key'], @bucket_name)
+
+      real_bucket = @raw_s3.directories.get(@bucket_name)
+      real_bucket.key.should == @bucket_name
+    end
+
+    it "uses a default name when the bucket name is not provided" do
+      setup_bucket(@config['aws_secret_id'], @config['aws_secret_key'], nil)
+      @bucket_minder.bucket.key.should == @bucket_name
+    end
+
+    describe "with a working bucket" do
+      before :each do
+        setup_bucket(@config['aws_secret_id'], @config['aws_secret_key'], @bucket_name)
+
+        @test_filename = 'somefile.tst'
+        @file_contents = 'i am file contents'
+        lambda { @bucket_minder.remove_object(@test_filename) }
+      end
+
+      it "can put, get and remove a file's contents to and from bucket" do
+        @bucket_minder.list(@test_filename).should be_empty
+        @bucket_minder.put(@test_filename, @file_contents)
+
+        remote_file = @bucket_minder.list(@test_filename).first
+        remote_file[:name].should == @test_filename
+
+        @bucket_minder.stream(@test_filename) do |chunk, remaining_size, total_size|
+          chunk.should == @file_contents
+        end
+
+        @bucket_minder.remove_object(@test_filename)
+        @bucket_minder.list(@test_filename).should be_empty
+      end
+
+      describe "when deleting backups" do
+        it "does not load the backup files into memory" do
+          @bucket_minder.files.should_not_receive(:get)
+
+          @bucket_minder.put('test_bucket_minder_file_1', 'foo')
+          @bucket_minder.remove_object('test_bucket_minder_file_1')
+
+          @bucket_minder.list('test_bucket_minder_file_1').should be_empty
+        end
+      end
+
+      describe "bucket minder merge operations" do
+        it "returns the number of files in a bucket" do
+          @bucket_minder.put('test_bucket_minder_file_1', 'foo')
+          @bucket_minder.put('test_bucket_minder_file_2', 'foo')
+
+          bucketfiles = @bucket_minder.list('test_bucket_minder_file')
+          bucketfiles.length.should == 2
+          bucketfiles.detect { |file| file[:name] == 'test_bucket_minder_file_1' }.should_not be_empty
+          bucketfiles.detect { |file| file[:name] == 'test_bucket_minder_file_2' }.should_not be_empty
+        end
+
+        it "collects file parts" do
+          @bucket_minder.put('test_bucket_minder_file_1', 'foo')
+          @bucket_minder.put('test_bucket_minder_file_2', 'foo')
+          @bucket_minder.put('test_bucket_minder_multi_file.part1', 'foo')
+          @bucket_minder.put('test_bucket_minder_multi_file.part2', 'foo')
+
+          bucketfiles = @bucket_minder.list('test_bucket_minder')
+          bucketfiles.length.should == 3
+          bucketfiles.detect { |file| file[:name] == 'test_bucket_minder_file_1' }.should_not be_empty
+          bucketfiles.detect { |file| file[:name] == 'test_bucket_minder_file_2' }.should_not be_empty
+
+          multi_part_file = bucketfiles.detect { |file| file[:name] == 'test_bucket_minder_multi_file' }
+          multi_part_file.should_not be_empty
+          multi_part_file[:keys].should == ["test_bucket_minder_multi_file.part1", "test_bucket_minder_multi_file.part2"]
+        end
+      end
+    end
+  end
+
+  describe "outside of us-east-1" do
+    before do
+      @config = spec_config
+      @bucket_name   = "ey-backup-non-us-#{Digest::SHA1.hexdigest(@config['aws_secret_id'])[0..11]}"
+      @region        = 'ap-northeast-1'
+
+      setup_s3(@config['aws_secret_id'], @config['aws_secret_key'], @region)
+    end
+
+    it "creates the bucket in the proper region" do
+      setup_bucket(@config['aws_secret_id'], @config['aws_secret_key'], @bucket_name, @region)
+      @bucket_minder.bucket.location.should == @region
+    end
+  end
+end

data/spec/config-example.yml

@@ -0,0 +1,11 @@
+---
+tmp_dir: 
+data_json: 
+mysql_user: root
+mysql_password:
+mysql_host: localhost
+postgresql_user: root
+postgresql_password:
+postgresql_host: localhost
+aws_secret_id: 'abcd123.id'
+aws_secret_key: 'abcd123.key'

data/spec/ey_api_spec.rb

@@ -0,0 +1,63 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe "EyApi" do
+  describe "#get_json" do
+    before(:each) do
+      @api = Object.new
+      @api.instance_variable_set("@rest", RestClient::Resource.new('http://example.org'))
+      @api.instance_variable_set("@env", @mock_environment_name)
+      @api.instance_variable_set("@keys", {})
+      @api.extend(EyApi)
+
+      @api.stub!(:sleep)
+    end
+
+    after(:each) do
+      FakeWeb.clean_registry
+    end
+
+    describe "when the JSON is available" do
+      before(:each) do
+        FakeWeb.register_uri(:post, "http://example.org/api/json_for_instance", :body => JSON.generate({:valid => "json"}))
+      end
+
+      it "should pass through exceptions that it receives" do
+        JSON.stub!(:parse).and_raise(Exception)
+        lambda {
+          @api.get_json("i-decafbad")
+        }.should raise_error
+      end
+
+      it "should return a valid JSON structure" do
+        presumably_valid_json = @api.get_json("i-feedface")
+        lambda {
+          JSON[presumably_valid_json]
+        }.should_not raise_error
+      end
+    end
+
+    describe "when the JSON is not yet available" do
+      
+      it "should retry until the 503 goes away" do
+        FakeWeb.register_uri(:post, "http://example.org/api/json_for_instance", [{
+            :status => ["503", "Service Unavailable"],
+            :body => "{}",
+          }, {
+            :status => ["503", "Service Unavailable"],
+            :body => "{}",
+          }, {
+            :status => ["200", "OK"],
+            :body => JSON.generate({:valid => 'json'}),
+          }]
+        )
+
+        lambda {
+          JSON[@api.get_json("i-robot")]
+        }.should_not raise_error
+      end
+      
+    end
+  end
+end
+
+

data/spec/ey_backup/backend_spec.rb

@@ -0,0 +1,12 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe EY::Backup do
+  before do
+    @backend = EY::Backup::Backend.new( spec_config['aws_secret_id'], spec_config['aws_secret_key'], 'ap-northeast-1', 'test-backup-bucket')
+  end
+
+  it "uses correct region" do
+    @backend.bucket_minder.bucket.location.should == 'ap-northeast-1'
+    @backend.bucket_minder.bucket_name == "test-backup-bucket"
+  end
+end

data/spec/ey_backup/backup_spec.rb

@@ -0,0 +1,54 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe EY::Backup do
+  before(:each) do
+    @db_name = create_mysql_database('first')
+  end
+
+  describe "#list" do
+
+    it 'prints the database to be listed first' do
+      EY::Backup.run([ "-c", backup_config_file ])
+
+      reset_logger
+
+      EY::Backup.run(["-c", backup_config_file, "-l", @db_name])
+
+      stdout.should =~ /\AListing database backups for #{@db_name}$/
+    end
+
+    it 'conforms to the /^\d:#{db_name} #{db_name}\.#{timestamp}\.#{ext}$/ line output' do
+      EY::Backup.run([ "-c", backup_config_file ])
+
+      reset_logger
+
+      EY::Backup.run(["-c", backup_config_file, "-l", @db_name])
+
+      stdout.should =~ /^\d+:[a-zA-Z0-9_\-]+\s+[a-zA-Z0-9_\-]+\.[T0-9\-]+\.[.a-zA-Z0-9]+$/
+    end
+  end
+
+  describe "#list_all" do
+    it 'prints all the database names' do
+      @db_name2 = create_mysql_database('second')
+
+      EY::Backup.run([ "-c", backup_config_file ])
+
+      reset_logger
+
+      EY::Backup.run(["-c", backup_config_file, '-l', ''])
+
+      stdout.should =~ /^\d+:#{@db_name}/
+      stdout.should =~ /^\d+:#{@db_name2}/
+    end
+  end
+
+  describe "cleanup" do
+    it "removes the last backup" do
+      set_keep 5
+      6.times do
+        EY::Backup.run([ "-c", backup_config_file ])
+      end
+    end
+  end
+end

data/spec/ey_backup/cli_spec.rb

@@ -0,0 +1,35 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe "MySQL Backups" do
+  before(:each) do
+    @db_name = create_mysql_database('first')
+  end
+
+  describe "--quiet" do
+    it "does not print anything to STDOUT" do
+      capturing_stdio do
+        reset_logger
+        EY::Backup.run(["-c", backup_config_file, '--quiet'])
+      end
+
+      last_stdout.should be_empty
+    end
+
+    it 'does print to STDOUT without the flag' do
+      EY::Backup.logger = nil
+
+      result = capturing_stdio do
+        EY::Backup.run(["-c", backup_config_file ])
+      end
+
+      last_stdout.should_not be_empty
+      EY::Backup.logger = nil
+
+      result = capturing_stdio do
+        EY::Backup.run(["-c", backup_config_file ])
+      end
+
+      last_stdout.should_not be_empty
+    end
+  end
+end