ey_cloud_server 1.4.26 → 1.4.28.pre

data/bin/binary_log_purge

@@ -20,10 +20,12 @@
 # Changelog 0.4 -> 1.0 
 # - Add automatic create user for master to connect to replica
 
-require 'rubygems'
+# Changelog 1.0 -> 1.1
+# - Remove unecessary require of mysql
+# - Remove -N from mysql command line (drops header row) for compatibility with 5.1 and 5.5
 
+require 'rubygems'
 require 'net/smtp'
-require 'mysql'
 require 'yaml'
 require 'open3'
 require 'getoptlong'
@@ -103,7 +105,7 @@ def run_query(host, user, password, query)
   if query == 'show processlist'
     stdin, stdout, stderr = Open3.popen3("mysql -u#{user} -p#{password} #{options} -h#{host} -N -e\"#{query}\"|grep 'Binlog'")
   else
-    stdin, stdout, stderr = Open3.popen3("mysql -u#{user} -p#{password} #{options} -h#{host} -N -e\"#{query}\"")
+    stdin, stdout, stderr = Open3.popen3("mysql -u#{user} -p#{password} #{options} -h#{host} -e\"#{query}\"")
   end
   query_error = stderr.read
   if query_error.length > 0

data/lib/ey_backup/cli.rb

@@ -46,6 +46,10 @@ module EY
             options[:config] = config
           end
 
+          opts.on("-b", "--bucket BUCKET", "Override default S3 bucket name. (Be Careful!)") do |bucket|
+            options[:backup_bucket] = bucket
+          end
+
           opts.on("-t", "--tmp_dir TMPDIR", "Use the given directory for temporary storage.") do |tmp_dir|
             options[:tmp_dir] = tmp_dir
           end
@@ -57,7 +61,7 @@ module EY
             options[:db] = db
           end
 
-          opts.on("-e", "--engine DATABASE_ENGINE", "The database engine. ex: mysql, postgres.") do |engine|
+          opts.on("-e", "--engine DATABASE_ENGINE", "The database engine. ex: mysql, postgres, mongodb.") do |engine|
             options[:engine] = engine
           end
 

data/lib/ey_backup/engines/mongodb_engine.rb

@@ -0,0 +1,64 @@
+module EY
+  module Backup
+    class Mongodb < Engine
+      register 'mongodb'
+
+      def dump(database_name, basename)
+        # Mongo dumps in directories. Compressing 
+        file = basename
+        
+        command = "mongodump --host #{host} -o #{file} --db #{database_name}; tar -cz #{file}"
+          
+        if gpg?
+          command << " | " << GPGEncryptor.command_for(key_id)
+          file << GPGEncryptor.extension
+        end
+
+        command << " > #{file}"
+
+        run(command)
+
+        file
+      end
+
+      def load(database_name, file)
+        if database_exists?(database_name)
+          cycle_database(database_name)
+        else
+          create_database(database_name)
+        end
+
+        command = "cat #{file}"
+
+        if gpg?
+          raise "Cannot load a GPG backup"
+        end
+
+        command << " | mongorestore --host #{host} --db #{database_name}"
+
+        run(command)
+      end
+
+      def database_exists?(database_name)
+        runs?("mongo --host #{host}  --quiet --eval \"for each(var db in db.runCommand('listDatabases').databases) if(db.sizeOnDisk > 1) print(db.name);\" admin | grep '#{database_name}'")
+      end
+
+      def drop_database(database_name)
+        runs?("mongo --host #{host} #{database_name} --quiet --eval 'printjson(db.dropDatabase());'")
+      end
+
+      def create_database(database_name) 
+        runs?("mongo --host #{host} #{database_name} --quiet --eval 'db.createCollection('ey_backup');'")
+      end
+
+      def cycle_database(database_name)
+        drop_database(database_name)
+        create_database(database_name)
+      end
+
+      def suffix
+        /\.(gz)$/
+      end
+    end
+  end
+end

data/lib/ey_backup/engines/postgresql_engine.rb

@@ -4,9 +4,9 @@ module EY
       register 'postgresql'
 
       def dump(database_name, basename)
-        file = basename + '.pgz'
+        file = basename + '.dump'
 
-        command = "PGPASSWORD='#{password}' pg_dump -h #{host} --format=c --no-owner --no-privileges -U#{username} #{database_name}"
+        command = "PGPASSWORD='#{password}' pg_dump -h #{host} --format=c --no-owner --no-privileges -Upostgres #{database_name}"
 
         if gpg?
           command << " | " << GPGEncryptor.command_for(key_id)
@@ -33,7 +33,7 @@ module EY
           raise "Cannot load a GPG backup"
         end
 
-        command << " | PGPASSWORD='#{password}' pg_restore -h #{host} --format=c --ignore-version -U#{username} -d #{database_name}"
+        command << " | PGPASSWORD='#{password}' pg_restore -h #{host} --format=c --ignore-version -Upostgres -d #{database_name}"
 
         run(command)
       end
@@ -56,7 +56,7 @@ module EY
       end
 
       def suffix
-        /pgz(\.gpz)?$/
+        /\.(dump|gpz)$/
       end
     end
   end

data/lib/ey_backup.rb

@@ -126,6 +126,7 @@ require lib_dir + '/backend'
 
 require lib_dir + '/engines/mysql_engine'
 require lib_dir + '/engines/postgresql_engine'
+require lib_dir + '/engines/mongodb_engine'
 
 require lib_dir + '/processors/gzipper'
 require lib_dir + '/processors/splitter'

data/lib/ey_cloud_server/version.rb

@@ -1,3 +1,3 @@
 module EY::CloudServer
-  VERSION = '1.4.26'
+  VERSION = '1.4.28.pre'
 end

data/spec/config-example.yml

@@ -7,5 +7,8 @@ mysql_host: localhost
 postgresql_user: root
 postgresql_password:
 postgresql_host: localhost
+mongodb_user: root
+mongodb_passwd: 
+mongodb_host: localhost
 aws_secret_id: 'abcd123.id'
 aws_secret_key: 'abcd123.key'

data/spec/ey_backup/mongo_backups_spec.rb

@@ -0,0 +1,106 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe "MongoDB Backups" do
+  before(:each) do
+    @db_name = create_mongodb_database('test')
+  end
+
+  it "makes a backup" do
+    EY::Backup.run(["-c", backup_config_file, "-e", "mongodb"])
+
+    reset_logger
+
+    EY::Backup.run(["-c", backup_config_file, "-l", @db_name, "-e", "mongodb"])
+
+    stdout.should match(/0:#{@db_name}/)
+  end
+
+  it "makes a split backup" do
+      EY::Backup.run(["-c", backup_config_file, "-e", "mongodb", "-s", "100"])
+  
+      backup = EY::Backup.run(["-c", backup_config_file, "-e", "mongodb", "-l", @db_name]).first
+      backup.parts.should > 1
+    end
+
+  it "makes GPG encrypted backups" do
+    import_gpg
+
+    EY::Backup.run(["-c", backup_config_file, '-e', 'mongodb', '-k', Helpers::PUBLIC_KEY_ID])
+
+    EY::Backup.run(["-c", backup_config_file, '-e', 'mongodb', "-l", @db_name])
+
+    stdout.should match(/0:#{@db_name}/)
+  end
+
+  it "restores a backup" do
+    run_mongocmd("db.foo.save({name: 'John Foo', age: 30});", @db_name).should be_true
+    run_mongocmd("db.bar.save({name: 'John Bar', age: 20});", @db_name).should be_true
+
+    EY::Backup.run(["-c", backup_config_file, '-e', 'mongodb'])
+
+    run_mongocmd("db.bar.drop();", @db_name).should be_true
+    run_mongocmd("db.bar.find();", @db_name).should be_false
+
+    reset_logger
+
+    EY::Backup.run(["-c", backup_config_file, "-l", @db_name, '-e', 'mongodb'])
+
+    stdout.should include("1 backup(s) found")
+
+    EY::Backup.run(["-c", backup_config_file, "-r", "0:#{@db_name}", '-e', 'mongodb'])
+
+    run_mongocmd("db.foo.find();", @db_name).should be_true
+    run_mongocmd("db.bar.find();", @db_name).should be_true
+  end
+
+  it "restores split backups" do
+    run_mongocmd("db.foo.save({name: 'John Foo', age: 30});", @db_name).should be_true
+    run_mongocmd("db.bar.save({name: 'John Bar', age: 20});", @db_name).should be_true
+
+    EY::Backup.run(["-c", backup_config_file, "-e", "MongoDB", "-s", "100"])
+    EY::Backup.run(["-c", backup_config_file, "-e", "MongoDB", "-l", @db_name]).first.parts.should > 1
+
+    run_mongocmd("db.bar.drop();", @db_name).should be_true
+    run_mongocmd("db.bar.find();", @db_name).should be_false
+
+    EY::Backup.run(["-c", backup_config_file, "-l", @db_name, '-e', 'mongodb'])
+
+    stdout.should include("1 backup(s) found")
+
+    EY::Backup.run(["-c", backup_config_file, "-r", "0:#{@db_name}", '-e', 'mongodb'])
+
+    run_mongocmd("db.foo.find();", @db_name).should be_true
+    run_mongocmd("db.bar.find();", @db_name).should be_true
+  end
+
+  it "downloads a split backup to 1 file" do
+    run_mongocmd("db.foo.save({name: 'John Foo', age: 30});", @db_name).should be_true
+    run_mongocmd("db.bar.save({name: 'John Bar', age: 20});", @db_name).should be_true
+
+    EY::Backup.run(["-c", backup_config_file, "-e", "mongodb", "-s", "100"])
+    EY::Backup.run(["-c", backup_config_file, "-e", "mongodb", "-l", @db_name]).first.parts.should > 1
+    EY::Backup.run(["-c", backup_config_file, "-e", 'mongodb', "--download", "0:#{@db_name}"])
+    files = Dir["#{EY::Backup.tmp_dir}/*#{@db_name}*"]
+
+    files.size.should == 1
+    FileUtils.rm(files.first)
+  end
+
+end
+
+describe "MongoDB Backups" do
+  before(:each) do
+    @dbs = [create_mongodb_database('first'), create_mongodb_database('second')]
+  end
+
+  it "makes multiple backup" do
+    EY::Backup.run(["-c", backup_config_file, "-e", "mongodb"])
+
+    reset_logger
+
+    @dbs.each do |db_name|
+      EY::Backup.run(["-c", backup_config_file, "-l", db_name, '-e', 'mongodb'])
+      stdout.should match(/0:#{db_name}/)
+    end
+  end
+end

data/spec/ey_backup/postgres_backups_spec.rb

@@ -12,7 +12,7 @@ describe "Postgres Backups" do
 
     EY::Backup.run(["-c", backup_config_file, "-l", @db_name, "-e", "postgresql"])
 
-    stdout.should match(/0:#{@db_name}.*\.pgz$/)
+    stdout.should match(/0:#{@db_name}.*\.dump.gz$/)
   end
 
   it "makes a split backup" do

data/spec/helpers.rb

@@ -112,7 +112,7 @@ module Helpers
   end
 
   def create_postgresql_database(db_key, region = 'us-east-1')
-    db_name = generate_database_name('postgresql')
+    db_name = generate_database_name('mongo')
     created_postgresql_dbs[db_key] = db_name
     drop_postgresql_database(db_key)
 # ssh -F /dev/null -o CheckHostIP=no -o StrictHostKeyChecking=no -i /home/$USER/.ssh/internal root@$DB_HOST "createdb -U postgres -h localhost -O $USER testdbreallytestingweee8"
@@ -122,6 +122,28 @@ module Helpers
     write_database_config('postgresql', postgresql_user, postgresql_password, postgresql_host, created_postgresql_dbs.values, region)
     db_name
   end
+  
+  def drop_mongodb_database(db_key)
+    db_name = created_mongodb_dbs[db_key]
+    command=%Q{ssh -F /dev/null -o CheckHostIP=no -o StrictHostKeyChecking=no -i /home/#{mongodb_user}/.ssh/internal root@#{mongodb_host} "mongo --host localhost #{db_name} --quiet --eval 'printjson(db.dropDatabase());'"}
+    system(command)
+  end
+
+  def check_mongodb_database(db_name)
+    command=%Q{ssh -F /dev/null -o CheckHostIP=no -o StrictHostKeyChecking=no -i /home/#{mongodb_user}/.ssh/internal root@#{mongodb_host} "mongo --host localhost --quiet --eval "for each(var db in db.runCommand('listDatabases').databases) if(db.sizeOnDisk > 1) print(db.name);" admin | grep '#{db_name}'"}
+    system(command) || raise("Could not find db: #{db_name} with command:\n #{command}")
+    db_name
+  end
+
+  def create_mongodb_database(db_key, region = 'us-east-1')
+    db_name = generate_database_name('test')
+    drop_mongodb_database(db_key)
+    command=%Q{ssh -F /dev/null -o CheckHostIP=no -o StrictHostKeyChecking=no -i /home/#{mongodb_user}/.ssh/internal root@#{mongodb_host} "mongo --host localhost #{db_name} --quiet --eval 'db.foo.save({ name : "derp"});'"}
+    system(command) || raise("Could not create db: #{db_name} with command:\n #{command}")
+    write_database_config('mongodb', mongodb_user, mongodb_password, mongodb_host, created_mongodb_dbs.values, region)
+    db_name
+  end  
+  
 
   def write_database_config(type, root_user, root_password, host, databases, region)
     @database_config = {
@@ -167,6 +189,10 @@ module Helpers
     system("PGPASSWORD='#{postgresql_password}' psql -c '#{command}' -h #{postgresql_host} -U #{postgresql_user} #{database}")
   end
 
+  def run_mongocmd(command, database)
+    system("mongo #{database} --quiet --eval \"#{command}\" --quiet")
+  end
+  
   def tmp_dir
     if spec_config['tmp_dir'].blank?
       File.dirname(__FILE__) + "/tmp/"
@@ -183,6 +209,10 @@ module Helpers
     spec_config['postgresql_user'] || raise("No postgresql user provided")
   end
 
+  def mongodb_user
+    spec_config['mongodb_user'] || raise("No mongodb user provided")
+  end
+
   def mysql_password
     spec_config['mysql_password']
   end
@@ -191,6 +221,10 @@ module Helpers
     spec_config['postgresql_password']
   end
 
+  def mongodb_password #<- is this really needed? Supported mongo will run in trusted mode
+    spec_config['mongodb_password']
+  end
+
   def mysql_host
     spec_config['mysql_host']
   end
@@ -199,6 +233,10 @@ module Helpers
     spec_config['postgresql_host']
   end
 
+  def mongodb_host
+    spec_config['mongodb_host']
+  end
+
   def created_mysql_dbs
     @created_mysql_dbs ||= {}
   end
@@ -207,6 +245,10 @@ module Helpers
     @created_postgresql_dbs ||= {}
   end
 
+  def created_mongodb_dbs
+    @created_mongodb_dbs ||= {}
+  end
+
   def spec_config
     Helpers.spec_config
   end

metadata

@@ -1,13 +1,14 @@
 --- !ruby/object:Gem::Specification 
 name: ey_cloud_server
 version: !ruby/object:Gem::Version 
-  hash: 51
-  prerelease: 
+  hash: 961915908
+  prerelease: 7
   segments: 
   - 1
   - 4
-  - 26
-  version: 1.4.26
+  - 28
+  - pre
+  version: 1.4.28.pre
 platform: ruby
 authors: 
 - Engine Yard
@@ -15,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-11-29 00:00:00 Z
+date: 2011-12-22 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: json
@@ -331,6 +332,7 @@ files:
 - lib/ey_backup/database.rb
 - lib/ey_backup/dumper.rb
 - lib/ey_backup/engine.rb
+- lib/ey_backup/engines/mongodb_engine.rb
 - lib/ey_backup/engines/mysql_engine.rb
 - lib/ey_backup/engines/postgresql_engine.rb
 - lib/ey_backup/loader.rb
@@ -350,6 +352,7 @@ files:
 - spec/ey_backup/backend_spec.rb
 - spec/ey_backup/backup_spec.rb
 - spec/ey_backup/cli_spec.rb
+- spec/ey_backup/mongo_backups_spec.rb
 - spec/ey_backup/mysql_backups_spec.rb
 - spec/ey_backup/postgres_backups_spec.rb
 - spec/ey_backup/spec_helper.rb
@@ -385,12 +388,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version 
-      hash: 3
+      hash: 25
       segments: 
-      - 0
-      version: "0"
+      - 1
+      - 3
+      - 1
+      version: 1.3.1
 requirements: []
 
 rubyforge_project: 
@@ -406,6 +411,7 @@ test_files:
 - spec/ey_backup/backend_spec.rb
 - spec/ey_backup/backup_spec.rb
 - spec/ey_backup/cli_spec.rb
+- spec/ey_backup/mongo_backups_spec.rb
 - spec/ey_backup/mysql_backups_spec.rb
 - spec/ey_backup/postgres_backups_spec.rb
 - spec/ey_backup/spec_helper.rb