mongolly 0.1.1 → 0.2.0

data/bin/mongolly

@@ -16,17 +16,17 @@ module Mongolly
     end
 
     desc "backup", "Snapshots the Database EBS Volumes"
+    method_option :dry_run, type: :boolean, desc: 'Step through command without changes'
     def backup
-      puts " ** Backing up ..."
-      Shepherd.new(@config).backup
+      Shepherd.new({dry_run: options[:dry_run]}.merge(@config)).backup
     end
 
     desc "clean", "Removes old Database EBS Snapshots"
     method_option :age, aliases: '-a', required: true
+    method_option :dry_run, type: :boolean, desc: 'Step through command without changes'
     def clean
       age = Time.parse(options[:age])
-      puts " ** Cleaning snapshots older than #{age}"
-      Shepherd.new(@config).cleanup(age)
+      Shepherd.new({dry_run: options[:dry_run]}.merge(@config)).cleanup(age)
     end
 
   private
@@ -39,7 +39,11 @@ module Mongolly
         db_password: nil,
         access_key_id: nil,
         secret_access_key: nil,
-        aws_region: 'us-east-1'
+        region: 'us-east-1',
+        log_level: 'info',
+        mongo_start_command: nil,
+        mongo_stop_command: nil,
+        config_server_ssh_user: nil
       }
 
       File.open( CONFIG_PATH, "w" ) do |f|

data/lib/mongolly/extensions/aws/ec2/instance_collection.rb

@@ -14,7 +14,7 @@ class AWS::EC2::InstanceCollection
 
       return instances.first
     rescue SocketError
-      raise RuntimeError.new("Unable to determine IP address from #{address}")
+      raise RuntimeError.new("Unable to determine IP address from #{address}:#{port}")
     end
 
 private

data/lib/mongolly/extensions/mongo/mongo_client.rb

@@ -1,58 +1,242 @@
 require 'mongo'
+require 'logger'
+require 'net/ssh'
+require 'retries'
+
+require 'debugger'
 
 class Mongo::MongoClient
+  MAX_DISABLE_BALANCER_WAIT = 60*5 # 5 Minutes
 
   def snapshot_ebs(options={})
+
+    @mongolly_dry_run = options[:dry_run] || false
+    @mongolly_logger = options[:logger] || Logger.new(STDOUT)
     options[:volume_tag] ||= 'mongolly'
-    ec2 = AWS::EC2.new(access_key_id: options[:access_key_id], secret_access_key: options[:secret_access_key], region: options[:region])
+    options[:backup_key] ||= (0...8).map{65.+(rand(25)).chr}.join
 
-    instance = ec2.instances.find_from_address(*snapshot_ebs_target.split(':'))
-    volumes = instance.volumes_with_tag(options[:volume_tag])
+    @ec2 = AWS::EC2.new(access_key_id: options[:access_key_id], secret_access_key: options[:secret_access_key], region: options[:region])
 
-    raise RuntimeError "no suitable volumes found"  unless volumes.length > 0
+    if mongos?
+      @mongolly_logger.info("Detected sharded cluster")
+      with_disabled_balancing do
+        stop_config_server(options[:config_server_ssh_user], options[:mongo_stop_command])
 
-    begin
-      if volumes.length >= 1
-        disable_profiling
-        lock!
+        backup_instance(config_server, options, false)
+
+        shards.each do |name,hosts|
+          @mongolly_logger.debug("Found Shard #{name} with hosts #{hosts}.")
+          replica_set_connection(hosts, options).snapshot_ebs(options)
+        end
+        start_config_server(options[:config_server_ssh_user], options[:mongo_start_command])
       end
+    else
+      backup_instance(snapshot_ebs_target, options, true )
+    end
+  end
+
+protected
+  def snapshot_ebs_target
+    host_port.join(':')
+  end
+
+  def backup_instance(address, options, lock = true)
+    host, port = address.split(':')
+    instance = @ec2.instances.find_from_address(host, port)
+
+    @mongolly_logger.info("Backing up instance #{instance.id} from #{host}:#{port}")
+
+    volumes = instance.volumes_with_tag(options[:volume_tag])
+
+    @mongolly_logger.debug("Found target volumes #{volumes.map(&:id).join(', ')} ")
 
-      backup_key = (0...8).map{65.+(rand(25)).chr}.join
+    raise RuntimeError.new "no suitable volumes found"  unless volumes.length > 0
+
+    backup_block = proc do
       volumes.each do |volume|
-        snapshot = volume.create_snapshot("#{backup_key} #{Time.now} mongolly backup")
-        snapshot.add_tag('created_at', value: Time.now)
-        snapshot.add_tag('backup_key', value: backup_key)
+        @mongolly_logger.debug("Snapshotting #{volume.id} with tag #{options[:backup_key]}")
+        unless @mongolly_dry_run
+          snapshot = volume.create_snapshot("#{options[:backup_key]} #{Time.now} mongolly #{host}")
+          snapshot.add_tag('created_at', value: Time.now)
+          snapshot.add_tag('backup_key', value: options[:backup_key])
+        end
+      end
+    end
+
+    if lock
+      with_database_locked &backup_block
+    else
+      backup_block.call
+    end
+  end
+
+  def disable_balancing
+    @mongolly_logger.debug "Disabling Shard Balancing"
+    self['config'].collection('settings').update({_id: 'balancer'}, {'$set' => {stopped: true}}, upsert: true)  unless @mongolly_dry_run
+  end
+
+  def enable_balancing
+    @mongolly_logger.debug "Enabling Shard Balancing"
+    retry_logger = Proc.new do |exception, attempt_number, total_delay|
+      @mongolly_logger.debug "Error enabling balancing (config server not up?); retry attempt #{attempt_number}; #{total_delay} seconds have passed."
+    end
+    with_retries(max_tries: 5, handler: retry_logger, rescue: Mongo::OperationFailure, base_sleep_seconds: 5, max_sleep_seconds: 120) do
+      self['config'].collection('settings').update({_id: 'balancer'}, {'$set' => {stopped: false}}, upsert: true)  unless @mongolly_dry_run
+    end
+  end
+
+  def balancer_active?
+    self['config'].collection('locks').find({_id: 'balancer'}).count > 1
+  end
+
+  def stop_config_server(user, stop_command)
+    raise RuntimeError.new "mongo ssh user not specified"  unless user.to_s.strip != ''
+    raise RuntimeError.new "mongo ssh user not specified"  unless stop_command.to_s.strip != ''
+    @mongolly_logger.debug "Stopping config server #{config_server}"
+    unless @mongolly_dry_run
+      exit_code = nil
+      output = ''
+      Net::SSH.start(config_server, user.strip) do |ssh|
+        channel = ssh.open_channel do |ch|
+          ch.request_pty
+          ch.exec(stop_command.strip) do |ch, success|
+            raise "Unable to stop config server on #{config_server}"  unless success
+          end
+        end
+        channel.on_request("exit-status") do |ch,data|
+          exit_code = data.read_long
+        end
+        channel.on_extended_data do |ch,type,data|
+          output += data
+        end
+        ssh.loop
+      end
+
+      if exit_code != 0
+        raise RuntimeError.new "Unable to stop config server, #{output}"
       end
+    end
+  end
 
+  def start_config_server(user, start_command )
+    raise RuntimeError.new "mongo ssh user not specified"  unless user.to_s.strip != ''
+    raise RuntimeError.new "mongo ssh user not specified"  unless start_command.to_s.strip != ''
+    @mongolly_logger.debug "Starting config server #{config_server}"
+    unless @mongolly_dry_run
+      exit_code = nil
+      output = ''
+      Net::SSH.start(config_server, user.strip) do |ssh|
+        channel = ssh.open_channel do |ch|
+          ch.request_pty
+          ch.exec(start_command.strip) do |ch, success|
+            raise "Unable to start config server on #{config_server}"  unless success
+          end
+        end
+        channel.on_request("exit-status") do |ch,data|
+          exit_code = data.read_long
+        end
+        channel.on_extended_data do |ch,type,data|
+          output += data
+        end
+        ssh.loop
+      end
+
+      if exit_code != 0
+        raise RuntimeError.new "Unable to start config server, #{output}"
+      end
+    end
+  end
+
+  def config_server
+    unless @config_server
+      @config_server = self['admin'].command( { getCmdLineOpts: 1 } )["parsed"]["configdb"].split(",").sort.first.split(":").first
+      @mongolly_logger.debug "Found config server #{@config_server}"
+    end
+    return @config_server
+  end
+
+  def with_disabled_balancing
+    begin
+      disable_balancing
+      term_time = Time.now + MAX_DISABLE_BALANCER_WAIT
+      while !@mongolly_dry_run && (Time.now < term_time) && balancer_active?
+        @mongolly_logger.info "Balancer active, sleeping for 10s (#{(term_time - Time.now).round}s remaining)"
+        sleep 10
+      end
+      if !@mongolly_dry_run && balancer_active?
+        raise RuntimeError.new "unable to disable balancer within #{MAX_DISABLE_BALANCER_WAIT}s"
+      end
+      @mongolly_logger.debug "With shard balancing disabled..."
+      yield
     ensure
-      unlock!  if locked?
-      enable_profiling
+      enable_balancing
     end
   end
 
-protected
-  def snapshot_ebs_target
-    host_port.join(':')
+  def with_database_locked
+    begin
+      @mongolly_logger.debug "Locking database..."
+      disable_profiling
+      lock!  unless @mongolly_dry_run || locked?
+      @mongolly_logger.debug "With database locked..."
+      yield
+    ensure
+      @mongolly_logger.debug "Unlocking database..."
+      unlock!  if !@mongolly_dry_run && locked?
+      enable_profiling
+    end
   end
 
   def disable_profiling
+    @mongolly_logger.debug("Disabling profiling... ")
+
     @profiled_dbs = {}
     database_names.each do |db|
       begin
-        @profiled_dbs[db] = self[db].profiling_level  unless self[db].profiling_level == :off
-        self[db].profiling_level = :off
+        unless self[db].profiling_level == :off
+          @mongolly_logger.debug("Disabling profiling for #{db}, level #{self[db].profiling_level}")
+          @profiled_dbs[db] = self[db].profiling_level
+          self[db].profiling_level = :off  unless @mongolly_dry_run
+        end
       rescue Mongo::InvalidNSName
+        @mongolly_logger.debug("Skipping database #{db} due to invalid name")
       end
     end
   end
 
   def enable_profiling
-    return false  if locked?
-    return true  unless @profiled_dbs
+    if locked?
+      @mongolly_logger.debug("Database locked, can't turn on profiling")
+      return false
+    end
+    unless @profiled_dbs
+      @monglly_logger.debug("No dbs in @profiled_dbs")
+      return true
+    end
+
     @profiled_dbs.each do |db,level|
-      self[db].profiling_level = level  rescue Mongo::InvalidNSName
+      begin
+        @mongolly_logger.debug("Enabling profiling for #{db}, level #{level}")
+        self[db].profiling_level = level  unless @mongolly_dry_run
+      rescue Mongo::InvalidNSName
+        @mongolly_logger.debug("Skipping database #{db} due to invalid name")
+      end
     end
     return true
   end
 
+  def shards
+    shards = {}
+    self['config']['shards'].find().each do |shard|
+      shards[shard['_id']] = shard['host'].split("/")[1].split(",")
+    end
+    shards
+  end
+
+  def replica_set_connection(hosts, options)
+    db = Mongo::MongoReplicaSetClient.new(hosts)
+    db['admin'].authenticate(options[:db_username], options[:db_password], true)
+    return db
+  end
+
 end

data/lib/mongolly/extensions/mongo/mongo_replica_set_client.rb

@@ -4,7 +4,9 @@ class Mongo::MongoReplicaSetClient
 
   def most_current_secondary
     replica = self['admin'].command( replSetGetStatus: 1 )
-    replica['members'].select { |m| m['state'] == 2 }.sort_by { |m| m['optime'] }.reverse.first['name']
+    current = replica['members'].select { |m| m['state'] == 2 }.sort_by { |m| m['optime'] }.reverse.first['name']
+    @mongolly_logger.debug("Found most current secondary #{current}")
+    current
   end
 
 protected

data/lib/mongolly/shepherd.rb

@@ -2,34 +2,56 @@ module Mongolly
   class Shepherd
 
     def initialize(options={})
+      @options           = options
       @access_key_id     = options[:access_key_id]
       @secret_access_key = options[:secret_access_key]
-      @region            = options[:aws_region] || 'us-east-1'
+      @region            = options[:region] || 'us-east-1'
       @database          = options[:database]
       @db_username       = options[:db_username]
       @db_password       = options[:db_password]
+      @dry_run           = options[:dry_run]
+      @logger            = options[:logger] || Logger.new(STDOUT)
+      @logger.level = case options[:log_level].strip
+                        when 'fatal'; then Logger::FATAL
+                        when 'error'; then Logger::ERROR
+                        when 'warn' ; then Logger::WARN
+                        when 'debug'; then Logger::DEBUG
+                        else Logger::INFO
+                      end
     end
 
     def backup
-      connection.snapshot_ebs(access_key_id: @access_key_id, secret_access_key: @secret_access_key, region: @region)
+      @logger.info "Starting backup..."
+      connection.snapshot_ebs({logger: @logger}.merge(@options))
+      @logger.info "Backup complete."
     end
 
     def cleanup(age)
-      raise ArgumentError.new("Must provide a Time object cleanup")  unless age.class <= Time
+      @logger.info "Starting cleanup..."
+      raise ArgumentError.new("Must provide a Time object to cleanup")  unless age.class <= Time
 
-      ec2 = AWS::EC2.new(access_key_id: @access_key_id, secret_access_key: @secret_access_key, region: @region)
+      ec2 = AWS::EC2.new(access_key_id: @access_key_id,
+                         secret_access_key: @secret_access_key,
+                         region: @region)
 
+      @logger.debug "deleting snapshots older than #{age}}"
       ec2.snapshots.with_owner(:self).each do |snapshot|
         unless snapshot.tags[:created_at].nil? || snapshot.tags[:backup_key].nil?
-          snapshot.delete  if Time.parse(snapshot.tags[:created_at]) < age
+          if Time.parse(snapshot.tags[:created_at]) < age
+            @logger.debug "deleting snapshot #{snapshot.id} tagged #{snapshot.tags[:backup_key]} created at #{snapshot.tags[:created_at]}, earlier than #{age}"
+            snapshot.delete  unless @dry_run
+          end
         end
       end
+      @logger.info "Cleanup complete."
     end
 
     def connection
       db = if @database.is_a? Array
+        @logger.debug "connecting to a replica set #{@database}"
         Mongo::MongoReplicaSetClient.new(@database)
       else
+        @logger.debug "connecting to a single instance #{@database}"
         Mongo::MongoClient.new(*@database.split(':'))
       end
       db['admin'].authenticate(@db_username, @db_password, true)

data/lib/mongolly/version.rb

@@ -1,3 +1,3 @@
 module Mongolly
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end

data/mongolly.gemspec

@@ -22,5 +22,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency("bson_ext", ["~> 1.8.3"])
   gem.add_dependency("aws-sdk", ["~> 1.9.5"])
   gem.add_dependency("ipaddress")
+  gem.add_dependency("net-ssh", ["~> 2.0"])
+  gem.add_dependency("retries")
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mongolly
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-18 00:00:00.000000000 Z
+date: 2013-05-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -91,6 +91,38 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: net-ssh
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: retries
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Easy backups for EBS-based MongoDB Databases
 email:
 - m@saffitz.com