mongolly 0.0.7 → 0.1.0

data/bin/mongolly

@@ -2,47 +2,31 @@
 require 'thor'
 require 'yaml'
 require 'time'
-require 'mongo'
 require 'mongolly'
 
 module Mongolly
   class Runner < Thor
 
+    CONFIG_PATH = File.expand_path '~/.mongolly'
+
     def initialize(*args)
       super
       @config = read_config
-      @db_name, @db_config = database_config
-      exit unless valid_config?
+      exit unless @config && valid_config?
     end
 
-    CONFIG_PATH = File.expand_path '~/.mongolly'
-
-    desc "snapshot", "takes an EBS snapshot of the given volumes"
-    method_option :database, aliases: '-d'
-    def snapshot
-      db = Mongo::Connection.new( @db_config["host"], @db_config["port"] )
-      db['admin'].authenticate( @db_config["user"], @db_config["pass"] )
-
-      SnapshotManager.take_snapshots( db, @config["aws_access_key_id"], @config["aws_secret_access_key"], @db_config["region"], @db_config["volumes"] )
+    desc "backup", "Snapshots the Database EBS Volumes"
+    def backup
+      puts " ** Backing up ..."
+      Shepherd.new(@config).backup
     end
 
-    desc "clean", "removes snapshots older than the given data"
-    method_option :maximum_age, aliases: '-a', required: true
-    method_option :database, aliases: '-d'
+    desc "clean", "Removes old Database EBS Snapshots"
+    method_option :age, aliases: '-a', required: true
     def clean
-      max_age = Time.parse(options[:maximum_age])
-
-      puts " ** Cleaning snapshots older than #{max_age}"
-
-      ec2 = AWS::EC2.new(access_key_id: @config["aws_access_key_id"], secret_access_key: @config["aws_secret_access_key"]).regions[@db_config["region"]]
-      ec2.snapshots.with_owner(:self).each do |snapshot|
-        unless snapshot.tags[:created_at].nil? || snapshot.tags[:backup_key].nil?
-          if Time.parse(snapshot.tags[:created_at]) < max_age
-            puts " ** Deleting #{snapshot.id} created on #{snapshot.tags[:created_at]} with key #{snapshot.tags[:backup_key]}"
-            snapshot.delete
-          end
-        end
-      end
+      age = Time.parse(options[:age])
+      puts " ** Cleaning snapshots older than #{age}"
+      Shepherd.new(@config).cleanup(age)
     end
 
   private
@@ -50,18 +34,12 @@ module Mongolly
       return true if File.exists? CONFIG_PATH
 
       empty_config = {
-        databases: {
-          dbname: {
-            host: nil,
-            port: nil,
-            user: nil,
-            pass: nil,
-            region: nil,
-            volumes: [nil]
-          }
-        },
-        aws_access_key_id: nil,
-        aws_secret_access_key: nil,
+        database: [],
+        db_username: nil,
+        db_password: nil,
+        access_key_id: nil,
+        secret_access_key: nil,
+        aws_region: 'us-east-1'
       }
 
       File.open( CONFIG_PATH, "w" ) do |f|
@@ -77,7 +55,7 @@ module Mongolly
     end
 
     def read_config
-      return unless seed_config
+      return false unless seed_config
       begin
         return YAML::load( File.read( CONFIG_PATH ) )
       rescue e
@@ -86,31 +64,9 @@ module Mongolly
       end
     end
 
-    def database_config
-      if options[:database].to_s.strip.empty? && @config["databases"].size > 1
-        raise ArgumentError.new("Database name not provided and more than database specified in the config file")
-      elsif ! @config["databases"].keys.include? options[:database].to_s.strip
-        raise ArgumentError.new("Database #{options[:database]} not defined in config")
-      end
-
-      db_name = options[:database].to_s.strip.empty? ? @config["databases"].keys.first : options[:database]
-      db_config = @config["databases"][db_name]
-
-      %w(host port user pass region).each do |arg|
-        if db_config[arg].to_s.strip.empty?
-          raise ArgumentError.new( "#{arg} for database #{db_name} cannot be empty" )
-        end
-      end
-      if db_config["volumes"].empty? or db_config["volumes"].map { |v| v.to_s.strip.empty? }.include?(true)
-        raise ArgumentError.new("volumes cannot be empty or include an empty string for database #{db_name}")
-      end
-
-      return db_name, db_config
-    end
-
     def valid_config?
-      %w(aws_access_key_id aws_secret_access_key).each do |arg|
-        raise ArgumentError.new("#{arg} cannot be empty") if @config[arg].to_s.strip.empty?
+      %w(database db_username db_password access_key_id secret_access_key).each do |arg|
+        raise ArgumentError.new("#{arg} cannot be empty") if @config[arg.to_sym].to_s.strip.empty?
       end
       return true
     end

data/lib/mongolly.rb

@@ -1,5 +1,11 @@
-require "mongolly/version"
-require "mongolly/snapshot_manager"
+require 'mongolly/extensions/aws/ec2/instance'
+require 'mongolly/extensions/aws/ec2/instance_collection'
+require 'mongolly/extensions/bson/timestamp'
+require 'mongolly/extensions/mongo/mongo_client'
+require 'mongolly/extensions/mongo/mongo_replica_set_client'
+
+require 'mongolly/version'
+require 'mongolly/shepherd'
 
 module Mongolly
 end

data/lib/mongolly/extensions.rb

@@ -0,0 +1,6 @@
+require 'extensions/aws/ec2/instance'
+require 'extensions/aws/ec2/instance_collection'
+require 'extensions/bson/timestamp'
+require 'extensions/mongo/mongo_client'
+require 'extensions/mongo/mongo_replica_set_client'
+

data/lib/mongolly/extensions/aws/ec2/instance.rb

@@ -0,0 +1,15 @@
+require 'aws-sdk'
+
+class AWS::EC2::Instance
+
+  def volumes_with_tag(key)
+    volumes = []
+    attachments.each do |name,attachment|
+      next  unless attachment.status == :attached
+      volume = attachment.volume
+      volumes << volume  if volume.status == :in_use && volume.tags.has_key?(key)
+    end
+    volumes
+  end
+
+end

data/lib/mongolly/extensions/aws/ec2/instance_collection.rb

@@ -0,0 +1,30 @@
+require 'aws-sdk'
+require 'socket'
+require 'ipaddress'
+
+class AWS::EC2::InstanceCollection
+
+    def find_from_address(address, port = 27107)
+      ip_address = convert_address_to_ip(address, port)
+
+      instances = select do |instance|
+        instance.public_ip_address == ip_address || instance.private_ip_address == ip_address
+      end
+      raise error_class "InstanceNotFound"  if instances.length != 1
+
+      return instances.first
+    rescue SocketError
+      raise RuntimeError.new("Unable to determine IP address from #{address}")
+    end
+
+private
+  def convert_address_to_ip(address, port)
+    return address if ::IPAddress.valid? address
+
+    ip_addresses = ::Addrinfo.getaddrinfo(address, port, nil, :STREAM)
+    raise error_class "MultipleIpAddressFound"  if ip_addresses.length > 1
+
+    return ip_addresses[0].ip_address
+  end
+
+end

data/lib/mongolly/extensions/bson/timestamp.rb

@@ -0,0 +1,12 @@
+require 'mongo'
+
+class BSON::Timestamp
+  include Comparable
+
+  def <=>(other)
+    s = seconds <=> other.seconds
+    return s  unless s == 0
+    increment <=> other.increment
+  end
+
+end

data/lib/mongolly/extensions/mongo/mongo_client.rb

@@ -0,0 +1,58 @@
+require 'mongo'
+
+class Mongo::MongoClient
+
+  def snapshot_ebs(options={})
+    options[:volume_tag] ||= 'mongolly'
+    ec2 = AWS::EC2.new(access_key_id: options[:access_key_id], secret_access_key: options[:secret_access_key], region: options[:region])
+
+    instance = ec2.instances.find_from_address(*snapshot_ebs_target.split(':'))
+    volumes = instance.volumes_with_tag(options[:volume_tag])
+
+    raise RuntimeError "no suitable volumes found"  unless volumes.length > 0
+
+    begin
+      if volumes.length >= 1
+        disable_profiling
+        lock!
+      end
+
+      backup_key = (0...8).map{65.+(rand(25)).chr}.join
+      volumes.each do |volume|
+        snapshot = volume.create_snapshot("#{backup_key} #{Time.now} mongolly backup")
+        snapshot.add_tag('created_at', value: Time.now)
+        snapshot.add_tag('backup_key', value: backup_key)
+      end
+
+    ensure
+      unlock!  if locked?
+      enable_profiling
+    end
+  end
+
+protected
+  def snapshot_ebs_target
+    host_port.join(':')
+  end
+
+  def disable_profiling
+    @profiled_dbs = {}
+    database_names.each do |db|
+      begin
+        @profiled_dbs[db] = self[db].profiling_level  unless self[db].profiling_level == :off
+        self[db].profiling_level = :off
+      rescue Mongo::InvalidNSName
+      end
+    end
+  end
+
+  def enable_profiling
+    return false  if locked?
+    return true  unless @profiled_dbs
+    @profiled_dbs.each do |db,level|
+      self[db].profiling_level = level  rescue Mongo::InvalidNSName
+    end
+    return true
+  end
+
+end

data/lib/mongolly/extensions/mongo/mongo_replica_set_client.rb

@@ -0,0 +1,14 @@
+require 'mongo'
+
+class Mongo::MongoReplicaSetClient
+
+  def most_current_secondary
+    replica = self['admin'].command( replSetGetStatus: 1 )
+    replica['members'].select { |m| m['state'] == 2 }.sort_by { |m| m['optime'] }.reverse.first['name']
+  end
+
+protected
+  def snapshot_ebs_target
+    most_current_secondary
+  end
+end

data/lib/mongolly/shepherd.rb

@@ -0,0 +1,39 @@
+module Mongolly
+  class Shepherd
+
+    def initialize(options={})
+      @access_key_id     = options[:access_key_id]
+      @secret_access_key = options[:secret_access_key]
+      @region            = options[:aws_region] || 'us-east-1'
+      @database          = options[:database]
+      @db_username       = options[:db_username]
+      @db_password       = options[:db_password]
+    end
+
+    def backup
+      connection.snapshot_ebs(access_key_id: @access_key_id, secret_access_key: @secret_access_key, region: @region)
+    end
+
+    def cleanup(age)
+      raise ArgumentError.new("Must provide a Time object cleanup")  unless age.class <= Time
+
+      ec2 = AWS::EC2.new(access_key_id: @access_key_id, secret_access_key: @secret_access_key, region: @region)
+
+      ec2.snapshots.with_owner(:self).each do |snapshot|
+        unless snapshot.tags[:created_at].nil? || snapshot.tags[:backup_key].nil?
+          snapshot.delete  if Time.parse(snapshot.tags[:created_at]) < age
+        end
+      end
+    end
+
+    def connection
+      db = if @database.is_a? Array
+        Mongo::MongoReplicaSetClient.new(@database)
+      else
+        Mongo::MongoClient.new(*@database.split(':'))
+      end
+      db['admin'].authenticate(@db_username, @db_password, true)
+      return db
+    end
+  end
+end

data/lib/mongolly/version.rb

@@ -1,3 +1,3 @@
 module Mongolly
-  VERSION = "0.0.7"
+  VERSION = "0.1.0"
 end

data/mongolly.gemspec

@@ -21,6 +21,6 @@ Gem::Specification.new do |gem|
   gem.add_dependency("mongo", ["~> 1.8.3"])
   gem.add_dependency("bson_ext", ["~> 1.8.3"])
   gem.add_dependency("aws-sdk", ["~> 1.5.8"])
-  gem.add_dependency("debugger")
+  gem.add_dependency("ipaddress")
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mongolly
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-21 00:00:00.000000000 Z
+date: 2013-05-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -76,7 +76,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.5.8
 - !ruby/object:Gem::Dependency
-  name: debugger
+  name: ipaddress
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -106,7 +106,13 @@ files:
 - Rakefile
 - bin/mongolly
 - lib/mongolly.rb
-- lib/mongolly/snapshot_manager.rb
+- lib/mongolly/extensions.rb
+- lib/mongolly/extensions/aws/ec2/instance.rb
+- lib/mongolly/extensions/aws/ec2/instance_collection.rb
+- lib/mongolly/extensions/bson/timestamp.rb
+- lib/mongolly/extensions/mongo/mongo_client.rb
+- lib/mongolly/extensions/mongo/mongo_replica_set_client.rb
+- lib/mongolly/shepherd.rb
 - lib/mongolly/version.rb
 - mongolly.gemspec
 homepage: http://www.github.com/msaffitz/mongolly

data/lib/mongolly/snapshot_manager.rb

@@ -1,59 +0,0 @@
-require 'aws-sdk'
-
-module Mongolly
-  class SnapshotManager
-
-    def self.take_snapshots(db, aws_key_id, aws_secret_key, region, volume_ids)
-      profile_levels = {}
-
-      unless db.locked?
-        puts " ** Locking Database"
-        db.database_names.each do |db_name|
-          begin
-            level = db[db_name].profiling_level
-            if level != :off
-              profile_levels[db_name] = level
-              db[db_name].profiling_level = :off
-            end
-          rescue Mongo::InvalidNSName
-            puts " ** Skiping #{db_name} with invalid database name"
-          end
-        end
-        db.lock!
-      end
-
-      begin
-        ec2 = AWS::EC2.new(access_key_id: aws_key_id, secret_access_key: aws_secret_key).regions[region]
-        backup_key = (0...8).map{65.+(rand(25)).chr}.join
-
-        puts " ** Starting Snapshot with key #{backup_key}"
-
-        volume_ids.map{ |v| v.to_s.strip }.each do |volume_id|
-          puts " ** Taking snapshot of volume #{volume_id}"
-          volume = ec2.volumes[volume_id]
-          raise RuntimeError.new("Volume #{volume_id} does not exist") unless volume.exists?
-
-          snapshot = volume.create_snapshot("#{backup_key} #{Time.now} mongo backup")
-          snapshot.add_tag('created_at', value: Time.now)
-          snapshot.add_tag('backup_key', value: backup_key)
-        end
-      ensure
-        if db.locked?
-          puts " ** Unlocking Database"
-          db.unlock!
-          db.database_names.each do |db_name|
-            begin
-               level = profile_levels[db_name]
-              unless level.nil? || level == :off
-                puts " ** Setting #{db_name} profile level to #{level.to_s}"
-                db[db_name].profiling_level = level
-              end
-            rescue Mongo::InvalidNSName
-              puts " ** Skiping #{db_name} with invalid database name"
-            end
-          end
-        end
-      end
-    end
-  end
-end