easy_e 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 37d1b662de1d5e139989991c7ddfe391a018e5a7
+  data.tar.gz: b73562bfc50c8fd642c4a310735b42cf81497c90
+SHA512:
+  metadata.gz: 97a6e29177667f468bd52e56ecfecce3de4135f383cac8e90d4f3f4c4c8e18422767571b469edc0c533a766802c3ed0064e5ff8f024a304d64228ad8c8b693fd
+  data.tar.gz: 34b82f63039da516b16f9ae1a17aa3f5c2e7f828f6a3376937fc761a679b661a36667950df5e6f9bccc89352b23c15a674b2f25cd0d97fc3c844ab76ac16908f

data/README.md

@@ -0,0 +1,42 @@
+easy-ec2-ebs-automatic-consistent-snapshot
+===
+
+Easier to use than it is to say, this project aims to provide easy, automatic,
+and consistent snapshots for AWS EBS volumes on EC2 instances.
+
+Some specific goals and how they are achieved:
+
+ - *Safety*: refuses to operate unless everything seems ok, and tries desperately to leave your system no worse than it started
+ - *Reliability*: comprehensive test sweet makes sure that Easy-E behaves as expected, even in unexpected conditions.
+ - *Visibility*: verbose logging options to inspect the decision-making process for any action
+ - *Ease of Installation*: just install the gem and add one line to your crontab
+ - *Ease of Use*: automatically detects volumes mounted to the machine
+ - *Ease of Monitoring*: 100% visibility of operation can be gained from off-the-shelf monitoring solution plugins
+ - *Maintainability*: well-organized code structure and a modern language 
+ - *Extensibility*: plugin architecture makes it easy to add lock support for services
+ - *Isolation*: plugin execution is isolated, so that an error in one is very unlikely to affect the others
+
+Install
+===
+```
+gem install easy-ec2-ebs-automatic-consistent-snapshot
+crontab -e
+```
+
+Testing
+===
+
+Because you'll be running this against production servers with critical data, it's important that the functionality is well-tested. A thorough, pessimistic, multi-layer test suite hopes to assuage your concerns about letting a computer handle such an important task. The test suite is unquestionably the most complex part of this project.
+
+Unit Tests
+---
+
+Like any good Ruby software, this tool has a unit test suite that seeks mostly to verify the plumbing and ensure that there are no runtime errors on the expected execution paths. This is only the tip of the iceberg...
+
+Vagrant Integration Testing
+---
+
+The integration layer contains an Ansible + Vagrant setup to configure clusters of services for live-fire testing (the AWS bits are mocked out via Easy-E's `--mock` flag). Simply running `vagrant up` will build a cluster of servers running MySQL, MongoDB, etc, configured in a master/slave architecture as approprite for the given system.
+
+There is also a set of Ansible tasks that verify the operation of each plugin under **both ideal and pathological** conditions. This means that Easy-E runs reliably, even when the services it operates on do not. Things like timeouts and service restart failures are modeled via `socat`, and assertions are made on the correct error output for each condition. For more info on how this is done, check `roles/integration-test/tasks/*.yml`
+

data/bin/easy-e

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require './lib/easy_e.rb'
+
+EasyE.new.execute

data/lib/easy_e/options.rb

@@ -0,0 +1,46 @@
+require 'optparse'
+
+class EasyE
+  module Options
+    def option_parser
+      unless @option_parser
+        @option_parser = OptionParser.new do |o|
+          o.banner = "Usage: #{$0} [options]"
+
+          o.on("-v", "--[no-]verbose", "Run verbosely") do |val|
+            options[:verbose] = val
+          end
+
+          o.on("-a", "--access-key <AWS ACCESS KEY>", "AWS access key") do |val|
+            options[:access_key] = val
+          end
+
+          o.on("-s", "--secret-key <AWS SECRET KEY>", "AWS secret key") do |val|
+            options[:secret_key] = val
+          end
+
+          o.on("-c", "--credentials-file <FILE>", "Load AWS credentials from the downloaded CSV file (overrides -a and -s)") do |val|
+            options[:credentials_file] = val
+          end
+
+          o.on("-m", "--[no-]mock", "Mock out AWS calls for testing in Vagrant") do |val|
+            options[:mock] = val
+          end
+
+          o.on("-l", "--logfile FILE", "Path to a file used for logging") do |filename|
+            options.logfile = filename
+            logger.debug filename
+          end
+        end
+
+        plugins.each { |plugin| plugin.collect_options @option_parser }
+      end
+
+      @option_parser
+    end
+  end
+
+  def options
+    @options ||= OpenStruct.new
+  end
+end

data/lib/easy_e/plugin.rb

@@ -0,0 +1,49 @@
+require 'ostruct'
+class EasyE::Plugin
+  @@registered_plugins = []
+
+  attr_reader :options, :logger
+
+  def self.inherited(klass)
+    registered_plugins.unshift klass
+  end
+
+  def self.registered_plugins
+    @@registered_plugins
+  end
+
+  def default_options
+    { }
+  end
+
+  def options
+    @options ||= OpenStruct.new default_options
+  end
+
+  def logger
+    EasyE.logger false
+  end
+
+  def collect_options option_parser
+    option_parser.on "--#{name.downcase}", "Enable the #{name} plugin" do
+      options.enable = true
+    end
+
+    defined_options.each do |option_name, description|
+      option_parser.on "--#{name.downcase}-#{option_name.to_s.gsub('_','-')} #{option_name.upcase}", description do |val|
+        options[option_name.to_sym] = val
+      end
+    end
+  end
+
+  def carefully msg
+    yield
+  rescue Exception => e
+    logger.error "Error while trying to #{msg}"
+    logger.error e
+    nil
+  end
+end
+
+require 'plugins/mysql_plugin'
+require 'plugins/mongo_plugin'

data/lib/easy_e/snapshotter.rb

@@ -0,0 +1,48 @@
+require 'csv'
+require 'httparty'
+require 'pp'
+module EasyE::Snapshotter
+  AWS_INSTANCE_ID_URL = 'http://169.254.169.254/latest/dynamic/instance-identity/document'
+
+  attr_writer :storage, :compute, :instance_id
+  def take_snapshots
+    attached_volumes.collect do |vol|
+      logger.debug "Snapping #{vol.volume_id}"
+      snapshot = compute.snapshots.new
+      snapshot.volume_id = vol.volume_id
+      snapshot.save
+      snapshot
+    end
+  end
+
+  # lazy loaders
+  def compute
+    require 'fog/aws'
+    Fog.mock! if options[:mock]
+    @compute ||= Fog::Compute.new({
+      :aws_access_key_id => access_key,
+      :aws_secret_access_key => secret_key,
+      :provider => "AWS"
+    }) 
+  end
+
+  def attached_volumes
+    @attached_volumes ||= compute.volumes.select { |vol| vol.server_id == instance_id }
+  end
+
+  def access_key
+    @access_key ||= if options[:credentials_file] then credentials.first["Access Key Id"] else options[:access_key] end
+  end
+
+  def secret_key
+    @secret_key ||= if options[:credentials_file] then credentials.first["Secret Access Key"] else options[:secret_key] end
+  end
+
+  def credentials
+    @credentials ||= CSV.parse(File.read(options[:credentials_file]), :headers => true)
+  end
+
+  def instance_id
+    @instance_id ||= JSON.parse(HTTParty.get(AWS_INSTANCE_ID_URL))["instanceId"]
+  end
+end

data/lib/easy_e.rb

@@ -0,0 +1,65 @@
+$:.unshift File.dirname __FILE__
+require 'logger'
+require 'ostruct'
+require 'easy_e/options'
+require 'easy_e/snapshotter'
+require 'easy_e/plugin'
+
+class EasyE
+  include EasyE::Options
+  include EasyE::Snapshotter
+
+  @@logger = nil
+  def self.logger logfile
+    unless @@logger
+      @@logger = Logger.new(logfile || STDOUT)
+      @@logger.level = Logger::DEBUG
+      @@logger.formatter = proc do |severity, datetime, progname, msg|
+        "[#{severity}] #{datetime.strftime("%Y-%m-%d %H:%M:%S")} #{msg}\n"
+      end
+    end
+
+    @@logger
+  end
+
+  def plugins
+    @plugins ||= registered_plugins.collect { |klass| klass.new }
+  end
+
+  def registered_plugins
+    EasyE::Plugin.registered_plugins
+  end
+
+  def run
+    plugins.each do |plugin|
+      begin
+        plugin.before if plugin.options.enable
+      rescue Exception => e
+        logger.error "Encountered error while running the #{plugin.name} plugin's before hook"
+        logger.error e
+      end
+    end
+
+    take_snapshots
+
+    plugins.each do |plugin|
+      begin
+        plugin.after if plugin.options.enable
+      rescue Exception => e
+        logger.error "Encountered error while running the #{plugin.name} plugin's after hook"
+        logger.error e
+      end
+    end
+  end
+
+  def execute
+    option_parser.parse!
+    logger.debug "Debug logging enabled"
+    run
+  end
+
+  def logger
+    # HACK -- the logfile argument only gets used on the first invocation
+    EasyE.logger options.logfile
+  end
+end

data/lib/plugins/mongo_plugin.rb

@@ -0,0 +1,125 @@
+require 'pp'
+class EasyE::Plugin::MongoPlugin < EasyE::Plugin
+  WIRED_TIGER_KEY = 'wiredTiger'
+  attr_accessor :client
+  def defined_options
+    {
+      service: 'Service to start after shutting down server',
+      shutdown: 'Shutdown mongodb server (this is required if your data and journal are on different volumes',
+      user: 'Mongo user',
+      password: 'Mongo password',
+      port: 'Mongo port',
+      host: 'Mongo host',
+      server_selection_timeout: 'Timeout in seconds while choosing a server to connect to (default 30)',
+      wait_queue_timeout: 'Timeout in seconds while waiting for a connection in the pool (default 1)',
+      connection_timeout: 'Timeout in seconds to wait for a socket to connect (default 5)',
+      socket_timeout: 'Timeout in seconds to wait for an operation to execute on a socket (default 5)'
+    }
+  end
+
+  def default_options
+    {
+      service: 'mongodb',
+      port: '27017',
+      shutdown: false,
+      host: 'localhost',
+      server_selection_timeout: 30,
+      wait_queue_timeout: 1,
+      connection_timeout: 5,
+      socket_timeout: 5
+    }
+  end
+
+  def client
+    @client ||= Mongo::Client.new [ "#{options.host}:#{options.port}" ], client_options
+  end
+
+  def client_options
+    {
+      user: options.user,
+      password: options.password,
+      server_selection_timeout: options.server_selection_timeout.to_i,
+      wait_queue_timeout: options.wait_queue_timeout.to_i,
+      connection_timeout: options.connection_timeout.to_i,
+      socket_timeout: options.socket_timeout.to_i
+    }
+  end
+
+  def before
+    require 'mongo'
+    Mongo::Logger.logger = logger
+    return logger.error "Refusing to operate" if carefully('check whether this node is a primary') { primary? }.nil?
+    return logger.error "This appears to be a primary member, refusing to operate" if primary?
+
+    if wired_tiger?
+      logger.info "Wired Tiger storage engine detected"
+      carefully('shutdown mongo') { shutdown_mongo } if options.shutdown
+    else
+      logger.info "MMAPv1 storage engine detected"
+      carefully('lock mongo') { lock_mongo }
+    end
+  end
+
+  def after
+    return logger.error "Refusing to operate" if carefully('check whether this node is a primary') { primary? }.nil?
+    return logger.error "This appears to be a primary member, refusing to operate" if primary?
+    
+    if wired_tiger?
+      carefully('start mongo') { start_mongo } if options.shutdown
+    else
+      carefully('unlock mongo') { unlock_mongo }
+    end
+
+    if carefully('check that mongo is still accessible') { client.command(serverStatus: 1).first }
+      logger.info "Received status from mongo, everything appears to be ok"
+    end
+  end
+
+  def name
+    "Mongo"
+  end
+
+  private
+
+  def wired_tiger?
+    if @wired_tiger.nil?
+      @wired_tiger = client.command(serverStatus: 1).first.has_key? WIRED_TIGER_KEY
+    end
+    @wired_tiger
+  end
+
+  def primary?
+    if @primary.nil?
+      @primary = client.command(isMaster: 1).first['ismaster']
+    end
+    @primary
+  end
+
+  def shutdown_mongo
+    logger.info 'Shutting down mongodb'
+    begin
+      # this will always raise an exception after it completes
+      client.command shutdown: 1
+    rescue Mongo::Error::SocketError => e
+      logger.debug "Received expected socket error after shutting down"
+    end
+
+    # we need a new connection now since the server has shut down
+    @client = nil
+  end
+
+  def start_mongo
+    logger.info "Starting mongodb via 'service #{options[:service]} start'"
+    system "service #{options[:service]} start"
+  end
+
+  def lock_mongo
+    logger.info "Locking mongo"
+    client.command(fsync: 1, lock: true)
+  end
+
+  def unlock_mongo
+    logger.info "Unlocking mongo"
+    client.database['$cmd.sys.unlock'].find().first
+  end
+end

data/lib/plugins/mysql_plugin.rb

@@ -0,0 +1,22 @@
+class EasyE::Plugin::MysqlPlugin < EasyE::Plugin
+  def defined_options
+    {
+      user: 'MySql Username',
+      pass: 'MySql Password',
+      port: 'MySql port',
+      host: 'MySql host'
+    }
+  end
+
+  def before
+    require 'mysql'
+    Mysql.new
+  end
+
+  def after
+  end
+
+  def name
+    "Mysql"
+  end
+end

metadata

@@ -0,0 +1,52 @@
+--- !ruby/object:Gem::Specification
+name: easy_e
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Bryan Conrad
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-04-21 00:00:00.000000000 Z
+dependencies: []
+description: Easy EBS snapshots that work
+email: bryan.conrad@synctree.com
+executables:
+- easy-e
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- bin/easy-e
+- lib/easy_e.rb
+- lib/easy_e/options.rb
+- lib/easy_e/plugin.rb
+- lib/easy_e/snapshotter.rb
+- lib/plugins/mongo_plugin.rb
+- lib/plugins/mysql_plugin.rb
+homepage: http://rubygems.org/gems/easy_e
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: Easy EBS snapshots that work
+test_files: []