snap-ebs 0.0.8

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6d2d9b7df8d19e53288b89d957cc9c56d7942d23
+  data.tar.gz: 07e9374eae9478236ef061a582a050d349015a49
+SHA512:
+  metadata.gz: 5f8986b7b2a0be56714219a524d8ef65fa6746b2a78b93fae5350c0e8893fe721a93c24550a10e40b4f9c13a482c02add2bdbd79fab5b0cf4ef4f667a2faefa7
+  data.tar.gz: c46faf076355911857de4ca88dedf6649ac5ebaf7e54a1df8c961ba8bc341d7ac3c64fc244ac3b7c56ee0150e6cbe20ee5832e7f64b7747318c95319b4dac8f4

data/README.md

@@ -0,0 +1,56 @@
+snap-ebsc2-ebs-automatic-consistent-snapshot
+===
+
+Easier to use than it is to say, this project aims to provide easy, automatic,
+and consistent snapshots for AWS EBS volumes on EC2 instances.
+
+Some specific goals and how they are achieved:
+
+ - *Safety*: refuses to operate unless everything seems ok, and tries desperately to leave your system no worse than it started
+ - *Reliability*: comprehensive test sweet makes sure that SnapEbs behaves as expected, even in unexpected conditions.
+ - *Visibility*: verbose logging options to inspect the decision-making process for any action
+ - *Ease of Installation*: just install the gem and add one line to your crontab
+ - *Ease of Use*: automatically detects volumes mounted to the machine
+ - *Ease of Monitoring*: 100% visibility of operation can be gained from off-the-shelf monitoring solution plugins
+ - *Maintainability*: well-organized code structure and a modern language 
+ - *Extensibility*: plugin architecture makes it easy to add lock support for services
+ - *Isolation*: plugin execution is isolated, so that an error in one is very unlikely to affect the others
+
+Install
+===
+
+Dependencies - Amazon Linux
+---
+
+```
+sudo yum install gcc \
+                 glibc-devel \
+                 make \
+                 mysql-devel \
+                 patch \
+                 ruby-devel \
+                 zlib-devel
+```
+
+```
+gem install snap-ebsc2-ebs-automatic-consistent-snapshot
+crontab -e
+```
+
+Testing
+===
+
+Because you'll be running this against production servers with critical data, it's important that the functionality is well-tested. A thorough, pessimistic, multi-layer test suite hopes to assuage your concerns about letting a computer handle such an important task. The test suite is unquestionably the most complex part of this project.
+
+Unit Tests
+---
+
+Like any good Ruby software, this tool has a unit test suite that seeks mostly to verify the plumbing and ensure that there are no runtime errors on the expected execution paths. This is only the tip of the iceberg...
+
+Vagrant Integration Testing
+---
+
+The integration layer contains an Ansible + Vagrant setup to configure clusters of services for live-fire testing (the AWS bits are mocked out via SnapEbs's `--mock` flag). Simply running `vagrant up` will build a cluster of servers running MySQL, MongoDB, etc, configured in a master/slave architecture as approprite for the given system.
+
+There is also a set of Ansible tasks that verify the operation of each plugin under **both ideal and pathological** conditions. This means that SnapEbs runs reliably, even when the services it operates on do not. Things like timeouts and service restart failures are modeled via `socat`, and assertions are made on the correct error output for each condition. For more info on how this is done, check `roles/integration-test/tasks/*.yml`
+

data/bin/snap-ebs

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+$:.unshift File.join File.dirname(__FILE__), '..'
+require 'lib/snap_ebs.rb'
+
+SnapEbs.new.execute

data/lib/plugins/mongo_plugin.rb

@@ -0,0 +1,140 @@
+require 'pp'
+class SnapEbs::Plugin::MongoPlugin < SnapEbs::Plugin
+  WIRED_TIGER_KEY = 'wiredTiger'
+  attr_accessor :client
+  def defined_options
+    {
+      service: 'Service to start after shutting down server',
+      shutdown: 'Shutdown mongodb server (this is required if your data and journal are on different volumes',
+      user: 'Mongo user',
+      password: 'Mongo password',
+      port: 'Mongo port',
+      host: 'Mongo host',
+      server_selection_timeout: 'Timeout in seconds while choosing a server to connect to (default 30)',
+      wait_queue_timeout: 'Timeout in seconds while waiting for a connection in the pool (default 1)',
+      connection_timeout: 'Timeout in seconds to wait for a socket to connect (default 5)',
+      socket_timeout: 'Timeout in seconds to wait for an operation to execute on a socket (default 5)'
+    }
+  end
+
+  def default_options
+    {
+      service: 'mongodb',
+      port: '27017',
+      shutdown: false,
+      host: 'localhost',
+      server_selection_timeout: 30,
+      wait_queue_timeout: 1,
+      connection_timeout: 5,
+      socket_timeout: 5
+    }
+  end
+
+  def client
+    @client ||= Mongo::Client.new [ "#{options.host}:#{options.port}" ], client_options
+  end
+
+  def client_options
+    {
+      connect: :direct,
+      user: options.user,
+      password: options.password,
+      server_selection_timeout: options.server_selection_timeout.to_i,
+      wait_queue_timeout: options.wait_queue_timeout.to_i,
+      connection_timeout: options.connection_timeout.to_i,
+      socket_timeout: options.socket_timeout.to_i
+    }
+  end
+
+  def before
+    require 'mongo'
+    Mongo::Logger.logger = logger
+    return unless safe_to_operate?
+
+    if wired_tiger?
+      logger.info "Wired Tiger storage engine detected"
+      carefully('stop mongo') { stop_mongo } if options.shutdown
+    else
+      logger.info "MMAPv1 storage engine detected"
+      carefully('lock mongo') { lock_mongo }
+    end
+  end
+
+  def after
+    if wired_tiger?
+      carefully('start mongo') { start_mongo } if options.shutdown
+    else
+      carefully('unlock mongo') { unlock_mongo }
+    end
+
+    if carefully('check that mongo is still accessible') { client.command(serverStatus: 1).first }
+      logger.info "Received status from mongo, everything appears to be ok"
+    end
+  end
+
+  def name
+    "Mongo"
+  end
+
+  private
+
+  def safe_to_operate?
+    # we check for strict equality with booleans here, because nil means an
+    # error occurred while checking, and it is unsafe to operate
+    return true if (primary? == false) or (standalone? == true)
+    logger.error "This appears to be a primary member, refusing to operate"
+    false
+  end
+
+  def wired_tiger?
+    if @wired_tiger.nil?
+      @wired_tiger = client.command(serverStatus: 1).first.has_key? WIRED_TIGER_KEY
+    end
+    @wired_tiger
+  end
+
+  def primary?
+    carefully 'check whether this node is a primary' do
+      if @primary.nil?
+        @primary = client.command(isMaster: 1).first['ismaster']
+      end
+      @primary
+    end
+  end
+
+  def standalone?
+    # this will raise an error on a non-RS mongod
+    client.command(replSetGetStatus: 1)
+    false
+  rescue
+    true
+  end
+
+  def stop_mongo
+    logger.info 'Stopping mongodb'
+    begin
+      # this will always raise an exception after it completes
+      client.command shutdown: 1
+    rescue Mongo::Error::SocketError => e
+      logger.debug "Received expected socket error after shutting down"
+    end
+
+    # we need a new connection now since the server has shut down
+    @client = nil
+  end
+
+  def start_mongo
+    logger.info "Starting mongodb via 'service #{options[:service]} start'"
+    system "service #{options[:service]} start"
+  end
+
+  def lock_mongo
+    logger.info "Locking mongo"
+    client.command(fsync: 1, lock: true)
+  end
+
+  def unlock_mongo
+    logger.info "Unlocking mongo"
+    client.database['$cmd.sys.unlock'].find().read
+  end
+end

data/lib/plugins/mysql_plugin.rb

@@ -0,0 +1,22 @@
+class SnapEbs::Plugin::MysqlPlugin < SnapEbs::Plugin
+  def defined_options
+    {
+      user: 'MySql Username',
+      pass: 'MySql Password',
+      port: 'MySql port',
+      host: 'MySql host'
+    }
+  end
+
+  def before
+    require 'mysql'
+    Mysql.new
+  end
+
+  def after
+  end
+
+  def name
+    "Mysql"
+  end
+end

data/lib/snap_ebs/options.rb

@@ -0,0 +1,50 @@
+require 'optparse'
+
+class SnapEbs
+  module Options
+    def option_parser
+      unless @option_parser
+        @option_parser = OptionParser.new do |o|
+          o.banner = "Usage: #{$0} [options]"
+
+          o.on("-v", "--[no-]verbose", "Run verbosely") do |val|
+            options[:verbose] = val
+          end
+
+          o.on("-a", "--access-key <AWS ACCESS KEY>", "AWS access key") do |val|
+            options[:access_key] = val
+          end
+
+          o.on("-s", "--secret-key <AWS SECRET KEY>", "AWS secret key") do |val|
+            options[:secret_key] = val
+          end
+
+          o.on("-c", "--credentials-file <FILE>", "Load AWS credentials from the downloaded CSV file (overrides -a and -s)") do |val|
+            options[:credentials_file] = val
+          end
+
+          o.on("-m", "--[no-]mock", "Mock out AWS calls for testing in Vagrant") do |val|
+            options[:mock] = val
+          end
+
+          o.on("-l", "--logfile FILE", "Path to a file used for logging") do |filename|
+            options.logfile = filename
+            logger.debug filename
+          end
+
+          o.on("-d", "--directory PATH", "Only snap volumes mounted to PATH, a comma-separated list of directories") do |d|
+            options.directory = d
+          end
+        end
+
+        plugins.each { |plugin| plugin.collect_options @option_parser }
+      end
+
+      @option_parser
+    end
+  end
+
+  def options
+    @options ||= OpenStruct.new
+  end
+end

data/lib/snap_ebs/plugin.rb

@@ -0,0 +1,49 @@
+require 'ostruct'
+class SnapEbs::Plugin
+  @@registered_plugins = []
+
+  attr_reader :options, :logger
+
+  def self.inherited(klass)
+    registered_plugins.unshift klass
+  end
+
+  def self.registered_plugins
+    @@registered_plugins
+  end
+
+  def default_options
+    { }
+  end
+
+  def options
+    @options ||= OpenStruct.new default_options
+  end
+
+  def logger
+    SnapEbs.logger false
+  end
+
+  def collect_options option_parser
+    option_parser.on "--#{name.downcase}", "Enable the #{name} plugin" do
+      options.enable = true
+    end
+
+    defined_options.each do |option_name, description|
+      option_parser.on "--#{name.downcase}-#{option_name.to_s.gsub('_','-')} #{option_name.upcase}", description do |val|
+        options[option_name.to_sym] = val
+      end
+    end
+  end
+
+  def carefully msg
+    yield
+  rescue Exception => e
+    logger.error "Error while trying to #{msg}"
+    logger.error e
+    nil
+  end
+end
+
+require 'plugins/mysql_plugin'
+require 'plugins/mongo_plugin'

data/lib/snap_ebs/snapshotter.rb

@@ -0,0 +1,80 @@
+require 'csv'
+require 'httparty'
+module SnapEbs::Snapshotter
+  AWS_INSTANCE_ID_URL = 'http://169.254.169.254/latest/dynamic/instance-identity/document'
+
+  def take_snapshots
+    attached_volumes.collect do |vol|
+      next unless should_snap vol
+      logger.debug "Snapping #{vol.id}"
+      snapshot = compute.snapshots.new
+      snapshot.volume_id = vol.id
+      snapshot.description = snapshot_name(vol)
+      snapshot.save
+      snapshot
+    end
+  end
+
+  # lazy loaders
+  def compute
+    require 'fog/aws'
+    if options[:mock]
+      Fog.mock!
+      @region = 'us-east-1'
+      @instance_id = 'i-deadbeef'
+      @instance_name = 'totally-not-the-cia'
+    end
+
+    @compute ||= Fog::Compute.new({
+      :aws_access_key_id => access_key,
+      :aws_secret_access_key => secret_key,
+      :region => region,
+      :provider => "AWS"
+    }) 
+  end
+
+  def attached_volumes
+    @attached_volumes ||= compute.volumes.select { |vol| vol.server_id == instance_id }
+  end
+
+  def access_key
+    @access_key ||= if options[:credentials_file] then credentials.first["Access Key Id"] else options[:access_key] end
+  end
+
+  def secret_key
+    @secret_key ||= if options[:credentials_file] then credentials.first["Secret Access Key"] else options[:secret_key] end
+  end
+
+  def credentials
+    @credentials ||= CSV.parse(File.read(options[:credentials_file]), :headers => true)
+  end
+
+  def instance_id
+    @instance_id ||= JSON.parse(HTTParty.get(AWS_INSTANCE_ID_URL))["instanceId"]
+  end
+
+  def instance_name
+    @instance_name ||= compute.servers.get(instance_id).tags['Name']
+  end
+
+  def region
+    @region ||= JSON.parse(HTTParty.get(AWS_INSTANCE_ID_URL))["region"]
+  end
+
+  def snapshot_name vol
+    id = instance_name
+    id = instance_id if id.nil? or id.empty?
+    "#{Time.now.strftime "%Y%m%d%H%M%S"}-#{id}-#{vol.device}"
+  end
+
+  def should_snap vol
+    normalized_device = vol.device.gsub('/dev/s', '/dev/xv') rescue vol.device
+    options.directory.nil? or devices_to_snap.include?(normalized_device)
+  end
+
+  def devices_to_snap
+    @devices_to_snap ||= options.directory.split(',').map { |dir| `df --output=source #{dir} | grep dev`.strip }
+    logger.debug @devices_to_snap
+    @devices_to_snap
+  end
+end

data/lib/snap_ebs.rb

@@ -0,0 +1,65 @@
+$:.unshift File.dirname __FILE__
+require 'logger'
+require 'ostruct'
+require 'snap_ebs/options'
+require 'snap_ebs/snapshotter'
+require 'snap_ebs/plugin'
+
+class SnapEbs
+  include SnapEbs::Options
+  include SnapEbs::Snapshotter
+
+  @@logger = nil
+  def self.logger logfile
+    unless @@logger
+      @@logger = Logger.new(logfile || STDOUT)
+      @@logger.level = Logger::DEBUG
+      @@logger.formatter = proc do |severity, datetime, progname, msg|
+        "[#{severity}] #{datetime.strftime("%Y-%m-%d %H:%M:%S")} #{msg}\n"
+      end
+    end
+
+    @@logger
+  end
+
+  def plugins
+    @plugins ||= registered_plugins.collect { |klass| klass.new }
+  end
+
+  def registered_plugins
+    SnapEbs::Plugin.registered_plugins
+  end
+
+  def run
+    plugins.each do |plugin|
+      begin
+        plugin.before if plugin.options.enable
+      rescue Exception => e
+        logger.error "Encountered error while running the #{plugin.name} plugin's before hook"
+        logger.error e
+      end
+    end
+
+    take_snapshots
+
+    plugins.each do |plugin|
+      begin
+        plugin.after if plugin.options.enable
+      rescue Exception => e
+        logger.error "Encountered error while running the #{plugin.name} plugin's after hook"
+        logger.error e
+      end
+    end
+  end
+
+  def execute
+    option_parser.parse!
+    logger.debug "Debug logging enabled"
+    run
+  end
+
+  def logger
+    # HACK -- the logfile argument only gets used on the first invocation
+    SnapEbs.logger options.logfile
+  end
+end

metadata

@@ -0,0 +1,108 @@
+--- !ruby/object:Gem::Specification
+name: snap-ebs
+version: !ruby/object:Gem::Version
+  version: 0.0.8
+platform: ruby
+authors:
+- Bryan Conrad
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-06-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fog
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.31'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.31'
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.13'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.13'
+- !ruby/object:Gem::Dependency
+  name: mysql
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.9'
+- !ruby/object:Gem::Dependency
+  name: mongo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+description: Easy EBS snapshots that work
+email: bryan.conrad@synctree.com
+executables:
+- snap-ebs
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- bin/snap-ebs
+- lib/plugins/mongo_plugin.rb
+- lib/plugins/mysql_plugin.rb
+- lib/snap_ebs.rb
+- lib/snap_ebs/options.rb
+- lib/snap_ebs/plugin.rb
+- lib/snap_ebs/snapshotter.rb
+homepage: http://rubygems.org/gems/snap-ebs
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: Easy EBS snapshots that work
+test_files: []