snapscatter 0.1.0

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in snapscatter.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Alex Escalante
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,95 @@
+# Snapscatter
+
+This script creates snapshots from EBS volumes and copies them across regions for disaster recovery.
+If your volume is used for database storage, this script can make sure your backup is consistent by
+flushing and stopping writes until the snapshot is complete.
+
+This script also purges old snapshots according to a simple retention policy specified by you, to keep
+your Amazon AWS bills under control.
+
+## Installation
+
+Just use the gem command to install:
+
+    $ gem install snapscatter
+
+## Usage
+
+You can have a look at the commands and options by just typing the name of the executable:
+
+    Commands:
+      snapscatter create          # Create snapshots, optionally copying them to destination region
+      snapscatter help [COMMAND]  # Describe available commands or one specific command
+      snapscatter list            # Show available snapshots
+      snapscatter purge           # purge snapshots older than the specified number of days
+      snapscatter targets         # Show volumes tagged for backup
+
+The best way to use this script is to make a shell wrapper for it that exports your AWS credentials
+as environment variables and then put it under the control of the cron demon.
+
+### Specifying volumes to backup
+
+You should use the AWS console to mark the volumes you want to be backed up using the tag `Backup` with a
+value of `true`. You can then check the list of these volumes using the command `targets`.
+
+### Taking snapshots
+
+Use the command `create` to take snapshots of all the tagged volumes. Snapshots will be taken to your default
+AWS region, but you can optionally supply the '--destination' flag to create a copy onto another
+region for disaster recovery.
+
+Every snapshot taken will have the `PurgeAllow` tag set with the value of `true`. If for some reason you want
+a snapshot not to be purged indefinitely, you can set this tag to any other value, or even remove the tag
+ altogether.
+
+### Purging snapshots
+
+You can call the `purge` command to delete any snapshots older than 30 days. This is the default retention policy
+but you can change it by using the optional '-d' flag (for `--days).
+
+You can also run this command with the `-n` (for `--noaction`) to only list the snapshots that would be purged
+under the specified retention policy, no snapshot will be purged.
+
+### Listing snapshots
+
+The `list` command will show all the snapshots subject to be purged, that is, all snapshots with the `PurgAllow`
+tag set to `true`. The `-f`option for this command gives more information on every snapshot: snapshot id, volume id
+and date of creation.
+
+### Consistent backups
+
+If the volumes your attempting to snapshot are being used by a database, then you want to force a flush to disk and
+stop writing so you can have a consistent backup. This script can do that for you (currently it only supports MongoDB).
+
+To use this feature, you can tag the volume with the `Consistent` tag. The value of this tag contains the connection
+information, as key-value pairs separated by commas, so that the script can have access to the database, flush and
+stop writes until the snapshot has been taken. Here's an example:
+
+    strategy: mongo, host: 127.0.0.1, port: 27017, usr: admin, pwd: 12345
+
+## Example
+
+Create a shell file like the following and put it under cron's control:
+
+    #!/bin/sh
+
+    export AWS_ACCESS_KEY_ID="YOURACCESSKEY"
+    export AWS_SECRET_ACCESS_KEY="YOURSECRETACCESSKEY"
+
+    snapscatter purge -d 20
+    snapscatter create --destination="us-west-1"
+
+You have to make two calls because the script won't purge and create snapshots on a single call.
+
+## TODO
+
+* Take consistency speficication out of the volume and put it in a configuration file
+* More database connectors
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/snapscatter

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'snapscatter/cli'
+Snapscatter::CLI.start

data/features/snapscatter.feature

@@ -0,0 +1,19 @@
+@announce-stdout
+Feature: snapscatter
+  In order to create and distribute snapshots
+  As a CLI
+  I can execute several different commands
+
+  Scenario: show available commands and options
+    When I run `snapscatter`
+    Then the output should contain "targets"
+    And the output should contain "list"
+    And the output should contain "go"
+
+  Scenario: show all volumes available for snapshot
+    When I run `snapscatter targets`
+    Then the output should match /vol-[0-9a-f]+/
+
+  Scenario: list all current snapshots
+    When I run `snapscatter list`
+    Then the output should match /snap-[0-9a-f]+/

data/features/support/setup.rb

@@ -0,0 +1,5 @@
+require 'aruba/cucumber'
+
+Before do
+  @aruba_timeout_seconds = 60
+end

data/lib/snapscatter/cli.rb

@@ -0,0 +1,80 @@
+require 'thor'
+require_relative '../snapscatter'
+
+module Snapscatter
+  class CLI < Thor
+
+    desc 'targets', 'Show volumes tagged for backup'
+    method_option :keys, type: :hash, banner: 'AWS security keys'
+    def targets
+      targets = Snapscatter.targets create_ec2
+      targets.each { |target| puts target.id }
+    end
+
+    desc 'list', 'Show available snapshots'
+    method_option :keys, type: :hash, banner: 'AWS security keys'
+    method_option :full, type: :boolean, aliases: '-f', banner: 'Show useful info about snapshots'
+    def list
+      snapshots = Snapscatter.list create_ec2
+      snapshots.each do |snapshot|
+        output = [ snapshot.id ]
+        if options[:full]
+          output << snapshot.volume_id
+          output << snapshot.start_time.strftime("%Y-%m-%d")
+        end
+        puts output.join(" ")
+      end
+    end
+
+    desc 'purge', 'purge snapshots older than the specified number of days'
+    method_option :keys, type: :hash, banner: 'AWS security keys'
+    method_option :days, type: :numeric, default: 30, aliases: '-d', banner: 'retention policy in days'
+    method_option :noaction, type: :boolean, default: false, aliases: '-n', banner: 'do not purge, just show'
+    def purge
+      purged = Snapscatter.purge create_ec2, options[:days], true # options[:noaction] # remove in production
+      purged.each { |snapshot| puts "#{snapshot.id}" }
+    end
+
+    desc 'create', 'Create snapshots, optionally copying them to destination region'
+    method_option :keys, type: :hash, banner: 'AWS security keys'
+    method_option :destination, type: :string, banner: 'region to copy snapshots to'
+    def create
+      source_ec2 = create_ec2
+      targets = Snapscatter.targets source_ec2
+      targets.each do |volume|
+        snapshot = nil
+        description = nil
+
+        Snapscatter.in_lock volume.tags['Consistent'] do
+          volume_name = volume.tags['Name']
+          date_as_string = Date.today.strftime("%Y-%m-%d")
+          description = "#{volume_name} #{date_as_string}"
+
+          snapshot = volume.create_snapshot description
+          snapshot.add_tag 'VolumeName', value: volume_name
+          snapshot.add_tag 'PurgeAllow', value: "true"
+
+          sleep 1 until [:completed, :error].include?(snapshot.status)
+        end
+
+        if snapshot.status == :completed
+          output = ["created", snapshot.id, description]
+          if options.has_key? 'destination'
+            destination_ec2 = create_ec2(region: options[:destination])
+            Snapscatter.copy destination_ec2, source_ec2.client.config.region, snapshot, description
+            output << "#{options[:destination]}"
+          end
+          puts output.join(" ")
+        else
+          puts "#{volume.id} (#{volume_name}): snapshot failed"
+        end
+      end
+    end
+
+    private
+      def create_ec2 ec2_options={}
+        ec2_options.merge! options[:keys] if options.has_key? :keys
+        ec2 = AWS::EC2.new(ec2_options)
+      end
+  end
+end

data/lib/snapscatter/locker.rb

@@ -0,0 +1,60 @@
+require 'mongo'
+
+module Snapscatter
+
+  class Locker
+
+    def initialize spec
+      strategy = spec[:strategy] && spec.delete(:strategy)
+      case strategy
+      when 'mongo'
+        @strategy = MongoLocker.new spec
+      else
+        @strategy = NoOpLocker.new
+      end
+    end
+
+    def lock
+      @strategy.lock
+    end
+
+    def unlock
+      @strategy.unlock
+    end
+  end
+
+  class NoOpLocker
+    def method_missing sym
+    end
+  end
+
+  class MongoLocker
+    def initialize spec
+      @host = spec[:host] && spec.delete(:host)
+      @port = spec[:port] && spec.delete(:port)
+      user = spec[:usr] && spec.delete(:usr)
+      password = spec[:pwd] && spec.delete(:pwd)
+
+      if @host
+        @client = Mongo::MongoClient.new @host, @port, spec # spec contains the options
+      else
+        @client = Mongo::MongoClient.new
+      end
+
+      if user
+        @client.add_auth 'admin', user, password, nil
+      end
+    end
+
+    def lock
+      @client.lock!
+      puts "locked mongo instance at #{@host}"
+    end
+
+    def unlock
+      @client.unlock!
+      puts "unlocked mongo instance at #{@host}"
+    end
+  end
+
+end

data/lib/snapscatter/version.rb

@@ -0,0 +1,3 @@
+module Snapscatter
+  VERSION = "0.1.0"
+end

data/lib/snapscatter.rb

@@ -0,0 +1,70 @@
+require 'snapscatter/version'
+require 'snapscatter/locker'
+require 'aws'
+
+module Snapscatter
+
+  private
+
+  def parse_spec str
+    str ||= ""
+    spec = {}
+    str.split(',').map do |i|
+      k, v = i.split(':').map { |i| i.strip }
+      spec[k.to_sym] = v
+    end
+    return spec
+  end
+
+  public
+
+  def targets ec2
+    ec2.volumes.tagged('Backup').tagged_values('true')
+  end
+
+  def list ec2
+    ec2.snapshots.tagged('PurgeAllow').tagged_values('true')
+  end
+
+  def copy ec2, region, snapshot, description
+    options = {
+      source_region: region,
+      source_snapshot_id: snapshot.id,
+      description: description
+    }
+
+    response = ec2.client.copy_snapshot options
+    copied_snapshot = ec2.snapshots[response.data[:snapshot_id]]
+    copied_snapshot.tags.set snapshot.tags
+  end
+
+  def purge ec2, purge_after_days, list_only
+    purged = []
+    snapshots = Snapscatter.list ec2
+    snapshots.each do |snapshot|
+      purge_date = snapshot.start_time.to_date + purge_after_days
+      # puts "#{Date.today} > #{purge_date} == #{Date.today > purge_date}"
+      if Date.today > purge_date
+        snapshot.delete if not list_only
+        purged << snapshot
+      end
+    end
+
+    return purged
+  end
+
+  # consistency spec should look like the following (all parameters but host, optional)
+  # strategy: mongo, host: 127.0.0.1, port: 27017, usr: admin, pwd: 12345
+  def in_lock consistency_spec
+    locker = Locker.new Snapscatter.parse_spec(consistency_spec)
+    locker.lock
+    begin
+      yield
+    ensure
+      locker.unlock
+    end
+  end
+
+  module_function :targets, :list, :copy, :purge, :in_lock, :parse_spec
+
+end

data/snapscatter.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'snapscatter/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "snapscatter"
+  spec.version       = Snapscatter::VERSION
+  spec.authors       = ["Alex Escalante"]
+  spec.email         = ["alex.escalante@gmail.com"]
+  spec.description   = %q{Geographically distributed and consistent AWS snapshots}
+  spec.summary       = %q{Creates consistent snapshots from EBS volumes and copies them across regions for disaster recovery}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "thor"
+  spec.add_dependency 'aws-sdk', '~> 1.0'
+  spec.add_dependency "mongo"
+  spec.add_dependency "bson_ext"
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "cucumber"
+  spec.add_development_dependency "aruba"
+end

metadata

@@ -0,0 +1,192 @@
+--- !ruby/object:Gem::Specification
+name: snapscatter
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Alex Escalante
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-10-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: mongo
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bson_ext
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: cucumber
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aruba
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Geographically distributed and consistent AWS snapshots
+email:
+- alex.escalante@gmail.com
+executables:
+- snapscatter
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/snapscatter
+- features/snapscatter.feature
+- features/support/setup.rb
+- lib/snapscatter.rb
+- lib/snapscatter/cli.rb
+- lib/snapscatter/locker.rb
+- lib/snapscatter/version.rb
+- snapscatter.gemspec
+homepage: ''
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Creates consistent snapshots from EBS volumes and copies them across regions
+  for disaster recovery
+test_files:
+- features/snapscatter.feature
+- features/support/setup.rb
+has_rdoc: