drebs 0.0.1 → 0.1.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    NDE5YTM0ZDEwMWJlNmZiMmY4NzQ2ZjhjOTI4NWQ0ODc4ZWExNzRmNg==
+  data.tar.gz: !binary |-
+    NDhlYmU3ZWFiYThiZTdiYmYxOTZiZjRiMWQ0NDM0YjEyN2VkZmM2Zg==
+SHA512:
+  metadata.gz: !binary |-
+    NTlhYzQ1NWVhYzliOWE3YTlkNmNlMDlkMjQ2ZTdjNjY1NzE1OTViZjA4NDRi
+    ZDQ0MTM1NDFiZGI1YTFmYTAwNDBkMWQwY2Y2NTJiODZjNTY3MmM2NzI0MzA1
+    MGRiOTE0M2VkMzAwYTY4ODlmM2IxMzFjYmZiMDUyZGE3OTNhY2Q=
+  data.tar.gz: !binary |-
+    ZTAxZTc1NzUwNjllNTlmMzgzYTMwODQ0NTNjOWE2ZGNlYTdlY2I3MzkyYWU2
+    NjJjOWY0NTE0ZGE3NWE4NDAyZmUxNjBlYjVlYWQxODM5MTVlNGRkZWQ3Yzg3
+    NjRhZTM0NDFmY2UwOTk2YTI0MGQyMTVhMDA0YzAwMzUxNmY0MTg=

data/README.md

@@ -8,14 +8,39 @@
 
 ## Installation & Setup
 1. Clone the repo or install the gem
-1. Currently configuration is located at the top of the drebs bin.  Add you ec2 key, etc.
-1. Add Crontab entry: 0 * * * * drebs
-
-## Issues
-* State including config is cached in drebs_state.json.  This file will need to be deleted to get drebs to pick up new config.  Doing so will orphan current snapshots which will need to be deleted manually.
+1. Output the example configuration to a file: drebs check_config example_config > your_config.yml
+1. Create an AWS account with authorization limited to create, list, & delete snapshots (Example comming soon)
+1. Add AWS API keys for above account to your_config.yml
+1. configure your_config.yml per your backup requirements
+1. test your configuration: drebs check_config your_config.yml && drebs check_cloud your_config.yml
+1. Add Crontab entry: 0 * * * * drebs execute your_config.yml
 
 ## Todo
-* Tests!
-* Refactor using main with db for state and external config
+* Improve test coverage
 * Use Whenever gem for crontab setup
 * Arbitrary execution intervals (Snapshots every 5 minutes instead of every hour)
+* AWS API keys and other config values from Instance Data: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AESDG-chapter-instancedata.html
+* Add example AWS user access configuration
+* Implement support for pruning to occur from some other host so that the keys to delete snapshots are not available on host.
+
+## Testing notes
+
+*  __shell command__:  If you do: `drebs shell some_config` you will end up at a shell with `@drebs` defined and you will be able to access `@drebs.db`, `@drebs.config`, & `@drebs.cloud`.  If you set `@drebs.cloud` to be an instance of TestCloud from the test suite you should be able to execute various functions without actually hitting AWS and so work from your dev box.
+
+* Due to the nature of drebs being designed to be run from an ec2 you will need to be on your ec2 instance to test many of the AWS interactions.
+
+* You should be able to verify data on a snapshot by creating an ebs volume from the snapshot, attaching the volume to your instance and then mounting its file system on some mount point - [aws docs on using volumes](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-using-volumes.html)
+
+## Copyright 2014 [dojo4](www.dojo4.com)
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/Rakefile

@@ -1,8 +1,8 @@
 This.rubyforge_project = 'DREBS'
-This.author = "Garett Shulman"
-This.email = "garett@dojo4.com"
+This.authors = ["Garett Shulman", "Miles Matthias"]
+This.email = "miles@dojo4.com"
 This.homepage = "https://github.com/dojo4/#{ This.lib }"
-
+This.licenses = "Apache-2.0"
 
 task :default do
   puts((Rake::Task.tasks.map{|task| task.name.gsub(/::/,':')} - ['default']).sort)
@@ -27,6 +27,7 @@ def run_tests!(which = nil)
   div = ('=' * 119)
   line = ('-' * 119)
 
+  puts "running #{ test_rbs.count } test files..."
   test_rbs.each_with_index do |test_rb, index|
     testno = index + 1
     command = "#{ File.basename(This.ruby) } -I ./lib -I ./test/lib #{ test_rb }"
@@ -56,7 +57,6 @@ def run_tests!(which = nil)
   end
 end
 
-
 task :gemspec do
   ignore_extensions = ['git', 'svn', 'tmp', /sw./, 'bak', 'gem']
   ignore_directories = ['pkg', 'db']
@@ -89,7 +89,7 @@ task :gemspec do
   #has_rdoc = true #File.exist?('doc')
   test_files = test(?e, "test/#{ lib }.rb") ? "test/#{ lib }.rb" : nil
   summary = object.respond_to?(:summary) ? object.summary : "summary: #{ lib } kicks the ass"
-  description = object.respond_to?(:description) ? object.description : "description: #{ lib } kicks the ass"
+  description = object.respond_to?(:description) ? object.description : "#{ lib }: Disaster Recovery for Elastic Block Store. An AWS EBS backup script."
 
   if This.extensions.nil?
     This.extensions = []
@@ -141,9 +141,10 @@ spec.add_dependency(*<%= Array(lib_version).flatten.inspect %>)
 spec.extensions.push(*<%= extensions.inspect %>)
 
 spec.rubyforge_project = <%= This.rubyforge_project.inspect %>
-spec.author = <%= This.author.inspect %>
+spec.authors = <%= This.authors.inspect %>
 spec.email = <%= This.email.inspect %>
 spec.homepage = <%= This.homepage.inspect %>
+spec.licenses = <%= This.licenses.inspect %>
 end
 __
       }

data/bin/drebs

@@ -1,249 +1,129 @@
 #!/usr/bin/env ruby
 
 require 'rubygems'
-require 'right_aws'
-require 'logger'
 require 'main'
+require 'logger'
 require 'systemu'
-require 'json'
+require 'yaml'
 require 'socket'
 require 'net/smtp'
+require 'pry'
 
-class DREBS
-
-  ###### Begin User Config
-  DREBS_HOST_NAME = 'An identifier for your host: www1'
-  AWS_ACCESS_KEY_ID = 'YOUR ACCESS KEY ID'
-  AWS_SECRET_ACCESS_KEY = 'YOUR SECRET ACCESS KEY'
-  REGION = 'us-west-1'
-  BACKUP_STRATEGY = [
-    {
-      'hours_between'=>1, 'num_to_keep'=>5,
-      'mount_point'=>'/dev/sdh',
-      'pre_snapshot_tasks'=> [
-        'pg_dump some_app_production > /path/to/backups/on/snapshoted/volume/some_app_production.sql',
-        'mongodump -d another_app-production -o /path/to/backups/on/snapshoted/volume/'
-      ]
-    },
-    {
-      'hours_between'=>6, 'num_to_keep'=>4,
-      'mount_point'=>'/dev/sdh',
-      'pre_snapshot_tasks'=> []
-    },
-    {
-      'hours_between'=>24, 'num_to_keep'=>4,
-      'mount_point'=>'/dev/sda1',
-      'pre_snapshot_tasks'=> []
-    },
-    {
-      'hours_between'=>96, 'num_to_keep'=>4,
-      'mount_point'=>'/dev/sda1',
-      'pre_snapshot_tasks'=> []
-    }
-  ]
-  LOG_PATH = '/usr/local/var/drebs.log'
-  BACKUP_STATE_FILE_PATH = '/usr/local/var/drebs_state.json'
-  EMAIL_ON_EXCEPTION = 'admin@your.org'
-  EMAIL_HOST = 'imap.gmail.com'
-  EMAIL_PORT = 993
-  EMAIL_USERNAME = 'your smpt username'
-  EMAIL_PASSWORD = 'your smpt password'
-  ###### End User Config
-
-  def initialize(options = {})
-    @drebs_host_name = options[:drebs_host_name] || options['drebs_host_name'] || DREBS_HOST_NAME
-    @aws_access_key_id = options[:aws_access_key_id] || options['aws_access_key_id'] || AWS_ACCESS_KEY_ID  
-    @aws_secret_access_key = options[:aws_secret_access_key] || options['aws_secret_access_key'] || AWS_SECRET_ACCESS_KEY   
-    @region = options[:region] || options['region'] || REGION   
-    @backup_strategy = options[:backup_strategy] || options['backup_strategy'] || BACKUP_STRATEGY   
-    @log_path = options[:log_path] || options['log_path'] || LOG_PATH   
-    @backup_state_file_path = options[:backup_state_file_path] || options['backup_state_file_path'] || BACKUP_STATE_FILE_PATH   
-    @email_on_exception =  options[:email_on_exception] || options['email_on_exception'] || EMAIL_ON_EXCEPTION   
-    @log = Logger.new(@log_path, 0, 10 * 1024 * 1024)
-  end
-  
-  def setup_backup_data(backup_strategy=@backup_strategy)
-    backup_data = Marshal.load(Marshal.dump(@backup_strategy))
-    backup_data.collect{|a_backup_strategy|
-      a_backup_strategy['hours_until_next_run'] = a_backup_strategy['hours_between']
-      a_backup_strategy['previous_snapshots'] = []
-    }
-    return backup_data
-  end
-  
-  def save_backup_data()
-    open @backup_state_file_path, "w" do |h|
-      h.write(@backup_data.to_json)
+Main {
+
+  argument('config_path'){
+    argument :optional
+    description "A configuration file path. Passing 'example_config' uses an example configuration.  Give it a try: >drebs check_config example_config"
+  }
+
+  mode 'execute' do
+    def run
+      setup
+      @drebs.execute
     end
   end
-  
-  def load_backup_data()
-    @backup_data = JSON.parse(open(@backup_state_file_path).read)
+
+#  mode 'install_crontab' do
+#    def run
+#      setup
+#      @drebs.install_crontab
+#    end
+#  end
+
+  mode 'check_config' do
+    def run
+      setup
+      puts("No config errors found!")
+    end
   end
-  
-  def ec2(key_id=@aws_access_key_id, key=@aws_secret_access_key, region=@region)
-    return RightAws::Ec2.new(key_id, key, {:region=>region})
+
+  mode 'check_cloud' do
+    def run
+      setup
+      @drebs.check_cloud
+    end
   end
-  
-  def find_local_instance()
-    private_ip = UDPSocket.open {|s| s.connect("8.8.8.8", 1); s.addr.last}
-    ec2.describe_instances.each {|instance|
-      return instance if instance[:private_ip_address] == private_ip
-    }
-    return nil
+
+  mode 'shell' do
+    def run
+      setup
+      binding.pry(:hooks => Pry::Hooks.new, :prompt => proc{|*a| "drebs>"})
+    end
   end
-  
-  def find_local_ebs(mount_point='/dev/sdh')
-    return nil if not local_instance = find_local_instance
-    local_instance[:block_device_mappings].each {|volume|
-      return volume if volume[:device_name] == mount_point
-    }
-    return nil
+
+  def run
+    help!
   end
-  
-  def get_snapshot(snapshot_id)
-    ec2.describe_snapshots {|a_snapshot|
-      return a_snapshot if a_snapshot[:aws_id] == snapshot_id
-    }
+
+  db {
+    create_table :strategies do
+      String :config
+      String :snapshots
+      String :status
+      String :time_til_next_run
+      String :time_between_runs
+      String :num_to_keep
+      String :pre_snapshot_tasks
+      String :post_snapshot_tasks
+      String :mount_point
+    end unless table_exists? :strategies
+  }
+
+  def setup()
+    unless config_path = params['config_path'].value
+      raise 'Please provide a value for config_path.'
+    end
+    @config = load_config(config_path)
+
+    require File.join(lib_dir(), 'drebs', 'cloud.rb')
+    require File.join(lib_dir(), 'drebs', 'main.rb')
+
+    config_errors = Drebs::Main.check_config(example_config('print' => false), @config)
+    if config_errors.length == 0
+      @drebs = Drebs::Main.new('config' => @config, 'db' => db)
+    else
+      config_errors.each{|config_error| puts(config_error)}
+    end
   end
-  
-  def create_local_snapshot(pre_snapshot_tasks=nil, post_snapshot_tasks=nil, mount_point='/dev/sdh')
-    local_instance=find_local_instance
-    ip = local_instance[:ip_address]
-    instance_id = local_instance[:aws_instance_id]
-    volume_id = local_instance[:block_device_mappings].select{|m| m[:device_name]==mount_point}.first[:ebs_volume_id]
-    return nil if not ebs = find_local_ebs(mount_point)
-    pre_snapshot_tasks.each do |task|
-      result = systemu(task)
-      unless result[0].exitstatus == 0
-        error_string = "Error while executing pre-snapshot task: #{task} on #{@drebs_host_name} #{ip}:#{mount_point} #{instance_id}:#{volume_id} #{result[1]} #{result[2]}"
-        @log.error(error_string)
-        send_email("DREBS Error!", error_string)
+
+  def base_dir(*args, &block)
+    basedir = File.dirname(File.expand_path(__FILE__))
+    File.join(basedir, *args)
+  ensure
+    if block
+      begin
+        $LOAD_PATH.unshift(basedir)
+        block.call()
+      ensure
+        $LOAD_PATH.shift()
       end
-    end if pre_snapshot_tasks
-    snapshot = ec2.create_snapshot(ebs[:ebs_volume_id], "DREBS #{@drebs_host_name} #{ip}:#{mount_point} #{instance_id}:#{volume_id}")
-    Thread.new(snapshot[:aws_id], post_snapshot_tasks) {|snapshot_id, post_snapshot_tasks|
-      1.upto(500) {|a|
-        sleep(3)
-        break if get_snapshot(snapshot_id)[:aws_status] == 'completed'
-      }
-      post_snapshot_tasks.each do |task|
-        result = systemu(task)
-        unless result[0].exitstatus == 0
-          error_string = "Error while executing post-snapshot task: #{task} on #{@drebs_host_name} #{ip}:#{mount_point} #{instance_id}:#{volume_id} #{result[1]} #{result[2]}"
-          @log.error(error_string)
-          send_email("DREBS Error!", error_string)
-        end
-      end if post_snapshot_tasks
-    }
-    return snapshot
-  end
-  
-  def find_local_snapshots(mount_point='/dev/sdh')
-    return nil if not ebs = find_local_ebs(mount_point)
-    snapshots = []
-    ec2.describe_snapshots.each {|snapshot|
-      snapshots.push(snapshot) if snapshot[:aws_volume_id] == ebs[:ebs_volume_id]
-    }
-    return snapshots
+    end
   end
-  
-  def prune_backups(backup_data)
-    to_prune = {}
-    backup_data.collect {|a_backup_strategy|
-      if a_backup_strategy['previous_snapshots'].count > a_backup_strategy['num_to_keep']
-        to_prune[a_backup_strategy['previous_snapshots'].shift] = nil
-      end
-    }
-    to_prune.each_key {|snapshot_to_prune|
-      ec2.delete_snapshot(snapshot_to_prune) unless backup_data.any? {|a_backup_strategy|
-        a_backup_strategy['previous_snapshots'].include?(snapshot_to_prune)
-      }
-    }
+
+  def lib_dir(&block)
+    base_dir(['..', 'lib'], &block)
   end
-  
-  def send_email(subject, body, options = {})
-    host = options[:email_host] || options['email_host'] || EMAIL_HOST
-    port = options[:email_port] || options['email_port'] || EMAIL_PORT
-    username = options[:email_username] || options['email_username'] || EMAIL_USERNAME
-    password = options[:email_password] || options['email_password'] || EMAIL_PASSWORD
-    
-
-    msg = "Subject: #{subject}\n\n#{body}"
-    smtp = Net::SMTP.new 'smtp.gmail.com', 587
-    smtp.enable_starttls
-    smtp.start('gmail.com', username, password, :login) {|smtp|
-      smtp.send_message(msg, username, @email_on_exception)
-    }
+
+  def config_dir(&block)
+    base_dir(['..', 'config'], &block)
   end
 
-  def run_drebs_cron
-    begin
-      unless File.exists?(@backup_state_file_path)
-        @backup_data = setup_backup_data()
-
-        mount_points = @backup_data.map {|strategy| strategy['mount_point']}.compact.flatten.uniq
-
-        mount_points.map do |mount_point|
-          strategies = @backup_data.select{|strategy| strategy['mount_point'] == mount_point}
-          pre_snapshot_tasks = strategies.map {|strategy| strategy['pre_snapshot_tasks']}.compact.flatten.uniq
-          post_snapshot_tasks = strategies.map {|strategy| strategy['post_snapshot_tasks']}.compact.flatten.uniq
-          unless strategies.empty?
-            @log.info("creating snapshot of #{mount_point}")
-            snapshot = create_local_snapshot(pre_snapshot_tasks, post_snapshot_tasks, mount_point)
-          end
-          strategies.collect {|strategy|
-            strategy['previous_snapshots'].push(snapshot[:aws_id])
-          }
-        end
-
-
-        save_backup_data
-      else
-        load_backup_data
-        @backup_data.collect {|strategy|
-          strategy['hours_until_next_run'] -= 1
-        }
-        backup_now = @backup_data.collect {|strategy| strategy if strategy['hours_until_next_run'] <= 0}.compact
-
-
-        mount_points = @backup_data.map {|strategy| strategy['mount_point']}.compact.flatten.uniq
-
-        mount_points.map do |mount_point|
-          strategies = @backup_data.select{|strategy| strategy['mount_point'] == mount_point}
-          pre_snapshot_tasks = strategies.map {|strategy| strategy['pre_snapshot_tasks']}.compact.flatten.uniq
-          post_snapshot_tasks = strategies.map {|strategy| strategy['post_snapshot_tasks']}.compact.flatten.uniq
-          unless strategies.empty?
-            @log.info("creating snapshot of #{mount_point}")
-            snapshot = create_local_snapshot(pre_snapshot_tasks, post_snapshot_tasks, mount_point)
-          end
-          strategies.collect {|strategy|
-            strategy['previous_snapshots'].push(snapshot[:aws_id])
-            strategy['hours_until_next_run'] = strategy['hours_between']
-          }
-
-        end
-
-        prune_backups(@backup_data)
-        save_backup_data
-      end
-    rescue Exception => error
-      @log.error("Exception occured during backup: #{error.message}\n#{error.backtrace.join("\n")}")
-      send_email("DREBS Error! on #{@drebs_host_name}", "Host: #{@drebs_host_name} AWS Instance: #{find_local_instance[:aws_instance_id]}\n#{error.message}\n#{error.backtrace.join("\n")}")
+  def example_config(params)
+    config_path = File.join(config_dir, 'example.yml')
+    example_config = IO.read(config_path)
+    if params['print']
+      puts("Example YAML Config File (#{config_path}):")
+      puts(example_config) 
+      puts()
     end
+    YAML::load(example_config)
   end
 
-end
-
-if __FILE__ == $0
-  status = DATA.flock(File::LOCK_EX|File::LOCK_NB)
-  exit(42) unless status == 0
-  Main {
-    def run
-      drebs = DREBS.new
-      drebs.run_drebs_cron
+  def load_config(config_path)
+    if config_path == 'example_config'
+      example_config('print' => true)
+    else
+      YAML::load(IO.read(config_path))
     end
-  }
-end
-
-__END__
+  end
+}

data/config/example.yml

@@ -0,0 +1,28 @@
+---
+aws_access_key_id: EXAMPLE_KEY_ID
+aws_secret_access_key: EXAMPLE_ACCESS_KEY
+region: us-west-1
+strategies:
+- hours_between: 1
+  num_to_keep: 5
+  mount_point: /dev/sdh
+  pre_snapshot_tasks:
+  - pg_dump some_app_prodction > /path/to/backups/on/volume/being/snapshoted
+  - mongodump --db some_app_prodction --out /path/to/backups/on/volume/being/snapshoted
+  post_snapshot_tasks:
+- hours_between: 6
+  num_to_keep: 4
+  mount_point: /dev/sdh
+- hours_between: 24
+  num_to_keep: 4
+  mount_point: /dev/sda1
+- hours_between: 96
+  num_to_keep: 4
+  mount_point: /dev/sda1
+log_path: /usr/local/var/drebs.log
+email_on_exception: admin@your.org
+email_host: 'smtp.gmail.com'
+email_port: 587
+email_domain: 'gmail.com'
+email_user: = 'Your smtp username'
+email_password: = 'Your smtp password'

data/drebs.gemspec

@@ -3,19 +3,32 @@
 
 Gem::Specification::new do |spec|
 spec.name = "drebs"
-spec.version = "0.0.1"
+spec.version = "0.1.0"
 spec.platform = Gem::Platform::RUBY
 spec.summary = "drebs"
-spec.description = "description: drebs kicks the ass"
+spec.description = "drebs: Disaster Recovery for Elastic Block Store. An AWS EBS backup script."
 
 spec.files =
 ["README.md",
  "Rakefile",
  "bin",
  "bin/drebs",
+ "config",
+ "config/example.yml",
  "drebs.gemspec",
  "lib",
- "lib/drebs.rb"]
+ "lib/drebs",
+ "lib/drebs.rb",
+ "lib/drebs/cloud.rb",
+ "lib/drebs/main.rb",
+ "test",
+ "test/helper.rb",
+ "test/unit",
+ "test/unit/drebs",
+ "test/unit/drebs/drebs_test.rb",
+ "test/unit/drebs/main_test.rb",
+ "tmp_test_data",
+ "tmp_test_data/db.sqlite"]
 
 spec.executables = ["drebs"]
 spec.require_path = "lib"
@@ -23,21 +36,24 @@ spec.require_path = "lib"
 spec.test_files = nil
 
 
-spec.add_dependency(*["right_aws", " >= 3.0.0 "])
+spec.add_dependency(*["right_aws", ">= 3.1.0"])
 
-spec.add_dependency(*["logger", " >= 1.2.8 "])
+spec.add_dependency(*["logger", ">= 1.2.8"])
 
-spec.add_dependency(*["main", " >= 5.0.0 "])
+spec.add_dependency(*["main", ">= 5.2.0"])
 
-spec.add_dependency(*["systemu", " >= 2.4.2 "])
+spec.add_dependency(*["systemu", ">= 2.4.2"])
 
-spec.add_dependency(*["json", " >= 1.5.1 "])
+spec.add_dependency(*["json", ">= 1.5.1"])
+
+spec.add_dependency(*["pry", ">= 0.9.12.6"])
 
 
 spec.extensions.push(*[])
 
 spec.rubyforge_project = "DREBS"
-spec.author = "Garett Shulman"
-spec.email = "garett@dojo4.com"
+spec.authors = ["Garett Shulman", "Miles Matthias"]
+spec.email = "miles@dojo4.com"
 spec.homepage = "https://github.com/dojo4/drebs"
+spec.licenses = "Apache-2.0"
 end

data/lib/drebs.rb

@@ -5,7 +5,7 @@
 # DREBS libs
 #
   module DREBS
-    Version = '0.0.1' unless defined?(Version)
+    Version = '0.1.0' unless defined?(Version)
 
     def version
       DREBS::Version
@@ -13,12 +13,13 @@
 
     def dependencies
       {
-        'right_aws' => [ 'right_aws' , ' >= 3.0.0 ' ] , 
-        'logger'    => [ 'logger'    , ' >= 1.2.8 ' ] , 
-        'main'      => [ 'main'      , ' >= 5.0.0 ' ] , 
-        'systemu'   => [ 'systemu'   , ' >= 2.4.2 ' ] , 
-        'json'      => [ 'json'      , ' >= 1.5.1 ' ] , 
-        }
+        'right_aws' => ['right_aws', '>= 3.1.0'],
+        'logger'    => ['logger'   , '>= 1.2.8'],
+        'main'      => ['main'     , '>= 5.2.0'],
+        'systemu'   => ['systemu'  , '>= 2.4.2'],
+        'json'      => ['json'     , '>= 1.5.1'],
+        'pry'       => ['pry'      , '>= 0.9.12.6'],
+      }
     end
 
     def libdir(*args, &block)

data/lib/drebs/cloud.rb

@@ -0,0 +1,91 @@
+require 'right_aws'
+
+module Drebs
+  class Cloud
+    def initialize(config)
+      @config = config
+    end
+
+    def check_cloud
+      ec2
+      find_local_instance
+    end
+
+    def ec2
+      key_id = @config["aws_access_key_id"]
+      key = @config["aws_secret_access_key"]
+      region = @config["region"]
+      return RightAws::Ec2.new(key_id, key, {:region=>region})
+    end
+    
+    def find_local_instance
+      #find a better way... right-aws?
+      private_ip = UDPSocket.open{|s| s.connect("8.8.8.8", 1); s.addr.last}
+      ec2.describe_instances.each do |instance|
+        return instance if instance[:private_ip_address] == private_ip
+      end
+      return nil
+    end
+    
+    def find_local_ebs(mount_point)
+      return nil if not local_instance = find_local_instance
+      local_instance[:block_device_mappings].each do |volume|
+        return volume if volume[:device_name] == mount_point
+      end
+      return nil
+    end
+
+    def local_ebs_ids
+      @ebs_ids ||= find_local_instance[:block_device_mappings].map do |volume| 
+        volume[:ebs_volume_id]
+      end rescue nil
+    end
+    
+    def get_snapshot(snapshot_id)
+      ec2.describe_snapshots {|a_snapshot|
+        return a_snapshot if a_snapshot[:aws_id] == snapshot_id
+      }
+    end
+    
+    def create_local_snapshot(pre_snapshot_tasks, post_snapshot_tasks, mount_point)
+      local_instance=find_local_instance
+      ip = local_instance[:ip_address]
+      instance_id = local_instance[:aws_instance_id]
+      volume_id = local_instance[:block_device_mappings].select{|m| m[:device_name]==mount_point}.first[:ebs_volume_id]
+      return nil if not ebs = find_local_ebs(mount_point)
+      pre_snapshot_tasks.each do |task|
+        result, stdout, stderr = systemu(task)
+        unless result.exitstatus == 0
+          raise Exception(
+            "Error while executing pre-snapshot task: #{task} on #{ip}:#{mount_point} #{instance_id}:#{volume_id} "
+          )
+        end
+      end if pre_snapshot_tasks
+      snapshot = ec2.create_snapshot(ebs[:ebs_volume_id], "DREBS #{ip}:#{mount_point} #{instance_id}:#{volume_id}")
+      Thread.new(snapshot[:aws_id], post_snapshot_tasks) do |snapshot_id, post_snapshot_tasks|
+        1.upto(500) do |a|
+          sleep(3)
+          break if get_snapshot(snapshot_id)[:aws_status] == 'completed'
+        end
+        post_snapshot_tasks.each do |task|
+          result = systemu(task)
+          unless result.exitstatus == 0
+            raise Exception(
+              "Error while executing post-snapshot task: #{task} on #{ip}:#{mount_point} #{instance_id}:#{volume_id} "
+            )
+          end
+        end if post_snapshot_tasks
+      end
+      return snapshot
+    end
+    
+    def find_local_snapshots(mount_point)
+      return nil if not ebs = find_local_ebs(mount_point)
+      snapshots = []
+      ec2.describe_snapshots.each {|snapshot|
+        snapshots.push(snapshot) if snapshot[:aws_volume_id] == ebs[:ebs_volume_id]
+      }
+      return snapshots
+    end
+  end
+end

data/lib/drebs/main.rb

@@ -0,0 +1,179 @@
+module Drebs
+  class Main
+    attr_reader :config
+    attr_reader :cloud
+    attr_reader :db
+
+    def initialize(params)
+      unless @config = params['config'].clone()
+        raise "No config_file_path passed!" 
+      end
+      unless @db = params['db']
+        raise "No db passed!" 
+      end
+      update_strategies(@config.delete("strategies"))
+      @cloud = Drebs::Cloud.new(@config)
+      @log = Logger.new(@config["log_path"])
+      @log.level = Logger::WARN
+    end
+
+    def check_cloud
+      @cloud.check_cloud()
+    end
+
+    def Main.check_config(reference_config, other_config)
+      reference_config = reference_config.clone()
+      reference_strategy = reference_config.delete('strategies').last
+  
+      errors = []
+      
+      config_ok = reference_config.keys.each do |key|
+        config_ok = other_config.has_key?(key) and other_config[key] != nil and other_config[key] != ""
+        errors.push("Missing key/value for key: #{key}") unless config_ok
+      end
+  
+      strategies = other_config['strategies']
+      if strategies.is_a?(Array) and strategies.first()
+        strategies.each_with_index do |strategy, i|
+          strategies_ok = reference_strategy.keys.all?{|key|
+            unless strategy.has_key?(key) and strategy[key] != nil and strategy[key] != ""
+              errors.push("Missing key/value for key: #{key}") unless config_ok
+            end
+          }
+        end
+      else
+        errors.push("Missing strategies array")
+      end
+  
+      return errors
+    end
+
+
+    def update_strategies(new_strategies)
+      new_strategies.each do |strategy|
+        exists = @db[:strategies].filter(:config=>strategy.to_yaml).update(:status=>"active")
+        if exists==0
+          pre_snapshot_tasks = strategy['pre_snapshot_tasks']
+          pre_snapshot_tasks = pre_snapshot_tasks ? pre_snapshot_tasks.join(",") : ""
+          post_snapshot_tasks = strategy['post_snapshot_tasks']
+          post_snapshot_tasks = post_snapshot_tasks ? post_snapshot_tasks.join(",") : ""
+          @db[:strategies].insert(
+            :config=>strategy.to_yaml,
+            :snapshots=>"",
+            :status=>"active",
+            :time_til_next_run => strategy['hours_between'],
+            :time_between_runs => strategy['hours_between'],
+            :num_to_keep => strategy['num_to_keep'],
+            :pre_snapshot_tasks => pre_snapshot_tasks,
+            :post_snapshot_tasks => post_snapshot_tasks,
+            :mount_point => strategy['mount_point']
+          )
+        end
+      end
+      deactivate_filter = new_strategies.map do |strategy|
+        "(config != '#{strategy.to_yaml}' and status == 'active')"
+      end.join(" and ")
+      @db[:strategies].filter(deactivate_filter).update(:status => 'inactive')
+    end
+
+    def send_email(subject, body)
+      host = @config['email_host']
+      port = @config['email_port']
+      domain = @config['email_domain']
+      username = @config['email_user']
+      password = @config['email_password']
+      
+      msg = "Subject: #{subject}\n\n#{body}"
+      smtp = Net::SMTP.new(host, port)
+      smtp.enable_starttls
+      smtp.start(domain, username, password, :login) {|smtp|
+        smtp.send_message(msg, username, @config['email_on_exception'])
+      }
+    end
+  
+    def prune_backups(strategies)
+      to_prune = []
+
+      strategies.each do |strategy|
+        snapshots = strategy[:snapshots].split(",")
+        if snapshots.uniq==[nil]
+          @db[:strategies].filter(:config=>strategy[:config]).update(:snapshots => "")
+        elsif snapshots == []
+        elsif snapshots.count > strategy[:num_to_keep].to_i
+          # only remove it from EC2 if there are no other strategies with this snapshot
+          to_prune.push(Map.for({
+            :snapshot => snapshots.first,
+            :strategy => strategy
+          }))
+        end
+      end
+
+      to_prune.each do |prune_obj|
+        snapshot = prune_obj.snapshot
+        strategy = prune_obj.strategy
+
+        # make sure that no other strategies have this snapshot
+        strategies_with_snapshot = @db[:strategies].all.select{|strategy| strategy[:snapshots].split(',').include?(snapshot)}
+
+        if strategies_with_snapshot.count == 1
+          begin
+            @cloud.ec2.delete_snapshot(snapshot.split(":")[0])
+          rescue RightAws::AwsError => e
+            type = e.errors.first.first rescue ''
+            raise unless type == "InvalidSnapshot.NotFound"
+          end
+        end
+
+        # update the strategy's snapshots to include all except given snapshot
+        new_snapshots = strategy[:snapshots].split(',').delete_if{|snap| snap == snapshot}.join(',')
+        @db[:strategies].filter(:config=>strategy[:config]).update(:snapshots => new_snapshots)
+      end
+    end
+
+    def execute
+      active_strategies = @db[:strategies].filter({:status=>"active"})
+  
+      #Decrement time_til_next_run, save
+      active_strategies.each do |s|
+        @db[:strategies].filter(:config=>s[:config]).update(
+          :time_til_next_run => (s[:time_til_next_run].to_i - 1)
+        )
+      end
+  
+      active_strategies = @db[:strategies].filter({:status=>"active"})
+
+      #backup_now = strategies where time_til_next_run <= 0
+      backup_now = active_strategies.to_a.select{|s| s[:time_til_next_run].to_i <= 0}
+  
+      #loop over strategies grouped by mount_point
+      backup_now.group_by{|s| s[:mount_point]}.each do |mount_point, strategies|
+        pre_snapshot_tasks = strategies.map{|s| s[:pre_snapshot_tasks].split(",")}.flatten.uniq
+        post_snapshot_tasks = strategies.map{|s| s[:pre_snapshot_tasks].split(",")}.flatten.uniq
+  
+        @log.info("creating snapshot of #{mount_point}")
+        begin
+          snapshot = @cloud.create_local_snapshot(pre_snapshot_tasks, post_snapshot_tasks, mount_point)
+  
+          strategies.collect {|s|
+            snapshots = s[:snapshots].split(",")
+            snapshots.select!{|snapshot| @cloud.local_ebs_ids.include? snapshot.split(":")[1]}
+            snapshots.push(
+              s[:status]=='active' ?
+                "#{snapshot[:aws_id]}:#{snapshot[:aws_volume_id]}" : nil
+            )
+            @db[:strategies].filter(:config=>s[:config]).update(
+              :snapshots => snapshots.join(","),
+              :time_til_next_run => s[:time_between_runs]
+            )
+          }
+    
+        rescue Exception => error
+          @log.error("Exception occured during backup: #{error.message}\n#{error.backtrace.join("\n")}")
+          send_email("DREBS Error!", "AWS Instance: #{@cloud.find_local_instance[:aws_instance_id]}\n#{error.message}\n#{error.backtrace.join("\n")}")
+        end
+      end
+
+      prune_backups(@db[:strategies])
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,126 @@
+require "main"
+require "./lib/drebs/main.rb"
+require "./lib/drebs/cloud.rb"
+require "test/unit"
+require "yaml"
+
+EXAMPLE_CONFIG_PATH = "./config/example.yml"
+TMP_TEST_DATA_PATH = "./tmp_test_data/"
+
+class TestEC2
+  def initialize(*args)
+    @snapshots = []
+    @instances = [
+      {
+        private_ip_address: UDPSocket.open{|s| s.connect("8.8.8.8", 1); s.addr.last},
+        ip_address: "127.0.0.1",
+        aws_instance_id: "fake_instance",
+        block_device_mappings: [
+          {device_name: "/dev/sda1", ebs_volume_id: "fake_sda1"},
+          {device_name: "/dev/sdh", ebs_volume_id: "fake_sdh"}
+        ]
+      }
+    ]
+  end
+
+  def describe_snapshots()
+    @snapshots
+  end
+
+  def describe_instances()
+    @instances
+  end
+
+  def create_snapshot(ebs_volume_id, snapshot_name)
+    new_snapshot = {
+      aws_id: "fake_snap-"+rand(36**6).to_s(36),
+      aws_status: 'completed',
+      aws_volume_id: ebs_volume_id
+    }
+    @snapshots.push(new_snapshot)
+    new_snapshot
+  end
+end
+
+class TestCloud < Drebs::Cloud
+  def ec2; @ec2 ||= TestEC2.new; end
+end
+
+class TestContext
+  def TestContext.main_context(&block)
+    Main.new() do |main|
+      config = YAML::load(IO.read(EXAMPLE_CONFIG_PATH))
+      main.dotdir(TMP_TEST_DATA_PATH)
+      main.db() do
+
+        drop_table :strategies if table_exists? :strategies
+        create_table :strategies do
+          String :config
+          String :snapshots
+          String :status
+          String :time_til_next_run
+          String :time_between_runs
+          String :num_to_keep
+          String :pre_snapshot_tasks
+          String :post_snapshot_tasks
+        end
+
+        drop_table :snapshots if table_exists? :snapshots
+        create_table :snapshots do
+          String :aws_id
+          String :volume
+        end 
+      end
+
+      if block
+        begin
+          block.call(config, main.db())
+        ensure
+          begin
+            #fake main cleanup
+            main.db() do
+              drop_table :strategies if table_exists? :strategies
+              drop_table :snapshots if table_exists? :snapshots
+            end
+          rescue => e
+          end
+        end
+      end
+    end
+  end
+
+  def TestContext.cloud_context(config, &block)
+    cloud = TestCloud.new(config)
+    main_context do |main|
+      if block
+        begin
+          block.call(cloud)
+        ensure
+          begin
+            #fake cloud cleanup
+          rescue => e
+          end
+        end
+      end
+    end
+  end
+
+  def TestContext.drebs_context(&block)
+    main_context() do |config, db|
+      cloud_context(config) do |cloud|
+        drebs = Drebs::Main.new('config' => config, 'db' => db)
+
+        if block
+          begin
+            block.call(config, db, cloud, drebs)
+          ensure
+            begin
+              #fake drebs cleanup
+            rescue => e
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/test/unit/drebs/drebs_test.rb

@@ -0,0 +1,11 @@
+require "./test/helper.rb"
+
+class DrebsMain < Test::Unit::TestCase
+
+  def test_backups_get_pruned
+    TestContext.drebs_context() do |config, db, cloud, drebs|
+require 'pry'; binding.pry
+    end
+  end
+
+end

data/test/unit/drebs/main_test.rb

@@ -0,0 +1,44 @@
+require "./test/helper.rb"
+
+class TestMain < Test::Unit::TestCase
+
+  def test_can_check_config
+    TestContext.drebs_context() do |config, db, cloud, drebs|
+      config_errors = drebs.class.check_config(config, config)
+      assert(config_errors == [])
+    end
+  end
+
+  def test_check_config_finds_missing_keys
+    TestContext.drebs_context() do |config, db, cloud, drebs|
+      bad_config = config.clone
+      bad_config.delete(bad_config.keys.first)
+      config_errors = drebs.class.check_config(config, bad_config)
+      assert(config_errors.length == 1)
+      assert(config_errors[0].include?("Missing key/value"))
+    end
+  end
+
+  def test_can_save_strategies
+    TestContext.main_context() do |config, db|
+      assert(db[:strategies].all == [])
+      drebs = Drebs::Main.new('config' => config, 'db' => db)
+      assert(db[:strategies].all.length > 0)
+    end
+  end
+
+  def test_removed_strategies_get_deactivated
+    TestContext.main_context() do |config, db|
+      drebs = Drebs::Main.new('config' => config, 'db' => db)
+      num_active_strategies = db[:strategies].where(:status=>"active").count
+      num_inactive_strategies = db[:strategies].where(:status=>"inactive").count
+      config['strategies'] = config['strategies'][1..-1]
+      drebs = Drebs::Main.new('config' => config, 'db' => db)
+      num_active_strategies_dec = db[:strategies].where(:status=>"active").count
+      num_inactive_strategies_inc = db[:strategies].where(:status=>"inactive").count
+      assert(num_active_strategies == num_active_strategies_dec + 1)
+      assert(num_inactive_strategies == num_inactive_strategies_inc - 1)
+    end
+  end
+
+end

metadata

@@ -1,73 +1,103 @@
 --- !ruby/object:Gem::Specification
 name: drebs
 version: !ruby/object:Gem::Version
-  version: 0.0.1
-  prerelease: 
+  version: 0.1.0
 platform: ruby
 authors:
 - Garett Shulman
+- Miles Matthias
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-01 00:00:00.000000000 Z
+date: 2014-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: right_aws
-  requirement: &2152789220 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 3.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *2152789220
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.1.0
 - !ruby/object:Gem::Dependency
   name: logger
-  requirement: &2152788720 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.2.8
   type: :runtime
   prerelease: false
-  version_requirements: *2152788720
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.2.8
 - !ruby/object:Gem::Dependency
   name: main
-  requirement: &2152788240 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 5.0.0
+        version: 5.2.0
   type: :runtime
   prerelease: false
-  version_requirements: *2152788240
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 5.2.0
 - !ruby/object:Gem::Dependency
   name: systemu
-  requirement: &2152787760 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 2.4.2
   type: :runtime
   prerelease: false
-  version_requirements: *2152787760
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.4.2
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &2152787280 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.5.1
   type: :runtime
   prerelease: false
-  version_requirements: *2152787280
-description: ! 'description: drebs kicks the ass'
-email: garett@dojo4.com
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.5.1
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.12.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.12.6
+description: ! 'drebs: Disaster Recovery for Elastic Block Store. An AWS EBS backup
+  script.'
+email: miles@dojo4.com
 executables:
 - drebs
 extensions: []
@@ -76,30 +106,37 @@ files:
 - README.md
 - Rakefile
 - bin/drebs
+- config/example.yml
 - drebs.gemspec
 - lib/drebs.rb
+- lib/drebs/cloud.rb
+- lib/drebs/main.rb
+- test/helper.rb
+- test/unit/drebs/drebs_test.rb
+- test/unit/drebs/main_test.rb
+- tmp_test_data/db.sqlite
 homepage: https://github.com/dojo4/drebs
-licenses: []
+licenses:
+- Apache-2.0
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: DREBS
-rubygems_version: 1.8.11
+rubygems_version: 2.1.8
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: drebs
 test_files: []