rds_backup_service 0.0.1

data/.gitignore

@@ -0,0 +1,12 @@
+*.gem
+.bundle
+config/accounts.yml
+pkg/*
+*~
+.DS_Store
+*.tmproj
+log/*.log
+tmp/**
+doc/**
+.yardoc
+1

data/.yardopts

@@ -0,0 +1 @@
+--no-private --protected lib/**/*.rb  - API.md

data/API.md

@@ -0,0 +1,29 @@
+RDS Backup Service REST API
+================
+The REST API exposes only one API call:
+
+    POST /api/v1/backups
+
+POST to here with parameter `rds_instance` set to an RDS instance name.
+
+----------------
+Backups
+----------------
+A Backup represents a long-running backup process for an RDS.
+
+**RESOURCE LOCATION**: `/api/v1/backups`
+
+**REQUEST PARAMETERS**:
+
+  * rds_instance:   the name of the RDS instance to dump to S3 - **REQUIRED**
+  * email:          an optional email address to send a message to on completion
+  
+
+**RESULT FIELDS**:
+
+  * rds_instance:   the name of the RDS instance being dumped to S3
+  * account_name:   the name of account from accounts.yml, if determined
+  * backup_status:  an HTTP-like status code for the process
+  * status_url:     a signed S3 URL to this job's JSON status as updated
+  * status_message: a descriptive progress message
+  * files:          an Array of output files for this job, including S3 URLs

data/Gemfile

@@ -0,0 +1,2 @@
+source "http://rubygems.org"
+gemspec

data/LICENSE.TXT

@@ -0,0 +1,10 @@
+----------------
+*MIT License*
+
+Copyright (c) 2012 Benton Roberts
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,125 @@
+RDS Backup Service
+================
+Fire-and-forget SQL backups of Amazon Web Services' RDS databases into S3.
+
+----------------
+What is it?
+----------------
+A REST-style web service and middleware library for safely dumping the contents
+of a live AWS Relational Database Service instance into a compressed SQL file.
+
+The service has only one API call (a POST to `/api/v1/backups`), which spawns
+a long-running worker process. The worker performs the following steps:
+
+1. Snapshots the original RDS
+2. Creates a new RDS instance based on the snapshot
+3. Configures the new RDS as needed, including rebooting for Parameter Group
+4. Connects to the RDS and dumps the database contents, compressing on the fly
+5. Uploads the compressed SQL file to S3, and optionally emails its URL
+6. Deletes up the snapshot, temporary instance, and local SQL dump
+
+----------------
+Why is it?
+----------------
+Safely and consistently grabbing the contents of a loaded, live RDS instance
+is a pain (if it has no existing slave). Though the steps are simple, they're
+brittle, slow, and involve lots of waiting for indeterminate time periods.
+
+----------------
+Installation
+----------------
+First install the dependencies:
+
+* Ruby 1.9, rake, and bundler
+* [Redis][] (for [Resque][] workers), or [DelayedJob][] (library only for now)
+
+The RDS Backup Service can be installed as a standalone application or as a
+Rack middleware library.
+
+###   To install as an application  ###
+
+Install project dependencies, fetch the code, and bundle up.
+
+    gem install rake bundler
+    git clone https://github.com/benton/rds_backup_service.git
+    cd rds_backup_service
+    bundle
+
+###   To install as a library   ###
+
+1) Install the gem, or add it as a Bundler dependency and `bundle`.
+
+      gem install rds_backup_service
+
+2) Require the middleware from your Rack application, then insert it
+  in the stack:
+
+      require 'rds_backup_service'
+      ...
+      config.middleware.use RDSBackup::Service  # (Rails application.rb)
+                                                # or
+      use RDSBackup::Service                    # (Sinatra)
+
+3) If desired, require the SecurityGroup setup task in your `Rakefile`:
+
+      require 'rds_backup_service/tasks'
+
+----------------
+Configuration and Setup
+----------------
+Two configuration files are required _(see included examples)_:
+
+* `./config/accounts.yml` or `ENV['RDS_ACCOUNTS_FILE']`
+
+  This file defines three different types of AWS accounts: the various RDS
+  accounts to grab SQL from; the S3 account where the SQL output
+  will be written; and an optional EC2 account, which is used by the
+  `setup:rds_backup_groups` rake task to perform post-configuration setup.
+
+* `./config/settings.yml` or `ENV['RDS_SETTINGS_FILE']`
+
+  This file defines the S3 bucket name for the output, plus some other options.
+
+Once these files have been edited, run `rake setup:rds_backup_groups`, which:
+
+* makes sure the configured Security Groups exist in all the RDS and EC2 accounts
+* opens the RDS Security Group in each RDS account to the EC2 Security Group
+* checks to see that the current host is in the EC2 Security Group (when in EC2)
+
+----------------
+Usage
+----------------
+The service is run in the standard Rack manner:
+
+    bundle exec rackup
+
+The entry point for the REST API is `/api/v1/backups`
+(See the {file:API.md API documentation})
+
+The Resque workers are run with:
+
+    QUEUE=backups rake resque:work
+
+
+----------------
+DelayedJob
+----------------
+The library (though not the service) can be used with DelayedJob.
+Place some code like this in your Controller or Model:
+
+    require 'rds_backup_service'
+    ...
+    job = RDSBackup::Job.new(params[:rds_id])
+    job.write_to_s3
+    Delayed::Job.enqueue RDSBackup::DelayedJob.new(job.rds_id, {
+        'backup_id' => job.backup_id,
+        'requested' => job.requested.to_s,
+        'email'     => params[:email],
+      })
+
+
+
+[Redis]: http://redis.io/
+[Resque]: https://github.com/defunkt/resque
+[DelayedJob]: https://github.com/collectiveidea/delayed_job
+

data/Rakefile

@@ -0,0 +1,18 @@
+# Set up bundler
+%w{rubygems bundler bundler/gem_tasks}.each {|dep| require dep}
+bundles = [:default]
+Bundler.setup(:default)
+case ENV['RACK_ENV']
+when 'development'  then Bundler.setup(:default, :development)
+when 'test'         then Bundler.setup(:default, :development, :test)
+end
+
+require 'rds_backup_service'
+require 'resque/tasks'
+ENV['TERM_CHILD'] = '1'
+
+# Load all tasks from 'lib/tasks'
+Dir["#{File.dirname(__FILE__)}/lib/tasks/*.rake"].sort.each {|ext| load ext}
+
+desc 'Default: runs all tests'
+task :default => :spec

data/config/accounts.example.yml

@@ -0,0 +1,46 @@
+# The S3 account is used to store all job output.
+s3account-name:
+  credentials:
+    aws_access_key_id: XXXXXXXXXXXXXXXXXXXX
+    aws_secret_access_key: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+  provider: AWS
+  service: Storage
+
+# This service should run in an EC2 account.
+# The creds here are used only for automated SecurityGroup setup.
+# If you don't run 'rake setup:rds_backup_groups',
+# then these won't be used -- but one-time EC2 setup must be done manually.
+ec2-account-for-service:
+  credentials:
+    aws_access_key_id: XXXXXXXXXXXXXXXXXXXX
+    aws_secret_access_key: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+  provider: AWS
+  service: Compute
+
+# The RDS instances in all the RDS accounts are avaliable for backup.
+
+rds-development:
+  credentials:
+    aws_access_key_id: XXXXXXXXXXXXXXXXXXXX
+    aws_secret_access_key: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+  delay: 120
+  provider: AWS
+  service: RDS
+  exclude_resources:
+  - parameters
+  - security_groups
+  - parameter_groups
+  - snapshots
+
+rds-production:
+  credentials:
+    aws_access_key_id: XXXXXXXXXXXXXXXXXXXX
+    aws_secret_access_key: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+  delay: 120
+  provider: AWS
+  service: RDS
+  exclude_resources:
+  - parameters
+  - security_groups
+  - parameter_groups
+  - snapshots

data/config/settings.yml

@@ -0,0 +1,26 @@
+# REQUIRED - The S3 bucket name where all backups will be saved
+backup_bucket: rdsbackups
+
+# Backups can be prefixed with backup_prefix if desired.
+# (Do not include a leading or trailing /)
+# default value = nil
+
+#backup_prefix: testing
+
+# The rds_security_group must be defined in each
+# RDS account, and be open to this server, so that the
+# mysqldump utlility can connect.
+# default value = rds-backup-service
+
+#rds_security_group: rds-backup-service
+
+# The ec2_security_group must be defined in the EC2 account
+# under which this service runs.
+# default value = rds-backup-service
+
+#ec2_security_group: rds-backup-service
+
+# Where to store DB data while compressing it, before upload
+# defaults to Ruby's Dir.tmpdir
+
+#tmp_dir: "/tmp"

data/config.ru

@@ -0,0 +1,2 @@
+require './lib/rds_backup_service/service'
+run RDSBackup::Service

data/lib/rds_backup_service/config.rb

@@ -0,0 +1,68 @@
+module RDSBackup
+  # models logic for post-configuration setup
+  module Config
+
+    # Attempts to set up the EC2 and RDS security groups as specified in the
+    # configuration. Raises an Exception on errors. Best if run from EC2.
+    def self.setup_security_groups(logger = nil)
+      log = logger || RDSBackup.default_logger(STDOUT)
+      # Configuration
+      log.info "Scanning system..."
+      (system = Ohai::System.new).all_plugins
+      log.info "Reading config files..."
+      settings = RDSBackup.settings
+      ec2_group_name = settings['ec2_security_group']
+      rds_group_name = settings['rds_security_group']
+      ec2 = RDSBackup.ec2
+
+      # EC2 Security Group creation
+      log.info "Checking EC2 for Security Group #{ec2_group_name}"
+      unless ec2_group = ec2.security_groups.get(ec2_group_name)
+        log.info "Creating EC2 Security group #{ec2_group_name}"
+        ec2_group = ec2.security_groups.create(:name => ec2_group_name,
+          :description => 'Created by rds_backup_service')
+      end
+
+      # RDS Security Group creation and authorization
+      RDSBackup.rds_accounts.each do |account_name, account_data|
+        log.info "Checking account #{account_name} for "+
+          "RDS Security group #{rds_group_name}"
+        rds = ::Fog::AWS::RDS.new(account_data[:credentials])
+        rds_group = rds.security_groups.get rds_group_name
+        unless rds_group
+          log.info "Creating security group #{rds_group_name} in #{account_name}"
+          rds_group = rds.security_groups.create(:id => rds_group_name,
+            :description => 'Created by rds_backup_service')
+        end
+        # Apply EC2 authorization to RDS Security Groups
+        owner = ec2.security_groups.first.owner_id
+        authorized = false
+        rds_group.ec2_security_groups.each do |authorization|
+          if (authorization['EC2SecurityGroupName'] == ec2_group_name) &&
+            (authorization['EC2SecurityGroupOwnerId'] == owner)
+              authorized = true
+          end
+        end
+        unless authorized
+          log.info "Authorizing EC2 Group for #{account_name}/#{rds_group_name}"
+          rds_group.authorize_ec2_security_group(ec2_group_name, owner)
+        end
+      end
+
+      # EC2 Security Group check for this host
+      unless system[:ec2]
+        log.warn "Not running in EC2 - open RDS groups to this host!"
+      else
+        unless this_host = ec2.servers.get(system[:ec2][:instance_id])
+          log.warn "Not running in EC2 account #{s3_acc_name}!"
+        else
+          log.info "Running in EC2. Current Security Groups = #{this_host.groups}"
+          unless this_host.groups.include? ec2_group_name
+            log.warn "This host is not in Security Group #{ec2_group_name}!"
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/rds_backup_service/model/backup_job.rb

@@ -0,0 +1,252 @@
+require 'fog_tracker'
+require 'fileutils'
+require 'json'
+module RDSBackup
+  # Backs up the contents of a single RDS database to S3
+  class Job
+
+    @queue = :backups
+
+    attr_reader :backup_id, :rds_id, :account_name, :options
+    attr_reader :status, :status_url, :message, :files, :requested
+
+    # Constructor.
+    # @param [String] rds_instance_id the ID of the RDS instance to backup
+    # @param [Hash] options optional additional parameters:
+    #  - backup_id - a unique ID for this job, if necessary
+    #  - requested - a Time when this job was requested
+    #  - email - an email address to be notified on completion
+    #  - logger - a Logger object, for printing this job's ongoing status
+    def initialize(rds_instance_id, options = {})
+      @rds_id, @options = rds_instance_id, options
+      @backup_id  = options['backup_id'] || "%016x" % (rand * 0xffffffffffffffff)
+      @requested  = options['requested'] ? Time.parse(options['requested']) : Time.now
+      @status     = 200
+      @message    = "queued"
+      @files      = []
+      @config     = RDSBackup.settings
+      @bucket     = @config['backup_bucket']
+      @s3_path    = (@config['backup_prefix'] ? "#{@config['backup_prefix']}/" : "")+
+                    "#{requested.strftime("%Y/%m/%d")}/#{rds_id}/#{backup_id}"
+      @snapshot_id  = "rds-backup-service-#{rds_id}-#{backup_id}"
+      @new_rds_id   = "rds-backup-service-#{backup_id}"
+      @new_password = "#{backup_id}"
+      @account_name = options['account_name']
+    end
+
+    # returns a JSON-format String representation of this backup job
+    def to_json
+      JSON.pretty_generate({
+        rds_instance:   rds_id,
+        account_name:   account_name,
+        backup_status:  status,
+        status_message: message,
+        status_url:     status_url,
+        files:          files,
+      })
+    end
+
+    # Writes this job's JSON representation to S3
+    def write_to_s3
+      status_path = "#{@s3_path}/status.json"
+      s3.put_object(@bucket, status_path, "#{to_json}\n")
+      unless @status_url
+        expire_date = Time.now + (3600 * 24)  # one day from now
+        @status_url = s3.get_object_http_url(@bucket, status_path, expire_date)
+        s3.put_object(@bucket, status_path, "#{to_json}\n")
+      end
+    end
+
+    # Entry point for the Resque framework.
+    # Parameters are the same as for #initialize()
+    def self.perform(rds_instance_id, options = {})
+      job = Job.new(rds_instance_id, options)
+      begin
+        job.perform_backup
+      rescue Resque::TermException => e
+        ::Resque.enqueue_to(:backups, Job, rds_instance_id,
+          options.merge(backup_id: job.backup_id, requested: job.requested.to_s))
+        job.update_status "Terminated on interrupt signal - requeued"
+      end
+    end
+
+    # Top-level, long-running method for performing the backup.
+    def perform_backup
+      begin
+        prepare_backup
+        update_status "Backing up #{rds_id} from account #{account_name}"
+        create_disconnected_rds
+        download_data_from_tmp_rds    # populates @sql_file
+        delete_disconnected_rds
+        upload_output_to_s3
+        update_status "Backup of #{rds_id} complete"
+        send_mail
+      rescue Exception => e
+        update_status "ERROR: #{e.message.split("\n").first}", 500
+        raise e
+      end
+    end
+
+    # Step 1 of the overall process - create a disconnected copy of the RDS
+    def create_disconnected_rds(new_rds_name = nil)
+      @new_rds_id = new_rds_name if new_rds_name
+      prepare_backup unless @original_server  # in case run as a convenience method
+      snapshot_original_rds
+      create_tmp_rds_from_snapshot
+      configure_tmp_rds
+      wait_for_new_security_group
+      wait_for_new_parameter_group # (reboots as needed)
+      destroy_snapshot
+    end
+
+    # Queries RDS for any pre-existing entities associated with this job.
+    # Also waits for the original RDS to become ready.
+    def prepare_backup
+      unless @original_server = RDSBackup.get_rds(rds_id)
+        names = RDSBackup.rds_accounts.map {|name, account| name }
+        raise "Unable to find RDS #{rds_id} in accounts #{names.join ", "}"
+      end
+      @account_name = @original_server.tracker_account[:name]
+      @rds = ::Fog::AWS::RDS.new(
+        RDSBackup.rds_accounts[@account_name][:credentials])
+      @snapshot         = @rds.snapshots.get @snapshot_id
+      @new_instance     = @rds.servers.get @new_rds_id
+    end
+
+    # Snapshots the original RDS
+    def snapshot_original_rds
+      unless @new_instance || @snapshot
+        update_status "Waiting for RDS instance #{@original_server.id}"
+        @original_server.wait_for { ready? }
+        update_status "Creating snapshot #{@snapshot_id} from RDS #{rds_id}"
+        @snapshot = @rds.snapshots.create(id: @snapshot_id, instance_id: rds_id)
+      end
+    end
+
+    # Creates a new RDS from the snapshot
+    def create_tmp_rds_from_snapshot
+      unless @new_instance
+        update_status "Waiting for snapshot #{@snapshot_id}"
+        @snapshot.wait_for { ready? }
+        update_status "Booting new RDS #{@new_rds_id} from snapshot #{@snapshot.id}"
+        @rds.restore_db_instance_from_db_snapshot(@snapshot.id,
+          @new_rds_id, 'DBInstanceClass' => @original_server.flavor_id)
+        @new_instance = @rds.servers.get @new_rds_id
+      end
+    end
+
+    # Destroys the snapshot if it exists
+    def destroy_snapshot
+      if (@snapshot = @rds.snapshots.get @snapshot_id)
+        update_status "Deleting snapshot #{@snapshot.id}"
+        @snapshot.destroy
+      end
+    end
+
+    # Updates the Master Password and applies the configured RDS Security Group
+    def configure_tmp_rds
+      update_status "Waiting for instance #{@new_instance.id}..."
+      @new_instance.wait_for { ready? }
+      update_status "Modifying RDS attributes for new RDS #{@new_instance.id}"
+      @rds.modify_db_instance(@new_instance.id, true, {
+        'DBParameterGroupName'  => @original_server.db_parameter_groups.
+                                    first['DBParameterGroupName'],
+        'DBSecurityGroups'      => [ @config['rds_security_group'] ],
+        'MasterUserPassword'    => @new_password,
+      })
+    end
+
+    # Wait for the new RDS Security Group to become 'active'
+    def wait_for_new_security_group
+      old_group_name = @config['rds_security_group']
+      update_status "Applying security group #{old_group_name}"+
+        " to #{@new_instance.id}"
+      @new_instance.wait_for {
+        new_group = (db_security_groups.select do |group|
+          group['DBSecurityGroupName'] == old_group_name
+        end).first
+        (new_group ? new_group['Status'] : 'Unknown') == 'active'
+      }
+    end
+
+    # Wait for the new RDS Parameter Group to become 'in-sync'
+    def wait_for_new_parameter_group
+      old_name = @original_server.db_parameter_groups.first['DBParameterGroupName']
+      update_status "Applying parameter group #{old_name} to #{@new_instance.id}"
+      job = self  # save local var for closure in wait_for, below
+      @new_instance.wait_for {
+        new_group = (db_parameter_groups.select do |group|
+          group['DBParameterGroupName'] == old_name
+        end).first
+        status = (new_group ? new_group['ParameterApplyStatus'] : 'Unknown')
+        if (status == "pending-reboot")
+          job.update_status "Rebooting RDS #{id} to apply ParameterGroup #{old_name}"
+          reboot and wait_for { ready? }
+        end
+        status == 'in-sync' && ready?
+      }
+    end
+
+    # Connects to the RDS server, and dumps the database to a temp dir
+    def download_data_from_tmp_rds
+      @new_instance.wait_for { ready? }
+      db_name = @original_server.db_name
+      db_user = @original_server.master_username
+      update_status "Dumping database #{db_name} from #{@new_instance.id}"
+      dump_time   = @snapshot ? Time.parse(@snapshot.created_at.to_s) : Time.now
+      date_stamp  = dump_time.strftime("%Y-%m-%d-%H%M%S")
+      @sql_file = "#{@config['tmp_dir']}/#{@s3_path}/#{db_name}.#{date_stamp}.sql.gz"
+      hostname  = @new_instance.endpoint['Address']
+      dump_cmd  = "mysqldump -u #{db_user} -h #{hostname} "+
+        "-p#{@new_password} #{db_name} | gzip >#{@sql_file}"
+      FileUtils.mkpath(File.dirname @sql_file)
+      @log.debug "Executing command: #{dump_cmd}"
+      `#{dump_cmd}`
+    end
+
+    # Destroys the temporary RDS instance
+    def delete_disconnected_rds
+      update_status "Deleting RDS instance #{@new_instance.id}"
+      @new_instance.destroy
+    end
+
+    # Uploads the compressed SQL file to S3
+    def upload_output_to_s3
+      update_status "Uploading output file #{::File.basename @sql_file}"
+      dump_path = "#{@s3_path}/#{::File.basename @sql_file}"
+      s3.put_object(@bucket, dump_path, File.read(@sql_file))
+      upload = s3.directories.get(@bucket).files.get dump_path
+      @files = [ {
+        name: ::File.basename(@sql_file),
+        size: upload.content_length,
+        url:  upload.url(Time.now + (3600 * 24 * 30))  # 30 days from now
+      } ]
+      @log.info "Deleting tmp directory #{File.dirname @sql_file}"
+      FileUtils.rm_rf(File.dirname @sql_file)
+    end
+
+    # Writes a new status message to the log, and writes the job info to S3
+    def update_status(message, new_status = nil)
+      @log      = @options['logger'] || RDSBackup.default_logger(STDOUT)
+      @message  = message
+      @status   = new_status if new_status
+      @status == 200 ? (@log.info message) : (@log.error message)
+      write_to_s3
+    end
+
+    # Sends a status email
+    def send_mail
+      return unless @options['email']
+      @log.info "Emailing #{@options['email']}..."
+      begin
+        RDSBackup::Email.new(self).send!
+      rescue Exception => e
+        @log.warn "Error sending email: #{e.message.split("\n").first}"
+      end
+    end
+
+    # lazily initializes and returns S3 connection
+    def s3 ; @s3 ||= RDSBackup.s3 end
+
+  end
+end

data/lib/rds_backup_service/model/delayed_job.rb

@@ -0,0 +1,10 @@
+module RDSBackup
+  # convenience wrapper class for DelayedJob.
+  # Parameters are the same as for RDSBackup::Job.initialize()
+  class DelayedJob < Struct.new(:rds_id, :options)
+    # Entry point for the DelayedJob framework.
+    def perform
+      RDSBackup::Job.new(rds_id, options).perform_backup
+    end
+  end
+end

data/lib/rds_backup_service/model/email.rb

@@ -0,0 +1,42 @@
+require 'mail'
+module RDSBackup
+  # an email representation of a Job, that can send itself to recipients.
+  class Email
+
+    attr_reader :job
+
+    # constructor - requires an RDSBackup::Job
+    def initialize(backup_job)
+      @job = backup_job
+    end
+
+    # Attempts to send email through local ESMTP port 25.
+    # Raises an Exception on failure.
+    def send!
+      raise "job #{job.backup_id} has no email option" unless job.options['email']
+      main_text = body_text # define local variables for closure over Mail.new
+      recipients, header = job.options['email'],
+      header = "Backup of RDS #{job.rds_id} (job ID #{job.backup_id})"
+      mail = Mail.new do
+        from    'rdsbackupservice@mdsol.com'
+        to      recipients
+        subject header
+        body    "#{main_text}\n"
+      end
+      mail.deliver!
+    end
+
+    # defines the body of a Job's status email
+    def body_text
+      msg = "Hello.\n\n"
+      if job.status == 200
+        msg += "Your backup of database #{job.rds_id} is complete.\n"+
+          (job.files.empty? ? "" : "Output is at #{job.files.first[:url]}\n")
+      else
+        msg += "Your backup is incomplete. (job ID #{job.backup_id})\n"
+      end
+      msg += "Job status: #{job.message}"
+    end
+
+  end
+end

data/lib/rds_backup_service/rds_backup_service.rb

@@ -0,0 +1,72 @@
+# The top-level project module. Contains some static helper methods.
+module RDSBackup
+  require 'fog'
+  require 'tmpdir'
+  require 'ohai'
+  PROJECT_DIR = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
+  require "#{PROJECT_DIR}/lib/rds_backup_service/version"
+
+  # Loads account information defined in config/accounts.yml, or
+  # ENV['RDS_ACCOUNTS_FILE'].
+  # @param account_file the path to a YAML file (see accounts.yml.example).
+  # @return [Hash] a Hash representing the account info.
+  # The keys in each account Hash are converted to Symbols.
+  def self.read_accounts(account_file = ENV['RDS_ACCOUNTS_FILE'])
+    YAML::load(File.read(account_file || "./config/accounts.yml")).
+      inject({}) {|a,b| a[b[0]] = b[1].symbolize_keys ; a }
+  end
+
+  # Loads account information, and returns only those
+  # entries that repesent RDS accounts.
+  def self.rds_accounts
+    RDSBackup.read_accounts.select{|id,acc| acc[:service] == 'RDS'}
+  end
+
+  # Returns a new connection to the AWS EC2 service (Fog::Compute::AWS)
+  def self.ec2
+    accts = RDSBackup.read_accounts.select{|id,acc| acc[:service] == 'Compute'}
+    raise "At least one S3 account must be defined" if accts.empty?
+    Fog::Compute::AWS.new(accts.first[1][:credentials])
+  end
+
+  # Returns a new connection to the AWS S3 service (Fog::Storage::AWS)
+  def self.s3
+    accts = RDSBackup.read_accounts.select{|id,acc| acc[:service] == 'Storage'}
+    raise "At least one S3 account must be defined" if accts.empty?
+    Fog::Storage::AWS.new(accts.first[1][:credentials])
+  end
+
+  # Returns the configuration Hash read from config/s3_account.yml
+  # or ENV['RDSDUMP_SETTINGS_FILE'].
+  def self.settings(settings_file = ENV['RDS_SETTINGS_FILE'])
+    { # here are some defaults
+      'rds_security_group'  => 'rds-backup-service',
+      'ec2_security_group'  => 'rds-backup-service',
+      'tmp_dir'             => Dir.tmpdir,
+    }.merge(YAML::load(
+      File.read(settings_file || "./config/settings.yml")))
+  end
+
+  # Defines the root URI path of the web service.
+  # @return [String] the root URI path of the web service
+  def self.root
+    "/api/v#{RDSBackup::API_VERSION}"
+  end
+
+  def self.default_logger(output = nil)
+    logger = ::Logger.new(output)
+    logger.sev_threshold = Logger::INFO
+    logger.formatter = proc {|lvl, time, prog, msg| "#{lvl}: #{msg}\n"}
+    logger
+  end
+
+  # Returns a Fog RDS entity for a given an RDS ID. Polls all accounts.
+  # The account name is attached to the result as 'tracker_account[:name]'.
+  # @param [String] the name of the desired RDS entity
+  # @return [Fog::AWS::RDS::Server] the RDS instance, or nil if not found
+  def self.get_rds(rds_id)
+    ::FogTracker::Tracker.new(RDSBackup.rds_accounts).update.
+      select{|rds| rds.identity == rds_id}.first
+  end
+
+end

data/lib/rds_backup_service/service.rb

@@ -0,0 +1,59 @@
+#!/usr/bin/env ruby
+require File.join(File.dirname(__FILE__), '..', 'rds_backup_service')
+require 'sinatra/base'
+require 'fog_tracker'
+require 'resque'
+
+module RDSBackup
+  # A RESTful web service for backing up RDS databases to S3.
+  # See the README for running this Rack-compliant service.
+  class Service < Sinatra::Base
+
+    # configure logging when not in test mode
+    configure :production, :development do
+      @log = RDSBackup.default_logger(STDOUT)
+      enable :logging
+    end
+
+    # on startup, load account information and start tracking RDS instances
+    configure do
+      @log.info "Loading account information..."
+      tracker = FogTracker::Tracker.new(RDSBackup.rds_accounts, :logger => @log)
+      @log.info "Starting tracker..."
+      tracker.update
+      tracker.start
+      set :tracker, tracker
+    end
+
+    before do ; content_type 'application/json' end   # serve JSON
+
+    ######## POST /api/vXXX/backups ########
+    # Queues a Job for a given :rds_instance
+    post "#{RDSBackup.root}/backups" do
+      rds_id  = params[:rds_instance]
+      servers = settings.tracker['*::AWS::RDS::servers']
+      rds     = (servers.select {|rds| rds.identity == rds_id}).first
+
+      # check for errors
+      if ! rds_id
+        return [ 400, { errors: ["Parameter 'rds_instance' required"]}.to_json ]
+      elsif ! (servers.map {|r| r.identity}).include?(rds_id)
+        return [ 404, { errors: ["RDS instance #{rds_id} not found"]}.to_json ]
+      end
+
+      # request is OK - queue up a BackupJob
+      job = Job.new(rds_id, params.merge(account_name: rds.tracker_account[:name]))
+      logger.info "Queuing backup of RDS #{rds_id} in account #{job.account_name}"
+      job.write_to_s3
+      ::Resque.enqueue_to(:backups, Job, job.rds_id, job.options.
+        merge({'backup_id' => job.backup_id, 'requested' => job.requested.to_s}))
+
+      [ 201,                                # return HTTP_CREATED, and
+        { 'Location' => job.status_url },   # point to the S3 document
+        "#{job.to_json}\n"                  # and return the job as JSON
+      ]
+    end
+
+    run! if app_file == $0  # start the server if ruby file executed directly
+  end
+end

data/lib/rds_backup_service/tasks.rb

@@ -0,0 +1,2 @@
+require 'rake'
+Dir["#{File.dirname(__FILE__)}/../tasks/*.rake"].sort.each {|ext| load ext}

data/lib/rds_backup_service/version.rb

@@ -0,0 +1,7 @@
+module RDSBackup
+  # The version of this Gem / Library.
+  VERSION = "0.0.1"
+
+  # This library's version of the REST API.
+  API_VERSION = "1"
+end

data/lib/rds_backup_service.rb

@@ -0,0 +1,8 @@
+%w{rubygems bundler}.each {|lib| require lib}
+Bundler.setup
+
+# Load all the other files in this library, except the service
+%w{ version rds_backup_service config
+    model/backup_job model/delayed_job model/email }.each do |file|
+  require File.join(File.dirname(__FILE__), "rds_backup_service/#{file}")
+end

data/lib/tasks/setup_groups.rake

@@ -0,0 +1,6 @@
+desc "Attempts to set up the rds_backup_service security groups"
+namespace :setup do
+  task :rds_backup_groups do
+    RDSBackup::Config.setup_security_groups
+  end
+end

data/rds_backup_service.gemspec

@@ -0,0 +1,41 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "rds_backup_service/version"
+
+Gem::Specification.new do |s|
+  s.name        = "rds_backup_service"
+  s.version     = RDSBackup::VERSION
+  s.authors     = ["Benton Roberts"]
+  s.email       = ["benton@bentonroberts.com"]
+  s.homepage    = "http://github.com/benton/rds_backup_service"
+  s.summary     = %q{Provides a REST API for backing up live RDS instances }+
+                  %q{to S3 as a compressed SQL file.}
+  s.description = %q{Provides a REST API for backing up live RDS instances }+
+                  %q{to S3 as a compressed SQL file.}
+  s.rubyforge_project = "rds_backup_service"
+
+  # This project is both a Gem and an Application,
+  # so the Gemfile.lock is included in the repo for application users,
+  # but excluded from the packaged Gem, for middleware use.
+  git_files       = `git ls-files`.split("\n")    # Read all files in the repo,
+  git_files.delete "Gemfile.lock"                 # remove Gemfile.lock, and
+  s.files         = git_files                     # use the result in the Gem.
+
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # Runtime dependencies
+  s.add_dependency "sinatra"
+  s.add_dependency "fog_tracker", ">=0.4.0"
+  s.add_dependency "resque"
+  s.add_dependency "mail"
+  s.add_dependency "ohai"
+
+  # Development / Test dependencies
+  s.add_development_dependency "rake"
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "guard"
+  s.add_development_dependency "guard-rspec"
+  s.add_development_dependency "ruby_gntp"
+end

metadata

@@ -0,0 +1,234 @@
+--- !ruby/object:Gem::Specification
+name: rds_backup_service
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Benton Roberts
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-09-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fog_tracker
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+- !ruby/object:Gem::Dependency
+  name: resque
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mail
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ohai
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby_gntp
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Provides a REST API for backing up live RDS instances to S3 as a compressed
+  SQL file.
+email:
+- benton@bentonroberts.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .yardopts
+- API.md
+- Gemfile
+- LICENSE.TXT
+- README.md
+- Rakefile
+- config.ru
+- config/accounts.example.yml
+- config/settings.yml
+- lib/rds_backup_service.rb
+- lib/rds_backup_service/config.rb
+- lib/rds_backup_service/model/backup_job.rb
+- lib/rds_backup_service/model/delayed_job.rb
+- lib/rds_backup_service/model/email.rb
+- lib/rds_backup_service/rds_backup_service.rb
+- lib/rds_backup_service/service.rb
+- lib/rds_backup_service/tasks.rb
+- lib/rds_backup_service/version.rb
+- lib/tasks/setup_groups.rake
+- rds_backup_service.gemspec
+homepage: http://github.com/benton/rds_backup_service
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 1151512606421308315
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 1151512606421308315
+requirements: []
+rubyforge_project: rds_backup_service
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Provides a REST API for backing up live RDS instances to S3 as a compressed
+  SQL file.
+test_files: []