automan 2.1.2

data/lib/automan/mixins/aws_caller.rb

@@ -0,0 +1,115 @@
+require 'aws-sdk'
+
+module Automan
+  module Mixins
+    module AwsCaller
+
+      def account
+        ENV['AWS_ACCOUNT_ID']
+      end
+
+      attr_reader :log_aws_calls
+      def log_aws_calls=(value)
+        if value == true
+          AWS.config(logger: @logger)
+        else
+          AWS.config(logger: nil)
+        end
+        @log_aws_calls = value
+      end
+
+      attr_writer :eb
+      def eb
+        if @eb.nil?
+          @eb = AWS::ElasticBeanstalk.new.client
+        end
+        @eb
+      end
+
+      attr_writer :s3
+      def s3
+        if @s3.nil?
+          @s3 = AWS::S3.new
+        end
+        @s3
+      end
+
+      S3_PROTO = 's3://'
+
+      def looks_like_s3_path?(path)
+        path.start_with? S3_PROTO
+      end
+
+      def parse_s3_path(path)
+        if !looks_like_s3_path? path
+          raise ArgumentError, "s3 path must start with '#{S3_PROTO}'"
+        end
+
+        rel_path = path[S3_PROTO.length..-1]
+        bucket = rel_path.split('/').first
+        key = rel_path.split('/')[1..-1].join('/')
+
+        return bucket, key
+      end
+
+      def s3_object_exists?(s3_path)
+        bucket, key = parse_s3_path s3_path
+        s3.buckets[bucket].objects[key].exists?
+      end
+
+      def s3_read(s3_path)
+        bucket, key = parse_s3_path s3_path
+        s3.buckets[bucket].objects[key].read
+      end
+
+      attr_writer :r53
+      def r53
+        if @r53.nil?
+          @r53 = AWS::Route53.new
+        end
+        @r53
+      end
+
+      attr_writer :elb
+      def elb
+        if @elb.nil?
+          @elb = AWS::ELB.new
+        end
+        @elb
+      end
+
+      attr_writer :cfn
+      def cfn
+        if @cfn.nil?
+          @cfn = AWS::CloudFormation.new
+        end
+        @cfn
+      end
+
+      attr_writer :as
+      def as
+        if @as.nil?
+          @as = AWS::AutoScaling.new
+        end
+        @as
+      end
+
+      attr_writer :rds
+      def rds
+        if @rds.nil?
+          @rds = AWS::RDS.new
+        end
+        @rds
+      end
+
+      attr_writer :ec2
+      def ec2
+        if @ec2.nil?
+          @ec2 = AWS::EC2.new
+        end
+        @ec2
+      end
+
+    end
+  end
+end

data/lib/automan/mixins/utils.rb

@@ -0,0 +1,18 @@
+module Automan::Mixins
+  module Utils
+
+    def region_from_az(availability_zone)
+      availability_zone[0..-2]
+    end
+
+    class ::String
+      def underscore
+        self.gsub(/::/, '/').
+        gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+        gsub(/([a-z\d])([A-Z])/,'\1_\2').
+        tr("-", "_").
+        downcase
+      end
+    end
+  end
+end

data/lib/automan/rds/errors.rb

@@ -0,0 +1,7 @@
+module Automan::RDS
+  class RequestFailedError < StandardError
+  end
+
+  class DatabaseDoesNotExistError < StandardError
+  end
+end

data/lib/automan/rds/snapshot.rb

@@ -0,0 +1,244 @@
+require 'automan'
+require 'time'
+require 'wait'
+
+module Automan::RDS
+  class Snapshot < Automan::Base
+    add_option :database,
+               :name,
+               :environment,
+               :prune,
+               :wait_for_completion
+
+    attr_accessor :max_snapshots
+
+    def initialize(options=nil)
+      @prune = true
+      @wait_for_completion = false
+      super
+      @wait = Wait.new({
+        delay: 30,
+        attempts: 20, # 20 x 30s == 10m
+        debug: true,
+        rescuer:  WaitRescuer.new(),
+        logger:   @logger
+      })
+
+      if ENV['MAX_SNAPSHOTS'].nil?
+        @max_snapshots = 50
+      else
+        @max_snapshots = ENV['MAX_SNAPSHOTS'].to_i
+      end
+    end
+
+    include Automan::Mixins::Utils
+
+    def find_db
+      db = nil
+      if !database.nil?
+        db = rds.db_instances[database]
+      elsif !environment.nil?
+        db = find_db_by_environment(environment)
+      end
+      db
+    end
+
+    def database_available?(database)
+      database.db_instance_status == 'available'
+    end
+
+    def wait_until_database_available(database)
+      state_wait = Wait.new({
+        delay:    10,
+        attempts: 20,
+        debug:    true,
+        rescuer:  WaitRescuer.new(),
+        logger:   @logger
+      })
+
+      state_wait.until do
+        logger.info "Waiting for database #{database.id} to be available"
+        database_available?(database)
+      end
+    end
+
+    def snapshot_count
+      rds.db_instances[database].snapshots.count
+    end
+
+    def create
+      log_options
+
+      db = find_db
+
+      if db.nil? || !db.exists?
+        raise DatabaseDoesNotExistError, "Database for #{environment} does not exist"
+      end
+
+      myname = name.nil? ? default_snapshot_name(db) : name.dup
+
+      wait_until_database_available(db)
+
+      if snapshot_count >= max_snapshots
+        logger.info "Too many snapshots (>= #{max_snapshots}), deleting oldest prunable."
+        old = nil
+        AWS.memoize do
+          old = oldest_prunable_snapshot
+        end
+        logger.info "Deleting #{old.id}"
+        old.delete
+      end
+
+      logger.info "Creating snapshot #{myname} for #{db.id}"
+      snap = db.create_snapshot(myname)
+
+      if prune == true
+        logger.info "Setting snapshot to be prunable"
+        set_prunable(snap)
+      end
+
+      if wait_for_completion == true
+        wait.until do
+          logger.info "Waiting for snapshot to complete..."
+          snapshot_finished?(snap)
+        end
+        logger.info "Snapshot finished (or timed out)."
+      end
+    end
+
+    def snapshot_finished?(snapshot)
+      unless snapshot.exists?
+        return false
+      end
+      snapshot.status == "available"
+    end
+
+    def db_environment(db)
+      arn = db_arn(db)
+      return tags(arn)['Name']
+    end
+
+    def default_snapshot_name(db)
+      env   = db_environment db
+      stime = Time.new.iso8601.gsub(/:/,'-')
+
+      return env + "-" + stime
+    end
+
+    def find_db_by_environment(environment)
+      rds.db_instances.each do |db|
+        if db_environment(db) == environment
+          return db
+        end
+      end
+      return nil
+    end
+
+    def db_arn(database)
+      region = region_from_az(database.availability_zone_name)
+      "arn:aws:rds:#{region}:#{account}:db:#{database.id}"
+    end
+
+    def snapshot_arn(snapshot)
+      region = region_from_az(snapshot.availability_zone_name)
+      "arn:aws:rds:#{region}:#{account}:snapshot:#{snapshot.id}"
+    end
+
+    # tag with CanPrune
+    def set_prunable(snapshot)
+      opts = {
+        resource_name: snapshot_arn(snapshot),
+        tags: [ {key: 'CanPrune', value: 'yes'} ]
+      }
+      response = rds.client.add_tags_to_resource opts
+
+      unless response.successful?
+        raise RequestFailedError "add_tags_to_resource failed: #{response.error}"
+      end
+    end
+
+    def delete
+      log_options
+
+      logger.info "Deleting snapshot #{name}"
+      rds.db_snapshots[name].delete
+    end
+
+    def tags(arn)
+      opts = {
+        resource_name: arn
+      }
+      response = rds.client.list_tags_for_resource opts
+
+      unless response.successful?
+        raise RequestFailedError "list_tags_for_resource failed: #{response.error}"
+      end
+
+      result = {}
+      response.data[:tag_list].each do |t|
+        result[ t[:key] ] = t[:value]
+      end
+      result
+    end
+
+    def can_prune?(snapshot)
+      tagged_can_prune?(snapshot) && available?(snapshot) && manual?(snapshot)
+    end
+
+    def tagged_can_prune?(snapshot)
+      arn = snapshot_arn(snapshot)
+      tags(arn)['CanPrune']  == 'yes'
+    end
+
+    def available?(snapshot)
+      snapshot.status == 'available'
+    end
+
+    def manual?(snapshot)
+      snapshot.snapshot_type == 'manual'
+    end
+
+    # older than a month?
+    def too_old?(time)
+      time.utc < (Time.now.utc - 60*60*24*30)
+    end
+
+    def get_all_snapshots
+      rds.db_snapshots
+    end
+
+    def prunable_snapshots
+      snapshots = get_all_snapshots
+      snapshots.select { |s| can_prune?(s) }
+    end
+
+    def oldest_prunable_snapshot
+      prunable_snapshots.sort_by { |s| s.created_at }.first
+    end
+
+    def prune_snapshots
+      logger.info "Pruning old db snapshots"
+
+      AWS.memoize do
+        prunable_snapshots.each do |snapshot|
+
+          timestamp = snapshot.created_at
+          snapshot_name = snapshot.db_snapshot_identifier
+
+          if too_old?(timestamp)
+            logger.info "Deleting #{snapshot_name} because it is too old."
+            snapshot.delete
+          end
+        end
+      end
+    end
+
+    def latest
+      log_options
+      db = find_db
+      logger.info "Finding most recent snapshot for #{db.id}"
+      s = db.snapshots.sort_by {|i| i.created_at }.last
+      logger.info "Most recent snapshot is #{s.id}"
+    end
+  end
+end

data/lib/automan/s3/downloader.rb

@@ -0,0 +1,25 @@
+require 'automan'
+
+module Automan::S3
+  class Downloader < Automan::Base
+    add_option :localfile, :s3file
+
+    include Automan::Mixins::Utils
+
+    def download
+      log_options
+
+      logger.info "uploading #{localfile} to #{s3file}"
+
+      bucket, key = parse_s3_path s3file
+
+      File.open(localfile, 'wb') do |file|
+        s3.buckets[bucket].objects[key].read do |chunk|
+          file.write(chunk)
+        end
+      end
+
+    end
+
+  end
+end

data/lib/automan/s3/uploader.rb

@@ -0,0 +1,20 @@
+require 'automan'
+
+module Automan::S3
+  class Uploader < Automan::Base
+    add_option :localfile, :s3file
+
+    include Automan::Mixins::Utils
+
+    def upload
+      log_options
+
+      logger.info "uploading #{localfile} to #{s3file}"
+
+      bucket, key = parse_s3_path s3file
+      s3.buckets[bucket].objects[key].write(:file => localfile)
+
+    end
+
+  end
+end

data/lib/automan/version.rb

@@ -0,0 +1,3 @@
+module Automan
+  VERSION = "2.1.2"
+end

data/lib/automan/wait_rescuer.rb

@@ -0,0 +1,17 @@
+require 'wait'
+
+class ::WaitRescuer < Wait::BaseRescuer
+  def initialize(exceptions=nil)
+    super
+  end
+  # Logs an exception.
+  def log(exception)
+    return if @logger.nil?
+
+    klass = exception.class.name
+    # We can omit the message if it's identical to the class name.
+    message = exception.message unless exception.message == klass
+
+    @logger.debug("Rescuer") { "rescued: #{klass}#{": #{message}" if message}" }
+  end
+end