qurd 0.0.1

data/lib/qurd/mixins.rb

@@ -0,0 +1,8 @@
+module Qurd
+  # Mixins for Qurd
+  module Mixins
+    autoload :AwsClients, 'qurd/mixins/aws_clients'
+    autoload :Configuration, 'qurd/mixins/configuration'
+    autoload :ConfigurationHelpers, 'qurd/mixins/configuration_helpers'
+  end
+end

data/lib/qurd/mixins/aws_clients.rb

@@ -0,0 +1,37 @@
+module Qurd
+  module Mixins
+    # Generic method for instantiating Aws clients
+    module AwsClients
+      # Memoize Aws clients, the caller must respond to +region+ and
+      # +aws_credentials+
+      # @param [String|Symbol] client the name of the client to instantiate
+      # @return [Object] an Aws client
+      # @example SQS
+      #   executor.aws_client(:SQS).list_queues
+      # @example EC2
+      #   executor.aws_client("EC2").describe_instances
+      # @raise [NameError] if the +client+ is not a valid Aws client class
+      def aws_client(client)
+        @qurd_aws_clients ||= {}
+        klass = Object.const_get("Aws::#{client}::Client")
+        @qurd_aws_clients[client.to_sym] ||= klass.new(
+          region: region,
+          credentials: aws_credentials)
+      end
+
+      # Wrap a block in a +begin+ +rescue+, which retries, if
+      # +Aws::Errors::ServiceError+ is raised. The method will retry the block
+      # immediately, up to n +tries+.
+      # @param [Fixnum] tries
+      # @raise [Aws::Errors::ServiceError] Any number of Aws error classes
+      def aws_retryable(tries = 2)
+        tries = [1, tries.to_i].max
+        begin
+          yield
+        rescue Aws::Errors::ServiceError => e
+          (tries -= 1).zero? ? raise(e) : retry
+        end
+      end
+    end
+  end
+end

data/lib/qurd/mixins/configuration.rb

@@ -0,0 +1,29 @@
+module Qurd
+  module Mixins
+    # Mixin the {#Qurd::Configuration} singleton
+    module Configuration
+      # Get the Qurd::Configuration singleton
+      # @return [Qurd::Configuration]
+      def qurd_config
+        @qurd_config ||= Qurd::Configuration.instance
+      end
+
+      # Get the parsed configuration for the daemon
+      # @return [Hashie::Mash]
+      def qurd_configuration
+        qurd_config.config
+      end
+
+      # Get the logger
+      # @return [Logger]
+      def qurd_logger
+        qurd_config.logger
+      end
+
+      # Log an error and raise an exception
+      def qurd_logger!(*a)
+        qurd_config.logger!(*a)
+      end
+    end
+  end
+end

data/lib/qurd/mixins/configuration_helpers.rb

@@ -0,0 +1,79 @@
+module Qurd
+  module Mixins
+    # Helpers for managing configuration details
+    module ConfigurationHelpers
+      private
+
+      def get_or_default(obj, method, default, cast = nil)
+        val = obj.send(method).nil? ? default : obj.send(method)
+        cast.nil? ? val : val.send(cast)
+      end
+
+      def verify_account!(name, monitor)
+        missing_keys = []
+        %w(credentials region queues).each do |key|
+          missing_keys << key if monitor[key].nil? || monitor[key].empty?
+        end
+
+        keys = missing_keys.join(', ')
+        logger! "Account #{name} missing keys: #{keys}" unless keys.empty?
+      end
+
+      def mkdir_p_file!(path)
+        return if File.exist?(path) && File.writable?(path)
+        dirname = File.dirname(path)
+        FileUtils.mkdir_p dirname
+        logger! "Directory not writable: #{dirname}" \
+          unless File.writable?(dirname)
+      end
+
+      def default_credentials
+        [['default',
+          Aws::InstanceProfileCredentials.new(http_open_timeout: 1,
+                                              http_read_timeout: 1,
+                                              retries: 1
+        )]]
+      end
+
+      def assume_role_credentials(cred)
+        opts = {}
+        %w(role_arn role_session_name policy duration_seconds
+           external_id).each do |key|
+          opts[key.to_sym] = cred.options[key] if cred.options.key?(key)
+        end
+        [cred.name, Aws::AssumeRoleCredentials.new(opts)]
+      end
+
+      def credentials(cred)
+        opts = [cred.options.access_key_id,
+                cred.options.secret_access_key,
+                cred.options.session_token]
+        [cred.name, Aws::Credentials.new(*opts)]
+      end
+
+      def shared_credentials(cred)
+        opts = {
+          profile_name: cred.options.profile_name
+        }
+        opts[:path] = cred.options.path if cred.options.key?(:path)
+        [cred.name, Aws::SharedCredentials.new(opts)]
+      end
+
+      def instance_profile_credentials(cred)
+        opts = {}
+        %w(retries ip_address port http_open_timeout http_read_timeout
+           delay http_debug_output).each do |key|
+          opts[key.to_sym] = cred.options[key] if cred.options.key?(key)
+        end
+        [cred.name, Aws::InstanceProfileCredentials.new(opts)]
+      end
+
+      def string2class(klass)
+        require klass.underscore
+        obj = Object.const_get(klass)
+        logger.debug("Found action #{klass}")
+        obj
+      end
+    end
+  end
+end

data/lib/qurd/processor.rb

@@ -0,0 +1,97 @@
+#  Gem module
+module Qurd
+  # Use a {#Qurd::Listener} to act on an AWS SQS message
+  class Processor
+    include Qurd::Mixins::Configuration
+    include Qurd::Mixins::AwsClients
+
+    # @!attribute listener [r]
+    #   @return [Qurd::Listener]
+    # @!attribute message [r]
+    #   @return [Qurd::Message]
+    attr_reader :listener, :message
+
+    # @param [Qurd::Listener] listener
+    # @param [Struct] msg An AWS SQS message
+    def initialize(listener, msg, listener_name, queue_url)
+      @listener = listener
+      @message = Message.new(
+        message: msg,
+        name: listener_name,
+        queue_url: queue_url,
+        aws_credentials: @listener.aws_credentials,
+        region: @listener.region
+      )
+    end
+
+    # Process an SQS message, by instantiating an instance of each action,
+    # calling +run_before+, +run+, and +run_after+, and deleting the message.
+    def process
+      qurd_logger.info("Processing #{listener.name} " \
+                       "action:#{message.action} " \
+                       "event:#{message.message.Event}")
+
+      if message.action
+        instantiate_actions
+        run_before
+        run_action
+        run_after
+      end
+
+      message.delete
+    end
+
+    # @private
+    def inspect
+      format('<Qurd::Processor:%x listener:%s message:%s>',
+             object_id,
+             listener.inspect,
+             message.inspect
+      )
+    end
+
+    private
+
+    def instantiate_actions
+      @actions = qurd_configuration.actions[message.action].map do |klass|
+        qurd_logger.debug("Instantiating #{klass}")
+        klass.new(message)
+      end
+    end
+
+    def run_before
+      run_actions('run_before') do |action|
+        action.send(:run_before)
+      end
+    end
+
+    def run_action
+      run_actions('run') do |action|
+        action.send(message.action) if message.action
+      end
+    end
+
+    def run_after
+      run_actions('run_after') do |action|
+        action.send(:run_after)
+      end
+    end
+
+    def run_actions(desc, &block)
+      @actions.each do |action|
+        qurd_logger.time("#{desc} #{action}") do
+          run(action, &block)
+        end
+      end
+    end
+
+    def run(action, &_block)
+      qurd_logger.debug("Running #{action}")
+      yield action
+    rescue StandardError => e
+      qurd_logger.error "#{action} raised #{e}"
+      qurd_logger.error e.backtrace.join("\n")
+      message.failed!(e)
+    end
+  end
+end

data/lib/qurd/version.rb

@@ -0,0 +1,5 @@
+# Gem module
+module Qurd
+  # The daemon version
+  VERSION = '0.0.1'
+end

data/lib/string.rb

@@ -0,0 +1,12 @@
+# rubocop:disable Documentation
+class String
+  def underscore
+    word = dup
+    word.gsub!(/::/, '/')
+    word.gsub!(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+    word.gsub!(/([a-z\d])([A-Z])/, '\1_\2')
+    word.tr!('-', '_')
+    word.downcase!
+    word
+  end
+end

data/qurd.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'qurd/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "qurd"
+  spec.version       = Qurd::VERSION
+  spec.authors       = ["Philip Champon"]
+  spec.email         = ["philip@adaptly.com"]
+  spec.summary       = %q{QUeue Resource Daemon: reaping and sowing your auto-scaled resources}
+  spec.description   = %q{Configure resources, based on auto scaling events, published to SQS. Qurd is extensible, simply create a plugin and let it rip.}
+  spec.homepage      = "https://github.com/Adaptly/qurd"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.8"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest"
+  spec.add_development_dependency "pry-nav"
+  spec.add_development_dependency "webmock"
+  spec.add_development_dependency "minitest-matchers_vaccine"
+
+  spec.add_runtime_dependency "cabin", "~> 0.7.0"
+  spec.add_runtime_dependency "hashie"
+  spec.add_runtime_dependency "aws-sdk", "~> 2.0.30"
+  spec.add_runtime_dependency "chef", ">= 11.16.0"
+end

data/test/action_test.rb

@@ -0,0 +1,115 @@
+require 'test_helper'
+describe Qurd::Action do
+  include WebMockStubs
+  class TestActionClass < Qurd::Action; end
+
+  describe 'class' do
+    def setup
+      aws_sqs_list_queues
+      aws_sqs_set_queue_attributes
+      aws_sqs_receive_message 'test/responses/aws/sqs-receive-message-1-launch.xml'
+      Qurd::Configuration.instance.configure('test/inputs/qurd.yml')
+    end
+    let(:subject) { TestActionClass }
+
+    it 'includes configuration mixin' do
+      %w(qurd_config qurd_configuration qurd_logger qurd_logger!).each do |m|
+        subject.must_respond_to m
+      end
+    end
+
+    it 'includes aws_client mixin' do
+      subject.must_respond_to :aws_client
+    end
+
+    describe '#configure' do
+      it 'logs something' do
+        mock = Minitest::Mock.new
+        mock.expect :debug, nil, ['Nothing to do']
+        subject.stub :qurd_logger, mock do
+          subject.configure('launch')
+        end
+        mock.verify
+      end
+    end
+  end
+
+  describe 'instance' do
+    def setup
+      aws_sqs_list_queues
+      aws_sqs_set_queue_attributes
+      aws_sqs_receive_message 'test/responses/aws/sqs-receive-message-1-launch.xml'
+      Qurd::Configuration.instance.configure('test/inputs/qurd.yml')
+    end
+    let(:sqs_client) { Aws::SQS::Client.new(region: 'us-west-2') }
+    let(:queue_url) { 'https://sqs.us-west-2.amazonaws.com/123456890/test2-ScalingNotificationsQueue-HPPYDAYSAGAI1' }
+    let(:sqs_message) { sqs_client.receive_message(queue_url: queue_url).messages.first }
+    let(:qurd_message) { Qurd::Message.new(message: sqs_message) }
+    let(:subject) { TestActionClass.new(qurd_message) }
+
+    it 'includes configuration mixin' do
+      %w(qurd_config qurd_configuration qurd_logger qurd_logger!).each do |m|
+        subject.must_respond_to m
+      end
+    end
+
+    it 'includes aws_client mixin' do
+      subject.must_respond_to :aws_client
+    end
+
+    describe '#new' do
+      it 'raises Qurd::Action::InvalidMessage' do
+        lambda do
+          TestActionClass.new(Object.new)
+        end.must_raise Qurd::Action::InvalidMessage
+      end
+
+      it 'sets various ivars' do
+        subject.message.must_equal qurd_message
+        subject.context.must_equal qurd_message.context
+        subject.instance_variable_get(:@message).must_equal subject.message
+        subject.instance_variable_get(:@context).must_equal subject.context
+      end
+    end
+
+    it 'responds to region' do
+      subject.region.must_equal qurd_message.region
+    end
+
+    it 'responds to aws_credentials' do
+      subject.aws_credentials.must_equal qurd_message.aws_credentials
+    end
+
+    %w[launch launch_error terminate terminate_error test].each do |action|
+      describe "##{action}" do
+        it 'raises RuntimeError' do
+          lambda do
+            subject.send action
+          end.must_raise RuntimeError
+        end
+      end
+    end
+
+    describe '#run_after' do
+      it 'logs a message' do
+        mock = Minitest::Mock.new
+        mock.expect :debug, nil, ['Nothing to do']
+        subject.stub :qurd_logger, mock do
+          subject.run_after
+        end
+        mock.verify
+      end
+    end
+
+    describe '#run_before' do
+      it 'logs a message' do
+        mock = Minitest::Mock.new
+        mock.expect :debug, nil, ['Nothing to do']
+        subject.stub :qurd_logger, mock do
+          subject.run_before
+        end
+        mock.verify
+      end
+    end
+  end
+end

data/test/chef_test.rb

@@ -0,0 +1,206 @@
+require 'test_helper'
+describe Qurd::Action::Chef do
+  include WebMockStubs
+  def setup
+    aws_sqs_list_queues
+    aws_sqs_set_queue_attributes
+    aws_ec2_describe_instances 'test/responses/aws/ec2-describe-instances-1.xml'
+    aws_sqs_receive_message 'test/responses/aws/sqs-receive-message-1-launch.xml'
+  end
+
+  let(:sqs_client) { Aws::SQS::Client.new(region: 'us-west-2') }
+  let(:queue_url) { 'https://sqs.us-west-2.amazonaws.com/123456890/test2-ScalingNotificationsQueue-HPPYDAYSAGAI1' }
+  let(:sqs_message) { sqs_client.receive_message(queue_url: queue_url).messages.first }
+  let(:qurd_message) { Qurd::Message.new(message: sqs_message, region: 'us-west-2', aws_credentials: Aws::Credentials.new('a', 'b')) }
+  let(:subject) { Qurd::Action::Chef.new(qurd_message) }
+
+  describe '#configure' do
+    it 'adds the Qurd::Message accessors chef_node, chef_client' do
+      Qurd::Configuration.instance.init('test/inputs/qurd_chef.yml')
+      Qurd::Action::Chef.configure('launch')
+      Qurd::Message.instance_methods.must_include :chef_node
+      Qurd::Message.instance_methods.must_include :chef_client
+    end
+
+    it 'sets the logger for chef' do
+      Qurd::Configuration.instance.init('test/inputs/qurd_chef.yml')
+      Qurd::Action::Chef.configure('launch')
+      ::Chef::Config[:log_location].path.must_equal 'tmp/qurd.log'
+    end
+
+    it 'sets the chef log level' do
+      expected = Qurd::Configuration.instance.config.log_level
+      Qurd::Configuration.instance.init('test/inputs/qurd_chef.yml')
+      Qurd::Action::Chef.configure('launch')
+      ::Chef::Config[:log_level].must_equal expected
+    end
+  end
+
+  describe '#chef_search' do
+    it 'memoizes Chef::Search::Query' do
+      subject.send(:chef_search).must_equal subject.send(:chef_search)
+    end
+  end
+
+  describe '#run_before' do
+    def setup
+      aws_sqs_list_queues
+      aws_sqs_set_queue_attributes
+      aws_ec2_describe_instances 'test/responses/aws/ec2-describe-instances-1.xml'
+      aws_sqs_receive_message 'test/responses/aws/sqs-receive-message-1-terminate.xml'
+      Qurd::Configuration.instance.configure('test/inputs/qurd_chef.yml')
+    end
+
+    it 'finds many nodes' do
+      chef_search(
+        'test/responses/chef/search-node-instance-n.json',
+        'node',
+        "instance_id:#{qurd_message.instance_id}"
+      )
+      chef_search(
+        'test/responses/chef/search-client-name-n.json',
+        'client',
+        'name:test-414.staging.example.com'
+      )
+      subject.run_before
+      subject.message.chef_node.must_equal nil
+      subject.message.context[:chef_name].must_equal nil
+    end
+
+    it 'finds a node and client' do
+      chef_search(
+        'test/responses/chef/search-node-instance-1.json',
+        'node',
+        "instance_id:#{qurd_message.instance_id}"
+      )
+      chef_search(
+        'test/responses/chef/search-client-name-1.json',
+        'client',
+        'name:test-414.staging.example.com'
+      )
+      subject.run_before
+      subject.message.chef_node.must_be_kind_of Chef::Node
+      subject.message.context[:chef_name].must_equal 'test-414.staging.example.com'
+      subject.message.chef_client.must_be_kind_of Chef::ApiClient
+      subject.message.context[:chef_client_name].must_equal 'test-414.staging.example.com'
+    end
+
+    it 'does not find a node' do
+      chef_search(
+        'test/responses/chef/search-node-instance-0.json',
+        'node',
+        "instance_id:#{qurd_message.instance_id}"
+      )
+      chef_search(
+        'test/responses/chef/search-client-name-0.json',
+        'client',
+        'name:test-414.staging.example.com'
+      )
+      subject.run_before
+      subject.message.chef_node.must_equal nil
+      subject.message.context[:chef_name].must_equal nil
+    end
+
+  end
+
+  describe '#terminate' do
+    def setup
+      aws_sqs_list_queues
+      aws_sqs_set_queue_attributes
+      aws_ec2_describe_instances 'test/responses/aws/ec2-describe-instances-1.xml'
+      aws_sqs_receive_message 'test/responses/aws/sqs-receive-message-1-terminate.xml'
+      chef_node_delete
+      chef_client_delete
+      chef_search(
+        'test/responses/chef/search-node-instance-1.json',
+        'node',
+        "instance_id:#{qurd_message.instance_id}"
+      )
+      chef_search(
+        'test/responses/chef/search-client-name-1.json',
+        'client',
+        'name:test-414.staging.example.com'
+      )
+      Qurd::Configuration.instance.configure('test/inputs/qurd_chef.yml')
+    end
+    let(:mock) { Minitest::Mock.new }
+    let(:node_mock) { Minitest::Mock.new }
+    let(:client_mock) { Minitest::Mock.new }
+
+    it 'saves a node; dry_run' do
+      mock.expect :debug, nil, ['Chef node found']
+      mock.expect :debug, nil, ['Dry run; missing node']
+
+      Qurd::Configuration.instance.config.dry_run = true
+      subject.run_before
+      subject.stub :qurd_logger, mock do
+        subject.terminate
+      end
+      mock.verify
+    end
+
+    it 'destroys a node; not dry_run; not failed' do
+      mock.expect :debug, nil, [String]
+      node_mock.expect :destroy, nil
+      node_mock.expect :nil?, false
+      client_mock.expect :destroy, nil
+      client_mock.expect :nil?, false
+
+      Qurd::Configuration.instance.config.dry_run = false
+      subject.run_before
+      qurd_message.stub :chef_client, client_mock do
+        qurd_message.stub :chef_node, node_mock do
+          subject.stub :qurd_logger, mock do
+            subject.terminate
+          end
+        end
+      end
+      mock.verify
+      node_mock.verify
+      client_mock.verify
+    end
+
+    it 'keeps a node; failed' do
+      mock.expect :warn, nil, ['Not deleting, message failed to process']
+      Qurd::Configuration.instance.config.dry_run = false
+      subject.run_before
+      qurd_message.stub :failed?, true do
+        subject.stub :qurd_logger, mock do
+          subject.terminate
+        end
+      end
+      mock.verify
+    end
+
+  end
+
+  describe '#test' do
+    def setup
+      aws_sqs_list_queues
+      aws_sqs_set_queue_attributes
+      aws_ec2_describe_instances 'test/responses/aws/ec2-describe-instances-1.xml'
+      aws_sqs_receive_message 'test/responses/aws/sqs-receive-message-1-test.xml'
+      chef_search(
+        'test/responses/chef/search-node-instance-1.json',
+        'node',
+        "instance_id:#{qurd_message.instance_id}"
+      )
+      chef_search(
+        'test/responses/chef/search-client-name-1.json',
+        'client',
+        'name:test-414.staging.example.com'
+      )
+      Qurd::Configuration.instance.configure('test/inputs/qurd_chef.yml')
+    end
+    let(:mock) { Minitest::Mock.new }
+
+    it 'logs Test' do
+      mock.expect :info, nil, ['Test']
+      subject.run_before
+      subject.stub :qurd_logger, mock do
+        subject.test
+      end
+      mock.verify
+    end
+  end
+end