qurd 0.0.1

data/lib/qurd/configuration.rb

@@ -0,0 +1,182 @@
+# rubocop:disable ClassLength
+module Qurd
+  # Parse a configuration file, create a logger and various data structures
+  class Configuration
+    include Singleton
+    include Mixins::ConfigurationHelpers
+    # @!attribute [r] config
+    #   Configuration options, ie
+    #   +aws_credentials+, +auto_scaling_queues+, +actions+, +daemonize+,
+    #   +dry_run+, +listen_timeout+, +log_file+, +log_level+, +pid_file+,
+    #   +save_failures+, +sqs_set_attributes_timeout+, +visibility_timeout+,
+    #   +wait_time+.
+    #   Additional configuration keys include +listeners+.
+    #   @return [Hashie::Mash] the config YAML as a Mash
+    # @!attribute [r] logger
+    #   The logger
+    #   @return [Cabin::Channel]
+    attr_reader :config, :logger
+
+    # Initialize Qurd
+    # @param [String] config_path The path to the config file, default
+    #   +/etc/qurd/config.yml+
+    def init(config_path)
+      config_path ||= '/etc/qurd/config.yml'
+      @config = Hashie::Mash.new YAML.load(File.read(config_path))
+      @sqs_queues = {}
+      @config.daemonize = false if @config.daemonize.nil?
+      @config.dry_run = get_or_default(@config, :dry_run, false)
+      @config.pid_file ||= '/var/run/qurd/qurd.pid'
+      @config.save_failures = get_or_default(@config, :save_failures, true)
+      @queues = []
+      @aws_credentials = []
+      vt = get_or_default(@config, :visibility_timeout, 300, :to_s)
+      wt = get_or_default(@config, :wait_time, 20, :to_s)
+      st = get_or_default(@config, :sqs_set_attributes_timeout, 10, :to_f)
+      lt = get_or_default(@config, :listen_timeout, vt, :to_f)
+      @config.visibility_timeout = vt
+      @config.wait_time = wt
+      @config.sqs_set_attributes_timeout = st
+      @config.listen_timeout = lt
+      %w[launch launch_error terminate terminate_error test].each do |action|
+        @config.actions[action] ||= []
+      end
+
+      configure_logger
+    end
+
+    # Configure Qurd
+    # @param [String] config_path The path to the config file, default
+    #   +/etc/qurd/config.yml+
+    def configure(config_path)
+      init(config_path)
+      mkdir_p_file!(@config.pid_file)
+
+      configure_credentials
+      configure_auto_scaling_queues
+      configure_actions
+    end
+
+    # Determine if the daemon is running in debug mode
+    # @return [Boolean]
+    def debug?
+      config.log_level == 'debug'
+    end
+
+    # Log an error and raise an exception
+    # @param [String] msg The error and exception message
+    # @param [Exception] e The exception to raise
+    def logger!(msg, e = RuntimeError)
+      logger.error msg
+      fail e, msg
+    end
+
+    # Get a logging context and optionally initialize it
+    # @param [Hash] attrs a hash of values to +merge+ into the context
+    # @return [Cabin::Context]
+    def get_context(attrs = {})
+      ctx = logger.context
+      attrs.each do |k, v|
+        ctx[k] = v
+      end
+      ctx
+    end
+
+    private
+
+    def configure_credentials
+      if config.aws_credentials.nil? || config.aws_credentials.empty?
+        creds = default_credentials
+      else
+        creds = config.aws_credentials.map do |cred|
+          cred.options ||= {}
+          case cred.type
+          when 'assume_role_credentials'
+            assume_role_credentials(cred)
+          when 'credentials'
+            credentials(cred)
+          when 'shared_credentials'
+            shared_credentials(cred)
+          when 'instance_profile_credentials'
+            instance_profile_credentials(cred)
+          else qurd_logger! "Credential type unknown: '#{cred.type}'"
+          end
+        end
+      end
+      config.aws_credentials = Hash[creds]
+    end
+
+    # Convert strings to objects
+    def configure_actions
+      missing = config.actions.inject([]) do |ary, mod|
+        action, klasses = mod
+        return ary if klasses.nil?
+        ctx = get_context(action: action)
+        logger! 'Action types must be an array' unless klasses.is_a?(Array)
+        klasses.map! do |klass|
+          begin
+            k = string2class(klass)
+            k.configure(action)
+            k
+          rescue NameError, LoadError => e
+            logger.error(e)
+            ary << klass
+          end
+        end
+        ctx.clear
+        ary
+      end
+
+      m = missing.uniq.join(', ')
+      logger! "Class undefined for actions: #{m}" if missing.any?
+    end
+
+    # Configure Cabin and Aws logging
+    def configure_logger
+      @logger = Cabin::Channel.new
+      if config.log_file || config.daemonize
+        path = config.log_file || '/var/log/qurd/qurd.log'
+        mkdir_p_file!(path)
+        config.log_file_io = open(path, 'w')
+        @ruby_logger = Logger.new(config.log_file_io)
+        @logger.level = (config.log_level || :info).to_sym
+      else
+        @logger.level = (config.log_level || :debug).to_sym
+        @ruby_logger = Logger.new(STDOUT)
+      end
+      @logger.subscribe(@ruby_logger)
+
+      Aws.config[:logger] = @ruby_logger
+      Aws.config[:http_wire_trace] = debug?
+
+      @logger.debug('Logging configured')
+    end
+
+    # Configure sqs clients and queues
+    def configure_auto_scaling_queues
+      config.listeners = config.auto_scaling_queues.map do |name, monitor|
+        if (config.auto_scaling_queues.nil? ||
+             config.auto_scaling_queues.empty?) &&
+           config.aws_credentials.default
+          creds = config.aws_credentials.default
+          monitor.credentials = 'default'
+        else
+          creds = config.aws_credentials[monitor.credentials]
+        end
+        verify_account!(name, monitor)
+        logger!("Undefined credential: '#{monitor.credentials}'") unless creds
+        vt = get_or_default(monitor, :visibility_timeout,
+                            config.visibility_timeout, :to_s)
+        wt = get_or_default(monitor, :wait_time, config.wait_time, :to_s)
+        Listener.new(
+          aws_credentials: creds,
+          name: name,
+          queues: monitor.queues,
+          region: monitor.region,
+          visibility_timeout: vt,
+          wait_time: wt
+        )
+      end
+    end
+  end
+end

data/lib/qurd/listener.rb

@@ -0,0 +1,207 @@
+# rubocop:disable ClassLength
+# Gem module
+module Qurd
+  # Provide an interface for interacting with configured queues and AWS.
+  class Listener
+    include Qurd::Mixins::AwsClients
+    include Qurd::Mixins::Configuration
+
+    # @!attribute aws_credentials [r]
+    #   The AWS credentials for the account
+    #   @return [Aws::Credentials]
+    # @!attribute message [r]
+    #   The message reveived
+    #   @return [Qurd::Message]
+    # @!attribute name [r]
+    #   The name of the executor
+    #   @return [String]
+    # @!attribute queues [r]
+    #   An array of AWS SQS URLs for the account
+    #   @return [Array<String>]
+    # @!attribute region [r]
+    #   The AWS region for the account
+    #   @return [String]
+    # @!attribute visibility_timeout [r]
+    #   @return [String]
+    # @!attribute wait_time [r]
+    #   @return [String]
+    attr_reader :aws_credentials,
+                :message,
+                :name,
+                :queues,
+                :region,
+                :visibility_timeout,
+                :wait_time
+
+    # @param [Hash] attrs
+    # @option attrs [Aws::Credentials] :aws_credentials
+    # @option attrs [String] :name
+    # @option attrs [Array<String|Regexp>] :queues An array of SQS names
+    #   and Regexps
+    # @option attrs [String] :region
+    # @option attrs [String] :visibility_timeout
+    # @option attrs [String] :wait_time
+    def initialize(attrs = {})
+      @aws_credentials = attrs[:aws_credentials]
+      @name = attrs[:name]
+      @region = attrs[:region]
+      @visibility_timeout = attrs[:visibility_timeout]
+      @wait_time = attrs[:wait_time]
+      @queues = convert_queues attrs[:queues]
+      configure_queues
+    end
+
+    # Create a thread for each queue URL, a context denoting the listener name
+    # and the queue URL.
+    # @param [Proc] _block the proc each thread should run
+    # @yieldparam [String] url the url of the queue
+    # @yieldparam [Cabin::Context] ctx the logging context
+    def queue_threads(&_block)
+      queues.map do |qurl|
+        qurd_logger.debug("Creating thread for #{qurl}")
+        Thread.new(qurl) do |url|
+          ctx = qurd_config.get_context(name: name, queue_name: url[/([^\/]+)$/])
+          qurd_logger.debug('Thread running')
+          yield url, ctx
+        end
+      end
+    end
+
+    # Create one thread per +queue+, receive messages from it and process each
+    # message received
+    # @return [Array<Thread>]
+    def listen
+      queue_threads do |qurl, _context|
+        loop do
+          begin
+            msgs = aws_client(:SQS).receive_message(
+              queue_url: qurl,
+              wait_time_seconds: wait_time,
+              visibility_timeout: visibility_timeout
+            )
+            threads = process_messages(qurl, msgs)
+            joins = threads.map do |thread|
+              thread.join(qurd_configuration.listen_timeout)
+            end
+            if joins.compact.count != threads.count
+              qurd_logger.warn('Some threads timed out')
+            end
+          rescue Aws::Errors::ServiceError => e
+            qurd_logger.error("Aws raised #{e}")
+          end
+        end
+      end
+    end
+
+    # @private
+    def inspect
+      format('<Qurd::Listener:%x name:%s>',  object_id, name)
+    end
+
+    private
+
+    def process_messages(qurl, msgs)
+      msgs.messages.map do |msg|
+        Thread.new(msg) do |m|
+          qurd_logger.debug("Found message #{msg}")
+          r = Processor.new self, m, name, qurl
+          r.process
+        end
+      end
+    end
+
+    def configure_queues
+      threads = configure_queues_threads
+      joins = threads.map do |thread|
+        thread.join(qurd_configuration.sqs_set_attributes_timeout)
+      end
+
+      qurd_logger! 'One or more threads timed out' \
+        if joins.compact.count != threads.count
+    end
+
+    def configure_queues_threads
+      queue_threads do |q, _context|
+        qurd_logger.debug("Setting wait_time:#{wait_time} " \
+                          "visibility_timeout:#{visibility_timeout} #{q}")
+        begin
+          aws_client(:SQS).set_queue_attributes(
+            queue_url: q,
+            attributes: {
+              ReceiveMessageWaitTimeSeconds: wait_time,
+              VisibilityTimeout: visibility_timeout
+            }
+          )
+        rescue Aws::SQS::Errors::ServiceError::QueueDoesNotExist => e
+          qurd_logger.error("SQS raised #{e}")
+        rescue Aws::SQS::Errors::ServiceError => e
+          qurd_logger.error("SQS raised #{e}")
+          raise e
+        end
+      end
+    end
+
+    # Convert a regex string to a regex, including modifiers
+    # @param [String] r String form of the regex
+    # @return [Regexp] The compiled regex
+    # @example With modifier
+    #   Qurd::Configuration.parse_regex("/foo/i")
+    def parse_regex(r)
+      # /foo/ or /foo/i
+      m = r.match %r{\A/(.*)/([a-z]*)\Z}mx
+      qurd_logger.debug("Found re: #{m[0]} 1: #{m[1]} 2: #{m[2]}")
+      args = modifier2int(m[2])
+      regex = Regexp.new(m[1], args)
+      qurd_logger.debug("Compiled regex #{regex}")
+      queue_url regex
+    end
+
+    def modifier2int(str)
+      args = 0
+      str.each_byte do |c|
+        args |= case c.chr
+                when 'i' then Regexp::IGNORECASE
+                when 'm' then Regexp::MULTILINE
+                when 'x' then Regexp::EXTENDED
+                when 'o' then 0
+                when 'e' then 16
+                else qurd_logger! "Unknown regex modifier #{c.chr}"
+                end
+      end
+      args
+    end
+
+    # Find the SQS URL for a named queue or a regex
+    # @overload queue_url(name)
+    #   @param [String] name The AWS SQS name
+    #   @return [String] AWS SQS URL
+    #
+    # @overload queue_url(name)
+    #   @param [Regexp] name regex of a queue name
+    #   @return [Array<String>]
+    def queue_url(name)
+      @sqs_queues ||= aws_client(:SQS).list_queues.queue_urls
+
+      if name.respond_to?(:upcase)
+        url = @sqs_queues.find { |u| u[/([^\/]+$)/] == name }
+      else
+        url = @sqs_queues.select { |u| u =~ name }
+      end
+      qurd_logger.debug("Queue #{name} found '#{url}'")
+      qurd_logger.warn("No queue found for '#{name}'") if url.nil? || url.empty?
+      url
+    rescue Aws::SQS::Errors::ServiceError => e
+      qurd_logger.error("SQS raised #{e}")
+      raise e
+    end
+
+    # Convert regexes to and strings to queue URLs
+    # @param [Array<String>] queues An array of queues to monitor
+    # @return [Array<String>] SQS URLs
+    def convert_queues(queues)
+      queues.map do |q|
+        q[0] == '/' ? parse_regex(q) : queue_url(q)
+      end.flatten.compact
+    end
+  end
+end

data/lib/qurd/message.rb

@@ -0,0 +1,231 @@
+# rubocop:disable Metrics/LineLength
+module Qurd
+  # Convert an SQS auto scaling message to a more usable object
+  # @example SQS auto scaling message
+  #   {
+  #     "Type" : "Notification",
+  #     "MessageId" : "e4379a5a-e119-53f7-b6ef-d7dbd32d31fe",
+  #     "TopicArn" : "arn:aws:sns:us-east-1:123456890:test-ScalingNotificationsTopic-HPPYDAYSAGAIN",
+  #     "Subject" : "Auto Scaling: termination for group \"test2-AutoScalingGroup-1QDX3CNO5SU3D\"",
+  #     "Message" : "{\"StatusCode\":\"InProgress\",\"Service\":\"AWS Auto Scaling\",\"AutoScalingGroupName\":\"test2-AutoScalingGroup-1QDX3CNO5SU3D\",\"Description\":\"Terminating EC2 instance: i-08e58cf8\",\"ActivityId\":\"93faaf3a-28cb-4982-a690-0a73c989ab1f\",\"Event\":\"autoscaling:EC2_INSTANCE_TERMINATE\",\"Details\":{\"Availability Zone\":\"us-east-1a\",\"Subnet ID\":\"subnet-3c3e0e14\"},\"AutoScalingGroupARN\":\"arn:aws:autoscaling:us-east-1:123456890:autoScalingGroup:4edb2535-5015-4b81-b668-88ecb0effcb7:autoScalingGroupName/test2-AutoScalingGroup-1QDX3CNO5SU3D\",\"Progress\":50,\"Time\":\"2015-03-16T19:33:08.181Z\",\"AccountId\":\"123456890\",\"RequestId\":\"93faaf3a-28cb-4982-a690-0a73c989ab1f\",\"StatusMessage\":\"\",\"EndTime\":\"2015-03-16T19:33:08.181Z\",\"EC2InstanceId\":\"i-08e58cf8\",\"StartTime\":\"2015-03-16T19:29:14.911Z\",\"Cause\":\"At 2015-03-16T19:29:14Z an instance was taken out of service in response to a ELB system health check failure.\"}",
+  #     "Timestamp" : "2015-03-16T19:33:08.242Z",
+  #     "SignatureVersion" : "1",
+  #     "Signature" : "I+SE8tMiq13/wDTPTJnJvHYi3jSjChhYByJAsnhY0wGa+0lxXc18vPIn9hIT0tYRNWMcR/Xn1AUNsgHrLjzB93xukyKA2CDff08zIuP0l4Xle/FSEJzfkJ0FDqZnzelFuZ2PMtO3lf5UY7CWZg/wKJv6I9CNJF4Ll9YgvC8Moe/31VwJwNy4TRAWdBhDuRXLjbEHoFNGjaGquiduOGySrgRmm74d0P0zWj7IfWbqO6ReNG2ADrqw+Bhn6dAkkeFH+9vJZeKdUCgsXX8XCBHcWX+yAb4WJH90hdosLN12DCdn2AvNgQfoTdpDPkTHC+QcwfRs52d3MD2WLrUfBMBy0A==",
+  #     "SigningCertURL" : "https://sns.us-east-1.amazonaws.com/SimpleNotificationService-d6d679a1d18e95c2f9ffcf11f4f9e198.pem",
+  #     "UnsubscribeURL" : "https://sns.us-east-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:us-east-1:123456890:test-ScalingNotificationsTopic-HPPYDAYSAGAIN:bd850bb2-1a69-4456-a517-c645a26f54b2"
+  #   }
+  class Message
+    extend Qurd::Mixins::Configuration
+
+    # Add setter and getter instances methods. If the get or set method is
+    # already defined, an exception will be raised.
+    # @param [Symbol] name the name of the method to add
+    def self.add_accessor(name)
+      if instance_methods.include?(name) ||
+         instance_methods.include?("#{name}=")
+        qurd_logger.warn "Can not replace a method! (#{name})"
+      end
+      attr_accessor name
+    end
+
+    include Qurd::Mixins::Configuration
+    include Qurd::Mixins::AwsClients
+    # @!attribute aws_credentials [r]
+    #   @return [Aws::Credentials]
+    # @!attribute context [r]
+    #   Cabin::Channel logs messages as well as context. Context is retained
+    #   until it is cleared.
+    #   @return [Cabin::Context] Context data
+    # @!attribute exceptions [r]
+    #   Action exceptions
+    #   @return [Array<Exception>]
+    # @!attribute name [r]
+    #   The Listener name
+    #   @return [String]
+    # @!attribute queue_url [r]
+    #   The SQS url the message came from
+    #   @return [String]
+    # @!attribute region [r]
+    #   AWS region
+    #   @return [String]
+    attr_reader :aws_credentials,
+                :context,
+                :exceptions,
+                :name,
+                :queue_url,
+                :region
+
+    #
+    # @param [Hash] attrs
+    # @option attrs [Aws::Credentials] :aws_credentials
+    # @option attrs [String] :name
+    # @option attrs [String] :queue_url
+    # @option attrs [String] :region msg AWS SQS message
+    # @option attrs [Struct] :message AWS SQS message
+    def initialize(attrs)
+      @aws_credentials = attrs[:aws_credentials]
+      @name = attrs[:name]
+      @queue_url = attrs[:queue_url]
+      @region = attrs[:region]
+      @sqs_message = attrs[:message]
+
+      @exceptions = []
+      @failed = false
+      @context = qurd_config.get_context(name: @name,
+                                         queue_name: (@queue_url[/[^\/]+$/] rescue nil),
+                                         instance_id: instance_id,
+                                         message_id: message_id,
+                                         action: action)
+
+      qurd_logger.info "Received #{body.Subject} Cause #{message.Cause} Event #{message.Event}"
+    end
+
+    # Convert the SQS message +body+ to a mash, keys include +Type+, +MessageId+,
+    # +TopicArn+, +Subject+, +Message+, +Timestamp+, +SignatureVersion+,
+    # +Signature+, +SigningCertURL+, +UnsubscribeURL+
+    # @return [Hashie::Mash]
+    def body
+      @body ||= Hashie::Mash.new JSON.load(@sqs_message.body)
+    rescue JSON::ParserError
+      @body = Hashie::Mash.new {}
+    end
+
+    # Convert +body.Message+ to a mash, keys include
+    # +StatusCode+, +Service+, +AutoScalingGroupName+, +Description+,
+    # +ActivityId+, +Event+, +Details+ +AutoScalingGroupARN+ +Progress+ +Time+
+    # +AccountId+ +RequestId+, +StatusMessage+ +EndTime+ +EC2InstanceId+
+    # +StartTime+ +Cause+
+    # @return [Hashie::Mash]
+    def message
+      @message ||= Hashie::Mash.new JSON.load(body.Message)
+    rescue JSON::ParserError
+      @message = Hashie::Mash.new {}
+    end
+
+    # The SQS message's +EC2InstanceId+
+    # @return [String]
+    def instance_id
+      @instance_id ||= message.EC2InstanceId
+    end
+
+    # The +body.MessageId+
+    # @return [String]
+    def message_id
+      @message_id ||= body.MessageId
+    end
+
+    # The SQS +receipt_handle+, used to delete a message
+    # @return [String]
+    def receipt_handle
+      @sqs_message.receipt_handle
+    end
+
+    # Record an action failure
+    # @param [Exception] e The exception
+    def failed!(e = nil)
+      qurd_logger.debug 'Failed'
+      @exceptions << e if e
+      @failed = true
+      nil
+    end
+
+    # Has processing the message failed
+    # @return [Boolean]
+    def failed?
+      @failed == true
+    end
+
+    # Memozied EC2 instance. Caller must anticipate +nil+ results, as instances
+    # may terminate before the message is received.
+    # @param [Fixnum] tries The number of times to retry the Aws API
+    # @return [Struct|nil]
+    def instance(tries = nil)
+      return @instance if @instance
+      @instance = aws_instance(tries)
+    end
+
+    # Memoize the instance's +Name+ tag
+    def instance_name
+      return @instance_name if @instance_name
+      @instance_name = instance.tags.find do |t|
+        t.key == 'Name'
+      end.value
+      qurd_logger.debug("Found instance name '#{@instance_name}'")
+      @instance_name
+    rescue NoMethodError
+      qurd_logger.debug('No instance found')
+      @instance_name = nil
+    end
+
+    # Convert the +message.Event+ to an action
+    # @return [String] +launch+, +launch_error+, +terminate+, +terminate_error+,
+    #   or +test+
+    def action
+      case message.Event
+      when 'autoscaling:EC2_INSTANCE_LAUNCH' then 'launch'
+      when 'autoscaling:EC2_INSTANCE_LAUNCH_ERROR' then 'launch_error'
+      when 'autoscaling:EC2_INSTANCE_TERMINATE' then 'terminate'
+      when 'autoscaling:EC2_INSTANCE_TERMINATE_ERROR' then 'terminate_error'
+      when 'autoscaling:TEST_NOTIFICATION' then 'test'
+      else
+        qurd_logger.info "Ignoring #{message.Event}"
+        failed!
+      end
+    end
+
+    # Delete an AWS SQS message
+    def delete
+      qurd_logger.debug('Preparing to delete message')
+      if failed? && qurd_configuration.save_failures
+        qurd_logger.error 'Message failed processing, not deleting'
+      elsif qurd_configuration.dry_run
+        qurd_logger.info 'Dry run'
+      else
+        delete_message
+      end
+      context.clear
+    end
+
+    # @private
+    def inspect
+      format('#<Qurd::Message message_id:%s subject:%s cause:%s ' \
+             'instance_id:%s instance:%s>',
+             message_id,
+             body.Subject,
+             message.Cause,
+             instance_id,
+             instance)
+    end
+
+    private
+
+    # Get the Aws EC2 instance, using +instance_id+
+    # @param [Fixnum] tries the number of retries
+    # @return [Struct|nil]
+    # @see instance_id
+    # @see instance
+    def aws_instance(tries = nil)
+      return unless instance_id
+      aws_retryable(tries) do
+        aws_client(:EC2).describe_instances(
+          instance_ids: [instance_id]
+        ).reservations.first.instances.first
+      end
+    rescue NoMethodError
+      nil
+    end
+
+    def delete_message
+      qurd_logger.info 'Deleting'
+      begin
+        aws_client(:SQS).delete_message(queue_url: queue_url,
+                                        receipt_handle: receipt_handle)
+      rescue Aws::SQS::Errors::ReceiptHandleIsInvalid
+        qurd_logger.info('SQS message deleted already or timed out')
+      rescue Aws::SQS::Errors::ServiceError => e
+        qurd_logger.error("SQS raised #{e}")
+        raise e
+      end
+    end
+  end
+end