qurd 0.0.2 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 94d9eff5cd0487e81357efd7bf1f070919d88a8e
-  data.tar.gz: 6e5ff2cbc80058aac96a37508708aaa4a7dea145
+  metadata.gz: 60d8e2710e6c78de59f55d37f0477ce3f1fd98a3
+  data.tar.gz: 63d106eb2c2817c6b1a7f21f77534eeb37ea639c
 SHA512:
-  metadata.gz: c0ba3fb9cf0cb2ac637e568ff3b3a27e9affd7be961169351fb38d64d2db8c7e346c6670790eddeef4d81888824bd21680309008571244154fce538f1c2bfb89
-  data.tar.gz: 4309198c61db3c1017178fe46c85b0535c5e667935fa31f6e8ec6ed17718aa46775930f713ad0b8a50272fe4b81de16f879f20f4673e97d92d904e8cef9879c9
+  metadata.gz: f1afebad665bf6e6b1586aadffa014ab7e201694d7104d373a8ddfe4e6ab945f9b887e5c776e5da41fde63199f762e87e40e38e03f848fb1e1115643252e7bce
+  data.tar.gz: 9a29b9618381985cb9396fdb388a1f14483027cc042731ddb264c2d6a67910b43b0730e9de2bfbe6ba935d31c2fe5cf5d44a40efd126ac277c42aa6f4cec8106

data/.gitignore

@@ -13,3 +13,4 @@
 *.a
 mkmf.log
 *.sw[a-z]
+test/reports

data/.ruby-version

@@ -1 +1 @@
-2.1.2
+2.3.1

data/CHANGELOG.md

@@ -0,0 +1,6 @@
+# qurd CHANGELOG
+
+## 0.0.3
+* Add periodic stats logging
+* Add configuration option `stats_interval`
+

data/README.md

@@ -19,15 +19,20 @@ Dummy is provided as a simple example, but, in a nutshell, inherit from
 Qurd::Action and override the actions you respond to.
 
 Your action class can configure itself, by overriding the class method
-`configure`. Instances must override the action methods launch, launch_error,
-terminate, terminate_error, and test. Action instances have two attributes,
-`message` and `context`. Message is a `Qurd::Message` instance. Context is a
-`Cabin::Context`, used for logging. Callbacks, to interact with the action
-before and after the instance are executed, can be overridden.
+`configure`. Instances must override the action methods `launch`,
+`launch_error`, `terminate`, `terminate_error`, and `test`. Action instances
+have two attributes, `message` and `context`. Message is a `Qurd::Message`
+instance. Context is a `Cabin::Context`, used for logging. Callbacks, to
+interact with the action before and after the instance are executed, can be
+overridden.
 
 The mixins for AwsClients and Configuration are also available at the class and
 instance level.
 
+Action instances will also respond to `aws_credentials`, 
+`chef_client`, `chef_client=`, `chef_node`, `chef_node=`, `failed!`, `failed?`,
+`instance`, `instance_id`, `instance_name`, `name`, `region`.
+
 ```ruby
 # This contrived example creates a file in s3 when an instance launches
 # It can be triggered by adding the class name to the list of actions in the
@@ -45,7 +50,7 @@ class Foo < Qurd::Action
     aws_retryable do
       aws_client(:S3).delete_object(
         bucket: qurd_configuration.bucket,
-        key: message.instance_id
+        key: instance_id
       )
     end
   end
@@ -54,8 +59,8 @@ class Foo < Qurd::Action
     aws_retryable do
       aws_client(:S3).put_object(
         bucket: qurd_configuration.bucket,
-        key: message.instance_id,
-        body: message.instance.private_ip_address
+        key: instance_id,
+        body: instance.private_ip_address
       )
     end
   end
@@ -64,7 +69,7 @@ class Foo < Qurd::Action
     aws_retryable do
       o = aws_client(:S3).get_object(
         bucket: qurd_configuration.bucket,
-        key: message.instance_id
+        key: instance_id
       )
       qurd_logger.debug("Found #{o.body}")
     end
@@ -121,7 +126,7 @@ If you are using the route53 action, you will also need
                 "route53:ChangeResourceRecordSets",
                 "route53:DeleteHostedZone",
                 "route53:GetHostedZone",
-                "route53:ListHostedZones",
+                "route53:ListHostedZonesByName",
                 "route53:ListResourceRecordSets"
             ],
             "Resource": [
@@ -215,6 +220,13 @@ To configure the daemon, edit the YAML configuration file. The default path is
     <td><tt>true</tt></td>
   </tr>
 
+  <tr>
+    <td><tt>stats_interval</tt></td>
+    <td>Control the interval at which stats are logged, in seconds</td>
+    <td><tt>Fixnum</tt></td>
+    <td><tt>600</tt></td>
+  </tr>
+
   <tr>
     <td><tt>sqs_set_attributes_timeout</tt></td>
     <td>Defines the timeout, in seconds, for a thread setting SQS attributes</td>

data/Rakefile

@@ -1,5 +1,7 @@
 require "bundler/gem_tasks"
 require "rake/testtask"
+require 'ci/reporter/rake/minitest'
+task :minitest => ['ci:setup:minitest', 'test']
 
 Rake::TestTask.new do |t|
   t.pattern = "test/*_test.rb"

data/lib/qurd/action.rb

@@ -1,3 +1,4 @@
+require 'forwardable'
 module Qurd
   # Subclass and override {#launch}, {#launch_error}, {#terminate},
   # {#terminate_error}, and {#test}, optionally override class method
@@ -12,12 +13,49 @@ module Qurd
     autoload :Chef, 'qurd/action/chef'
     autoload :Route53, 'qurd/action/route53'
 
+    extend Forwardable
+
     extend Qurd::Mixins::Configuration
     include Qurd::Mixins::Configuration
 
     extend Qurd::Mixins::AwsClients
     include Qurd::Mixins::AwsClients
 
+    # @!attribute aws_credentials [r]
+    #   @return [Aws::Credentials]
+    def_delegator :@message, :aws_credentials, :aws_credentials
+    # @!attribute chef_client
+    #   @return [Chef::ApiClient]
+    def_delegator :@message, :chef_client, :chef_client
+    def_delegator :@message, :chef_client=, :chef_client=
+    # @!attribute chef_node
+    #   @return [Chef::Node]
+    def_delegator :@message, :chef_node, :chef_node
+    def_delegator :@message, :chef_node=, :chef_node=
+    # @!method failed!(e) [r]
+    #   Log an action failure, setting the Qurd::Message
+    #   @return [Aws::Credentials]
+    #   @see Qurd::Message.failed!
+    def_delegator :@message, :failed!, :failed!
+    # @!attribute failed? [r]
+    #   @return [Boolean]
+    def_delegator :@message, :failed?, :failed?
+    # @!attribute instance_id [r]
+    #   @return [String]
+    def_delegator :@message, :instance_id, :instance_id
+    # @!attribute instance_name [r]
+    #   @return [String]
+    def_delegator :@message, :instance_name, :instance_name
+    # @!attribute instance [r]
+    #   @return [Struct]
+    def_delegator :@message, :instance, :instance
+    # @!attribute name [r]
+    #   @return [String]
+    def_delegator :@message, :name, :name
+    # @!attribute region [r]
+    #   @return [String]
+    def_delegator :@message, :region, :region
+
     # @!attribute context [r]
     #   The logging context
     #   @return [Cabin::Context]
@@ -46,18 +84,6 @@ module Qurd
       @context = message.context
     end
 
-    # Aws region for the message
-    # @return [String]
-    def region
-      message.region
-    end
-
-    # Aws credentials for the message
-    # @return [Aws::Credentials]
-    def aws_credentials
-      message.aws_credentials
-    end
-
     # Executed before the processor runs the plugins for an action
     # @see Qurd::Processor
     def run_before

data/lib/qurd/action/chef.rb

@@ -47,14 +47,14 @@ module Qurd
       # dry_run is not true
       # @see {#Qurd::Message}
       def terminate
-        if message.failed?
+        if failed?
           qurd_logger.warn('Not deleting, message failed to process')
         elsif qurd_configuration.dry_run
           check_dry_run
         else
           qurd_logger.debug('Deleting')
-          message.chef_node.destroy unless message.chef_node.nil?
-          message.chef_client.destroy unless message.chef_client.nil?
+          chef_node.destroy unless chef_node.nil?
+          chef_client.destroy unless chef_client.nil?
         end
       end
 
@@ -68,26 +68,27 @@ module Qurd
       # Set the +message+ +chef_node+ and +context+ +chef_name+
       # @see chef_search_node
       def find_chef_node
-        node = chef_search_node
-        message.chef_node = node
+        node = chef_search_node("instance_id", instance_id)
+        node ||= chef_search_node("name", instance_name)
+        self.chef_node = node
         message.context[:chef_name] = node.name
         qurd_logger.debug('Chef node found')
       rescue NoMethodError
         qurd_logger.warn('Chef node not found')
-        message.chef_node = nil
+        self.chef_node = nil
         message.context[:chef_name] = nil
       end
 
       # Set the +message+ +chef_client+ and +context+ +chef_client_name+
       # @see chef_search_client
       def find_chef_client
-        client = chef_search_client(message.chef_node.name)
-        message.chef_client = client
+        client = chef_search_client(chef_node.name)
+        self.chef_client = client
         message.context[:chef_client_name] = client.name
         qurd_logger.debug('Chef client found')
       rescue NoMethodError
         qurd_logger.warn('Chef client not found')
-        message.chef_client = nil
+        self.chef_client = nil
         message.context[:chef_client_name] = nil
       end
 
@@ -98,10 +99,11 @@ module Qurd
       end
 
       # Search for a Chef node, based on the +instance_id+
+      # @param [String] key the chef key to search
+      # @param [String] value the value of the key to search
       # @return [Chef::Node|nil]
-      # @see instance_id
-      def chef_search_node
-        res = chef_search.search(:node, "instance_id:#{message.instance_id}")
+      def chef_search_node(key, value)
+        res = chef_search.search(:node, "#{key}:#{value}")
         res.last == 1 ? res[0][0] : nil
       end
 

data/lib/qurd/action/route53.rb

@@ -39,7 +39,7 @@ module Qurd
       # dry_run is not true
       # @see {#Qurd::Message}
       def terminate
-        if message.failed?
+        if failed?
           qurd_logger.warn('Not deleting, message failed to process')
         elsif qurd_configuration.dry_run
           if !hosted_zone
@@ -63,21 +63,17 @@ module Qurd
 
       private
 
-      def instance_name
-        message.instance_name
-      end
-
       def route53
         @route53 ||= aws_client(:Route53)
       end
 
       def qurd_route53
-        @config ||= qurd_configuration.route53[message.name]
+        @config ||= qurd_configuration.route53[name]
       end
 
       def chef_node_name
         return @chef_node_name if @chef_node_name
-        @chef_node_name = message.chef_node.name
+        @chef_node_name = chef_node.name
         qurd_logger.debug("Found chef name '#{@chef_node_name}'")
         @chef_node_name
       rescue NoMethodError
@@ -127,7 +123,7 @@ module Qurd
         end
       rescue Qurd::Action::Route53::Errors => e
         qurd_logger.error("Failed to delete: #{e}")
-        message.failed!(e)
+        failed!(e)
       end
 
       def hosted_zone(tries = nil)

data/lib/qurd/configuration.rb

@@ -8,8 +8,8 @@ module Qurd
     #   Configuration options, ie
     #   +aws_credentials+, +auto_scaling_queues+, +actions+, +daemonize+,
     #   +dry_run+, +listen_timeout+, +log_file+, +log_level+, +pid_file+,
-    #   +save_failures+, +sqs_set_attributes_timeout+, +visibility_timeout+,
-    #   +wait_time+.
+    #   +save_failures+, +stats_interval+, +sqs_set_attributes_timeout+,
+    #   +visibility_timeout+, +wait_time+.
     #   Additional configuration keys include +listeners+.
     #   @return [Hashie::Mash] the config YAML as a Mash
     # @!attribute [r] logger
@@ -30,10 +30,12 @@ module Qurd
       @config.save_failures = get_or_default(@config, :save_failures, true)
       @queues = []
       @aws_credentials = []
+      st = get_or_default(@config, :sqs_set_attributes_timeout, 10, :to_f)
+      si = get_or_default(@config, :stats_interval, 600, :to_i)
       vt = get_or_default(@config, :visibility_timeout, 300, :to_s)
       wt = get_or_default(@config, :wait_time, 20, :to_s)
-      st = get_or_default(@config, :sqs_set_attributes_timeout, 10, :to_f)
       lt = get_or_default(@config, :listen_timeout, vt, :to_f)
+      @config.stats_interval = si
       @config.visibility_timeout = vt
       @config.wait_time = wt
       @config.sqs_set_attributes_timeout = st

data/lib/qurd/listener.rb

@@ -58,6 +58,9 @@ module Qurd
     # @yieldparam [Cabin::Context] ctx the logging context
     def queue_threads(&_block)
       queues.map do |qurl|
+        @mutex = Mutex.new
+        @counter = Hashie::Mash.new({thread_timeouts: 0, aws_service_errors: 0, successes: 0, failures: 0, messages: 0})
+
         qurd_logger.debug("Creating thread for #{qurl}")
         Thread.new(qurl) do |url|
           ctx = qurd_config.get_context(name: name, queue_name: url[/([^\/]+)$/])
@@ -71,7 +74,7 @@ module Qurd
     # message received
     # @return [Array<Thread>]
     def listen
-      queue_threads do |qurl, _context|
+      threads = queue_threads do |qurl, _context|
         loop do
           begin
             msgs = aws_client(:SQS).receive_message(
@@ -84,13 +87,17 @@ module Qurd
               thread.join(qurd_configuration.listen_timeout)
             end
             if joins.compact.count != threads.count
+              nthreads = threads.count - joins.compact.count
+              lock_counter { @counter.thread_timeouts += nthreads }
               qurd_logger.warn('Some threads timed out')
             end
           rescue Aws::Errors::ServiceError => e
+            lock_counter { @counter.aws_service_errors += 1 }
             qurd_logger.error("Aws raised #{e}")
           end
         end
       end
+      threads << stats_thread
     end
 
     # @private
@@ -100,12 +107,39 @@ module Qurd
 
     private
 
+    def stats_thread
+      Thread.new do
+        loop do
+          sleep(qurd_configuration.stats_interval)
+          lock_counter { qurd_logger.info("STATS", @counter) }
+        end
+      end
+    end
+
+    def lock_counter(&_block)
+      @mutex.synchronize {
+        begin
+          yield
+        rescue ThreadError => e
+          @mutex.sleep(0.1)
+          qurd_logger.debug("ThreadError: #{e}")
+          retry
+        end
+      }
+    end
+
     def process_messages(qurl, msgs)
+      lock_counter { @counter.messages += msgs.messages.count }
       msgs.messages.map do |msg|
         Thread.new(msg) do |m|
           qurd_logger.debug("Found message #{msg}")
           r = Processor.new self, m, name, qurl
           r.process
+          lock_counter { 
+            r.message.failed? ?
+              @counter.failures += 1 :
+              @counter.successes += 1
+          }
         end
       end
     end
@@ -134,6 +168,7 @@ module Qurd
           )
         rescue Aws::SQS::Errors::ServiceError::QueueDoesNotExist => e
           qurd_logger.error("SQS raised #{e}")
+          Thread.terminate
         rescue Aws::SQS::Errors::ServiceError => e
           qurd_logger.error("SQS raised #{e}")
           raise e

data/lib/qurd/version.rb

@@ -1,5 +1,5 @@
 # Gem module
 module Qurd
   # The daemon version
-  VERSION = '0.0.2'
+  VERSION = '0.0.6'
 end

data/qurd.gemspec

@@ -21,6 +21,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.8"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "minitest"
+  spec.add_development_dependency "ci_reporter_minitest"
   spec.add_development_dependency "pry-nav"
   spec.add_development_dependency "webmock"
   spec.add_development_dependency "minitest-matchers_vaccine"

data/scripts/artifact.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+bundle package --all
+bundle install --binstubs bundler-bin --deployment --local --without development test
+echo ${BUILD_NUMBER} > .release.txt
+git rev-parse --verify HEAD >> .release.txt
+rm -rf release
+mkdir -p release
+tar -cvzf release/qurd-${BUILD_NUMBER}.tgz --exclude release * .release.txt .bundle .ruby-version

data/test/action_test.rb

@@ -39,12 +39,12 @@ describe Qurd::Action do
       aws_sqs_list_queues
       aws_sqs_set_queue_attributes
       aws_sqs_receive_message 'test/responses/aws/sqs-receive-message-1-launch.xml'
-      Qurd::Configuration.instance.configure('test/inputs/qurd.yml')
+      Qurd::Configuration.instance.configure('test/inputs/qurd_chef.yml')
     end
     let(:sqs_client) { Aws::SQS::Client.new(region: 'us-west-2') }
     let(:queue_url) { 'https://sqs.us-west-2.amazonaws.com/123456890/test2-ScalingNotificationsQueue-HPPYDAYSAGAI1' }
     let(:sqs_message) { sqs_client.receive_message(queue_url: queue_url).messages.first }
-    let(:qurd_message) { Qurd::Message.new(message: sqs_message) }
+    let(:qurd_message) { Qurd::Message.new(message: sqs_message, region: 'us-west-2', aws_credentials: Aws::Credentials.new('a', 'b')) }
     let(:subject) { TestActionClass.new(qurd_message) }
 
     it 'includes configuration mixin' do
@@ -72,12 +72,12 @@ describe Qurd::Action do
       end
     end
 
-    it 'responds to region' do
-      subject.region.must_equal qurd_message.region
-    end
-
-    it 'responds to aws_credentials' do
-      subject.aws_credentials.must_equal qurd_message.aws_credentials
+    %w[aws_credentials instance chef_client chef_node failed! failed? 
+       instance_id instance_name name region].each do |method|
+      it "responds to #{method}" do
+        aws_ec2_describe_instances 'test/responses/aws/ec2-describe-instances-1.xml'
+       subject.send(method).must_equal qurd_message.send(method)
+      end
     end
 
     %w[launch launch_error terminate terminate_error test].each do |action|

data/test/chef_test.rb

@@ -57,6 +57,11 @@ describe Qurd::Action::Chef do
         'node',
         "instance_id:#{qurd_message.instance_id}"
       )
+      chef_search(
+        'test/responses/chef/search-node-instance-n.json',
+        'node',
+        "name:test-414.staging.example.com"
+      )
       chef_search(
         'test/responses/chef/search-client-name-n.json',
         'client',
@@ -67,7 +72,7 @@ describe Qurd::Action::Chef do
       subject.message.context[:chef_name].must_equal nil
     end
 
-    it 'finds a node and client' do
+    it 'finds a node (instance_id) and client' do
       chef_search(
         'test/responses/chef/search-node-instance-1.json',
         'node',
@@ -85,12 +90,40 @@ describe Qurd::Action::Chef do
       subject.message.context[:chef_client_name].must_equal 'test-414.staging.example.com'
     end
 
+    it 'finds a node (name) and client' do
+      chef_search(
+        'test/responses/chef/search-node-instance-0.json',
+        'node',
+        "instance_id:#{qurd_message.instance_id}"
+      )
+      chef_search(
+        'test/responses/chef/search-node-instance-1.json',
+        'node',
+        "name:test-414.staging.example.com"
+      )
+      chef_search(
+        'test/responses/chef/search-client-name-1.json',
+        'client',
+        'name:test-414.staging.example.com'
+      )
+      subject.run_before
+      subject.message.chef_node.must_be_kind_of Chef::Node
+      subject.message.context[:chef_name].must_equal 'test-414.staging.example.com'
+      subject.message.chef_client.must_be_kind_of Chef::ApiClient
+      subject.message.context[:chef_client_name].must_equal 'test-414.staging.example.com'
+    end
+
     it 'does not find a node' do
       chef_search(
         'test/responses/chef/search-node-instance-0.json',
         'node',
         "instance_id:#{qurd_message.instance_id}"
       )
+      chef_search(
+        'test/responses/chef/search-node-instance-0.json',
+        'node',
+        "name:test-414.staging.example.com"
+      )
       chef_search(
         'test/responses/chef/search-client-name-0.json',
         'client',

data/test/configuration_test.rb

@@ -61,6 +61,7 @@ describe Qurd::Configuration do
         subject.config.pid_file.must_equal '/var/run/qurd/qurd.pid'
         subject.config.save_failures.must_equal true
         subject.config.sqs_set_attributes_timeout.must_equal 10.0
+        subject.config.stats_interval.must_equal 600
         subject.config.visibility_timeout.must_equal '300'
         subject.config.wait_time.must_equal '20'
       end
@@ -74,6 +75,7 @@ describe Qurd::Configuration do
         pid_file: 'tmp/qurd.pid',
         save_failures: false,
         sqs_set_attributes_timeout: 3,
+        stats_interval: 100,
         visibility_timeout: 0,
         wait_time: 1
       )
@@ -85,6 +87,7 @@ describe Qurd::Configuration do
         subject.config.pid_file.must_equal 'tmp/qurd.pid'
         subject.config.save_failures.must_equal false
         subject.config.sqs_set_attributes_timeout.must_equal 3.0
+        subject.config.stats_interval.must_equal 100
         subject.config.visibility_timeout.must_equal '0'
         subject.config.wait_time.must_equal '1'
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: qurd
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.6
 platform: ruby
 authors:
 - Philip Champon
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-17 00:00:00.000000000 Z
+date: 2017-11-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ci_reporter_minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry-nav
   requirement: !ruby/object:Gem::Requirement
@@ -161,6 +175,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".ruby-version"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -183,6 +198,7 @@ files:
 - lib/qurd/version.rb
 - lib/string.rb
 - qurd.gemspec
+- scripts/artifact.sh
 - test/action_test.rb
 - test/chef_test.rb
 - test/configuration_test.rb
@@ -251,7 +267,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: 'QUeue Resource Daemon: reaping and sowing your auto-scaled resources'
@@ -304,4 +320,3 @@ test_files:
 - test/route53_test.rb
 - test/support/web_mock_stubs.rb
 - test/test_helper.rb
-has_rdoc: