asger 0.2.1 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: abc4ad19cb57955e168d9f3c1931eec3ba3f4835
-  data.tar.gz: d9c71640218a1819c020207e04b962be00875325
+  metadata.gz: 5c91487b1cd2ad1031fc3d9b1e72a789dd446ca4
+  data.tar.gz: fc47bf1f5b779029ef2a69d8219777c653422462
 SHA512:
-  metadata.gz: f08683fa0fdaff1e6ab811d79d46dc855b6817063815748b1889a3dd28f404341e1089f8a0c5d7056867355a598807c649845aa19a2c39c1640ee210f1263886
-  data.tar.gz: 58804ae9a19d5e4edab5b77fba9fa9aaf6057b57bc63fe59ca1b507b3c17db433aff00a3df7667ea14f43c20247e5235498fbfa20eee10f7cd1ed2456f212387
+  metadata.gz: daad5b8b810429b6989cfda98b67da0384022edb68970071ec00a339ec6cee7a7013edde5d7ce27fc60990ea63e7337112aede22ba23c5d85915f77023a591e8
+  data.tar.gz: 090c4f37a0f6e461e74f4933fcb5fe921f5c3a0f65e7b5e908de355dbd52ea640ee510c5e78906c8ad04ffc4d29cf5940a658201a599293bcc88f52bcfc15a2d

data/.ruby-version

@@ -0,0 +1 @@
+2.3.0

data/README.md

@@ -1,6 +1,6 @@
 # `asger` #
 
-`asger` is a tool designed to field notifications from Amazon EC2 auto-scaling groups via a SNS topic subscribed to an SQS queue. (Which probably sounds alarmingly specific, but it's the most common way to do this!) Once a notification is fielded, the user can define Tasks that then perform actions on instance creation ("up" functions) and termination ("down" functions).
+`asger` is a tool designed to field notifications from Amazon EC2 auto-scaling groups via a SNS topic subscribed to an SQS queue. (Which probably sounds alarmingly specific, but it's the most common way to do this!) Once a notification is fielded, the user can define Tasks that then perform actions on instance creation ("up" functions) and termination ("down" functions), as well as their associated failure events.
 
 ### Important Notes ###
 - When multiple tasks are running in a single `asger` instance, they will be run in order on instance creation and _in reverse order_ on instance termination.

data/asger.gemspec

@@ -21,8 +21,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake",             "~> 10.0"
   spec.add_development_dependency "pry"
 
-  spec.add_runtime_dependency     'aws-sdk',          '~> 2.0.27'
-  spec.add_runtime_dependency     'trollop',          '~> 2.1.1'
+  spec.add_runtime_dependency     'aws-sdk',            '~> 2.2.22'
+  spec.add_runtime_dependency     'trollop',            '~> 2.1.1'
   spec.add_runtime_dependency     "hashie",             "~> 3.3"
+  spec.add_runtime_dependency     'ice_nine',           '~> 0.11.2'
   spec.add_runtime_dependency     'activesupport',      '~> 4.2.0'
 end

data/lib/asger/cli.rb

@@ -20,17 +20,19 @@ module Asger
                               :type => :string, :multi => true
         opt :queue_url,       "URL of the SQS queue to read from",
                               :type => :string
-        opt :pause_time,      "Time (in seconds) to pause between polls.",
-                              :default => 0
         opt :verbose,         "enables verbose logging",
                               :default => false
         opt :die_on_error,    "Terminates if an exception is thrown within the task runner.",
                               :default => true
 
+        opt :delete_messages, 'Delete messages from the SQS queue after processing (off is useful for development).',
+                              :default => true
+
         opt :aws_logging,     "Provides the Asger logger to AWS (use for deep debugging).", :default => false
 
         opt :shared_credentials, "Tells Asger to use shared credentials from '~/.aws/credentials'.", :type => :string
         opt :iam,             "Tells Asger to use IAM credentials.", :default => false
+        opt :region,          'Specifies an AWS region.', :type => :string
       end
 
       logger = Logger.new($stderr)
@@ -47,9 +49,10 @@ module Asger
         exit(1)
       end
 
-      logger.warn "No tasks configured; Asger will run, but won't do much." unless (opts[:task_file] && !opts[:task_file].empty?)
+      logger.warn "No tasks configured; Asger will run, but won't do much." \
+        unless (opts[:task_file] && !opts[:task_file].empty?)
 
-      param_files = 
+      param_files =
         opts[:parameter_file].map do |pf|
           logger.debug "Parsing parameter file '#{pf}'."
           case File.extname(pf)
@@ -76,8 +79,12 @@ module Asger
         end
 
       aws_logger = opts[:aws_logging] ? logger : nil
-      sqs_client = Aws::SQS::Client.new(logger: aws_logger, credentials: credentials)
-      ec2_client = Aws::EC2::Client.new(logger: aws_logger, credentials: credentials)
+      sqs_client = Aws::SQS::Client.new(logger: aws_logger,
+        region: opts[:region], credentials: credentials)
+      ec2_client = Aws::EC2::Client.new(logger: aws_logger,
+        region: opts[:region], credentials: credentials)
+      asg_client = Aws::AutoScaling::Client.new(logger: aws_logger,
+        region: opts[:region], credentials: credentials)
 
 
       stock_scripts_dir = File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "stock_scripts"))
@@ -85,12 +92,17 @@ module Asger
 
       logger.info "Using task files:"
       task_files.each { |tf| logger.info " - #{tf}" }
-      runner = Runner.new(logger, sqs_client, ec2_client, opts[:queue_url], parameters, task_files)
-
-      logger.info "Beginning run loop. Sleeping between steps for #{opts[:pause_time]} seconds."
+      runner = Runner.new(logger: logger, aws_logger: aws_logger,
+                          region: opts[:region], credentials: credentials,
+                          queue_url: opts[:queue_url],
+                          parameters: parameters,
+                          task_files: task_files,
+                          no_delete_messages: !opts[:delete_messages])
+
+      logger.info "Beginning poll loop."
       loop do
         begin
-          runner.step()
+          runner.poll
         rescue StandardError => err
           logger.error "Encountered an error."
           logger.error "#{err.class.name}: #{err.message}"
@@ -98,10 +110,11 @@ module Asger
 
           if opts[:die_on_error]
             raise err
+          else
+            logger.error "re-entering poll."
           end
         end
-        sleep opts[:pause_time] unless opts[:pause_time] == 0
       end
     end
   end
-end
+end

data/lib/asger/runner.rb

@@ -1,6 +1,7 @@
 require 'logger'
 require 'aws-sdk'
-require 'hashie'
+require 'ice_nine'
+require 'active_support/all'
 
 require 'asger/task'
 
@@ -9,66 +10,102 @@ module Asger
     # @param logger [Logger] the logger for Asger to use
     # @param sqs_client [Aws::SQS::Client] the SQS client to use for polling
     # @param ec2_client [Aws::EC2::Client] the EC2 client to use to get instance information
+    # @param asg_client [Aws::AutoScaling::Client] the ASG client to use to get ASG information
     # @param queue_url [String] the queue URL to poll
     # @param parameters [Hash] a hash of parameters to pass to {Task}s
     # @param task_files [Array<String>] list of file paths to load as {Task}s
-    def initialize(logger, sqs_client, ec2_client, queue_url, parameters, task_files)
+    # @param no_delete_messages [TrueClass, FalseClass] if true, don't call sqs:DeleteMessage
+    def initialize(logger:, aws_logger:, credentials:,
+                   region:, queue_url:,
+                   parameters:, task_files:, no_delete_messages:)
       @logger = logger
-      @sqs_client = sqs_client
-      @ec2_client = ec2_client
+      @region = region
+      @parameters = IceNine.deep_freeze(parameters.merge(
+        region: region, credentials: credentials
+      ).deep_symbolize_keys)
+
+      @sqs_client = Aws::SQS::Client.new(logger: aws_logger,
+        region: region, credentials: credentials)
+      @ec2_client = Aws::EC2::Client.new(logger: aws_logger,
+        region: region, credentials: credentials)
+      @asg_client = Aws::AutoScaling::Client.new(logger: aws_logger,
+        region: region, credentials: credentials)
       @ec2_resource_client = Aws::EC2::Resource.new(client: @ec2_client)
+      @asg_resource_client = Aws::AutoScaling::Resource.new(client: @asg_client)
       @queue_url = queue_url
-      @parameters = Hashie::Mash.new(parameters)
       @tasks = task_files.map { |tf| Task.from_file(@logger, tf) }
+      @no_delete_messages = no_delete_messages
 
       @logger.info "#{@tasks.length} task(s) set up."
+      @logger.warn('no_delete_messages is set; will not clear SQS messages!') \
+        if @no_delete_messages
 
-      @tasks.each { |t| t.invoke_sanity_check(@parameters) }
+      @tasks.each { |t| t.invoke_init(@parameters) }
     end
 
 
-    def step()
-      messages = @sqs_client.receive_message(queue_url: @queue_url)[:messages]
-      messages.each do |msg|
-        notification = JSON.parse(JSON.parse(msg[:body])["Message"])
-        if notification["Event"] != nil
-          case notification["Event"].gsub("autoscaling:", "")
+    def poll()
+      poller = Aws::SQS::QueuePoller.new(@queue_url, client: @sqs_client,
+        max_number_of_messages: 10, skip_delete: true)
+
+      poller.poll do |msgs|
+        [ msgs ].flatten.each do |msg|
+          notification = JSON.parse(JSON.parse(msg.body)["Message"])
+          if notification["Event"] != nil
+            asg = @asg_resource_client.group(notification['AutoScalingGroupName'])
+            instance_id = notification["EC2InstanceId"]
+
+            @logger.warn("ASG '#{asg}' has fired event, but does not exist - already cleaned up?") \
+              unless asg.exists?
+
+            case notification["Event"].gsub("autoscaling:", "")
             when "EC2_INSTANCE_LAUNCH"
-              instance_id = notification["EC2InstanceId"]
-              @logger.info "Instance launched: #{instance_id}"
+              @logger.info "Instance launched in '#{asg.name}': #{instance_id}"
 
               instance = @ec2_resource_client.instance(instance_id)
               @tasks.each do |task|
-                task.invoke_up(instance, @parameters)
+                task.invoke_up(instance, asg, @parameters)
               end
 
-              delete_message(msg)
+              delete_message(msg) unless @no_delete_messages
             when "EC2_INSTANCE_LAUNCH_ERROR"
-              @logger.warn "Instance launch error received."
-              delete_message(msg)
+              @logger.warn "Instance failed to launch in '#{asg.name}'."
+
+              @tasks.each do |task|
+                task.invoke_up_failed(asg, @parameters)
+              end
+
+              delete_message(msg) unless @no_delete_messages
             when "EC2_INSTANCE_TERMINATE"
-              instance_id = notification["EC2InstanceId"]
-              @logger.info "Instance terminated: #{instance_id}"
+              @logger.info "Instance terminated in '#{asg.name}': #{instance_id}"
 
               @tasks.reverse_each do |task|
-                task.invoke_down(instance_id, @parameters)
+                task.invoke_down(instance_id, asg, @parameters)
               end
-              delete_message(msg)
+
+              delete_message(msg) unless @no_delete_messages
             when "EC2_INSTANCE_TERMINATE_ERROR"
-              @logger.warn "Instance terminate error received."
-              delete_message(msg)
+              @logger.warn "Instance failed to terminate in '#{asg.name}': #{instance_id}"
+
+              @tasks.reverse_each do |task|
+                task.invoke_down_failed(instance_id, asg, @parameters)
+              end
+              delete_message(msg) unless @no_delete_messages
             when "TEST_NOTIFICATION"
-              @logger.debug "Found test notification in queue."
-              delete_message(msg)
+              @logger.info "Found test notification in queue."
+              delete_message(msg) unless @no_delete_messages
             else
               @logger.debug "Unrecognized notification '#{notification["Event"]}', ignoring."
+            end
           end
         end
       end
     end
 
     private
+
     def delete_message(msg)
+      @logger.debug "Deleting message '#{msg[:receipt_handle]}'"
       @sqs_client.delete_message(queue_url: @queue_url, receipt_handle: msg[:receipt_handle])
     end
   end

data/lib/asger/task.rb

@@ -15,64 +15,103 @@ module Asger
       instance_eval(code, filename, 1)
     end
 
-    def invoke_sanity_check(parameters)
-      if @sanity_check_proc
-        logger.debug "Sanity checking for '#{@name}'..."
-        @sanity_check_proc.call(parameters)
+    def invoke_init(parameters)
+      if @init_proc
+        logger.debug "Initializing for '#{@name}'..."
+        @init_proc.call(parameters)
       else
-        logger.debug "No sanity check for '#{@name}'."
+        logger.debug "No init for '#{@name}'."
       end
     end
 
-    def invoke_up(instance, parameters)
+    def invoke_up(instance, asg, parameters)
       if @up_proc
         logger.debug "Invoking up for '#{@name}'..."
-        @up_proc.call(instance, parameters)
+        @up_proc.call(instance, asg, parameters)
         logger.debug "Up invoked for '#{@name}'..."
       else
         logger.debug "No up for '#{@name}'."
       end
     end
 
-    def invoke_down(instance_id, parameters)
+    def invoke_down(instance_id, asg, parameters)
       if @down_proc
         logger.debug "Invoking down for '#{@name}'..."
-        @down_proc.call(instance_id, parameters)
+        @down_proc.call(instance_id, asg, parameters)
         logger.debug "Down invoked for '#{@name}'..."
       else
         logger.debug "No down for '#{@name}'."
       end
     end
 
+    def invoke_up_failed(asg, parameters)
+      if @up_failed_proc
+        logger.debug "Invoking up_failed for '#{@name}'..."
+        @up_failed_proc.call(asg, parameters)
+        logger.debug "up_failed invoked for '#{@name}'..."
+      else
+        logger.debug "No up_failed for '#{@name}'."
+      end
+    end
+
+    def invoke_down_failed(instance_id, asg, parameters)
+      if @down_failed_proc
+        logger.debug "Invoking down_failed for '#{@name}'..."
+        @down_failed_proc.call(instance_id, asg, parameters)
+        logger.debug "down_failed invoked for '#{@name}'..."
+      else
+        logger.debug "No down_failed for '#{@name}'."
+      end
+    end
+
     def self.from_file(logger, file)
       Task.new(logger, File.read(file), file)
     end
 
     private
-    # Defines a sanity check function, which should raise and fail (which will halt
-    # Asger before it does anything with the actual queue) if there's a problem with
-    # the parameter set.
-    # 
+    # Defines an init function, which should set member vars. Raise and fail (which
+    # will halt Asger before it does anything with the actual queue) if there's a
+    # problem with the parameter set.
+    #
     # @yield [parameters]
     # @yieldparam parameters [Hash] the parameters passed in to Asger
-    def sanity_check(&block)
-      @sanity_check_proc = block
+    def init(&block)
+      @init_proc = block
     end
 
-    # Defines an 'up' function.
+    # Defines an 'up' function, addressing `EC2_INSTANCE_LAUNCH`.
     # @yield [instance, parameters]
     # @yieldparam instance [Aws::EC2::Instance] the instance that has been created
+    # @yieldparam asg [nil, Aws::AutoScaling::AutoScalingGroup] the ASG resource of the launched instance
     # @yieldparam parameters [Hash] the parameters passed in to Asger
     def up(&block)
       @up_proc = block
     end
 
-    # Defines a 'down' function.
+    # Defines a 'down' function, addressing `EC2_INSTANCE_TERMINATE`.
     # @yield [instance_id, parameters]
     # @yieldparam instance_id [String] the ID of the recently terminated instance
+    # @yieldparam asg [nil, Aws::AutoScaling::AutoScalingGroup] the ASG resource of the terminated instance
     # @yieldparam parameters [Hash] the parameters passed in to Asger
     def down(&block)
       @down_proc = block
     end
+
+    # Defines an 'up_failed' function, addressing `EC2_INSTANCE_LAUNCH_ERROR`.
+    # @yield [asg, parameters]
+    # @yieldparam asg [nil, Aws::AutoScaling::AutoScalingGroup] the ASG resource of the failed instance
+    # @yieldparam parameters [Hash] the parameters passed in to Asger
+    def up_failed(&block)
+      @up_failed_proc = block
+    end
+
+    # Defines an 'up_failed' function, addressing `EC2_INSTANCE_TERMINATE_ERROR`.
+    # @yield [asg, parameters]
+    # @yieldparam instance_id [String] the ID of the instance that failed to terminate
+    # @yieldparam asg [nil, Aws::AutoScaling::AutoScalingGroup] the ASG resource of the failed instance
+    # @yieldparam parameters [Hash] the parameters passed in to Asger
+    def down_failed(&block)
+      @down_failed_proc = block
+    end
   end
 end

data/lib/asger/version.rb

@@ -1,4 +1,4 @@
 module Asger
   # The current version of Asger.
-  VERSION = "0.2.1"
+  VERSION = "1.0.2"
 end

data/stock_scripts/chef_deregister.rb

@@ -2,31 +2,31 @@
 # It will perform a Chef search for an 'instance_id' tag that matches the instance ID
 # provided by the AWS notification. To tag a node with the instance_id from bash, do
 # something like this on-node (probably in cloud-init):
-# 
+#
 # ```bash
 # knife tag create -c /path/to/client.rb $NODE_NAME $INSTANCE_ID
 # ```
-# 
+#
 # Which, in our naming scheme at Leaf, becomes:
-# 
+#
 # ```bash
 # knife tag create -c /etc/chef/client.rb vpn.test-cloud.infra.06f1d39c i-06f1d39c
 # ```
-# 
+#
 # PARAMETERS:
-# 
+#
 # chef_deregister.knife_config: the path to the knife config to use for search/node delete
 
 require 'json'
 
-sanity_check do |parameters|
+init do |parameters|
   knife_config = parameters[:chef_deregister][:knife_config]
   raise "parameters[:chef_deregister][:knife_config] is not set" unless knife_config
 
   raise "file '#{knife_config}' does not exist" unless File.exist?(knife_config)
 end
 
-down do |instance_id, parameters|
+down do |instance_id, asg, parameters|
   knife_config = parameters[:chef_deregister][:knife_config]
 
   search_result = Asger::Util::run_command("knife search: #{instance_id}",

data/stock_scripts/echo.rb

@@ -1,9 +1,20 @@
 # Just logs the instances being created and destroyed to the logger for testing.
+init do |parameters|
+  logger.info 'echo - init'
+end
+
+up do |instance, asg, parameters|
+  logger.info "echo - upping instance in '#{asg.name}': #{instance}"
+end
 
-up do |instance, parameters|
-  logger.info "upping instance: #{instance}"
+up_failed do |asg, parameters|
+  logger.warn "echo - failed to up instance in '#{asg.name}'"
 end
 
-down do |instance_id, parameters|
-  logger.info "downing instance: #{instance_id}"
-end
+down do |instance_id, asg, parameters|
+  logger.info "echo - downing instance in '#{asg.name}': #{instance_id}"
+end
+
+down_failed do |instance_id, asg, parameters|
+  logger.warn "echo - failed to down instance in '#{asg.name}': #{instance_id}"
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: asger
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 1.0.2
 platform: ruby
 authors:
 - Ed Ropple
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-26 00:00:00.000000000 Z
+date: 2016-03-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.27
+        version: 2.2.22
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.27
+        version: 2.2.22
 - !ruby/object:Gem::Dependency
   name: trollop
   requirement: !ruby/object:Gem::Requirement
@@ -94,6 +94,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: ice_nine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.11.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.11.2
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -117,6 +131,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".ruby-version"
 - ".yardopts"
 - Gemfile
 - LICENSE.txt
@@ -152,7 +167,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: A persistent daemon that watches an AWS autoscaling group for changes and