asger 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1d7e123884650c6770de89781c7fb27cd17106c1
+  data.tar.gz: 85913ef76261482b872a1d87ec892c6869bbc212
+SHA512:
+  metadata.gz: ddee73b770fb2fc1b69b59f5e89d230ec95d135264dd697920f453756dfd0e883a69d02f25eb477905538a424a9d99ca12edcf67a1b9ca2d2a07bfa286b4e99b
+  data.tar.gz: f0ac3696d8f16ece760f270a9cf0d5a9ba887f58bfc20218fff238562ed58ed648c81c09b9d2a454e4c379a23cafd6b67c2c835885e4e150f37f8a66f7986384

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.yardopts

@@ -0,0 +1 @@
+--private --list-undoc --markup markdown - LICENSE.md

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in asger.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Ed Ropple
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,51 @@
+# `asger` #
+
+`asger` is a tool designed to field notifications from Amazon EC2 auto-scaling groups via a SNS topic subscribed to an SQS queue. (Which probably sounds alarmingly specific, but it's the most common way to do this!) Once a notification is fielded, the user can define Tasks that then perform actions on instance creation ("up" functions) and termination ("down" functions).
+
+### Important Notes ###
+- When multiple tasks are running in a single `asger` instance, they will be run in order on instance creation and _in reverse order_ on instance termination.
+
+## Contributors ##
+`asger` was built primarily at [Leaf](http://leaf.me) by [Ed Ropple](mailto:ed+asger@edropple.com) ([twitter](https://twitter.com/edropple)).
+
+## Standalone ##
+
+`asger` is designed primarily to be run as a daemon, accepting "tasks" in the form of Ruby files. Tasks are [fairly simple](https://github.com/eropple/asger/blob/master/samples/echo.rb); more documentation will be forthcoming.
+
+Sample usage:
+
+```bash
+./bin/asger --queue-url 'https://sqs.us-east-1.amazonaws.com/ACCOUNT_ID/QUEUE_NAME' --shared-credentials=CREDS --parameter-file /tmp/some_params.yaml --task-file %/echo.rb --task-file %/chef_deregister.rb
+```
+
+(Note: `%` is a special path character when passed to `--task-file`; it refers to the `stock_scripts` directory within the gem, so you can get started right away with the stock scripts.)
+
+## Embedded ##
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'asger'
+```
+
+And then execute:
+
+```bash
+bundle
+```
+
+Or install it yourself as:
+
+```bash
+gem install asger
+```
+
+Yardocs are available with `yard`, though they aren't complete. Nothing in `asger` is particularly complicated, though, so I recommend just taking a look at the source.
+
+## Contributing ##
+
+1. Fork it ( https://github.com/[my-github-username]/asger/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/asger.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'asger/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "asger"
+  spec.version       = Asger::VERSION
+  spec.authors       = ["Ed Ropple"]
+  spec.email         = ["ed@edropple.com"]
+  spec.summary       = %q{A persistent daemon that watches an AWS autoscaling group for changes and dispatches your code.}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler",          "~> 1.7"
+  spec.add_development_dependency "rake",             "~> 10.0"
+  spec.add_development_dependency "pry"
+
+  spec.add_runtime_dependency     'aws-sdk',          '~> 2.0.27'
+  spec.add_runtime_dependency     'trollop',          '~> 2.1.1'
+  spec.add_runtime_dependency     "hashie",             "~> 3.3"
+  spec.add_runtime_dependency     'activesupport',      '~> 4.2.0'
+end

data/bin/asger

@@ -0,0 +1,7 @@
+#! /usr/bin/env ruby
+
+lib = File.expand_path(File.dirname(__FILE__) + '/../lib')
+$LOAD_PATH.unshift(lib) if File.directory?(lib) && !$LOAD_PATH.include?(lib)
+
+require 'asger/cli'
+Asger::CLI::main()

data/lib/asger.rb

@@ -0,0 +1,5 @@
+require "asger/version"
+
+# The root namespace for Asger.
+module Asger
+end

data/lib/asger/cli.rb

@@ -0,0 +1,97 @@
+require 'logger'
+require 'json'
+require 'yaml'
+require 'trollop'
+
+require 'aws-sdk'
+require 'active_support'
+
+require 'asger/runner'
+
+module Asger
+  # Command line functionality for Asger.
+  module CLI
+    # Entry point called from `bin/asger`.
+    def self.main()
+      opts = Trollop::options do
+        opt :task_file,       "path to a task (Ruby file; pass in order of execution; % refers to the stock_scripts directory)",
+                              :type => :string, :multi => true
+        opt :parameter_file,  "path to a params file (YAML or JSON; later files override earlier ones)",
+                              :type => :string, :multi => true
+        opt :queue_url,       "URL of the SQS queue to read from",
+                              :type => :string
+        opt :pause_time,      "Time (in seconds) to pause between polls.",
+                              :default => 0
+        opt :verbose,         "enables verbose logging",
+                              :default => false
+        opt :die_on_error,    "Terminates if an exception is thrown within the task runner.",
+                              :default => true
+
+        opt :aws_logging,     "Provides the Asger logger to AWS (use for deep debugging).", :default => false
+
+        opt :shared_credentials, "Tells Asger to use shared credentials from '~/.aws/credentials'.", :type => :string
+      end
+
+      logger = Logger.new($stderr)
+      logger.info "Initializing Asger."
+
+      logger.level = opts[:verbose] ? Logger::DEBUG : Logger::INFO
+
+      if !opts[:queue_url]
+        logger.error "--queue-url is required."
+        exit(1)
+      end
+      logger.warn "No tasks configured; Asger will run, but won't do much." unless (opts[:task_file] && !opts[:task_file].empty?)
+
+      param_files = 
+        opts[:parameter_file].map do |pf|
+          logger.debug "Parsing parameter file '#{pf}'."
+          case File.extname(pf)
+            when ".json"
+              JSON.parse(File.read(pf))
+            when ".yaml"
+              YAML.load(File.read(pf))
+            else
+              raise "Unrecognized parameter file: '#{pf}'."
+          end
+        end
+
+      parameters = {}
+      param_files.each { |p| parameters.deep_merge!(p) }
+
+      credentials = nil
+      if opts[:shared_credentials]
+        logger.info "Using shared credentials '#{opts[:shared_credentials]}'."
+        credentials = Aws::SharedCredentials.new(profile_name: opts[:shared_credentials])
+      end
+
+      aws_logger = opts[:aws_logging] ? logger : nil
+      sqs_client = Aws::SQS::Client.new(logger: aws_logger, credentials: credentials)
+      ec2_client = Aws::EC2::Client.new(logger: aws_logger, credentials: credentials)
+
+
+      stock_scripts_dir = File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "stock_scripts"))
+      task_files = opts[:task_file].map { |f| f.gsub("%", stock_scripts_dir)}
+
+      logger.info "Using task files:"
+      task_files.each { |tf| logger.info " - #{tf}" }
+      runner = Runner.new(logger, sqs_client, ec2_client, opts[:queue_url], parameters, task_files)
+
+      logger.info "Beginning run loop. Sleeping between steps for #{opts[:pause_time]} seconds."
+      loop do
+        begin
+          runner.step()
+        rescue StandardError => err
+          logger.error "Encountered an error."
+          logger.error "#{err.class.name}: #{err.message}"
+          err.backtrace.each { |bt| logger.error bt }
+
+          if opts[:die_on_error]
+            raise err
+          end
+        end
+        sleep opts[:pause_time] unless opts[:pause_time] == 0
+      end
+    end
+  end
+end

data/lib/asger/runner.rb

@@ -0,0 +1,75 @@
+require 'logger'
+require 'aws-sdk'
+require 'hashie'
+
+require 'asger/task'
+
+module Asger
+  class Runner
+    # @param logger [Logger] the logger for Asger to use
+    # @param sqs_client [Aws::SQS::Client] the SQS client to use for polling
+    # @param ec2_client [Aws::EC2::Client] the EC2 client to use to get instance information
+    # @param queue_url [String] the queue URL to poll
+    # @param parameters [Hash] a hash of parameters to pass to {Task}s
+    # @param task_files [Array<String>] list of file paths to load as {Task}s
+    def initialize(logger, sqs_client, ec2_client, queue_url, parameters, task_files)
+      @logger = logger
+      @sqs_client = sqs_client
+      @ec2_client = ec2_client
+      @ec2_resource_client = Aws::EC2::Resource.new(client: @ec2_client)
+      @queue_url = queue_url
+      @parameters = Hashie::Mash.new(parameters)
+      @tasks = task_files.map { |tf| Task.from_file(@logger, tf) }
+
+      @logger.info "#{@tasks.length} task(s) set up."
+
+      @tasks.each { |t| t.invoke_sanity_check(@parameters) }
+    end
+
+
+    def step()
+      messages = @sqs_client.receive_message(queue_url: @queue_url)[:messages]
+      messages.each do |msg|
+        notification = JSON.parse(JSON.parse(msg[:body])["Message"])
+        if notification["Event"] != nil
+          case notification["Event"].gsub("autoscaling:", "")
+            when "EC2_INSTANCE_LAUNCH"
+              instance_id = notification["EC2InstanceId"]
+              @logger.info "Instance launched: #{instance_id}"
+
+              instance = @ec2_resource_client.instance(instance_id)
+              @tasks.each do |task|
+                task.invoke_up(instance, @parameters)
+              end
+
+              delete_message(msg)
+            when "EC2_INSTANCE_LAUNCH_ERROR"
+              @logger.warn "Instance launch error received."
+              delete_message(msg)
+            when "EC2_INSTANCE_TERMINATE"
+              instance_id = notification["EC2InstanceId"]
+              @logger.info "Instance terminated: #{instance_id}"
+
+              @tasks.reverse_each do |task|
+                task.invoke_down(instance_id, @parameters)
+              end
+              delete_message(msg)
+            when "EC2_INSTANCE_TERMINATE_ERROR"
+              @logger.warn "Instance terminate error received."
+              delete_message(msg)
+            when "TEST_NOTIFICATION"
+              @logger.debug "Found test notification in queue."
+              delete_message(msg)
+            else
+              @logger.debug "Unrecognized notification '#{notification["Event"]}', ignoring."
+          end
+        end
+      end
+    end
+
+    private
+    def delete_message(msg)
+      @sqs_client.delete_message(queue_url: @queue_url, receipt_handle: msg[:receipt_handle])
+    end
+  end
+end

data/lib/asger/task.rb

@@ -0,0 +1,78 @@
+require 'asger/util'
+
+module Asger
+  # A `Task` is a wrapper around an `up` and a `down` function. Up functions
+  # are called when an auto-scaling group adds an instance, and Asger retrieves
+  # the instance data for the task. Down functions are called when an
+  # auto-scaling group downs an instance, and Asger passes along only the
+  # instance ID (because it's all we've got).
+  class Task
+    attr_reader :logger
+
+    def initialize(logger, code, filename = "unknown_file.rb")
+      @logger = logger
+      @name = File.basename(filename)
+      instance_eval(code, filename, 1)
+    end
+
+    def invoke_sanity_check(parameters)
+      if @sanity_check_proc
+        logger.debug "Sanity checking for '#{@name}'..."
+        @sanity_check_proc.call(parameters)
+      else
+        logger.debug "No sanity check for '#{@name}'."
+      end
+    end
+
+    def invoke_up(instance, parameters)
+      if @up_proc
+        logger.debug "Invoking up for '#{@name}'..."
+        @up_proc.call(instance, parameters)
+        logger.debug "Up invoked for '#{@name}'..."
+      else
+        logger.debug "No up for '#{@name}'."
+      end
+    end
+
+    def invoke_down(instance_id, parameters)
+      if @down_proc
+        logger.debug "Invoking down for '#{@name}'..."
+        @down_proc.call(instance_id, parameters)
+        logger.debug "Down invoked for '#{@name}'..."
+      else
+        logger.debug "No down for '#{@name}'."
+      end
+    end
+
+    def self.from_file(logger, file)
+      Task.new(logger, File.read(file), file)
+    end
+
+    private
+    # Defines a sanity check function, which should raise and fail (which will halt
+    # Asger before it does anything with the actual queue) if there's a problem with
+    # the parameter set.
+    # 
+    # @yield [parameters]
+    # @yieldparam parameters [Hash] the parameters passed in to Asger
+    def sanity_check(&block)
+      @sanity_check_proc = block
+    end
+
+    # Defines an 'up' function.
+    # @yield [instance, parameters]
+    # @yieldparam instance [Aws::EC2::Instance] the instance that has been created
+    # @yieldparam parameters [Hash] the parameters passed in to Asger
+    def up(&block)
+      @up_proc = block
+    end
+
+    # Defines a 'down' function.
+    # @yield [instance_id, parameters]
+    # @yieldparam instance_id [String] the ID of the recently terminated instance
+    # @yieldparam parameters [Hash] the parameters passed in to Asger
+    def down(&block)
+      @down_proc = block
+    end
+  end
+end

data/lib/asger/util.rb

@@ -0,0 +1,31 @@
+require 'open3'
+
+module Asger
+  module Util
+
+    def self.run_command(subcommand_name, command, logger = nil, directory = nil)
+      directory = directory || Dir.getwd()
+      logger.info "[#{subcommand_name}] => #{command}" unless !logger
+
+      lines = []
+
+      Dir.chdir(directory) do
+        # TODO: this should probably raise on nonzero.
+        thread = Open3::popen3(command) do |stdin, stdout, stderr, wait_thr|
+          stdout.read.split("\n").each do |line|
+            lines << line 
+            logger.debug "[#{subcommand_name}] O: #{line}" unless !logger
+          end
+          stderr.read.split("\n").each do |line|
+            logger.debug "[#{subcommand_name}] E: #{line}" unless !logger
+          end
+
+          wait_thr
+        end
+
+        [ thread.value.exitstatus, lines.join("\n") ]
+      end
+    end
+
+  end
+end

data/lib/asger/version.rb

@@ -0,0 +1,4 @@
+module Asger
+  # The current version of Asger.
+  VERSION = "0.2.0"
+end

data/stock_scripts/chef_deregister.rb

@@ -0,0 +1,61 @@
+# This script will deregister a node from the Chef server given in the parameters files.
+# It will perform a Chef search for an 'instance_id' tag that matches the instance ID
+# provided by the AWS notification. To tag a node with the instance_id from bash, do
+# something like this on-node (probably in cloud-init):
+# 
+# ```bash
+# knife tag create -c /path/to/client.rb $NODE_NAME $INSTANCE_ID
+# ```
+# 
+# Which, in our naming scheme at Leaf, becomes:
+# 
+# ```bash
+# knife tag create -c /etc/chef/client.rb vpn.test-cloud.infra.06f1d39c i-06f1d39c
+# ```
+# 
+# PARAMETERS:
+# 
+# chef_deregister.knife_config: the path to the knife config to use for search/node delete
+
+require 'json'
+
+sanity_check do |parameters|
+  knife_config = parameters[:chef_deregister][:knife_config]
+  raise "parameters[:chef_deregister][:knife_config] is not set" unless knife_config
+
+  raise "file '#{knife_config}' does not exist" unless File.exist?(knife_config)
+end
+
+down do |instance_id, parameters|
+  knife_config = parameters[:chef_deregister][:knife_config]
+
+  search_result = Asger::Util::run_command("knife search: #{instance_id}",
+                                           "knife search node -c '#{knife_config}' -i -F json 'tags:#{instance_id}'",
+                                          logger)
+
+  raise "knife search for '#{instance_id}' failed with exit code #{search_result[0]}." unless search_result[0] == 0
+
+  search_json = JSON.parse(search_result[1])
+
+  case search_json["rows"].length
+  when 0
+    logger.warn "No Chef entry found for instance '#{instance_id}'."
+    logger.warn "Make sure the instance was Chef-tagged and came up successfully."
+  when 1
+    node_name = search_json["rows"][0]
+    delete_node_result = Asger::Util::run_command("knife node delete: #{node_name}",
+                                                  "knife node delete -y -c '#{knife_config}' #{node_name}",
+                                                  logger)
+    raise "knife node delete for '#{instance_id}' failed with exit code #{search_result[0]}." unless delete_node_result[0] == 0
+    delete_client_result = Asger::Util::run_command("knife client delete: #{node_name}",
+                                                    "knife client delete -y -c '#{knife_config}' #{node_name}",
+                                                    logger)
+
+    raise "knife client delete for '#{instance_id}' failed with exit code #{search_result[0]}." unless delete_client_result[0] == 0
+  else
+    logger.error "#{search_json["rows"].length} entries found in Chef for '#{instance_id}."
+    logger.error "I don't know what to safely do; you should handle this yourself."
+  end
+
+  logger.info "Removed instance '#{instance_id}' from Chef."
+end

data/stock_scripts/echo.rb

@@ -0,0 +1,9 @@
+# Just logs the instances being created and destroyed to the logger for testing.
+
+up do |instance, parameters|
+  logger.info "upping instance: #{instance}"
+end
+
+down do |instance_id, parameters|
+  logger.info "downing instance: #{instance_id}"
+end

metadata

@@ -0,0 +1,161 @@
+--- !ruby/object:Gem::Specification
+name: asger
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Ed Ropple
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-02-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.27
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.27
+- !ruby/object:Gem::Dependency
+  name: trollop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.1
+- !ruby/object:Gem::Dependency
+  name: hashie
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.2.0
+description: 
+email:
+- ed@edropple.com
+executables:
+- asger
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".yardopts"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- asger.gemspec
+- bin/asger
+- lib/asger.rb
+- lib/asger/cli.rb
+- lib/asger/runner.rb
+- lib/asger/task.rb
+- lib/asger/util.rb
+- lib/asger/version.rb
+- stock_scripts/chef_deregister.rb
+- stock_scripts/echo.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: A persistent daemon that watches an AWS autoscaling group for changes and
+  dispatches your code.
+test_files: []
+has_rdoc: