cluster-discovery 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 52dee8af31d3c92628213fc8dc3466aff557c9a1
+  data.tar.gz: 0d3d7aef4253ee018c4be0f05c8b08fba9a24a88
+SHA512:
+  metadata.gz: 74624cd578d290a6023685cd4161272640fd8de962877cbe4941838218c62a462d2e5bcc5c7045ca00ed8aeef53beaa0453ffc9bebe1c57f3b019abc450c366e
+  data.tar.gz: 90b73cbcdb2f4c9af9eb5b5c1f404267966bcc27946c0f058ece3b00c9ed48a9826a6e19d7dfea2f30a3a3e560bcf235de70853c2d1ef22dd31ccfd40e1331fd

data/.gitignore

@@ -0,0 +1,13 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,9 @@
+inherit_from: .rubocop_todo.yml
+
+Metrics/MethodLength:
+  Exclude:
+    - 'Rakefile'
+
+Metrics/LineLength:
+  Exclude:
+    - 'Rakefile'

data/.rubocop_todo.yml

@@ -0,0 +1,62 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2016-01-06 15:20:00 -0500 using RuboCop version 0.35.1.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 1
+# Cop supports --auto-correct.
+
+Style/BlockComments:
+  Exclude:
+    - 'spec/spec_helper.rb'
+
+# Offense count: 2
+# Configuration parameters: Exclude.
+Style/Documentation:
+  Exclude:
+    - 'spec/**/*'
+    - 'test/**/*'
+    - 'lib/cluster/discovery.rb'
+    - 'lib/cluster/discovery/version.rb'
+    - 'lib/**/*'
+
+# Offense count: 2
+# Cop supports --auto-correct.
+Style/LeadingCommentSpace:
+  Exclude:
+    - 'Gemfile'
+
+# Offense count: 2
+# Cop supports --auto-correct.
+# Configuration parameters: PreferredDelimiters.
+Style/PercentLiteralDelimiters:
+  Exclude:
+    - 'cluster-discovery.gemspec'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles, AllowInnerSlashes.
+Style/RegexpLiteral:
+  Exclude:
+    - 'Guardfile'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+Style/RescueModifier:
+  Exclude:
+    - 'lib/cluster/discovery/version.rb'
+
+# Offense count: 3
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+#Style/StringLiterals:
+#  Enabled: false
+
+# Offense count: 2
+# Cop supports --auto-correct.
+Style/UnneededPercentQ:
+  Exclude:
+    - 'cluster-discovery.gemspec'

data/Gemfile

@@ -0,0 +1,26 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in cluster-discovery.gemspec
+gemspec
+
+group :development do
+  gem 'yard', '~> 0.8'
+  gem 'thor', '~> 0.19.1'
+  gem 'thor-scmversion', '= 1.7.0'
+
+  group :guard do
+    gem 'guard', '~> 2.13'
+    gem 'guard-rspec', '~> 4.6'
+    gem 'guard-rubocop', '~> 1.2'
+    gem 'terminal-notifier-guard', '~> 1.6'
+  end
+end
+
+group :development, :test do
+  gem 'rubocop', '~> 0.34'
+  gem 'simplecov', '~> 0.11'
+  gem 'vcr', '~> 3.0'
+  gem 'webmock', '~> 1.22'
+  gem 'rspec', '~> 3.4'
+  gem 'httparty', '~> 0.13'
+end

data/Gemfile.lock

@@ -0,0 +1,141 @@
+PATH
+  remote: .
+  specs:
+    cluster-discovery (0.0.0)
+      aws-sdk (~> 2.2)
+      diplomat (~> 0.15)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.3.8)
+    ast (2.2.0)
+    astrolabe (1.3.1)
+      parser (~> 2.2)
+    aws-sdk (2.2.9)
+      aws-sdk-resources (= 2.2.9)
+    aws-sdk-core (2.2.9)
+      jmespath (~> 1.0)
+    aws-sdk-resources (2.2.9)
+      aws-sdk-core (= 2.2.9)
+    coderay (1.1.0)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    diff-lcs (1.2.5)
+    diplomat (0.15.0)
+      faraday (~> 0.9)
+      json (~> 1.8)
+    docile (1.1.5)
+    faraday (0.9.2)
+      multipart-post (>= 1.2, < 3)
+    ffi (1.9.10)
+    formatador (0.2.5)
+    guard (2.13.0)
+      formatador (>= 0.2.4)
+      listen (>= 2.7, <= 4.0)
+      lumberjack (~> 1.0)
+      nenv (~> 0.1)
+      notiffany (~> 0.0)
+      pry (>= 0.9.12)
+      shellany (~> 0.0)
+      thor (>= 0.18.1)
+    guard-compat (1.2.1)
+    guard-rspec (4.6.4)
+      guard (~> 2.1)
+      guard-compat (~> 1.1)
+      rspec (>= 2.99.0, < 4.0)
+    guard-rubocop (1.2.0)
+      guard (~> 2.0)
+      rubocop (~> 0.20)
+    hashdiff (0.2.3)
+    httparty (0.13.7)
+      json (~> 1.8)
+      multi_xml (>= 0.5.2)
+    jmespath (1.1.3)
+    json (1.8.3)
+    listen (3.0.5)
+      rb-fsevent (>= 0.9.3)
+      rb-inotify (>= 0.9)
+    lumberjack (1.0.10)
+    method_source (0.8.2)
+    mixlib-shellout (2.2.5)
+    multi_xml (0.5.5)
+    multipart-post (2.0.0)
+    nenv (0.2.0)
+    notiffany (0.0.8)
+      nenv (~> 0.1)
+      shellany (~> 0.0)
+    parser (2.2.3.0)
+      ast (>= 1.1, < 3.0)
+    powerpack (0.1.1)
+    pry (0.10.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    rainbow (2.0.0)
+    rake (10.4.2)
+    rb-fsevent (0.9.7)
+    rb-inotify (0.9.5)
+      ffi (>= 0.5.0)
+    rspec (3.4.0)
+      rspec-core (~> 3.4.0)
+      rspec-expectations (~> 3.4.0)
+      rspec-mocks (~> 3.4.0)
+    rspec-core (3.4.1)
+      rspec-support (~> 3.4.0)
+    rspec-expectations (3.4.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.4.0)
+    rspec-mocks (3.4.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.4.0)
+    rspec-support (3.4.1)
+    rubocop (0.35.1)
+      astrolabe (~> 1.3)
+      parser (>= 2.2.3.0, < 3.0)
+      powerpack (~> 0.1)
+      rainbow (>= 1.99.1, < 3.0)
+      ruby-progressbar (~> 1.7)
+      tins (<= 1.6.0)
+    ruby-progressbar (1.7.5)
+    safe_yaml (1.0.4)
+    shellany (0.0.1)
+    simplecov (0.11.1)
+      docile (~> 1.1.0)
+      json (~> 1.8)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
+    slop (3.6.0)
+    terminal-notifier-guard (1.6.4)
+    thor (0.19.1)
+    thor-scmversion (1.7.0)
+      mixlib-shellout
+      thor
+    tins (1.6.0)
+    vcr (3.0.1)
+    webmock (1.22.5)
+      addressable (< 2.4.0)
+      crack (>= 0.3.2)
+      hashdiff
+    yard (0.8.7.6)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.7)
+  cluster-discovery!
+  guard (~> 2.13)
+  guard-rspec (~> 4.6)
+  guard-rubocop (~> 1.2)
+  httparty (~> 0.13)
+  rake (~> 10.0)
+  rspec (~> 3.4)
+  rubocop (~> 0.34)
+  simplecov (~> 0.11)
+  terminal-notifier-guard (~> 1.6)
+  thor (~> 0.19.1)
+  thor-scmversion (= 1.7.0)
+  vcr (~> 3.0)
+  webmock (~> 1.22)
+  yard (~> 0.8)

data/Guardfile

@@ -0,0 +1,47 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+## Uncomment and set this to only include directories you want to watch
+# directories %w(app lib config test spec features) \
+#  .select{|d| Dir.exists?(d) ? d : UI.warning("Directory #{d} does not exist")}
+
+## Note: if you are using the `directories` clause above and you are not
+## watching the project directory ('.'), then you will want to move
+## the Guardfile to a watched dir and symlink it back, e.g.
+#
+#  $ mkdir config
+#  $ mv Guardfile config/
+#  $ ln -s config/Guardfile .
+#
+# and, you'll have to watch "config/Guardfile" instead of "Guardfile"
+
+# Note: The cmd option is now required due to the increasing number of ways
+#       rspec may be run, below are examples of the most common uses.
+#  * bundler: 'bundle exec rspec'
+#  * bundler binstubs: 'bin/rspec'
+#  * spring: 'bin/rspec' (This will use spring if running and you have
+#                          installed the spring binstubs per the docs)
+#  * zeus: 'zeus rspec' (requires the server to be started separately)
+#  * 'just' rspec: 'rspec'
+
+guard :rspec, cmd: 'bundle exec rspec' do
+  require 'guard/rspec/dsl'
+  dsl = Guard::RSpec::Dsl.new(self)
+
+  # Feel free to open issues for suggestions and improvements
+
+  # RSpec files
+  rspec = dsl.rspec
+  watch(rspec.spec_helper) { rspec.spec_dir }
+  watch(rspec.spec_support) { rspec.spec_dir }
+  watch(rspec.spec_files)
+
+  # Ruby files
+  ruby = dsl.ruby
+  dsl.watch_spec_files_for(ruby.lib_files)
+end
+
+guard :rubocop do
+  watch(%r{.+\.rb$})
+  watch(%r{(?:.+/)?\.rubocop\.yml$}) { |m| File.dirname(m[0]) }
+end

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2016 Andrew Thompson, Rapid7 LLC.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,89 @@
+# Cluster::Discovery
+
+This is a library that provides a generic interface to multiple cluster providers.
+
+## Cluster Providers
+Cluster providers are something that provides metadata about a collection of hosts in a cluster.  Supported providers are as follows:
+
+* [x] EC2 Tags
+* [x] EC2 AutoScaling Groups
+* [ ] Consul Services
+* [ ] Static Lists
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'cluster-discovery'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install cluster-discovery
+
+## Usage
+
+### EC2 Tags
+
+To discover cluster instances with EC2 Tags use something like the following example.  The keys `aws_region`, `aws_tags`(`key`, `value`) are all required.
+
+This returns an Array of EC2 Instance Objects
+
+```ruby
+instances = Cluster::Discovery.discover(
+  'ec2_tag',
+  aws_region: 'us-east-1',
+  aws_tags: [{ key: 'Service', value: 'router' }])
+
+instances.map(&:instance_id)
+```
+
+### AutoScaling Groups
+
+To discover cluster instances by AutoScaling Group use something like the following example.  The keys `aws_region`, `aws_asg` are all required.
+
+```ruby
+instances = Cluster::Discovery.discover(
+  'ec2_asg',
+  aws_region: 'us-east-1',
+  aws_asg: 'foo-prod-v000')
+
+instances.map(&:instance_id)
+```
+
+### Consul
+
+To discover cluster instances using Consul use something like the following example. The keys `consul_url` and `consul_service` are required, `leader` and `tags` are optional.
+
+```ruby
+instances = Cluster::Discovery.discover(
+  'consul',
+  consul_url: 'http://my.consul.cluster:8500',
+  consul_service: 'redis',
+  leader: true,
+  tags: 'master')
+instances.map(&:Address)
+```
+
+## Contributing
+
+### Running the tests
+
+```bash
+TEST_CONSUL_HOST=my.consul.cluster rake spec
+```
+Or just:
+```bash
+rake spec
+```
+
+1. Fork it ( https://github.com/rapid7/cluster-discovery/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,51 @@
+require 'bundler/gem_tasks'
+require 'rubocop/rake_task'
+require 'rspec/core/rake_task'
+require 'httparty'
+require 'logger'
+
+@my_logger = Logger.new(STDOUT)
+
+RuboCop::RakeTask.new
+RSpec::Core::RakeTask.new(:spec)
+
+def register_service(id:, ip:, tag:, status:)
+  json_doc = format(%(
+    {
+      "Datacenter": "dc1",
+      "Node": "foobar%{id}",
+      "Address": "%{ip}",
+      "Service": {
+        "ID": "redis%{id}",
+        "Service": "redis",
+        "Tags": ["%{tag}", "v1"],
+        "Address": "127.0.0.1",
+        "Port": 8000
+      },
+      "Check": {
+        "Node": "foobar%{id}",
+        "CheckID": "service:redis1",
+        "Name": "Redis health check",
+        "Notes": "Script based health check",
+        "Status": "%{status}",
+        "ServiceID": "redis%{id}"
+      }
+    }
+  ), id: id, ip: ip, tag: tag, status: status)
+
+  consul_host = ENV['TEST_CONSUL_HOST']
+  options = { logger: @my_logger, log_level: :debug, log_format: :curl }
+  options[:body] = json_doc
+  HTTParty.put("http://#{consul_host}:8500/v1/catalog/register", options)
+end
+
+task :refresh_fixtures do
+  register_service(id: '1', ip: '192.168.10.10', tag: 'master', status: 'passing')
+  register_service(id: '2', ip: '192.168.10.11', tag: 'slave', status: 'passing')
+  register_service(id: '3', ip: '192.168.10.12', tag: 'master', status: 'passing')
+  register_service(id: '4', ip: '192.168.10.13', tag: 'slave', status: 'passing')
+  register_service(id: '5', ip: '192.168.10.14', tag: 'master', status: 'passing')
+  register_service(id: '6', ip: '192.168.10.15', tag: 'slave', status: 'passing')
+end
+
+task default: [:spec, :rubocop]

data/Thorfile

@@ -0,0 +1,3 @@
+require 'bundler'
+require 'bundler/setup'
+require 'thor/scmversion'

data/cluster-discovery.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'cluster/discovery/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'cluster-discovery'
+  spec.version       = Cluster::Discovery::VERSION
+  spec.authors       = ['Andrew Thompson']
+  spec.email         = ['Andrew_Thompson@rapid7.com']
+  spec.summary       = 'Cluster Discovery Library'
+  spec.description   = %q{This is a library that provides a generic interface to multiple cluster providers.  Cluster providers include: EC2 Tags, EC2 AutoScaling Groups, and Consul}
+  spec.homepage      = ''
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.7'
+  spec.add_development_dependency 'rake', '~> 10.0'
+
+  spec.add_dependency 'aws-sdk', '~> 2.2'
+  spec.add_dependency 'diplomat', '~> 0.15'
+end

data/lib/cluster/discovery/consul.rb

@@ -0,0 +1,46 @@
+module Cluster
+  module Discovery
+    class Consul
+      def initialize(consul_url: nil)
+        unless consul_url.nil?
+          Diplomat.configure do |config|
+            config.url = consul_url
+          end
+        end
+        @configuration = Diplomat.configuration
+      end
+
+      # Discovery nodes using a Consul cluster
+      #
+      # @param [String] consul_service: Name of the Consul service
+      # @param [Boolean] leader: (false) Fetch :all or or :first(the leader)
+      # @param [Array] tags: ([]) List of tags to limit discovery by
+      # @return [Struct] Object representing the Node in the service
+      def discover(consul_service:, leader: false, tags: [])
+        scope, options = build_extra_opts(leader: leader, tags: tags)
+        nodes = Diplomat::Service.get(
+          consul_service,
+          scope,
+          options
+        )
+        [nodes].flatten
+      end
+
+      # Build args for Diplomat::Service.get
+      #
+      # @param [Hash] args the options to pass to Diplomat::Service.get
+      # @option opts [Symbol] :leader Wheather or not to get only the leader
+      # @option opts [Symbol] :tags The tags to discover by
+      # @return [Array] The options to pass to Diplomat::Service.get
+      def build_extra_opts(args)
+        opts = []
+        leader = args[:leader] == true ? :first : :all
+        opts << leader
+        opts << { tag: args[:tags] } unless args[:tags].empty?
+        opts
+      end
+    end
+  end
+end
+
+require 'diplomat'

data/lib/cluster/discovery/ec2/auto_scaling.rb

@@ -0,0 +1,27 @@
+module Cluster
+  module Discovery
+    module EC2
+      class AutoScaling
+        def initialize(aws_region:)
+          @aws_region = aws_region
+        end
+
+        # Discover EC2 Instances by AutoScaling Group
+        #
+        # @param [String] aws_auto_scaling_group AutoScaling Group to limit
+        #  discovery by
+        # @return [Array<Aws::EC2::Types::Instance>] Discovery result
+        def discover(aws_asg:)
+          fail EmptyASGError if aws_asg.nil?
+          tags = Cluster::Discovery::EC2::Tag.new(aws_region: @aws_region)
+          tags.discover(aws_tags: [
+            {
+              key: 'aws:autoscaling:groupName',
+              value: aws_asg
+            }
+          ])
+        end
+      end
+    end
+  end
+end

data/lib/cluster/discovery/ec2/tag.rb

@@ -0,0 +1,64 @@
+module Cluster
+  module Discovery
+    module EC2
+      class Tag
+        # @!attribute [r]
+        # The EC2 Client Object
+        attr_reader :ec2_client
+
+        # Initialize the EC2 Client object
+        #
+        # @param [String] aws_region: The aws region
+        # @return [Aws::EC2::Client] The EC2 Client object
+        def initialize(aws_region:)
+          @ec2_client ||= Aws::EC2::Client.new(region: aws_region)
+        end
+
+        # Discover EC2 Instances by Tag
+        #
+        # @param [Array<Hash>] aws_tags AWS Tags to limit discovery
+        # @option aws_tags [String] :key The name of the tag
+        # @option aws_tags [Array<String>] :values The value(s) of the tag
+        # @example For example, aws_tags might look something like this:
+        #  [
+        #    { key: "tag:Service", values: ["MyService"] },
+        #    { key: "tag:Purpose", values: ["MyPurpose"] }
+        #  ]
+        # @return [Array<Aws::EC2::Types::Instance>] Array of EC2
+        #  Instance objects
+        def discover(aws_tags: [])
+          fail EmptyTagsError if aws_tags.empty?
+          discover_instances_by_tags(build_tags(aws_tags))
+        end
+
+        private
+
+        def default_tags
+          { name: 'instance-state-name', values: ['running'] }
+        end
+
+        def build_tags(tags = [])
+          unless tags.empty?
+            tag_keys = tags.map(&:keys).flatten.uniq
+            fail MalformedTagsError unless [:key, :value] == tag_keys
+            tags.map! do |t|
+              { name: "tag:#{t[:key]}", values: [t[:value]] }
+            end
+          end
+          tags << default_tags
+          tags.flatten
+        end
+
+        def discover_instances_by_tags(tags)
+          instances = ec2_client.describe_instances(
+            filters: tags).inject([]) do |a, page|
+            a << page.reservations.map(&:instances)
+          end
+          instances.flatten.compact
+        end
+      end
+    end
+  end
+end
+
+require 'aws-sdk'

data/lib/cluster/discovery/errors.rb

@@ -0,0 +1,21 @@
+module Cluster
+  module Discovery
+    class EmptyTagsError < RuntimeError
+      def message
+        'Tags cannot be empty'
+      end
+    end
+
+    class MalformedTagsError < RuntimeError
+      def message
+        'Missing or invald keys'
+      end
+    end
+
+    class EmptyASGError < RuntimeError
+      def message
+        'Must be a vaild AutoScaling Group'
+      end
+    end
+  end
+end

data/lib/cluster/discovery/version.rb

@@ -0,0 +1,6 @@
+module Cluster
+  module Discovery
+    PATH = File.expand_path('../../../../VERSION', __FILE__)
+    VERSION = IO.read(PATH) rescue '0.0.1'
+  end
+end