MovableInkAWS 1.3.0 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7021c3e1d0f994fe76adec836d62f1fa50454389c9dc9d07c0e754a1f05f9ac5
-  data.tar.gz: 47fa2194267bb7f8e163ae1e815d708588832789cd1857184d581c5725570d04
+  metadata.gz: 9748092eb913e843641b0ac9d68106cb8ae3dc141c5f3ef0cf2794c7b52a41f5
+  data.tar.gz: 1bcbc2de4d41898654cf8df5e09fb1a43465d338f73b64a36fcf785db882fb15
 SHA512:
-  metadata.gz: 8c1c045dbc080b60e3101796494ed88cc6dd4e3f4ae697342a41771dc194c9c4c3526ea4fefd1f4c70ce0647bf3593e8c13d0832c057974d8f60221a3e3cc2f7
-  data.tar.gz: 9b553f0f33f02d1456eafb8e06ee1ff65b60adc02f6fc1b0f17f6c94eae56b9774fdbcb547619474fa4773f3b2d8c36af344cb27a3bf91168284a970183ca53c
+  metadata.gz: ed19423e404a705e9caf74762f6bd1cd08b5f552af5b24d6c355364d59c422d2cfba5f225e10cd021a79e77ae5d65ee8773c86eda5340e32d69d088cfb4789c4
+  data.tar.gz: 6c5edc3a00e2d798140c30d2cce8279c03099e9b30aa1c1630e14a10ae075756328d9327ebfab94171bea47394503d4f9a592c9a8d5e706be37fe6c1dfbbd7f6

data/.travis.yml

@@ -1,7 +1,6 @@
 language: ruby
 
 rvm:
-  - 2.5
   - 2.6
 
 bundler_args: "--retry=3"

data/Gemfile

@@ -3,4 +3,5 @@ gemspec
 
 group :test do
   gem 'rspec',    '~> 3.6'
+  gem 'webmock'
 end

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    MovableInkAWS (1.3.0)
+    MovableInkAWS (2.0.1)
       aws-sdk-athena (~> 1)
       aws-sdk-autoscaling (~> 1)
       aws-sdk-cloudwatch (~> 1)
@@ -15,66 +15,80 @@ PATH
       aws-sdk-sns (~> 1)
       aws-sdk-ssm (~> 1)
       aws-sigv4 (~> 1.1)
+      diplomat (= 2.4.2)
       httparty (= 0.16.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
     aws-eventstream (1.1.0)
-    aws-partitions (1.364.0)
-    aws-sdk-athena (1.31.0)
-      aws-sdk-core (~> 3, >= 3.99.0)
+    aws-partitions (1.390.0)
+    aws-sdk-athena (1.33.0)
+      aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-autoscaling (1.45.0)
-      aws-sdk-core (~> 3, >= 3.99.0)
+    aws-sdk-autoscaling (1.48.0)
+      aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-cloudwatch (1.43.0)
-      aws-sdk-core (~> 3, >= 3.99.0)
+    aws-sdk-cloudwatch (1.46.0)
+      aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-core (3.105.0)
+    aws-sdk-core (3.109.2)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-ec2 (1.193.0)
-      aws-sdk-core (~> 3, >= 3.99.0)
+    aws-sdk-ec2 (1.207.0)
+      aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-eks (1.41.0)
-      aws-sdk-core (~> 3, >= 3.99.0)
+    aws-sdk-eks (1.45.0)
+      aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-elasticache (1.41.0)
-      aws-sdk-core (~> 3, >= 3.99.0)
+    aws-sdk-elasticache (1.45.0)
+      aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-kms (1.37.0)
-      aws-sdk-core (~> 3, >= 3.99.0)
+    aws-sdk-kms (1.39.0)
+      aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-rds (1.97.0)
-      aws-sdk-core (~> 3, >= 3.99.0)
+    aws-sdk-rds (1.105.0)
+      aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-route53 (1.42.0)
-      aws-sdk-core (~> 3, >= 3.99.0)
+    aws-sdk-route53 (1.44.0)
+      aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.79.1)
-      aws-sdk-core (~> 3, >= 3.104.3)
+    aws-sdk-s3 (1.83.1)
+      aws-sdk-core (~> 3, >= 3.109.0)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.1)
-    aws-sdk-sns (1.30.0)
-      aws-sdk-core (~> 3, >= 3.99.0)
+    aws-sdk-sns (1.35.0)
+      aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-ssm (1.88.0)
-      aws-sdk-core (~> 3, >= 3.99.0)
+    aws-sdk-ssm (1.95.0)
+      aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
     aws-sigv4 (1.2.2)
       aws-eventstream (~> 1, >= 1.0.2)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    deep_merge (1.2.1)
     diff-lcs (1.3)
+    diplomat (2.4.2)
+      deep_merge (~> 1.0, >= 1.0.1)
+      faraday (>= 0.9, < 1.1.0)
+    faraday (1.0.1)
+      multipart-post (>= 1.2, < 3)
+    hashdiff (1.0.0)
     httparty (0.16.3)
       mime-types (~> 3.0)
       multi_xml (>= 0.5.2)
     jmespath (1.4.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2020.0512)
+    mime-types-data (3.2020.1104)
     multi_xml (0.6.0)
+    multipart-post (2.1.1)
+    public_suffix (4.0.5)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -88,6 +102,11 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-support (3.9.3)
+    safe_yaml (1.0.5)
+    webmock (3.7.6)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
 
 PLATFORMS
   ruby
@@ -95,6 +114,7 @@ PLATFORMS
 DEPENDENCIES
   MovableInkAWS!
   rspec (~> 3.6)
+  webmock
 
 BUNDLED WITH
    2.1.3

data/MovableInkAWS.gemspec

@@ -23,6 +23,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'aws-sdk-ssm', '~> 1'
   s.add_runtime_dependency 'aws-sigv4', '~> 1.1'
   s.add_runtime_dependency 'httparty',  '0.16.3'
+  s.add_runtime_dependency 'diplomat',  '2.4.2'
 
   all_files  = `git ls-files`.split("\n")
   test_files = `git ls-files -- {test,spec,features}/*`.split("\n")

data/lib/movable_ink/aws.rb

@@ -66,6 +66,7 @@ module MovableInk
                Aws::EC2::Errors::Unavailable,
                Aws::EKS::Errors::TooManyRequestsException,
                Aws::SNS::Errors::ThrottledException,
+               Aws::SNS::Errors::Throttling,
                Aws::AutoScaling::Errors::Throttling,
                Aws::AutoScaling::Errors::ThrottledException,
                Aws::S3::Errors::SlowDown,

data/lib/movable_ink/aws/ec2.rb

@@ -1,4 +1,5 @@
 require 'aws-sdk-ec2'
+require 'diplomat'
 
 module MovableInk
   class AWS
@@ -75,7 +76,7 @@ module MovableInk
         @me ||= all_instances.select{|instance| instance.instance_id == instance_id}.first rescue nil
       end
 
-      def instances(role:, exclude_roles: [], region: my_region, availability_zone: nil, exact_match: false, use_cache: true)
+      def instances_with_ec2_discovery(role:, exclude_roles: [], region: my_region, availability_zone: nil, exact_match: false, use_cache: true)
         roles = role.split(/\s*,\s*/)
         if use_cache == false
           filter = default_filter.push({
@@ -106,6 +107,65 @@ module MovableInk
         end
       end
 
+      def instances_with_consul_discovery(role:, region: my_region, availability_zone: nil)
+        if role == nil || role == ''
+          raise MovableInk::AWS::Errors::RoleNameRequiredError
+        end
+
+        if role.include?('_')
+          raise MovableInk::AWS::Errors::RoleNameInvalidError
+        end
+
+        Diplomat.configure do |config|
+          config.url = "https://localhost:8501"
+          config.options = { ssl: { verify: false } }
+        end
+
+        consul_service_options = Hash.new
+
+        consul_service_options[:dc] = datacenter(region: region)
+        consul_service_options[:stale] = true
+        consul_service_options[:cached] = true
+        consul_service_options[:passing] = true
+
+        consul_service_options[:node_meta] = "availability_zone:#{availability_zone}" unless availability_zone.nil?
+
+        Diplomat::Health.service(role, consul_service_options).map { |endpoint|
+          OpenStruct.new (
+          {
+            private_ip_address: endpoint.Node['Address'],
+            instance_id: endpoint.Node['Meta']['instance_id'],
+            tags: [
+              {
+                key: 'Name',
+                value: endpoint.Node['Node']
+              },
+              {
+                key: 'mi:roles',
+                value: endpoint.Node['Meta']['mi_roles']
+              },
+              {
+                key: 'mi:monitoring_roles',
+                value: endpoint.Node['Meta']['mi_monitoring_roles']
+              }
+            ],
+            placement: {
+              availability_zone: endpoint.Node['Meta']['availability_zone']
+            }
+          })
+        }
+      end
+
+      def instances(role:, exclude_roles: [], region: my_region, availability_zone: nil, exact_match: false, use_cache: true, discovery_type: 'ec2')
+        if discovery_type == 'ec2'
+          instances_with_ec2_discovery(role: role, exclude_roles: exclude_roles, region: region, availability_zone: availability_zone, exact_match: exact_match, use_cache: use_cache)
+        elsif discovery_type == 'consul'
+          instances_with_consul_discovery(role: role, region: region, availability_zone: availability_zone)
+        else
+          raise MovableInk::AWS::Errors::InvalidDiscoveryTypeError
+        end
+      end
+
       def thopter
         private_ip_addresses(thopter_instance).first
       end

data/lib/movable_ink/aws/errors.rb

@@ -5,6 +5,9 @@ module MovableInk
       class FailedWithBackoff < StandardError; end
       class EC2Required < StandardError; end
       class NoEnvironmentTagError < StandardError; end
+      class InvalidDiscoveryTypeError < StandardError; end
+      class RoleNameRequiredError < StandardError; end
+      class RoleNameInvalidError < StandardError; end
     end
   end
 end

data/lib/movable_ink/aws/sns.rb

@@ -11,29 +11,20 @@ module MovableInk
       end
 
       def sns_slack_topic_arn
-        run_with_backoff do
-          sns.list_topics.each do |resp|
-            resp.topics.each do |topic|
-              return topic.topic_arn if topic.topic_arn.include? "slack-aws-alerts"
-            end
-          end
-        end
+        @sns_slack_topic_arn ||= sns_topics.detect { |topic| topic.topic_arn.include? "slack-aws-alerts" }.topic_arn rescue nil
       end
 
       def sns_pagerduty_topic_arn
-        run_with_backoff do
-          sns.list_topics.each do |resp|
-            resp.topics.each do |topic|
-              return topic.topic_arn if topic.topic_arn.include? "pagerduty-custom-alerts"
-            end
-          end
-        end
+        @sns_pagerduty_topic_arn ||= sns_topics.detect { |topic| topic.topic_arn.include? "pagerduty-custom-alerts" }.topic_arn rescue nil
+      end
+
+      def sns_topics
+        @sns_topics ||= run_with_backoff { sns.list_topics.flat_map(&:topics) }
       end
 
       def notify_and_sleep(seconds, error_class)
         message = "Throttled by AWS. Sleeping #{seconds} seconds, (#{error_class})"
-        notify_slack(subject: 'API Throttled',
-                     message: message)
+        notify_slack(subject: 'API Throttled', message: message)
         puts message
         sleep seconds
       end

data/lib/movable_ink/version.rb

@@ -1,5 +1,5 @@
 module MovableInk
   class AWS
-    VERSION = '1.3.0'
+    VERSION = '2.0.1'
   end
 end

data/spec/ec2_spec.rb

@@ -1,4 +1,5 @@
 require_relative '../lib/movable_ink/aws'
+require 'webmock/rspec'
 
 describe MovableInk::AWS::EC2 do
   context "outside EC2" do
@@ -21,6 +22,7 @@ describe MovableInk::AWS::EC2 do
   end
 
   context "inside EC2" do
+    WebMock.allow_net_connect!
     let(:aws) { MovableInk::AWS.new }
     let(:ec2) { Aws::EC2::Client.new(stub_responses: true) }
     let(:tag_data) { ec2.stub_data(:describe_tags, tags: [
@@ -305,6 +307,150 @@ describe MovableInk::AWS::EC2 do
       end
     end
 
+    context "instances with consul discovery" do
+      let(:my_availability_zone) { 'us-east-1a' }
+      let(:my_datacenter) { 'iad' }
+      let(:other_availability_zone) { 'us-east-1b' }
+      let(:consul_app_service_instances) {
+        [{
+            Node: {
+              Node: 'app_instance1',
+              Address: '10.0.0.1',
+              Datacenter: my_datacenter,
+              Meta: {
+                availability_zone: my_availability_zone,
+                instance_id: 'i-12345',
+                mi_monitoring_roles: 'app',
+                mi_roles: 'app'
+              },
+              ServiceID: 'app',
+              ServiceName: 'app',
+              ServiceTags: ['foo', 'bar'],
+              ServicePort: 80,
+            }
+        },
+        {
+          Node: {
+              Node: 'app_instance2',
+              Address: '10.0.0.2',
+              Datacenter: my_datacenter,
+              Meta: {
+                availability_zone: my_availability_zone,
+                instance_id: 'i-54321',
+                mi_monitoring_roles: 'app',
+                mi_roles: 'app'
+              },
+              ServiceID: 'app',
+              ServiceName: 'app',
+              ServiceTags: ['foo', 'bar'],
+              ServicePort: 80,
+            }
+       }]
+      }
+
+      let(:consul_ojos_service_instances) {
+        [
+        {
+            Node: {
+              Node: 'ojos_instance2',
+              Address: '10.0.0.4',
+              Datacenter: my_datacenter,
+              Meta: {
+                availability_zone: other_availability_zone,
+                instance_id: 'i-zyx987',
+                mi_monitoring_roles: 'ojos',
+                mi_roles: 'ojos'
+              },
+              ServiceID: 'ojos',
+              ServiceName: 'ojos',
+              ServiceTags: ['foo', 'bar'],
+              ServicePort: 2702,
+            },
+        },
+        {
+            Node: {
+              Node: 'ojos_instance3',
+              Address: '10.0.0.5',
+              Datacenter: my_datacenter,
+              Meta: {
+                availability_zone: other_availability_zone,
+                instance_id: 'i-987654',
+                mi_monitoring_roles: 'ojos',
+                mi_roles: 'ojos'
+              },
+              ServiceID: 'ojos',
+              ServiceName: 'ojos',
+              ServiceTags: ['foo', 'bar'],
+              ServicePort: 2702,
+            }
+        }]
+      }
+
+      before(:each) do
+        allow(aws).to receive(:mi_env).and_return('test')
+        allow(aws).to receive(:availability_zone).and_return(my_availability_zone)
+        allow(aws).to receive(:my_region).and_return('us-east-1')
+        allow(aws).to receive(:ec2).and_return(ec2)
+      end
+
+      it "returns an error if no role is defined" do
+        expect{ aws.instances(role: nil, discovery_type: 'consul') }.to raise_error(MovableInk::AWS::Errors::RoleNameRequiredError)
+      end
+
+      it "returns an error if an invalid role is passed" do
+        expect{ aws.instances(role: 'asset_proxy', discovery_type: 'consul') }.to raise_error(MovableInk::AWS::Errors::RoleNameInvalidError)
+      end
+
+      it "returns all instances matching a consul service" do
+        miaws = double(MovableInk::AWS)
+        allow(miaws).to receive(:my_region).and_return('us-east-1')
+
+        json = JSON.generate(consul_app_service_instances)
+        stub_request(:get, "https://localhost:8501/v1/health/service/app?cached&dc=#{my_datacenter}&passing&stale").
+         with(
+            headers: {
+            'Accept'=>'*/*',
+            'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
+            'User-Agent'=>'Faraday v1.0.1'
+           }).
+         to_return(status: 200, body: json, headers: {})
+
+        app_instances = aws.instances(role: 'app', discovery_type: 'consul')
+        expect(app_instances.map{|i| i.tags.first[:value]}).to eq(['app_instance1', 'app_instance2'])
+
+        json = JSON.generate(consul_ojos_service_instances)
+        stub_request(:get, "https://localhost:8501/v1/health/service/ojos?cached&dc=#{my_datacenter}&passing&stale").
+         with(
+           headers: {
+          'Accept'=>'*/*',
+          'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
+          'User-Agent'=>'Faraday v1.0.1'
+           }).
+         to_return(status: 200, body: json, headers: {})
+
+        ojos_instances = aws.instances(role: 'ojos', discovery_type: 'consul')
+        expect(ojos_instances.map{|i| i.tags.first[:value ]}).to eq(['ojos_instance2', 'ojos_instance3'])
+      end
+
+      it "returns all instances matching a consul service filtered by availability_zone" do
+        miaws = double(MovableInk::AWS)
+        allow(miaws).to receive(:my_region).and_return('us-east-1')
+
+        json = JSON.generate(consul_ojos_service_instances)
+        stub_request(:get, "https://localhost:8501/v1/health/service/ojos?cached&dc=#{my_datacenter}&node-meta=availability_zone:#{other_availability_zone}&passing&stale").
+         with(
+            headers: {
+            'Accept'=>'*/*',
+            'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
+            'User-Agent'=>'Faraday v1.0.1'
+           }).
+         to_return(status: 200, body: json, headers: {})
+
+        ojos_instances = aws.instances(role: 'ojos', availability_zone: other_availability_zone, discovery_type: 'consul')
+        expect(ojos_instances.map{|i| i.tags.first[:value]}).to eq(['ojos_instance2', 'ojos_instance3'])
+      end
+    end
+
     context "ordered roles" do
       let(:my_availability_zone) { 'us-east-1a' }
       let(:other_availability_zone) { 'us-east-1b' }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: MovableInkAWS
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 2.0.1
 platform: ruby
 authors:
 - Matt Chesler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-03 00:00:00.000000000 Z
+date: 2020-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-core
@@ -206,6 +206,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.16.3
+- !ruby/object:Gem::Dependency
+  name: diplomat
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.4.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.4.2
 description: AWS Utility methods for MovableInk
 email: mchesler@movableink.com
 executables: []