MovableInkAWS 1.3.1 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cd2763ea3f172fa051b72e65e9a3ef722ef961e67ca43fac2195f5a9acfeb881
-  data.tar.gz: 7100767e992eacbd67763ea5ab3c221cf70787d6e36aa9732ee6372d0a27937e
+  metadata.gz: cf12982879cf6c6fbbd4a1027aa6de717c3084431339ad583cd4ae38d934bc91
+  data.tar.gz: e1baa1af0dc4a999bcba1ee421edc4204465afaadbc72a3190e97c57a948c1de
 SHA512:
-  metadata.gz: 64807199ef1e1aba116ebd9bfac37a0395e32935cca99172a0a82865249b6a4586f452ffe9ccd45733e9354cedef6479967f95d35375c4baf530835eb6b4823c
-  data.tar.gz: aeb957ad8595c852a7f97ca771d766f9f943663dc7db3eb2cdb2e0c9189c627bac4c197609ef2706b499dd33f11deedf9f45a19b99633d9b77261235ded530e5
+  metadata.gz: 567ef63e82f5d5bbf5b57b9b2aa5a853a63af0d572f710e813c322d135fef345456fc9cc3e3f103666f05065d244f67055ea78350f9b4ecec111b159663d2616
+  data.tar.gz: 2712730cb9852affc31a1586d36eba62b76a621d8e82dad39d8439bb0f47db72e897147666e5ea01ca2f75c7c4aeb3f5363f60b531dabd07f90e96813a846cb1

data/.travis.yml

@@ -1,7 +1,6 @@
 language: ruby
 
 rvm:
-  - 2.5
   - 2.6
 
 bundler_args: "--retry=3"

data/Gemfile

@@ -3,4 +3,5 @@ gemspec
 
 group :test do
   gem 'rspec',    '~> 3.6'
+  gem 'webmock'
 end

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    MovableInkAWS (1.3.1)
+    MovableInkAWS (2.1.0)
       aws-sdk-athena (~> 1)
       aws-sdk-autoscaling (~> 1)
       aws-sdk-cloudwatch (~> 1)
@@ -15,66 +15,80 @@ PATH
       aws-sdk-sns (~> 1)
       aws-sdk-ssm (~> 1)
       aws-sigv4 (~> 1.1)
+      diplomat (= 2.4.2)
       httparty (= 0.16.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
     aws-eventstream (1.1.0)
-    aws-partitions (1.378.0)
+    aws-partitions (1.416.0)
     aws-sdk-athena (1.33.0)
       aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-autoscaling (1.47.0)
+    aws-sdk-autoscaling (1.53.0)
       aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-cloudwatch (1.45.0)
+    aws-sdk-cloudwatch (1.47.0)
       aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-core (3.109.0)
+    aws-sdk-core (3.111.0)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-ec2 (1.198.0)
+    aws-sdk-ec2 (1.220.0)
       aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-eks (1.44.0)
+    aws-sdk-eks (1.46.0)
       aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-elasticache (1.43.0)
+    aws-sdk-elasticache (1.50.0)
       aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-kms (1.39.0)
+    aws-sdk-kms (1.41.0)
       aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-rds (1.101.0)
+    aws-sdk-rds (1.110.0)
       aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-route53 (1.44.0)
+    aws-sdk-route53 (1.45.0)
       aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.82.0)
+    aws-sdk-s3 (1.87.0)
       aws-sdk-core (~> 3, >= 3.109.0)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.1)
-    aws-sdk-sns (1.32.0)
+    aws-sdk-sns (1.36.0)
       aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-ssm (1.93.0)
+    aws-sdk-ssm (1.102.0)
       aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
     aws-sigv4 (1.2.2)
       aws-eventstream (~> 1, >= 1.0.2)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    deep_merge (1.2.1)
     diff-lcs (1.3)
+    diplomat (2.4.2)
+      deep_merge (~> 1.0, >= 1.0.1)
+      faraday (>= 0.9, < 1.1.0)
+    faraday (1.0.1)
+      multipart-post (>= 1.2, < 3)
+    hashdiff (1.0.0)
     httparty (0.16.3)
       mime-types (~> 3.0)
       multi_xml (>= 0.5.2)
     jmespath (1.4.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2020.0512)
+    mime-types-data (3.2020.1104)
     multi_xml (0.6.0)
+    multipart-post (2.1.1)
+    public_suffix (4.0.5)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -88,6 +102,11 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-support (3.9.3)
+    safe_yaml (1.0.5)
+    webmock (3.7.6)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
 
 PLATFORMS
   ruby
@@ -95,6 +114,7 @@ PLATFORMS
 DEPENDENCIES
   MovableInkAWS!
   rspec (~> 3.6)
+  webmock
 
 BUNDLED WITH
    2.1.3

data/MovableInkAWS.gemspec

@@ -23,6 +23,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'aws-sdk-ssm', '~> 1'
   s.add_runtime_dependency 'aws-sigv4', '~> 1.1'
   s.add_runtime_dependency 'httparty',  '0.16.3'
+  s.add_runtime_dependency 'diplomat',  '2.4.2'
 
   all_files  = `git ls-files`.split("\n")
   test_files = `git ls-files -- {test,spec,features}/*`.split("\n")

data/lib/movable_ink/aws.rb

@@ -10,6 +10,7 @@ require_relative 'aws/s3'
 require_relative 'aws/eks'
 require_relative 'aws/elasticache'
 require_relative 'aws/api_gateway'
+require_relative 'consul/consul'
 require 'aws-sdk-cloudwatch'
 
 

data/lib/movable_ink/aws/ec2.rb

@@ -1,4 +1,5 @@
 require 'aws-sdk-ec2'
+require 'diplomat'
 
 module MovableInk
   class AWS
@@ -75,7 +76,7 @@ module MovableInk
         @me ||= all_instances.select{|instance| instance.instance_id == instance_id}.first rescue nil
       end
 
-      def instances(role:, exclude_roles: [], region: my_region, availability_zone: nil, exact_match: false, use_cache: true)
+      def instances_with_ec2_discovery(role:, exclude_roles: [], region: my_region, availability_zone: nil, exact_match: false, use_cache: true)
         roles = role.split(/\s*,\s*/)
         if use_cache == false
           filter = default_filter.push({
@@ -106,6 +107,59 @@ module MovableInk
         end
       end
 
+      def instances_with_consul_discovery(role:, region: my_region, availability_zone: nil)
+        if role == nil || role == ''
+          raise MovableInk::AWS::Errors::RoleNameRequiredError
+        end
+
+        if role.include?('_')
+          raise MovableInk::AWS::Errors::RoleNameInvalidError
+        end
+
+        consul_service_options = {
+          dc: datacenter(region: region),
+          stale: true,
+          cached: true,
+          passing: true,
+        }
+        consul_service_options[:node_meta] = "availability_zone:#{availability_zone}" unless availability_zone.nil?
+
+        Diplomat::Health.service(role, consul_service_options).map { |endpoint|
+          OpenStruct.new (
+          {
+            private_ip_address: endpoint.Node['Address'],
+            instance_id: endpoint.Node['Meta']['instance_id'],
+            tags: [
+              {
+                key: 'Name',
+                value: endpoint.Node['Node']
+              },
+              {
+                key: 'mi:roles',
+                value: endpoint.Node['Meta']['mi_roles']
+              },
+              {
+                key: 'mi:monitoring_roles',
+                value: endpoint.Node['Meta']['mi_monitoring_roles']
+              }
+            ],
+            placement: {
+              availability_zone: endpoint.Node['Meta']['availability_zone']
+            }
+          })
+        }
+      end
+
+      def instances(role:, exclude_roles: [], region: my_region, availability_zone: nil, exact_match: false, use_cache: true, discovery_type: 'ec2')
+        if discovery_type == 'ec2'
+          instances_with_ec2_discovery(role: role, exclude_roles: exclude_roles, region: region, availability_zone: availability_zone, exact_match: exact_match, use_cache: use_cache)
+        elsif discovery_type == 'consul'
+          instances_with_consul_discovery(role: role, region: region, availability_zone: availability_zone)
+        else
+          raise MovableInk::AWS::Errors::InvalidDiscoveryTypeError
+        end
+      end
+
       def thopter
         private_ip_addresses(thopter_instance).first
       end

data/lib/movable_ink/aws/errors.rb

@@ -5,6 +5,9 @@ module MovableInk
       class FailedWithBackoff < StandardError; end
       class EC2Required < StandardError; end
       class NoEnvironmentTagError < StandardError; end
+      class InvalidDiscoveryTypeError < StandardError; end
+      class RoleNameRequiredError < StandardError; end
+      class RoleNameInvalidError < StandardError; end
     end
   end
 end

data/lib/movable_ink/aws/sns.rb

@@ -11,29 +11,20 @@ module MovableInk
       end
 
       def sns_slack_topic_arn
-        run_with_backoff do
-          sns.list_topics.each do |resp|
-            resp.topics.each do |topic|
-              return topic.topic_arn if topic.topic_arn.include? "slack-aws-alerts"
-            end
-          end
-        end
+        @sns_slack_topic_arn ||= sns_topics.detect { |topic| topic.topic_arn.include? "slack-aws-alerts" }.topic_arn rescue nil
       end
 
       def sns_pagerduty_topic_arn
-        run_with_backoff do
-          sns.list_topics.each do |resp|
-            resp.topics.each do |topic|
-              return topic.topic_arn if topic.topic_arn.include? "pagerduty-custom-alerts"
-            end
-          end
-        end
+        @sns_pagerduty_topic_arn ||= sns_topics.detect { |topic| topic.topic_arn.include? "pagerduty-custom-alerts" }.topic_arn rescue nil
+      end
+
+      def sns_topics
+        @sns_topics ||= run_with_backoff { sns.list_topics.flat_map(&:topics) }
       end
 
       def notify_and_sleep(seconds, error_class)
         message = "Throttled by AWS. Sleeping #{seconds} seconds, (#{error_class})"
-        notify_slack(subject: 'API Throttled',
-                     message: message)
+        notify_slack(subject: 'API Throttled', message: message)
         puts message
         sleep seconds
       end

data/lib/movable_ink/consul/consul.rb

@@ -0,0 +1,21 @@
+require 'diplomat'
+
+Diplomat.configure do |config|
+  config.url = "https://localhost:8501"
+  config.options = { ssl: { verify: false } }
+end
+
+module MovableInk
+  module Consul
+    module Kv
+      def self.get(*args)
+        begin
+          result = Diplomat::Kv.get(*args)
+          JSON.parse(result) if !result.nil?
+        rescue JSON::ParserError
+          result
+        end
+      end
+    end
+  end
+end

data/lib/movable_ink/version.rb

@@ -1,5 +1,5 @@
 module MovableInk
   class AWS
-    VERSION = '1.3.1'
+    VERSION = '2.1.0'
   end
 end

data/spec/consul_spec.rb

@@ -0,0 +1,22 @@
+require_relative '../lib/movable_ink/aws'
+require 'webmock/rspec'
+
+describe MovableInk::Consul do
+  describe MovableInk::Consul::Kv do
+    it 'get returns a hash of json content' do
+      allow(Diplomat::Kv).to receive(:get).and_return({ hello: 'world' }.to_json)
+      expect(MovableInk::Consul::Kv.get('ham')).to eq({ 'hello' => 'world' })
+    end
+
+    it 'returns a string if the stored value is not json' do
+      value = 'some-database-password'
+      allow(Diplomat::Kv).to receive(:get).and_return(value)
+      expect(MovableInk::Consul::Kv.get('db_password')).to eq(value)
+    end
+
+    it 'forwards all arguments to Diplomat' do
+      expect(Diplomat::Kv).to receive(:get).with('foo/', recurse: true)
+      MovableInk::Consul::Kv.get('foo/', recurse: true)
+    end
+  end
+end

data/spec/ec2_spec.rb

@@ -1,4 +1,5 @@
 require_relative '../lib/movable_ink/aws'
+require 'webmock/rspec'
 
 describe MovableInk::AWS::EC2 do
   context "outside EC2" do
@@ -21,6 +22,7 @@ describe MovableInk::AWS::EC2 do
   end
 
   context "inside EC2" do
+    WebMock.allow_net_connect!
     let(:aws) { MovableInk::AWS.new }
     let(:ec2) { Aws::EC2::Client.new(stub_responses: true) }
     let(:tag_data) { ec2.stub_data(:describe_tags, tags: [
@@ -305,6 +307,150 @@ describe MovableInk::AWS::EC2 do
       end
     end
 
+    context "instances with consul discovery" do
+      let(:my_availability_zone) { 'us-east-1a' }
+      let(:my_datacenter) { 'iad' }
+      let(:other_availability_zone) { 'us-east-1b' }
+      let(:consul_app_service_instances) {
+        [{
+            Node: {
+              Node: 'app_instance1',
+              Address: '10.0.0.1',
+              Datacenter: my_datacenter,
+              Meta: {
+                availability_zone: my_availability_zone,
+                instance_id: 'i-12345',
+                mi_monitoring_roles: 'app',
+                mi_roles: 'app'
+              },
+              ServiceID: 'app',
+              ServiceName: 'app',
+              ServiceTags: ['foo', 'bar'],
+              ServicePort: 80,
+            }
+        },
+        {
+          Node: {
+              Node: 'app_instance2',
+              Address: '10.0.0.2',
+              Datacenter: my_datacenter,
+              Meta: {
+                availability_zone: my_availability_zone,
+                instance_id: 'i-54321',
+                mi_monitoring_roles: 'app',
+                mi_roles: 'app'
+              },
+              ServiceID: 'app',
+              ServiceName: 'app',
+              ServiceTags: ['foo', 'bar'],
+              ServicePort: 80,
+            }
+       }]
+      }
+
+      let(:consul_ojos_service_instances) {
+        [
+        {
+            Node: {
+              Node: 'ojos_instance2',
+              Address: '10.0.0.4',
+              Datacenter: my_datacenter,
+              Meta: {
+                availability_zone: other_availability_zone,
+                instance_id: 'i-zyx987',
+                mi_monitoring_roles: 'ojos',
+                mi_roles: 'ojos'
+              },
+              ServiceID: 'ojos',
+              ServiceName: 'ojos',
+              ServiceTags: ['foo', 'bar'],
+              ServicePort: 2702,
+            },
+        },
+        {
+            Node: {
+              Node: 'ojos_instance3',
+              Address: '10.0.0.5',
+              Datacenter: my_datacenter,
+              Meta: {
+                availability_zone: other_availability_zone,
+                instance_id: 'i-987654',
+                mi_monitoring_roles: 'ojos',
+                mi_roles: 'ojos'
+              },
+              ServiceID: 'ojos',
+              ServiceName: 'ojos',
+              ServiceTags: ['foo', 'bar'],
+              ServicePort: 2702,
+            }
+        }]
+      }
+
+      before(:each) do
+        allow(aws).to receive(:mi_env).and_return('test')
+        allow(aws).to receive(:availability_zone).and_return(my_availability_zone)
+        allow(aws).to receive(:my_region).and_return('us-east-1')
+        allow(aws).to receive(:ec2).and_return(ec2)
+      end
+
+      it "returns an error if no role is defined" do
+        expect{ aws.instances(role: nil, discovery_type: 'consul') }.to raise_error(MovableInk::AWS::Errors::RoleNameRequiredError)
+      end
+
+      it "returns an error if an invalid role is passed" do
+        expect{ aws.instances(role: 'asset_proxy', discovery_type: 'consul') }.to raise_error(MovableInk::AWS::Errors::RoleNameInvalidError)
+      end
+
+      it "returns all instances matching a consul service" do
+        miaws = double(MovableInk::AWS)
+        allow(miaws).to receive(:my_region).and_return('us-east-1')
+
+        json = JSON.generate(consul_app_service_instances)
+        stub_request(:get, "https://localhost:8501/v1/health/service/app?cached&dc=#{my_datacenter}&passing&stale").
+         with(
+            headers: {
+            'Accept'=>'*/*',
+            'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
+            'User-Agent'=>'Faraday v1.0.1'
+           }).
+         to_return(status: 200, body: json, headers: {})
+
+        app_instances = aws.instances(role: 'app', discovery_type: 'consul')
+        expect(app_instances.map{|i| i.tags.first[:value]}).to eq(['app_instance1', 'app_instance2'])
+
+        json = JSON.generate(consul_ojos_service_instances)
+        stub_request(:get, "https://localhost:8501/v1/health/service/ojos?cached&dc=#{my_datacenter}&passing&stale").
+         with(
+           headers: {
+          'Accept'=>'*/*',
+          'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
+          'User-Agent'=>'Faraday v1.0.1'
+           }).
+         to_return(status: 200, body: json, headers: {})
+
+        ojos_instances = aws.instances(role: 'ojos', discovery_type: 'consul')
+        expect(ojos_instances.map{|i| i.tags.first[:value ]}).to eq(['ojos_instance2', 'ojos_instance3'])
+      end
+
+      it "returns all instances matching a consul service filtered by availability_zone" do
+        miaws = double(MovableInk::AWS)
+        allow(miaws).to receive(:my_region).and_return('us-east-1')
+
+        json = JSON.generate(consul_ojos_service_instances)
+        stub_request(:get, "https://localhost:8501/v1/health/service/ojos?cached&dc=#{my_datacenter}&node-meta=availability_zone:#{other_availability_zone}&passing&stale").
+         with(
+            headers: {
+            'Accept'=>'*/*',
+            'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
+            'User-Agent'=>'Faraday v1.0.1'
+           }).
+         to_return(status: 200, body: json, headers: {})
+
+        ojos_instances = aws.instances(role: 'ojos', availability_zone: other_availability_zone, discovery_type: 'consul')
+        expect(ojos_instances.map{|i| i.tags.first[:value]}).to eq(['ojos_instance2', 'ojos_instance3'])
+      end
+    end
+
     context "ordered roles" do
       let(:my_availability_zone) { 'us-east-1a' }
       let(:other_availability_zone) { 'us-east-1b' }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: MovableInkAWS
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 2.1.0
 platform: ruby
 authors:
 - Matt Chesler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-01 00:00:00.000000000 Z
+date: 2021-01-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-core
@@ -206,6 +206,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.16.3
+- !ruby/object:Gem::Dependency
+  name: diplomat
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.4.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.4.2
 description: AWS Utility methods for MovableInk
 email: mchesler@movableink.com
 executables: []
@@ -232,9 +246,11 @@ files:
 - lib/movable_ink/aws/s3.rb
 - lib/movable_ink/aws/sns.rb
 - lib/movable_ink/aws/ssm.rb
+- lib/movable_ink/consul/consul.rb
 - lib/movable_ink/version.rb
 - spec/autoscaling_spec.rb
 - spec/aws_spec.rb
+- spec/consul_spec.rb
 - spec/ec2_spec.rb
 - spec/elasticache_spec.rb
 - spec/metadata_spec.rb