MovableInkAWS 1.2.2 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 17ceb1cd0d9da92a94c3e5f7eb4d46d9303546bae02c68355f9257e26719e7d9
-  data.tar.gz: b70e2dcf79443ee2c2b8cdc700880de3976efe52af15517d7852ed69e2c444c0
+  metadata.gz: ca127d84b2a782fac6a45b9f2ab7d39891ea9b58c1f72897198363fdec217b03
+  data.tar.gz: 1bead2930395e8a2f53b7dce303a5904790072e78f7abf965514f29af736d7ac
 SHA512:
-  metadata.gz: 1eaced59806f774de531934dc18217f9fa699f8ca5a02829260f10bf1a0c293bb640c97dbb7270c884a7c0e217fc58a506c69cdd44cf5cc567e70f25b7f8fc4d
-  data.tar.gz: 8654770ca2b03ab180af4bcd9727aaa94c4cad1090f4e5002172f621b8c5c6ab90dfd49b90039312de7e86b68fdd2135d78d9882c643330d47f3f7fe0c861bd5
+  metadata.gz: 6225008398613de6fd9d239394029bf141252ed6adca815068bd1a88be2df8e7a6947a3a7055aaab41d4833c00edeb770e90b191395f139597fb7a5455de47c9
+  data.tar.gz: 5bf8debaf1f7afafb249b494d26c5142c5d5ce147721e112183c5c6f66a5d7a543c1af7d5f4d3cf4fbe701763a50042c5ed47d6a288db357b0b364c51fbb3713

data/.travis.yml

@@ -1,7 +1,6 @@
 language: ruby
 
 rvm:
-  - 2.5
   - 2.6
 
 bundler_args: "--retry=3"

data/Gemfile

@@ -3,4 +3,5 @@ gemspec
 
 group :test do
   gem 'rspec',    '~> 3.6'
+  gem 'webmock'
 end

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    MovableInkAWS (1.2.2)
+    MovableInkAWS (1.3.3)
       aws-sdk-athena (~> 1)
       aws-sdk-autoscaling (~> 1)
       aws-sdk-cloudwatch (~> 1)
@@ -15,58 +15,70 @@ PATH
       aws-sdk-sns (~> 1)
       aws-sdk-ssm (~> 1)
       aws-sigv4 (~> 1.1)
+      diplomat (= 2.4.2)
       httparty (= 0.16.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
     aws-eventstream (1.1.0)
-    aws-partitions (1.353.0)
-    aws-sdk-athena (1.30.0)
-      aws-sdk-core (~> 3, >= 3.99.0)
+    aws-partitions (1.383.0)
+    aws-sdk-athena (1.33.0)
+      aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-autoscaling (1.44.0)
-      aws-sdk-core (~> 3, >= 3.99.0)
+    aws-sdk-autoscaling (1.47.0)
+      aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-cloudwatch (1.42.0)
-      aws-sdk-core (~> 3, >= 3.99.0)
+    aws-sdk-cloudwatch (1.45.0)
+      aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-core (3.104.3)
+    aws-sdk-core (3.109.1)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-ec2 (1.183.0)
-      aws-sdk-core (~> 3, >= 3.99.0)
+    aws-sdk-ec2 (1.200.0)
+      aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-eks (1.39.0)
-      aws-sdk-core (~> 3, >= 3.99.0)
+    aws-sdk-eks (1.45.0)
+      aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-elasticache (1.40.0)
-      aws-sdk-core (~> 3, >= 3.99.0)
+    aws-sdk-elasticache (1.44.0)
+      aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-kms (1.36.0)
-      aws-sdk-core (~> 3, >= 3.99.0)
+    aws-sdk-kms (1.39.0)
+      aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-rds (1.95.0)
-      aws-sdk-core (~> 3, >= 3.99.0)
+    aws-sdk-rds (1.104.0)
+      aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-route53 (1.40.0)
-      aws-sdk-core (~> 3, >= 3.99.0)
+    aws-sdk-route53 (1.44.0)
+      aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.76.0)
-      aws-sdk-core (~> 3, >= 3.104.1)
+    aws-sdk-s3 (1.83.1)
+      aws-sdk-core (~> 3, >= 3.109.0)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.1)
-    aws-sdk-sns (1.29.0)
-      aws-sdk-core (~> 3, >= 3.99.0)
+    aws-sdk-sns (1.33.0)
+      aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-ssm (1.86.0)
-      aws-sdk-core (~> 3, >= 3.99.0)
+    aws-sdk-ssm (1.95.0)
+      aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sigv4 (1.2.1)
+    aws-sigv4 (1.2.2)
       aws-eventstream (~> 1, >= 1.0.2)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    deep_merge (1.2.1)
     diff-lcs (1.3)
+    diplomat (2.4.2)
+      deep_merge (~> 1.0, >= 1.0.1)
+      faraday (>= 0.9, < 1.1.0)
+    faraday (1.0.1)
+      multipart-post (>= 1.2, < 3)
+    hashdiff (1.0.0)
     httparty (0.16.3)
       mime-types (~> 3.0)
       multi_xml (>= 0.5.2)
@@ -75,6 +87,8 @@ GEM
       mime-types-data (~> 3.2015)
     mime-types-data (3.2020.0512)
     multi_xml (0.6.0)
+    multipart-post (2.1.1)
+    public_suffix (4.0.5)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -88,6 +102,11 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-support (3.9.3)
+    safe_yaml (1.0.5)
+    webmock (3.7.6)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
 
 PLATFORMS
   ruby
@@ -95,6 +114,7 @@ PLATFORMS
 DEPENDENCIES
   MovableInkAWS!
   rspec (~> 3.6)
+  webmock
 
 BUNDLED WITH
    2.1.3

data/MovableInkAWS.gemspec

@@ -23,6 +23,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'aws-sdk-ssm', '~> 1'
   s.add_runtime_dependency 'aws-sigv4', '~> 1.1'
   s.add_runtime_dependency 'httparty',  '0.16.3'
+  s.add_runtime_dependency 'diplomat',  '2.4.2'
 
   all_files  = `git ls-files`.split("\n")
   test_files = `git ls-files -- {test,spec,features}/*`.split("\n")

data/lib/movable_ink/aws.rb

@@ -66,6 +66,7 @@ module MovableInk
                Aws::EC2::Errors::Unavailable,
                Aws::EKS::Errors::TooManyRequestsException,
                Aws::SNS::Errors::ThrottledException,
+               Aws::SNS::Errors::Throttling,
                Aws::AutoScaling::Errors::Throttling,
                Aws::AutoScaling::Errors::ThrottledException,
                Aws::S3::Errors::SlowDown,

data/lib/movable_ink/aws/ec2.rb

@@ -1,4 +1,5 @@
 require 'aws-sdk-ec2'
+require 'diplomat'
 
 module MovableInk
   class AWS
@@ -9,13 +10,13 @@ module MovableInk
       end
 
       def mi_env_cache_file_path
-        '/etc/movableink/mi_env'
+        '/etc/movableink/environments.json'
       end
 
       def mi_env
         @mi_env ||= if File.exist?(mi_env_cache_file_path)
-          value = File.read(mi_env_cache_file_path)
-          value.empty? ? load_mi_env : value
+          environments = JSON.parse(File.read(mi_env_cache_file_path))
+          environments[my_region] || load_mi_env
         else
           load_mi_env
         end
@@ -75,7 +76,7 @@ module MovableInk
         @me ||= all_instances.select{|instance| instance.instance_id == instance_id}.first rescue nil
       end
 
-      def instances(role:, exclude_roles: [], region: my_region, availability_zone: nil, exact_match: false, use_cache: true)
+      def instances_with_ec2_discovery(role:, exclude_roles: [], region: my_region, availability_zone: nil, exact_match: false, use_cache: true)
         roles = role.split(/\s*,\s*/)
         if use_cache == false
           filter = default_filter.push({
@@ -106,6 +107,64 @@ module MovableInk
         end
       end
 
+      def instances_with_consul_discovery(role:, region: my_region, availability_zone: nil)
+        if role == nil || role == ''
+          raise MovableInk::AWS::Errors::RoleNameRequiredError
+        end
+
+        if role.include?('_')
+          raise MovableInk::AWS::Errors::RoleNameInvalidError
+        end
+
+        Diplomat.configure do |config|
+          config.url = "https://localhost:8501"
+          config.options = { ssl: { verify: false } }
+        end
+
+        consul_instances = Diplomat::Service.get(role, :all, { :dc => datacenter(region: region), :stale => true, :cached => true }).map { |node|
+          OpenStruct.new (
+            {
+            private_ip_address:  node.Address,
+            instance_id: node.NodeMeta['instance_id'],
+            tags: [
+              {
+                key: 'Name',
+                value: node.Node
+              },
+              {
+                key: 'mi:roles',
+                value: node.NodeMeta['mi_roles']
+              },
+              {
+                key: 'mi:monitoring_roles',
+                value: node.NodeMeta['mi_monitoring_roles']
+              }
+            ],
+            placement: {
+              availability_zone: node.NodeMeta['availability_zone']
+            }
+          })
+        }
+
+        if availability_zone
+          consul_instances.select { |consul_instance|
+            consul_instance.placement[:availability_zone] == availability_zone
+          }
+        else
+          consul_instances
+        end
+      end
+
+      def instances(role:, exclude_roles: [], region: my_region, availability_zone: nil, exact_match: false, use_cache: true, discovery_type: 'ec2')
+        if discovery_type == 'ec2'
+          instances_with_ec2_discovery(role: role, exclude_roles: exclude_roles, region: region, availability_zone: availability_zone, exact_match: exact_match, use_cache: use_cache)
+        elsif discovery_type == 'consul'
+          instances_with_consul_discovery(role: role, region: region, availability_zone: availability_zone)
+        else
+          raise MovableInk::AWS::Errors::InvalidDiscoveryTypeError
+        end
+      end
+
       def thopter
         private_ip_addresses(thopter_instance).first
       end

data/lib/movable_ink/aws/errors.rb

@@ -5,6 +5,9 @@ module MovableInk
       class FailedWithBackoff < StandardError; end
       class EC2Required < StandardError; end
       class NoEnvironmentTagError < StandardError; end
+      class InvalidDiscoveryTypeError < StandardError; end
+      class RoleNameRequiredError < StandardError; end
+      class RoleNameInvalidError < StandardError; end
     end
   end
 end

data/lib/movable_ink/aws/sns.rb

@@ -11,29 +11,20 @@ module MovableInk
       end
 
       def sns_slack_topic_arn
-        run_with_backoff do
-          sns.list_topics.each do |resp|
-            resp.topics.each do |topic|
-              return topic.topic_arn if topic.topic_arn.include? "slack-aws-alerts"
-            end
-          end
-        end
+        @sns_slack_topic_arn ||= sns_topics.detect { |topic| topic.topic_arn.include? "slack-aws-alerts" }.topic_arn rescue nil
       end
 
       def sns_pagerduty_topic_arn
-        run_with_backoff do
-          sns.list_topics.each do |resp|
-            resp.topics.each do |topic|
-              return topic.topic_arn if topic.topic_arn.include? "pagerduty-custom-alerts"
-            end
-          end
-        end
+        @sns_pagerduty_topic_arn ||= sns_topics.detect { |topic| topic.topic_arn.include? "pagerduty-custom-alerts" }.topic_arn rescue nil
+      end
+
+      def sns_topics
+        @sns_topics ||= run_with_backoff { sns.list_topics.flat_map(&:topics) }
       end
 
       def notify_and_sleep(seconds, error_class)
         message = "Throttled by AWS. Sleeping #{seconds} seconds, (#{error_class})"
-        notify_slack(subject: 'API Throttled',
-                     message: message)
+        notify_slack(subject: 'API Throttled', message: message)
         puts message
         sleep seconds
       end

data/lib/movable_ink/version.rb

@@ -1,5 +1,5 @@
 module MovableInk
   class AWS
-    VERSION = '1.2.2'
+    VERSION = '2.0.0'
   end
 end

data/spec/ec2_spec.rb

@@ -1,4 +1,5 @@
 require_relative '../lib/movable_ink/aws'
+require 'webmock/rspec'
 
 describe MovableInk::AWS::EC2 do
   context "outside EC2" do
@@ -21,6 +22,7 @@ describe MovableInk::AWS::EC2 do
   end
 
   context "inside EC2" do
+    WebMock.allow_net_connect!
     let(:aws) { MovableInk::AWS.new }
     let(:ec2) { Aws::EC2::Client.new(stub_responses: true) }
     let(:tag_data) { ec2.stub_data(:describe_tags, tags: [
@@ -32,26 +34,14 @@ describe MovableInk::AWS::EC2 do
     }
 
     it 'will read mi_env from disk when the cache file exists' do
+      allow(aws).to receive(:my_region).and_return('us-east-2')
       f = Tempfile.new('cache')
-      f.write('staging')
+      f.write(JSON.generate({ "us-east-1": "production", "us-west-2": "production", "us-east-2": "staging" }))
       f.close
       allow(aws).to receive(:mi_env_cache_file_path).and_return(f.path)
       expect(aws.mi_env).to eq('staging')
     end
 
-    it 'will read tag data if the cache file is empty' do
-      ec2.stub_responses(:describe_tags, tag_data)
-      allow(aws).to receive(:my_region).and_return('us-east-1')
-      allow(aws).to receive(:instance_id).and_return('i-12345')
-      allow(aws).to receive(:ec2).and_return(ec2)
-
-      f = Tempfile.new('cache')
-      f.write('')
-      f.close
-      allow(aws).to receive(:mi_env_cache_file_path).and_return(f.path)
-      expect(aws.mi_env).to eq('test')
-    end
-
     it "should find the environment from the current instance's tags" do
       ec2.stub_responses(:describe_tags, tag_data)
       allow(aws).to receive(:my_region).and_return('us-east-1')
@@ -317,6 +307,165 @@ describe MovableInk::AWS::EC2 do
       end
     end
 
+    context "instances with consul discovery" do
+      let(:my_availability_zone) { 'us-east-1a' }
+      let(:my_datacenter) { 'iad' }
+      let(:other_availability_zone) { 'us-east-1b' }
+      let(:consul_app_service_instances) {
+        [
+          {
+            Node: 'app_instance1',
+            Address: '10.0.0.1',
+            Datacenter: my_datacenter,
+            NodeMeta:
+            {
+              availability_zone: my_availability_zone,
+              instance_id: 'i-12345',
+              mi_monitoring_roles: 'app',
+              mi_roles: 'app'
+            },
+            ServiceID: 'app',
+            ServiceName: 'app',
+            ServiceTags: ['foo', 'bar'],
+            ServicePort: 80,
+          },
+          {
+            Node: 'app_instance2',
+            Address: '10.0.0.2',
+            Datacenter: my_datacenter,
+            NodeMeta:
+            {
+              availability_zone: my_availability_zone,
+              instance_id: 'i-54321',
+              mi_monitoring_roles: 'app',
+              mi_roles: 'app'
+            },
+            ServiceID: 'app',
+            ServiceName: 'app',
+            ServiceTags: ['foo', 'bar'],
+            ServicePort: 80,
+          }
+        ]
+      }
+
+      let(:consul_ojos_service_instances) {
+        [
+          {
+            Node: 'ojos_instance1',
+            Address: '10.0.0.3',
+            Datacenter: my_datacenter,
+            NodeMeta:
+            {
+              availability_zone: my_availability_zone,
+              instance_id: 'i-123abc',
+              mi_monitoring_roles: 'ojos',
+              mi_roles: 'ojos'
+            },
+            ServiceID: 'ojos',
+            ServiceName: 'ojos',
+            ServiceTags: ['foo', 'bar'],
+            ServicePort: 2702,
+          },
+          {
+            Node: 'ojos_instance2',
+            Address: '10.0.0.4',
+            Datacenter: my_datacenter,
+            NodeMeta:
+            {
+              availability_zone: other_availability_zone,
+              instance_id: 'i-zyx987',
+              mi_monitoring_roles: 'ojos',
+              mi_roles: 'ojos'
+            },
+            ServiceID: 'ojos',
+            ServiceName: 'ojos',
+            ServiceTags: ['foo', 'bar'],
+            ServicePort: 2702,
+          },
+          {
+            Node: 'ojos_instance3',
+            Address: '10.0.0.5',
+            Datacenter: my_datacenter,
+            NodeMeta:
+            {
+              availability_zone: other_availability_zone,
+              instance_id: 'i-987654',
+              mi_monitoring_roles: 'ojos',
+              mi_roles: 'ojos'
+            },
+            ServiceID: 'ojos',
+            ServiceName: 'ojos',
+            ServiceTags: ['foo', 'bar'],
+            ServicePort: 2702,
+          }
+        ]
+      }
+
+      before(:each) do
+        allow(aws).to receive(:mi_env).and_return('test')
+        allow(aws).to receive(:availability_zone).and_return(my_availability_zone)
+        allow(aws).to receive(:my_region).and_return('us-east-1')
+        allow(aws).to receive(:ec2).and_return(ec2)
+      end
+
+      it "returns an error if no role is defined" do
+        expect{ aws.instances(role: nil, discovery_type: 'consul') }.to raise_error(MovableInk::AWS::Errors::RoleNameRequiredError)
+      end
+
+      it "returns an error if an invalid role is passed" do
+        expect{ aws.instances(role: 'asset_proxy', discovery_type: 'consul') }.to raise_error(MovableInk::AWS::Errors::RoleNameInvalidError)
+      end
+
+      it "returns all instances matching a consul service" do
+        miaws = double(MovableInk::AWS)
+        allow(miaws).to receive(:my_region).and_return('us-east-1')
+
+        json = JSON.generate(consul_app_service_instances)
+        stub_request(:get, "https://localhost:8501/v1/catalog/service/app?dc=#{my_datacenter}&stale=true&cached=true").
+         with(
+            headers: {
+            'Accept'=>'*/*',
+            'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
+            'User-Agent'=>'Faraday v1.0.1'
+           }).
+         to_return(status: 200, body: json, headers: {})
+
+        app_instances = aws.instances(role: 'app', discovery_type: 'consul')
+        expect(app_instances.map{|i| i.tags.first[:value]}).to eq(['app_instance1', 'app_instance2'])
+
+        json = JSON.generate(consul_ojos_service_instances)
+        stub_request(:get, "https://localhost:8501/v1/catalog/service/ojos?dc=#{my_datacenter}&stale=true&cached=true").
+         with(
+           headers: {
+          'Accept'=>'*/*',
+          'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
+          'User-Agent'=>'Faraday v1.0.1'
+           }).
+         to_return(status: 200, body: json, headers: {})
+
+        ojos_instances = aws.instances(role: 'ojos', discovery_type: 'consul')
+        expect(ojos_instances.map{|i| i.tags.first[:value ]}).to eq(['ojos_instance1', 'ojos_instance2', 'ojos_instance3'])
+      end
+
+      it "returns all instances matching a consul service filtered by availability_zone" do
+        miaws = double(MovableInk::AWS)
+        allow(miaws).to receive(:my_region).and_return('us-east-1')
+
+        json = JSON.generate(consul_ojos_service_instances)
+        stub_request(:get, "https://localhost:8501/v1/catalog/service/ojos?dc=iad&stale=true&cached=true").
+         with(
+            headers: {
+            'Accept'=>'*/*',
+            'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
+            'User-Agent'=>'Faraday v1.0.1'
+           }).
+         to_return(status: 200, body: json, headers: {})
+
+        ojos_instances = aws.instances(role: 'ojos', availability_zone: other_availability_zone, discovery_type: 'consul')
+        expect(ojos_instances.map{|i| i.tags.first[:value]}).to eq(['ojos_instance2', 'ojos_instance3'])
+      end
+    end
+
     context "ordered roles" do
       let(:my_availability_zone) { 'us-east-1a' }
       let(:other_availability_zone) { 'us-east-1b' }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: MovableInkAWS
 version: !ruby/object:Gem::Version
-  version: 1.2.2
+  version: 2.0.0
 platform: ruby
 authors:
 - Matt Chesler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-01 00:00:00.000000000 Z
+date: 2020-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-core
@@ -206,6 +206,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.16.3
+- !ruby/object:Gem::Dependency
+  name: diplomat
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.4.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.4.2
 description: AWS Utility methods for MovableInk
 email: mchesler@movableink.com
 executables: []