MovableInkAWS 1.3.3 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a474efe41182db10e521fd3df11b259261591f6113f2ab71cf66b0e603199578
-  data.tar.gz: 527f38ea7a18d93366b1fe9cbf16f75c3d8f79c8d6f29d6b44e8a4b7eb92d82f
+  metadata.gz: ca127d84b2a782fac6a45b9f2ab7d39891ea9b58c1f72897198363fdec217b03
+  data.tar.gz: 1bead2930395e8a2f53b7dce303a5904790072e78f7abf965514f29af736d7ac
 SHA512:
-  metadata.gz: 778adf6177659e449ee6a4c4fea6007120d7397bd5acd109111dfaf3f673af6289e1e9b82f8b8c2a34583c86d9b567283836cedbaa822a183bdd730b074fb9e7
-  data.tar.gz: 6f94b899d09790472b843e2fd69f895ed349f97a4b21443d413b61713d464c8eb864cdfce1a495139a2eb67c31cd36ab3fe0f485d333ab2b5f58e178c87ee167
+  metadata.gz: 6225008398613de6fd9d239394029bf141252ed6adca815068bd1a88be2df8e7a6947a3a7055aaab41d4833c00edeb770e90b191395f139597fb7a5455de47c9
+  data.tar.gz: 5bf8debaf1f7afafb249b494d26c5142c5d5ce147721e112183c5c6f66a5d7a543c1af7d5f4d3cf4fbe701763a50042c5ed47d6a288db357b0b364c51fbb3713

data/Gemfile

@@ -3,4 +3,5 @@ gemspec
 
 group :test do
   gem 'rspec',    '~> 3.6'
+  gem 'webmock'
 end

data/Gemfile.lock

@@ -15,11 +15,14 @@ PATH
       aws-sdk-sns (~> 1)
       aws-sdk-ssm (~> 1)
       aws-sigv4 (~> 1.1)
+      diplomat (= 2.4.2)
       httparty (= 0.16.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
     aws-eventstream (1.1.0)
     aws-partitions (1.383.0)
     aws-sdk-athena (1.33.0)
@@ -66,7 +69,16 @@ GEM
       aws-sigv4 (~> 1.1)
     aws-sigv4 (1.2.2)
       aws-eventstream (~> 1, >= 1.0.2)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    deep_merge (1.2.1)
     diff-lcs (1.3)
+    diplomat (2.4.2)
+      deep_merge (~> 1.0, >= 1.0.1)
+      faraday (>= 0.9, < 1.1.0)
+    faraday (1.0.1)
+      multipart-post (>= 1.2, < 3)
+    hashdiff (1.0.0)
     httparty (0.16.3)
       mime-types (~> 3.0)
       multi_xml (>= 0.5.2)
@@ -75,6 +87,8 @@ GEM
       mime-types-data (~> 3.2015)
     mime-types-data (3.2020.0512)
     multi_xml (0.6.0)
+    multipart-post (2.1.1)
+    public_suffix (4.0.5)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -88,6 +102,11 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-support (3.9.3)
+    safe_yaml (1.0.5)
+    webmock (3.7.6)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
 
 PLATFORMS
   ruby
@@ -95,6 +114,7 @@ PLATFORMS
 DEPENDENCIES
   MovableInkAWS!
   rspec (~> 3.6)
+  webmock
 
 BUNDLED WITH
    2.1.3

data/MovableInkAWS.gemspec

@@ -23,6 +23,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'aws-sdk-ssm', '~> 1'
   s.add_runtime_dependency 'aws-sigv4', '~> 1.1'
   s.add_runtime_dependency 'httparty',  '0.16.3'
+  s.add_runtime_dependency 'diplomat',  '2.4.2'
 
   all_files  = `git ls-files`.split("\n")
   test_files = `git ls-files -- {test,spec,features}/*`.split("\n")

data/lib/movable_ink/aws/ec2.rb

@@ -1,4 +1,5 @@
 require 'aws-sdk-ec2'
+require 'diplomat'
 
 module MovableInk
   class AWS
@@ -75,7 +76,7 @@ module MovableInk
         @me ||= all_instances.select{|instance| instance.instance_id == instance_id}.first rescue nil
       end
 
-      def instances(role:, exclude_roles: [], region: my_region, availability_zone: nil, exact_match: false, use_cache: true)
+      def instances_with_ec2_discovery(role:, exclude_roles: [], region: my_region, availability_zone: nil, exact_match: false, use_cache: true)
         roles = role.split(/\s*,\s*/)
         if use_cache == false
           filter = default_filter.push({
@@ -106,6 +107,64 @@ module MovableInk
         end
       end
 
+      def instances_with_consul_discovery(role:, region: my_region, availability_zone: nil)
+        if role == nil || role == ''
+          raise MovableInk::AWS::Errors::RoleNameRequiredError
+        end
+
+        if role.include?('_')
+          raise MovableInk::AWS::Errors::RoleNameInvalidError
+        end
+
+        Diplomat.configure do |config|
+          config.url = "https://localhost:8501"
+          config.options = { ssl: { verify: false } }
+        end
+
+        consul_instances = Diplomat::Service.get(role, :all, { :dc => datacenter(region: region), :stale => true, :cached => true }).map { |node|
+          OpenStruct.new (
+            {
+            private_ip_address:  node.Address,
+            instance_id: node.NodeMeta['instance_id'],
+            tags: [
+              {
+                key: 'Name',
+                value: node.Node
+              },
+              {
+                key: 'mi:roles',
+                value: node.NodeMeta['mi_roles']
+              },
+              {
+                key: 'mi:monitoring_roles',
+                value: node.NodeMeta['mi_monitoring_roles']
+              }
+            ],
+            placement: {
+              availability_zone: node.NodeMeta['availability_zone']
+            }
+          })
+        }
+
+        if availability_zone
+          consul_instances.select { |consul_instance|
+            consul_instance.placement[:availability_zone] == availability_zone
+          }
+        else
+          consul_instances
+        end
+      end
+
+      def instances(role:, exclude_roles: [], region: my_region, availability_zone: nil, exact_match: false, use_cache: true, discovery_type: 'ec2')
+        if discovery_type == 'ec2'
+          instances_with_ec2_discovery(role: role, exclude_roles: exclude_roles, region: region, availability_zone: availability_zone, exact_match: exact_match, use_cache: use_cache)
+        elsif discovery_type == 'consul'
+          instances_with_consul_discovery(role: role, region: region, availability_zone: availability_zone)
+        else
+          raise MovableInk::AWS::Errors::InvalidDiscoveryTypeError
+        end
+      end
+
       def thopter
         private_ip_addresses(thopter_instance).first
       end

data/lib/movable_ink/aws/errors.rb

@@ -5,6 +5,9 @@ module MovableInk
       class FailedWithBackoff < StandardError; end
       class EC2Required < StandardError; end
       class NoEnvironmentTagError < StandardError; end
+      class InvalidDiscoveryTypeError < StandardError; end
+      class RoleNameRequiredError < StandardError; end
+      class RoleNameInvalidError < StandardError; end
     end
   end
 end

data/lib/movable_ink/version.rb

@@ -1,5 +1,5 @@
 module MovableInk
   class AWS
-    VERSION = '1.3.3'
+    VERSION = '2.0.0'
   end
 end

data/spec/ec2_spec.rb

@@ -1,4 +1,5 @@
 require_relative '../lib/movable_ink/aws'
+require 'webmock/rspec'
 
 describe MovableInk::AWS::EC2 do
   context "outside EC2" do
@@ -21,6 +22,7 @@ describe MovableInk::AWS::EC2 do
   end
 
   context "inside EC2" do
+    WebMock.allow_net_connect!
     let(:aws) { MovableInk::AWS.new }
     let(:ec2) { Aws::EC2::Client.new(stub_responses: true) }
     let(:tag_data) { ec2.stub_data(:describe_tags, tags: [
@@ -305,6 +307,165 @@ describe MovableInk::AWS::EC2 do
       end
     end
 
+    context "instances with consul discovery" do
+      let(:my_availability_zone) { 'us-east-1a' }
+      let(:my_datacenter) { 'iad' }
+      let(:other_availability_zone) { 'us-east-1b' }
+      let(:consul_app_service_instances) {
+        [
+          {
+            Node: 'app_instance1',
+            Address: '10.0.0.1',
+            Datacenter: my_datacenter,
+            NodeMeta:
+            {
+              availability_zone: my_availability_zone,
+              instance_id: 'i-12345',
+              mi_monitoring_roles: 'app',
+              mi_roles: 'app'
+            },
+            ServiceID: 'app',
+            ServiceName: 'app',
+            ServiceTags: ['foo', 'bar'],
+            ServicePort: 80,
+          },
+          {
+            Node: 'app_instance2',
+            Address: '10.0.0.2',
+            Datacenter: my_datacenter,
+            NodeMeta:
+            {
+              availability_zone: my_availability_zone,
+              instance_id: 'i-54321',
+              mi_monitoring_roles: 'app',
+              mi_roles: 'app'
+            },
+            ServiceID: 'app',
+            ServiceName: 'app',
+            ServiceTags: ['foo', 'bar'],
+            ServicePort: 80,
+          }
+        ]
+      }
+
+      let(:consul_ojos_service_instances) {
+        [
+          {
+            Node: 'ojos_instance1',
+            Address: '10.0.0.3',
+            Datacenter: my_datacenter,
+            NodeMeta:
+            {
+              availability_zone: my_availability_zone,
+              instance_id: 'i-123abc',
+              mi_monitoring_roles: 'ojos',
+              mi_roles: 'ojos'
+            },
+            ServiceID: 'ojos',
+            ServiceName: 'ojos',
+            ServiceTags: ['foo', 'bar'],
+            ServicePort: 2702,
+          },
+          {
+            Node: 'ojos_instance2',
+            Address: '10.0.0.4',
+            Datacenter: my_datacenter,
+            NodeMeta:
+            {
+              availability_zone: other_availability_zone,
+              instance_id: 'i-zyx987',
+              mi_monitoring_roles: 'ojos',
+              mi_roles: 'ojos'
+            },
+            ServiceID: 'ojos',
+            ServiceName: 'ojos',
+            ServiceTags: ['foo', 'bar'],
+            ServicePort: 2702,
+          },
+          {
+            Node: 'ojos_instance3',
+            Address: '10.0.0.5',
+            Datacenter: my_datacenter,
+            NodeMeta:
+            {
+              availability_zone: other_availability_zone,
+              instance_id: 'i-987654',
+              mi_monitoring_roles: 'ojos',
+              mi_roles: 'ojos'
+            },
+            ServiceID: 'ojos',
+            ServiceName: 'ojos',
+            ServiceTags: ['foo', 'bar'],
+            ServicePort: 2702,
+          }
+        ]
+      }
+
+      before(:each) do
+        allow(aws).to receive(:mi_env).and_return('test')
+        allow(aws).to receive(:availability_zone).and_return(my_availability_zone)
+        allow(aws).to receive(:my_region).and_return('us-east-1')
+        allow(aws).to receive(:ec2).and_return(ec2)
+      end
+
+      it "returns an error if no role is defined" do
+        expect{ aws.instances(role: nil, discovery_type: 'consul') }.to raise_error(MovableInk::AWS::Errors::RoleNameRequiredError)
+      end
+
+      it "returns an error if an invalid role is passed" do
+        expect{ aws.instances(role: 'asset_proxy', discovery_type: 'consul') }.to raise_error(MovableInk::AWS::Errors::RoleNameInvalidError)
+      end
+
+      it "returns all instances matching a consul service" do
+        miaws = double(MovableInk::AWS)
+        allow(miaws).to receive(:my_region).and_return('us-east-1')
+
+        json = JSON.generate(consul_app_service_instances)
+        stub_request(:get, "https://localhost:8501/v1/catalog/service/app?dc=#{my_datacenter}&stale=true&cached=true").
+         with(
+            headers: {
+            'Accept'=>'*/*',
+            'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
+            'User-Agent'=>'Faraday v1.0.1'
+           }).
+         to_return(status: 200, body: json, headers: {})
+
+        app_instances = aws.instances(role: 'app', discovery_type: 'consul')
+        expect(app_instances.map{|i| i.tags.first[:value]}).to eq(['app_instance1', 'app_instance2'])
+
+        json = JSON.generate(consul_ojos_service_instances)
+        stub_request(:get, "https://localhost:8501/v1/catalog/service/ojos?dc=#{my_datacenter}&stale=true&cached=true").
+         with(
+           headers: {
+          'Accept'=>'*/*',
+          'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
+          'User-Agent'=>'Faraday v1.0.1'
+           }).
+         to_return(status: 200, body: json, headers: {})
+
+        ojos_instances = aws.instances(role: 'ojos', discovery_type: 'consul')
+        expect(ojos_instances.map{|i| i.tags.first[:value ]}).to eq(['ojos_instance1', 'ojos_instance2', 'ojos_instance3'])
+      end
+
+      it "returns all instances matching a consul service filtered by availability_zone" do
+        miaws = double(MovableInk::AWS)
+        allow(miaws).to receive(:my_region).and_return('us-east-1')
+
+        json = JSON.generate(consul_ojos_service_instances)
+        stub_request(:get, "https://localhost:8501/v1/catalog/service/ojos?dc=iad&stale=true&cached=true").
+         with(
+            headers: {
+            'Accept'=>'*/*',
+            'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
+            'User-Agent'=>'Faraday v1.0.1'
+           }).
+         to_return(status: 200, body: json, headers: {})
+
+        ojos_instances = aws.instances(role: 'ojos', availability_zone: other_availability_zone, discovery_type: 'consul')
+        expect(ojos_instances.map{|i| i.tags.first[:value]}).to eq(['ojos_instance2', 'ojos_instance3'])
+      end
+    end
+
     context "ordered roles" do
       let(:my_availability_zone) { 'us-east-1a' }
       let(:other_availability_zone) { 'us-east-1b' }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: MovableInkAWS
 version: !ruby/object:Gem::Version
-  version: 1.3.3
+  version: 2.0.0
 platform: ruby
 authors:
 - Matt Chesler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-20 00:00:00.000000000 Z
+date: 2020-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-core
@@ -206,6 +206,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.16.3
+- !ruby/object:Gem::Dependency
+  name: diplomat
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.4.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.4.2
 description: AWS Utility methods for MovableInk
 email: mchesler@movableink.com
 executables: []