MovableInkAWS 1.3.3 → 2.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a474efe41182db10e521fd3df11b259261591f6113f2ab71cf66b0e603199578
-  data.tar.gz: 527f38ea7a18d93366b1fe9cbf16f75c3d8f79c8d6f29d6b44e8a4b7eb92d82f
+  metadata.gz: 6de21c0fdb46ad5c3506f242a6a2732abe711831fbde4b70c72ad24147e37bce
+  data.tar.gz: 035f21cb39b733f31bc05b86e877cf81cc62b7c2e7a5a80832df2d2ce147053b
 SHA512:
-  metadata.gz: 778adf6177659e449ee6a4c4fea6007120d7397bd5acd109111dfaf3f673af6289e1e9b82f8b8c2a34583c86d9b567283836cedbaa822a183bdd730b074fb9e7
-  data.tar.gz: 6f94b899d09790472b843e2fd69f895ed349f97a4b21443d413b61713d464c8eb864cdfce1a495139a2eb67c31cd36ab3fe0f485d333ab2b5f58e178c87ee167
+  metadata.gz: 65af327d97a4fddc422eac4b212ff9444b4d70e82c045baf4c96627e37152415fa183417399a189aa6c47f26176fab07640662e7a092063f48665bf214b9f13c
+  data.tar.gz: b91d80649d4f32bd8b878125e1c2d4a9c7fd6825194b82d563c43017897a85b3e784ff2cc6e6c3cb28f714f17a49d0bec57da754b67eea67698a4663cdc9c4a8

data/Gemfile

@@ -3,4 +3,5 @@ gemspec
 
 group :test do
   gem 'rspec',    '~> 3.6'
+  gem 'webmock'
 end

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    MovableInkAWS (1.3.3)
+    MovableInkAWS (2.1.2)
       aws-sdk-athena (~> 1)
       aws-sdk-autoscaling (~> 1)
       aws-sdk-cloudwatch (~> 1)
@@ -15,66 +15,80 @@ PATH
       aws-sdk-sns (~> 1)
       aws-sdk-ssm (~> 1)
       aws-sigv4 (~> 1.1)
+      diplomat (= 2.4.2)
       httparty (= 0.16.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
     aws-eventstream (1.1.0)
-    aws-partitions (1.383.0)
+    aws-partitions (1.416.0)
     aws-sdk-athena (1.33.0)
       aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-autoscaling (1.47.0)
+    aws-sdk-autoscaling (1.53.0)
       aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-cloudwatch (1.45.0)
+    aws-sdk-cloudwatch (1.47.0)
       aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-core (3.109.1)
+    aws-sdk-core (3.111.0)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-ec2 (1.200.0)
+    aws-sdk-ec2 (1.220.0)
       aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-eks (1.45.0)
+    aws-sdk-eks (1.46.0)
       aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-elasticache (1.44.0)
+    aws-sdk-elasticache (1.50.0)
       aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-kms (1.39.0)
+    aws-sdk-kms (1.41.0)
       aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-rds (1.104.0)
+    aws-sdk-rds (1.110.0)
       aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-route53 (1.44.0)
+    aws-sdk-route53 (1.45.0)
       aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.83.1)
+    aws-sdk-s3 (1.87.0)
       aws-sdk-core (~> 3, >= 3.109.0)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.1)
-    aws-sdk-sns (1.33.0)
+    aws-sdk-sns (1.36.0)
       aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-ssm (1.95.0)
+    aws-sdk-ssm (1.102.0)
       aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
     aws-sigv4 (1.2.2)
       aws-eventstream (~> 1, >= 1.0.2)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    deep_merge (1.2.1)
     diff-lcs (1.3)
+    diplomat (2.4.2)
+      deep_merge (~> 1.0, >= 1.0.1)
+      faraday (>= 0.9, < 1.1.0)
+    faraday (1.0.1)
+      multipart-post (>= 1.2, < 3)
+    hashdiff (1.0.0)
     httparty (0.16.3)
       mime-types (~> 3.0)
       multi_xml (>= 0.5.2)
     jmespath (1.4.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2020.0512)
+    mime-types-data (3.2020.1104)
     multi_xml (0.6.0)
+    multipart-post (2.1.1)
+    public_suffix (4.0.5)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -88,6 +102,11 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-support (3.9.3)
+    safe_yaml (1.0.5)
+    webmock (3.7.6)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
 
 PLATFORMS
   ruby
@@ -95,6 +114,7 @@ PLATFORMS
 DEPENDENCIES
   MovableInkAWS!
   rspec (~> 3.6)
+  webmock
 
 BUNDLED WITH
    2.1.3

data/MovableInkAWS.gemspec

@@ -23,6 +23,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'aws-sdk-ssm', '~> 1'
   s.add_runtime_dependency 'aws-sigv4', '~> 1.1'
   s.add_runtime_dependency 'httparty',  '0.16.3'
+  s.add_runtime_dependency 'diplomat',  '2.4.2'
 
   all_files  = `git ls-files`.split("\n")
   test_files = `git ls-files -- {test,spec,features}/*`.split("\n")

data/lib/movable_ink/aws.rb

@@ -10,6 +10,7 @@ require_relative 'aws/s3'
 require_relative 'aws/eks'
 require_relative 'aws/elasticache'
 require_relative 'aws/api_gateway'
+require_relative 'consul/consul'
 require 'aws-sdk-cloudwatch'
 
 

data/lib/movable_ink/aws/ec2.rb

@@ -1,4 +1,5 @@
 require 'aws-sdk-ec2'
+require 'diplomat'
 
 module MovableInk
   class AWS
@@ -75,7 +76,7 @@ module MovableInk
         @me ||= all_instances.select{|instance| instance.instance_id == instance_id}.first rescue nil
       end
 
-      def instances(role:, exclude_roles: [], region: my_region, availability_zone: nil, exact_match: false, use_cache: true)
+      def instances_with_ec2_discovery(role:, exclude_roles: [], region: my_region, availability_zone: nil, exact_match: false, use_cache: true)
         roles = role.split(/\s*,\s*/)
         if use_cache == false
           filter = default_filter.push({
@@ -106,6 +107,58 @@ module MovableInk
         end
       end
 
+      def instances_with_consul_discovery(role:, region: my_region, availability_zone: nil)
+        if role == nil || role == ''
+          raise MovableInk::AWS::Errors::RoleNameRequiredError
+        end
+
+        consul_service_options = {
+          dc: datacenter(region: region),
+          stale: true,
+          cached: true,
+          passing: true,
+        }
+        consul_service_options[:node_meta] = "availability_zone:#{availability_zone}" unless availability_zone.nil?
+
+        # We replace underscores with dashes in the role name in order to comply with
+        # consul service naming conventions while still retaining the role name we use
+        # within MI configuration
+        Diplomat::Health.service(role.gsub('_', '-'), consul_service_options).map { |endpoint|
+          OpenStruct.new (
+          {
+            private_ip_address: endpoint.Node['Address'],
+            instance_id: endpoint.Node['Meta']['instance_id'],
+            tags: [
+              {
+                key: 'Name',
+                value: endpoint.Node['Node']
+              },
+              {
+                key: 'mi:roles',
+                value: endpoint.Node['Meta']['mi_roles']
+              },
+              {
+                key: 'mi:monitoring_roles',
+                value: endpoint.Node['Meta']['mi_monitoring_roles']
+              }
+            ],
+            placement: {
+              availability_zone: endpoint.Node['Meta']['availability_zone']
+            }
+          })
+        }
+      end
+
+      def instances(role:, exclude_roles: [], region: my_region, availability_zone: nil, exact_match: false, use_cache: true, discovery_type: 'ec2')
+        if discovery_type == 'ec2'
+          instances_with_ec2_discovery(role: role, exclude_roles: exclude_roles, region: region, availability_zone: availability_zone, exact_match: exact_match, use_cache: use_cache)
+        elsif discovery_type == 'consul'
+          instances_with_consul_discovery(role: role, region: region, availability_zone: availability_zone)
+        else
+          raise MovableInk::AWS::Errors::InvalidDiscoveryTypeError
+        end
+      end
+
       def thopter
         private_ip_addresses(thopter_instance).first
       end

data/lib/movable_ink/aws/errors.rb

@@ -5,6 +5,9 @@ module MovableInk
       class FailedWithBackoff < StandardError; end
       class EC2Required < StandardError; end
       class NoEnvironmentTagError < StandardError; end
+      class InvalidDiscoveryTypeError < StandardError; end
+      class RoleNameRequiredError < StandardError; end
+      class RoleNameInvalidError < StandardError; end
     end
   end
 end

data/lib/movable_ink/consul/consul.rb

@@ -0,0 +1,21 @@
+require 'diplomat'
+
+Diplomat.configure do |config|
+  config.url = "https://localhost:8501"
+  config.options = { ssl: { verify: false } }
+end
+
+module MovableInk
+  module Consul
+    module Kv
+      def self.get(*args)
+        begin
+          result = Diplomat::Kv.get(*args)
+          JSON.parse(result) if !result.nil?
+        rescue JSON::ParserError
+          result
+        end
+      end
+    end
+  end
+end

data/lib/movable_ink/version.rb

@@ -1,5 +1,5 @@
 module MovableInk
   class AWS
-    VERSION = '1.3.3'
+    VERSION = '2.1.2'
   end
 end

data/spec/consul_spec.rb

@@ -0,0 +1,22 @@
+require_relative '../lib/movable_ink/aws'
+require 'webmock/rspec'
+
+describe MovableInk::Consul do
+  describe MovableInk::Consul::Kv do
+    it 'get returns a hash of json content' do
+      allow(Diplomat::Kv).to receive(:get).and_return({ hello: 'world' }.to_json)
+      expect(MovableInk::Consul::Kv.get('ham')).to eq({ 'hello' => 'world' })
+    end
+
+    it 'returns a string if the stored value is not json' do
+      value = 'some-database-password'
+      allow(Diplomat::Kv).to receive(:get).and_return(value)
+      expect(MovableInk::Consul::Kv.get('db_password')).to eq(value)
+    end
+
+    it 'forwards all arguments to Diplomat' do
+      expect(Diplomat::Kv).to receive(:get).with('foo/', recurse: true)
+      MovableInk::Consul::Kv.get('foo/', recurse: true)
+    end
+  end
+end

data/spec/ec2_spec.rb

@@ -1,4 +1,5 @@
 require_relative '../lib/movable_ink/aws'
+require 'webmock/rspec'
 
 describe MovableInk::AWS::EC2 do
   context "outside EC2" do
@@ -21,6 +22,7 @@ describe MovableInk::AWS::EC2 do
   end
 
   context "inside EC2" do
+    WebMock.allow_net_connect!
     let(:aws) { MovableInk::AWS.new }
     let(:ec2) { Aws::EC2::Client.new(stub_responses: true) }
     let(:tag_data) { ec2.stub_data(:describe_tags, tags: [
@@ -305,6 +307,146 @@ describe MovableInk::AWS::EC2 do
       end
     end
 
+    context "instances with consul discovery" do
+      let(:my_availability_zone) { 'us-east-1a' }
+      let(:my_datacenter) { 'iad' }
+      let(:other_availability_zone) { 'us-east-1b' }
+      let(:consul_app_service_instances) {
+        [{
+            Node: {
+              Node: 'app_instance1',
+              Address: '10.0.0.1',
+              Datacenter: my_datacenter,
+              Meta: {
+                availability_zone: my_availability_zone,
+                instance_id: 'i-12345',
+                mi_monitoring_roles: 'app',
+                mi_roles: 'app'
+              },
+              ServiceID: 'app',
+              ServiceName: 'app',
+              ServiceTags: ['foo', 'bar'],
+              ServicePort: 80,
+            }
+        },
+        {
+          Node: {
+              Node: 'app_instance2',
+              Address: '10.0.0.2',
+              Datacenter: my_datacenter,
+              Meta: {
+                availability_zone: my_availability_zone,
+                instance_id: 'i-54321',
+                mi_monitoring_roles: 'app',
+                mi_roles: 'app'
+              },
+              ServiceID: 'app',
+              ServiceName: 'app',
+              ServiceTags: ['foo', 'bar'],
+              ServicePort: 80,
+            }
+       }]
+      }
+
+      let(:consul_ojos_service_instances) {
+        [
+        {
+            Node: {
+              Node: 'ojos_instance2',
+              Address: '10.0.0.4',
+              Datacenter: my_datacenter,
+              Meta: {
+                availability_zone: other_availability_zone,
+                instance_id: 'i-zyx987',
+                mi_monitoring_roles: 'ojos',
+                mi_roles: 'ojos'
+              },
+              ServiceID: 'ojos',
+              ServiceName: 'ojos',
+              ServiceTags: ['foo', 'bar'],
+              ServicePort: 2702,
+            },
+        },
+        {
+            Node: {
+              Node: 'ojos_instance3',
+              Address: '10.0.0.5',
+              Datacenter: my_datacenter,
+              Meta: {
+                availability_zone: other_availability_zone,
+                instance_id: 'i-987654',
+                mi_monitoring_roles: 'ojos',
+                mi_roles: 'ojos'
+              },
+              ServiceID: 'ojos',
+              ServiceName: 'ojos',
+              ServiceTags: ['foo', 'bar'],
+              ServicePort: 2702,
+            }
+        }]
+      }
+
+      before(:each) do
+        allow(aws).to receive(:mi_env).and_return('test')
+        allow(aws).to receive(:availability_zone).and_return(my_availability_zone)
+        allow(aws).to receive(:my_region).and_return('us-east-1')
+        allow(aws).to receive(:ec2).and_return(ec2)
+      end
+
+      it "returns an error if no role is defined" do
+        expect{ aws.instances(role: nil, discovery_type: 'consul') }.to raise_error(MovableInk::AWS::Errors::RoleNameRequiredError)
+      end
+
+      it "returns all instances matching a consul service" do
+        miaws = double(MovableInk::AWS)
+        allow(miaws).to receive(:my_region).and_return('us-east-1')
+
+        json = JSON.generate(consul_app_service_instances)
+        stub_request(:get, "https://localhost:8501/v1/health/service/app?cached&dc=#{my_datacenter}&passing&stale").
+         with(
+            headers: {
+            'Accept'=>'*/*',
+            'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
+            'User-Agent'=>'Faraday v1.0.1'
+           }).
+         to_return(status: 200, body: json, headers: {})
+
+        app_instances = aws.instances(role: 'app', discovery_type: 'consul')
+        expect(app_instances.map{|i| i.tags.first[:value]}).to eq(['app_instance1', 'app_instance2'])
+
+        json = JSON.generate(consul_ojos_service_instances)
+        stub_request(:get, "https://localhost:8501/v1/health/service/ojos?cached&dc=#{my_datacenter}&passing&stale").
+         with(
+           headers: {
+          'Accept'=>'*/*',
+          'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
+          'User-Agent'=>'Faraday v1.0.1'
+           }).
+         to_return(status: 200, body: json, headers: {})
+
+        ojos_instances = aws.instances(role: 'ojos', discovery_type: 'consul')
+        expect(ojos_instances.map{|i| i.tags.first[:value ]}).to eq(['ojos_instance2', 'ojos_instance3'])
+      end
+
+      it "returns all instances matching a consul service filtered by availability_zone" do
+        miaws = double(MovableInk::AWS)
+        allow(miaws).to receive(:my_region).and_return('us-east-1')
+
+        json = JSON.generate(consul_ojos_service_instances)
+        stub_request(:get, "https://localhost:8501/v1/health/service/ojos?cached&dc=#{my_datacenter}&node-meta=availability_zone:#{other_availability_zone}&passing&stale").
+         with(
+            headers: {
+            'Accept'=>'*/*',
+            'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
+            'User-Agent'=>'Faraday v1.0.1'
+           }).
+         to_return(status: 200, body: json, headers: {})
+
+        ojos_instances = aws.instances(role: 'ojos', availability_zone: other_availability_zone, discovery_type: 'consul')
+        expect(ojos_instances.map{|i| i.tags.first[:value]}).to eq(['ojos_instance2', 'ojos_instance3'])
+      end
+    end
+
     context "ordered roles" do
       let(:my_availability_zone) { 'us-east-1a' }
       let(:other_availability_zone) { 'us-east-1b' }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: MovableInkAWS
 version: !ruby/object:Gem::Version
-  version: 1.3.3
+  version: 2.1.2
 platform: ruby
 authors:
 - Matt Chesler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-20 00:00:00.000000000 Z
+date: 2021-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-core
@@ -206,6 +206,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.16.3
+- !ruby/object:Gem::Dependency
+  name: diplomat
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.4.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.4.2
 description: AWS Utility methods for MovableInk
 email: mchesler@movableink.com
 executables: []
@@ -232,9 +246,11 @@ files:
 - lib/movable_ink/aws/s3.rb
 - lib/movable_ink/aws/sns.rb
 - lib/movable_ink/aws/ssm.rb
+- lib/movable_ink/consul/consul.rb
 - lib/movable_ink/version.rb
 - spec/autoscaling_spec.rb
 - spec/aws_spec.rb
+- spec/consul_spec.rb
 - spec/ec2_spec.rb
 - spec/elasticache_spec.rb
 - spec/metadata_spec.rb