MovableInkAWS 2.2.0 → 2.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d7b1d77334dfeeb0283ce569cb62b7bce8a2518de96c73156f2b25693f0a2168
-  data.tar.gz: 56b60538a821c7fab3e55ad7ee0a80ac85312ff003521764e0721dcf86d5577c
+  metadata.gz: d5b1e204235285ad22ef1eabe26e765a93cd4025680d9f5eaa1167a053510e3d
+  data.tar.gz: f28f6c72bf9b1b954a9400aeb1ffe1dc62b4f7984ccb36ec1ca3d30b3bb9c681
 SHA512:
-  metadata.gz: 4132433dd73377d07d141297180ec17f983c4f124811ef45e474a78aa31d493d44ec2df98ed393707d13b095821182e54fcbcf23eff61f924d915aaf4eec1edd
-  data.tar.gz: e317ad0ac916a54ffd6042310906310ea5be40f3b6b88f83b3f8c6b45364bcea3df1282c5106a5d9032cf1804137e1c03002fd0cbc7942142ca2f7ec45245798
+  metadata.gz: fcd12a5ab67c2d0c409d3689d9e3daafa942fbaa3c679ae693355b1db4293ecabd00150b67706adc65250bb438e7f021e5d436261bb607d6d2bcb319eadb3758
+  data.tar.gz: f2e78bbfc703c0b82c2d1678d363031a9da107c800b5dc8c62b17e7c2e432ffdcaf376e99a6e62e6d2916ed7bead4d409aad07ff0e257e56562c3b0b5d35cf55

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    MovableInkAWS (2.2.0)
+    MovableInkAWS (2.3.2)
       aws-sdk-athena (~> 1)
       aws-sdk-autoscaling (~> 1)
       aws-sdk-cloudwatch (~> 1)
@@ -23,51 +23,51 @@ GEM
   specs:
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
-    aws-eventstream (1.1.0)
-    aws-partitions (1.416.0)
-    aws-sdk-athena (1.33.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-eventstream (1.1.1)
+    aws-partitions (1.445.0)
+    aws-sdk-athena (1.37.0)
+      aws-sdk-core (~> 3, >= 3.112.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-autoscaling (1.53.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-sdk-autoscaling (1.60.0)
+      aws-sdk-core (~> 3, >= 3.112.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-cloudwatch (1.47.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-sdk-cloudwatch (1.51.0)
+      aws-sdk-core (~> 3, >= 3.112.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-core (3.111.0)
+    aws-sdk-core (3.114.0)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-ec2 (1.220.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-sdk-ec2 (1.234.0)
+      aws-sdk-core (~> 3, >= 3.112.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-eks (1.46.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-sdk-eks (1.51.0)
+      aws-sdk-core (~> 3, >= 3.112.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-elasticache (1.50.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-sdk-elasticache (1.55.0)
+      aws-sdk-core (~> 3, >= 3.112.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-kms (1.41.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-sdk-kms (1.43.0)
+      aws-sdk-core (~> 3, >= 3.112.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-rds (1.110.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-sdk-rds (1.118.0)
+      aws-sdk-core (~> 3, >= 3.112.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-route53 (1.45.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-sdk-route53 (1.49.0)
+      aws-sdk-core (~> 3, >= 3.112.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.87.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-sdk-s3 (1.93.1)
+      aws-sdk-core (~> 3, >= 3.112.0)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.1)
-    aws-sdk-sns (1.36.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-sdk-sns (1.39.0)
+      aws-sdk-core (~> 3, >= 3.112.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-ssm (1.102.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-sdk-ssm (1.108.0)
+      aws-sdk-core (~> 3, >= 3.112.0)
       aws-sigv4 (~> 1.1)
-    aws-sigv4 (1.2.2)
+    aws-sigv4 (1.2.3)
       aws-eventstream (~> 1, >= 1.0.2)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
@@ -85,7 +85,7 @@ GEM
     jmespath (1.4.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2020.1104)
+    mime-types-data (3.2021.0225)
     multi_xml (0.6.0)
     multipart-post (2.1.1)
     public_suffix (4.0.5)

data/lib/movable_ink/aws/ec2.rb

@@ -124,31 +124,60 @@ module MovableInk
         # consul service naming conventions while still retaining the role name we use
         # within MI configuration
         Diplomat::Health.service(role.gsub('_', '-'), consul_service_options).map { |endpoint|
-          OpenStruct.new (
-          {
-            private_ip_address: endpoint.Node['Address'],
-            instance_id: endpoint.Node['Meta']['instance_id'],
-            tags: [
-              {
-                key: 'Name',
-                value: endpoint.Node['Node']
-              },
-              {
-                key: 'mi:roles',
-                value: endpoint.Node['Meta']['mi_roles']
-              },
-              {
-                key: 'mi:monitoring_roles',
-                value: endpoint.Node['Meta']['mi_monitoring_roles']
-              }
-            ],
-            placement: {
-              availability_zone: endpoint.Node['Meta']['availability_zone']
-            }
-          })
+          if endpoint.Node.dig('Meta', 'external-source') == 'kubernetes'
+            map_k8s_consul_endpoint(endpoint, role)
+          else
+            map_ec2_consul_endpoint(endpoint)
+          end
         }
       end
 
+      def map_k8s_consul_endpoint(endpoint, role)
+        OpenStruct.new ({
+          private_ip_address: endpoint.Service.dig('Address'),
+          instance_id: nil,
+          tags: [
+            {
+              key: 'Name',
+              value: endpoint.Service.dig('ID')
+            },
+            {
+              key: 'mi:roles',
+              value: role
+            },
+            {
+              key: 'mi:monitoring_roles',
+              value: role
+            }
+          ],
+          placement: { availability_zone: nil }
+        })
+      end
+
+      def map_ec2_consul_endpoint(endpoint)
+        OpenStruct.new ({
+          private_ip_address: endpoint.Node.dig('Address'),
+          instance_id: endpoint.Node.dig('Meta', 'instance_id'),
+          tags: [
+            {
+              key: 'Name',
+              value: endpoint.Node.dig('Node')
+            },
+            {
+              key: 'mi:roles',
+              value: endpoint.Node.dig('Meta', 'mi_roles')
+            },
+            {
+              key: 'mi:monitoring_roles',
+              value: endpoint.Node.dig('Meta', 'mi_monitoring_roles')
+            }
+          ],
+          placement: {
+            availability_zone: endpoint.Node.dig('Meta', 'availability_zone')
+          }
+        })
+      end
+
       def instances(role:, exclude_roles: [], region: my_region, availability_zone: nil, exact_match: false, use_cache: true, discovery_type: 'ec2')
         if discovery_type == 'ec2'
           instances_with_ec2_discovery(role: role, exclude_roles: exclude_roles, region: region, availability_zone: availability_zone, exact_match: exact_match, use_cache: use_cache)

data/lib/movable_ink/aws/errors.rb

@@ -3,7 +3,7 @@ module MovableInk
     module Errors
       class ServiceError < StandardError; end
       class FailedWithBackoff < StandardError; end
-      class EC2Required < StandardError; end
+      class MetadataTimeout < StandardError; end
       class NoEnvironmentTagError < StandardError; end
       class InvalidDiscoveryTypeError < StandardError; end
       class RoleNameRequiredError < StandardError; end

data/lib/movable_ink/aws/metadata.rb

@@ -3,59 +3,55 @@ require 'net/http'
 module MovableInk
   class AWS
     module Metadata
-      def http
-        @http ||= begin
+      def http(timeout_seconds: 1)
+        @http = begin
           http = Net::HTTP.new("169.254.169.254", 80)
-          http.open_timeout = 1
-          http.read_timeout = 1
+          http.open_timeout = timeout_seconds
+          http.read_timeout = timeout_seconds
           http
         end
       end
 
-      def retrieve_metadata(key)
-        request = Net::HTTP::Get.new("/latest/meta-data/#{key}")
-        request['X-aws-ec2-metadata-token'] = imds_token
-        response = http.request(request)
-        response.body
-      rescue
-        ""
+      def retrieve_metadata(key, tries: 3)
+        tries.times do |num|
+          num += 1
+          request = Net::HTTP::Get.new("/latest/meta-data/#{key}")
+          request['X-aws-ec2-metadata-token'] = imds_token
+          response = http(timeout_seconds: num * 3).request(request)
+          return response.body
+        rescue Net::OpenTimeout, Net::ReadTimeout, Errno::EHOSTDOWN
+          sleep(num * 2)
+        end
+
+        raise MovableInk::AWS::Errors::MetadataTimeout
       end
 
       def availability_zone
-        @availability_zone ||= begin
-          az = retrieve_metadata('placement/availability-zone')
-          raise(MovableInk::AWS::Errors::EC2Required) if az.empty?
-          az
-        end
+        @availability_zone ||= retrieve_metadata('placement/availability-zone')
       end
 
       def instance_id
-        @instance_id ||= begin
-          id = retrieve_metadata('instance-id')
-          raise(MovableInk::AWS::Errors::EC2Required) if id.empty?
-          id
-        end
+        @instance_id ||= retrieve_metadata('instance-id')
       end
 
       def private_ipv4
-        @ipv4 ||= begin
-          ipv4 = retrieve_metadata('local-ipv4')
-          raise(MovableInk::AWS::Errors::EC2Required) if ipv4.empty?
-          ipv4
-        end
+        @ipv4 ||= retrieve_metadata('local-ipv4')
       end
 
       private
 
-      def imds_token
-        begin
+      def imds_token(tries: 3)
+        tries.times do |num|
+          num += 1
           request = Net::HTTP::Put.new('/latest/api/token')
           request['X-aws-ec2-metadata-token-ttl-seconds'] = 120
-          response = http.request(request)
-          response.body
-        rescue
-          nil
+          response = http(timeout_seconds: num * 3).request(request)
+          return response.body
+        rescue Net::OpenTimeout, Net::ReadTimeout, Errno::EHOSTDOWN
+          sleep(num * 2)
         end
+
+        raise MovableInk::AWS::Errors::MetadataTimeout
       end
     end
   end

data/lib/movable_ink/aws.rb

@@ -103,7 +103,11 @@ module MovableInk
     end
 
     def my_region
-      @my_region ||= availability_zone.chop
+      @my_region ||= if ENV['AWS_REGION'].nil?
+        availability_zone.chop
+      else
+        ENV['AWS_REGION']
+      end
     end
 
     def datacenter(region: my_region)

data/lib/movable_ink/version.rb

@@ -1,5 +1,5 @@
 module MovableInk
   class AWS
-    VERSION = '2.2.0'
+    VERSION = '2.3.2'
   end
 end

data/spec/aws_spec.rb

@@ -1,17 +1,37 @@
 require_relative '../lib/movable_ink/aws'
+require 'webmock/rspec'
 
 describe MovableInk::AWS do
   context "outside EC2" do
+    before(:each) do
+      allow_any_instance_of(MovableInk::AWS).to receive(:sleep).and_return(true)
+    end
+
+    after(:each) { ENV['AWS_REGION'] = nil }
+
     it 'doesnt raise an error if instance_id is set' do
       aws = MovableInk::AWS.new(instance_id: 'i-987654321')
       expect(aws.instance_id).to eq('i-987654321')
     end
+
+    it 'doesnt raise if AWS_REGION is set' do
+      ENV['AWS_REGION'] = 'us-east-1'
+      miaws = MovableInk::AWS.new
+      expect(miaws.my_region).to eq('us-east-1')
+    end
+
+    it 'raises when AWS_REGION is not set and the metadata service is not available' do
+      miaws = MovableInk::AWS.new
+      # stub an error making a request to the metadata api
+      stub_request(:put, 'http://169.254.169.254/latest/api/token').to_raise(Net::OpenTimeout)
+      expect { miaws.my_region }.to raise_error(MovableInk::AWS::Errors::MetadataTimeout)
+    end
   end
 
   context "inside EC2" do
     it "should find the datacenter by region" do
       aws = MovableInk::AWS.new
-      expect(aws).to receive(:retrieve_metadata).with('placement/availability-zone').and_return("us-east-1a")
+      expect(aws).to receive(:availability_zone).and_return("us-east-1a")
       expect(aws.datacenter).to eq('iad')
     end
 

data/spec/ec2_spec.rb

@@ -2,12 +2,16 @@ require_relative '../lib/movable_ink/aws'
 require 'webmock/rspec'
 
 describe MovableInk::AWS::EC2 do
+  before(:each) do
+    allow_any_instance_of(MovableInk::AWS).to receive(:sleep).and_return(true)
+  end
+
   context "outside EC2" do
     it "should raise an error if trying to load mi_env outside of EC2" do
       aws = MovableInk::AWS.new
-      allow(aws).to receive(:retrieve_metadata).with('instance-id').and_return("")
-      allow(aws).to receive(:retrieve_metadata).with('placement/availability-zone').and_return("")
-      expect{ aws.mi_env }.to raise_error(MovableInk::AWS::Errors::EC2Required)
+      # stub an error making a request to the metadata api
+      stub_request(:put, 'http://169.254.169.254/latest/api/token').to_raise(Net::OpenTimeout)
+      expect{ aws.mi_env }.to raise_error(MovableInk::AWS::Errors::MetadataTimeout)
     end
 
     it "should use the provided environment" do
@@ -17,12 +21,16 @@ describe MovableInk::AWS::EC2 do
 
     it "should not find a 'me'" do
       aws = MovableInk::AWS.new
+      # stub an error making a request to the metadata api
+      stub_request(:put, 'http://169.254.169.254/latest/api/token').to_raise(Net::OpenTimeout)
       expect(aws.me).to eq(nil)
     end
   end
 
   context "inside EC2" do
-    WebMock.allow_net_connect!
+    before(:each) { WebMock.allow_net_connect! }
+    after(:each) { WebMock.disable_net_connect! }
+
     let(:aws) { MovableInk::AWS.new }
     let(:ec2) { Aws::EC2::Client.new(stub_responses: true) }
     let(:tag_data) { ec2.stub_data(:describe_tags, tags: [
@@ -386,6 +394,36 @@ describe MovableInk::AWS::EC2 do
         }]
       }
 
+      let(:consul_kubernetes_service_instances) do
+        [
+          {
+            "Node": {
+              "ID": "",
+              "Node": "k8s-sync",
+              "Address": "127.0.0.1",
+              "Datacenter": "my_datacenter",
+              "TaggedAddresses": nil,
+              "Meta": {
+                "external-source": "kubernetes",
+                "CreateIndex": 987654321,
+                "ModifyIndex": 123456789,
+              }
+            },
+            "Service": {
+              "ID": "pod-name-ac8f920827af",
+              "Service": "kubernetes-service-name",
+              "Tags": ["k8s"],
+              "Address": "10.0.0.1",
+              "Port": 8080,
+              "Meta": {
+                "external-source": "kubernetes",
+                "external-k8s-ns": "default",
+              }
+            }
+          }
+        ]
+      end
+
       before(:each) do
         allow(aws).to receive(:mi_env).and_return('test')
         allow(aws).to receive(:availability_zone).and_return(my_availability_zone)
@@ -444,6 +482,34 @@ describe MovableInk::AWS::EC2 do
 
         ojos_instances = aws.instances(role: 'ojos', availability_zone: other_availability_zone, discovery_type: 'consul')
         expect(ojos_instances.map{|i| i.tags.first[:value]}).to eq(['ojos_instance2', 'ojos_instance3'])
+        expect(ojos_instances.map{|i| i.tags[1][:value]}).to eq(['ojos', 'ojos'])
+      end
+
+      it "returns backends that are synced from consul-k8s" do
+        miaws = double(MovableInk::AWS)
+        allow(miaws).to receive(:my_region).and_return('us-east-1')
+
+        json = JSON.generate(consul_kubernetes_service_instances)
+        stub_request(:get, "https://localhost:8501/v1/health/service/kubernetes-service-name?cached&dc=#{my_datacenter}&passing&stale")
+          .with({
+            headers: {
+              'Accept'=>'*/*',
+              'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
+              'User-Agent'=>'Faraday v1.0.1'
+            }
+          })
+          .to_return(status: 200, body: json, headers: {})
+
+         resp_instances = aws.instances(role: 'kubernetes-service-name', discovery_type: 'consul')
+         backend = resp_instances.first
+         name_tag = backend.tags.find {|t| t[:key] == 'Name' }
+         roles_tag = backend.tags.find {|t| t[:key] == 'mi:roles' }
+
+         expect(name_tag[:value]).to eq('pod-name-ac8f920827af')
+         expect(roles_tag[:value]).to eq('kubernetes-service-name')
+         expect(backend.instance_id).to eq(nil)
+         expect(backend.private_ip_address).to eq('10.0.0.1')
+         expect(backend.private_ip_address).to eq('10.0.0.1')
       end
     end
 

data/spec/metadata_spec.rb

@@ -1,20 +1,25 @@
 require_relative '../lib/movable_ink/aws'
+require 'webmock/rspec'
 
 describe MovableInk::AWS::Metadata do
+  before(:each) do
+    allow_any_instance_of(MovableInk::AWS).to receive(:sleep).and_return(true)
+  end
+
   context 'outside ec2' do
-    it 'should raise an error if EC2 is required' do
+    it 'should raise an error if the metadata service times out' do
       aws = MovableInk::AWS.new
-      allow(aws).to receive(:retrieve_metadata).with('instance-id').and_return("")
-      allow(aws).to receive(:retrieve_metadata).with('placement/availability-zone').and_return("")
-
-      expect{ aws.instance_id }.to raise_error(MovableInk::AWS::Errors::EC2Required)
-      expect{ aws.availability_zone }.to raise_error(MovableInk::AWS::Errors::EC2Required)
+      # stub an error making a request to the metadata api
+      stub_request(:put, 'http://169.254.169.254/latest/api/token').to_raise(Net::OpenTimeout)
+      expect{ aws.instance_id }.to raise_error(MovableInk::AWS::Errors::MetadataTimeout)
+      expect{ aws.availability_zone }.to raise_error(MovableInk::AWS::Errors::MetadataTimeout)
     end
 
     it 'should raise an error if trying to load private_ipv4 outside of EC2' do
       aws = MovableInk::AWS.new
-      allow(aws).to receive(:retrieve_metadata).with('local-ipv4').and_return('')
-      expect{ aws.private_ipv4 }.to raise_error(MovableInk::AWS::Errors::EC2Required)
+      # stub an error making a request to the metadata api
+      stub_request(:put, 'http://169.254.169.254/latest/api/token').to_raise(Net::OpenTimeout)
+      expect{ aws.private_ipv4 }.to raise_error(MovableInk::AWS::Errors::MetadataTimeout)
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: MovableInkAWS
 version: !ruby/object:Gem::Version
-  version: 2.2.0
+  version: 2.3.2
 platform: ruby
 authors:
 - Matt Chesler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-23 00:00:00.000000000 Z
+date: 2021-11-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-core
@@ -276,7 +276,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: AWS Utility methods for MovableInk