MovableInkAWS 1.0.16 → 1.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8da5fcf1d95cc4642ae47bc180040671d9baeb9bbabb3c92bdf8c7a1a387b33b
-  data.tar.gz: ebb0966bf640735e8e5d2e2b0903ac246a8e61f85eb3f21a2950afd935ca338d
+  metadata.gz: 135c3f868d20fde5a59ac85a4e899cbd008fdcf6e4adb52e87ce0ce890305c6a
+  data.tar.gz: e5c2db50def4842fd9d70278603aec613d2bb3e13874740ba4d2dc8cb088819f
 SHA512:
-  metadata.gz: 635c4fe35a5c3bb404ec5ea1621345a5dca26bb8e85a911e772ad2420cbd6469a0ac896a1311f4f86326948fc25298d006981451521df244333d50d2654c82b1
-  data.tar.gz: af18dbf13eab9fc4b06a3006c593289f7a6323afc779e08fe881f6b3e93a2ce76a0091ef36855653e848b17887d369745750c9d857085ea4816d6070e6b62053
+  metadata.gz: 31b99276587e8b25f7cfd3af5f7852542d969c675ce168236b1d2a1c33f683844151897e946eb6edebc4f1a3c70900e7c6d0aef204b389702b031b5652f7f3bd
+  data.tar.gz: 22d3b8dfb489a985f10b70f615be7055759882f52ca818c345d546f9f64291cd787b341430d1df72636df73ffb51ef370d85b66be883a45d2d081cc2739e95c0

data/.travis.yml

@@ -1,7 +1,8 @@
 language: ruby
 
 rvm:
-  - 2.2
+  - 2.5
+  - 2.6
 
 bundler_args: "--retry=3"
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    MovableInkAWS (1.0.16)
+    MovableInkAWS (1.1.5)
       aws-sdk-athena (~> 1)
       aws-sdk-autoscaling (~> 1)
       aws-sdk-cloudwatch (~> 1)
@@ -20,47 +20,47 @@ GEM
   remote: https://rubygems.org/
   specs:
     aws-eventstream (1.1.0)
-    aws-partitions (1.322.0)
-    aws-sdk-athena (1.25.0)
+    aws-partitions (1.325.0)
+    aws-sdk-athena (1.27.0)
       aws-sdk-core (~> 3, >= 3.71.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-autoscaling (1.36.0)
+    aws-sdk-autoscaling (1.37.0)
       aws-sdk-core (~> 3, >= 3.71.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-cloudwatch (1.37.0)
+    aws-sdk-cloudwatch (1.38.0)
       aws-sdk-core (~> 3, >= 3.71.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-core (3.96.1)
+    aws-sdk-core (3.97.1)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-ec2 (1.163.0)
+    aws-sdk-ec2 (1.165.0)
       aws-sdk-core (~> 3, >= 3.71.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-elasticache (1.34.0)
+    aws-sdk-elasticache (1.36.0)
       aws-sdk-core (~> 3, >= 3.71.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-kms (1.31.0)
+    aws-sdk-kms (1.33.0)
       aws-sdk-core (~> 3, >= 3.71.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-rds (1.84.0)
+    aws-sdk-rds (1.85.0)
       aws-sdk-core (~> 3, >= 3.71.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-route53 (1.34.0)
+    aws-sdk-route53 (1.35.0)
       aws-sdk-core (~> 3, >= 3.71.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.66.0)
+    aws-sdk-s3 (1.67.1)
       aws-sdk-core (~> 3, >= 3.96.1)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.1)
-    aws-sdk-sns (1.23.0)
+    aws-sdk-sns (1.24.0)
       aws-sdk-core (~> 3, >= 3.71.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-ssm (1.78.0)
+    aws-sdk-ssm (1.80.0)
       aws-sdk-core (~> 3, >= 3.71.0)
       aws-sigv4 (~> 1.1)
-    aws-sigv4 (1.1.3)
+    aws-sigv4 (1.1.4)
       aws-eventstream (~> 1.0, >= 1.0.2)
     diff-lcs (1.3)
     httparty (0.16.3)
@@ -75,15 +75,15 @@ GEM
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
       rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.0)
-      rspec-support (~> 3.9.0)
-    rspec-expectations (3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.0)
+    rspec-mocks (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
-    rspec-support (3.9.0)
+    rspec-support (3.9.3)
 
 PLATFORMS
   ruby
@@ -93,4 +93,4 @@ DEPENDENCIES
   rspec (~> 3.6)
 
 BUNDLED WITH
-   1.17.3
+   2.1.3

data/lib/movable_ink/aws.rb

@@ -1,4 +1,5 @@
 require_relative 'aws/errors'
+require_relative 'aws/metadata'
 require_relative 'aws/ec2'
 require_relative 'aws/sns'
 require_relative 'aws/autoscaling'
@@ -13,6 +14,7 @@ require 'aws-sdk-cloudwatch'
 
 module MovableInk
   class AWS
+    include Metadata
     include EC2
     include SNS
     include Autoscaling
@@ -61,6 +63,7 @@ module MovableInk
                Aws::EC2::Errors::ResourceAlreadyAssociated,
                Aws::EC2::Errors::Unavailable,
                Aws::SNS::Errors::ThrottledException,
+               Aws::AutoScaling::Errors::Throttling,
                Aws::AutoScaling::Errors::ThrottledException,
                Aws::S3::Errors::SlowDown,
                Aws::Route53::Errors::Throttling,
@@ -93,14 +96,6 @@ module MovableInk
       self.class.regions
     end
 
-    def availability_zone
-      @availability_zone ||= begin
-        az = `ec2metadata --availability-zone 2>/dev/null`.chomp
-        raise(MovableInk::AWS::Errors::EC2Required) if az.empty?
-        az
-      end
-    end
-
     def my_region
       @my_region ||= availability_zone.chop
     end

data/lib/movable_ink/aws/ec2.rb

@@ -63,14 +63,6 @@ module MovableInk
         end
       end
 
-      def instance_id
-        @instance_id ||= begin
-          id = `ec2metadata --instance-id 2>/dev/null`.chomp
-          raise(MovableInk::AWS::Errors::EC2Required) if id.empty?
-          id
-        end
-      end
-
       def instance_tags
         @instance_tags ||= run_with_backoff(quiet: true) do
           ec2.describe_tags({
@@ -79,30 +71,31 @@ module MovableInk
         end
       end
 
-      def private_ipv4
-        @ipv4 ||= begin
-          ipv4 = `ec2metadata --local-ipv4 2>/dev/null`.chomp
-          raise(MovableInk::AWS::Errors::EC2Required) if ipv4.empty?
-          ipv4
-        end
-      end
-
       def me
         @me ||= all_instances.select{|instance| instance.instance_id == instance_id}.first rescue nil
       end
 
-      def instances(role:, exclude_roles: [], region: my_region, availability_zone: nil, exact_match: false)
-        instances = all_instances(region: region).select { |instance|
-          instance.tags.select{ |tag| tag.key == 'mi:roles' }.detect { |tag|
-            roles = tag.value.split(/\s*,\s*/)
-            if exact_match
-              roles == [role]
-            else
-              exclude_roles.push('decommissioned')
-              roles.include?(role) && !roles.any? { |role| exclude_roles.include?(role) }
-            end
+      def instances(role:, exclude_roles: [], region: my_region, availability_zone: nil, exact_match: false, use_cache: true)
+        roles = role.split(/\s*,\s*/)
+        if use_cache == false
+          filter = default_filter.push({
+            name: 'tag:mi:roles',
+            values: roles
+          })
+          instances = load_all_instances(region, filter: filter)
+        else
+          instances = all_instances(region: region).select { |instance|
+            instance.tags.select{ |tag| tag.key == 'mi:roles' }.detect { |tag|
+              tag_roles = tag.value.split(/\s*,\s*/)
+              if exact_match
+                tag_roles == roles
+              else
+                exclude_roles.push('decommissioned')
+                tag_roles.any? { |tag_role| roles.include?(tag_role) } && !tag_roles.any? { |role| exclude_roles.include?(role) }
+              end
+            }
           }
-        }
+        end
 
         if availability_zone
           instances.select { |instance|
@@ -118,15 +111,15 @@ module MovableInk
       end
 
       def statsd_host
-        instance_ip_addresses_by_role(role: 'statsd', availability_zone: availability_zone).sample
+        instance_ip_addresses_by_role(role: 'statsd', availability_zone: availability_zone, use_cache: false).sample
       end
 
       def private_ip_addresses(instances)
         instances.map(&:private_ip_address)
       end
 
-      def instance_ip_addresses_by_role(role:, exclude_roles: [], region: my_region, availability_zone: nil, exact_match: false)
-        private_ip_addresses(instances(role: role, exclude_roles: exclude_roles, region: region, availability_zone: availability_zone, exact_match: exact_match))
+      def instance_ip_addresses_by_role(role:, exclude_roles: [], region: my_region, availability_zone: nil, exact_match: false, use_cache: true)
+        private_ip_addresses(instances(role: role, exclude_roles: exclude_roles, region: region, availability_zone: availability_zone, exact_match: exact_match, use_cache: use_cache))
       end
 
       def instance_ip_addresses_by_role_ordered(role:, exclude_roles: [], region: my_region, exact_match: false)

data/lib/movable_ink/aws/metadata.rb

@@ -0,0 +1,62 @@
+require 'net/http'
+
+module MovableInk
+  class AWS
+    module Metadata
+      def http
+        @http ||= begin
+          http = Net::HTTP.new("169.254.169.254", 80)
+          http.open_timeout = 1
+          http.read_timeout = 1
+          http
+        end
+      end
+
+      def retrieve_metadata(key)
+        request = Net::HTTP::Get.new("/latest/meta-data/#{key}")
+        request['X-aws-ec2-metadata-token'] = imds_token
+        response = http.request(request)
+        response.body
+      rescue
+        ""
+      end
+
+      def availability_zone
+        @availability_zone ||= begin
+          az = retrieve_metadata('placement/availability-zone')
+          raise(MovableInk::AWS::Errors::EC2Required) if az.empty?
+          az
+        end
+      end
+
+      def instance_id
+        @instance_id ||= begin
+          id = retrieve_metadata('instance-id')
+          raise(MovableInk::AWS::Errors::EC2Required) if id.empty?
+          id
+        end
+      end
+
+      def private_ipv4
+        @ipv4 ||= begin
+          ipv4 = retrieve_metadata('local-ipv4')
+          raise(MovableInk::AWS::Errors::EC2Required) if ipv4.empty?
+          ipv4
+        end
+      end
+
+      private
+
+      def imds_token
+        begin
+          request = Net::HTTP::Put.new('/latest/api/token')
+          request['X-aws-ec2-metadata-token-ttl-seconds'] = 120
+          response = http.request(request)
+          response.body
+        rescue
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/movable_ink/version.rb

@@ -1,5 +1,5 @@
 module MovableInk
   class AWS
-    VERSION = '1.0.16'
+    VERSION = '1.1.5'
   end
 end

data/spec/aws_spec.rb

@@ -2,15 +2,6 @@ require_relative '../lib/movable_ink/aws'
 
 describe MovableInk::AWS do
   context "outside EC2" do
-    it "should raise an error if EC2 is required" do
-      aws = MovableInk::AWS.new
-      allow(aws).to receive(:`).with('ec2metadata --instance-id 2>/dev/null').and_return("")
-      allow(aws).to receive(:`).with('ec2metadata --availability-zone 2>/dev/null').and_return("")
-
-      expect{ aws.instance_id }.to raise_error(MovableInk::AWS::Errors::EC2Required)
-      expect{ aws.availability_zone }.to raise_error(MovableInk::AWS::Errors::EC2Required)
-    end
-
     it 'doesnt raise an error if instance_id is set' do
       aws = MovableInk::AWS.new(instance_id: 'i-987654321')
       expect(aws.instance_id).to eq('i-987654321')
@@ -18,21 +9,9 @@ describe MovableInk::AWS do
   end
 
   context "inside EC2" do
-    it "should call ec2metadata to get the instance ID" do
-      aws = MovableInk::AWS.new
-      expect(aws).to receive(:`).with('ec2metadata --instance-id 2>/dev/null').and_return("i-12345\n")
-      expect(aws.instance_id).to eq('i-12345')
-    end
-
-    it "should call ec2metadata to get the availability zone" do
-      aws = MovableInk::AWS.new
-      expect(aws).to receive(:`).with('ec2metadata --availability-zone 2>/dev/null').and_return("us-east-1a\n")
-      expect(aws.availability_zone).to eq('us-east-1a')
-    end
-
     it "should find the datacenter by region" do
       aws = MovableInk::AWS.new
-      expect(aws).to receive(:`).with('ec2metadata --availability-zone 2>/dev/null').and_return("us-east-1a\n")
+      expect(aws).to receive(:retrieve_metadata).with('placement/availability-zone').and_return("us-east-1a")
       expect(aws.datacenter).to eq('iad')
     end
 

data/spec/ec2_spec.rb

@@ -4,17 +4,11 @@ describe MovableInk::AWS::EC2 do
   context "outside EC2" do
     it "should raise an error if trying to load mi_env outside of EC2" do
       aws = MovableInk::AWS.new
-      allow(aws).to receive(:`).with('ec2metadata --instance-id 2>/dev/null').and_return("")
-      allow(aws).to receive(:`).with('ec2metadata --availability-zone 2>/dev/null').and_return("")
+      allow(aws).to receive(:retrieve_metadata).with('instance-id').and_return("")
+      allow(aws).to receive(:retrieve_metadata).with('placement/availability-zone').and_return("")
       expect{ aws.mi_env }.to raise_error(MovableInk::AWS::Errors::EC2Required)
     end
 
-    it 'should raise an error if trying to load private_ipv4 outside of EC2' do
-      aws = MovableInk::AWS.new
-      allow(aws).to receive(:`).with('ec2metadata --local-ipv4 2>/dev/null').and_return('')
-      expect{ aws.private_ipv4 }.to raise_error(MovableInk::AWS::Errors::EC2Required)
-    end
-
     it "should use the provided environment" do
       aws = MovableInk::AWS.new(environment: 'test')
       expect(aws.mi_env).to eq('test')
@@ -67,12 +61,6 @@ describe MovableInk::AWS::EC2 do
       expect(aws.mi_env).to eq('test')
     end
 
-    it 'calls ec2metadata to get the private ipv4 address of the instance' do
-      aws = MovableInk::AWS.new
-      allow(aws).to receive(:`).with('ec2metadata --local-ipv4 2>/dev/null').and_return('10.0.0.1')
-      expect(aws.private_ipv4).to eq('10.0.0.1')
-    end
-
     context 'instance_tags' do
       it 'returns the tags of the current instance' do
         ec2.stub_responses(:describe_tags, tag_data)
@@ -153,18 +141,6 @@ describe MovableInk::AWS::EC2 do
               availability_zone: availability_zone
             }
           },
-          {
-            tags: [
-              {
-                key: 'mi:roles',
-                value: 'something_else'
-              }
-            ],
-            private_ip_address: '10.0.0.3',
-            placement: {
-              availability_zone: availability_zone
-            }
-          }
         ]])
       }
 
@@ -183,8 +159,8 @@ describe MovableInk::AWS::EC2 do
     context "instances" do
       let(:my_availability_zone) { 'us-east-1a' }
       let(:other_availability_zone) { 'us-east-1b' }
-      let(:instance_data) { ec2.stub_data(:describe_instances, reservations: [
-        instances: [
+      let(:instances) {
+        [
           {
             tags: [
               {
@@ -252,12 +228,54 @@ describe MovableInk::AWS::EC2 do
             placement: {
               availability_zone: other_availability_zone
             }
+          },
+          {
+            tags: [
+              {
+                key: 'mi:name',
+                value: 'instance5'
+              },
+              {
+                key: 'mi:roles',
+                value: 'app_db'
+              }
+            ],
+            instance_id: 'i-321cba',
+            private_ip_address: '10.0.0.5',
+            placement: {
+              availability_zone: other_availability_zone
+            }
           }
-        ]])
+        ]
+      }
+      let(:single_role_instance_data) { ec2.stub_data(:describe_instances, reservations: [
+        instances: instances.select { |instance|
+          instance[:tags].detect { |tag| tag[:key] == 'mi:roles' && tag[:value] == 'app_db_replica' }
+        }
+      ])}
+      let(:multi_role_instance_data) { ec2.stub_data(:describe_instances, reservations: [
+        instances: instances.select { |instance|
+          instance[:tags].detect { |tag|
+            tag[:key] == 'mi:roles' && ['app_db', 'app_db_replica'].any?(tag[:value])
+          }
+        }
+      ])}
+      let(:all_roles_instance_data) { ec2.stub_data(:describe_instances, reservations: [ instances: instances ])
       }
 
       before(:each) do
-        ec2.stub_responses(:describe_instances, instance_data)
+        ec2.stub_responses(:describe_instances, -> (context) {
+          if (context.params[:filters].length == 2)
+            all_roles_instance_data
+          else
+            role_filter = context.params[:filters].detect { |filter| filter[:name] == 'tag:mi:roles' }
+            if role_filter[:values].length == 1
+              single_role_instance_data
+            else
+              multi_role_instance_data
+            end
+          end
+        })
         allow(aws).to receive(:mi_env).and_return('test')
         allow(aws).to receive(:availability_zone).and_return(my_availability_zone)
         allow(aws).to receive(:my_region).and_return('us-east-1')
@@ -283,6 +301,16 @@ describe MovableInk::AWS::EC2 do
         expect(instances.map{|i| i.tags.first.value }).to eq(['instance4'])
       end
 
+      it "returns roles with exactly the specified role, letting the API to the filtering" do
+        instances = aws.instances(role: 'app_db_replica', use_cache: false)
+        expect(instances.map{|i| i.tags.first.value }).to eq(['instance4'])
+      end
+
+      it "returns roles with any of the specified roles, letting the API to the filtering" do
+        instances = aws.instances(role: 'app_db,app_db_replica', use_cache: false)
+        expect(instances.map{|i| i.tags.first.value }).to eq(['instance4', 'instance5'])
+      end
+
       it "excludes requested roles" do
         instances = aws.instances(role: 'app_db_replica', exclude_roles: ['db'])
         expect(instances.map{|i| i.tags.first.value }).to eq(['instance1', 'instance4'])

data/spec/metadata_spec.rb

@@ -0,0 +1,40 @@
+require_relative '../lib/movable_ink/aws'
+
+describe MovableInk::AWS::Metadata do
+  context 'outside ec2' do
+    it 'should raise an error if EC2 is required' do
+      aws = MovableInk::AWS.new
+      allow(aws).to receive(:retrieve_metadata).with('instance-id').and_return("")
+      allow(aws).to receive(:retrieve_metadata).with('placement/availability-zone').and_return("")
+
+      expect{ aws.instance_id }.to raise_error(MovableInk::AWS::Errors::EC2Required)
+      expect{ aws.availability_zone }.to raise_error(MovableInk::AWS::Errors::EC2Required)
+    end
+
+    it 'should raise an error if trying to load private_ipv4 outside of EC2' do
+      aws = MovableInk::AWS.new
+      allow(aws).to receive(:retrieve_metadata).with('local-ipv4').and_return('')
+      expect{ aws.private_ipv4 }.to raise_error(MovableInk::AWS::Errors::EC2Required)
+    end
+  end
+
+  context 'inside ec2' do
+    it 'calls the EC2 metadata service to get the private ipv4 address of the instance' do
+      aws = MovableInk::AWS.new
+      allow(aws).to receive(:retrieve_metadata).with('local-ipv4').and_return('10.0.0.1')
+      expect(aws.private_ipv4).to eq('10.0.0.1')
+    end
+
+    it 'calls the EC2 metadata service to get the instance ID' do
+      aws = MovableInk::AWS.new
+      expect(aws).to receive(:retrieve_metadata).with('instance-id').and_return("i-12345")
+      expect(aws.instance_id).to eq('i-12345')
+    end
+
+    it 'calls the EC2 metadata service to get the availability zone' do
+      aws = MovableInk::AWS.new
+      expect(aws).to receive(:retrieve_metadata).with('placement/availability-zone').and_return("us-east-1a")
+      expect(aws.availability_zone).to eq('us-east-1a')
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: MovableInkAWS
 version: !ruby/object:Gem::Version
-  version: 1.0.16
+  version: 1.1.5
 platform: ruby
 authors:
 - Matt Chesler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-28 00:00:00.000000000 Z
+date: 2020-08-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-core
@@ -212,6 +212,7 @@ files:
 - lib/movable_ink/aws/ec2.rb
 - lib/movable_ink/aws/elasticache.rb
 - lib/movable_ink/aws/errors.rb
+- lib/movable_ink/aws/metadata.rb
 - lib/movable_ink/aws/route53.rb
 - lib/movable_ink/aws/s3.rb
 - lib/movable_ink/aws/sns.rb
@@ -221,6 +222,7 @@ files:
 - spec/aws_spec.rb
 - spec/ec2_spec.rb
 - spec/elasticache_spec.rb
+- spec/metadata_spec.rb
 - spec/route53_spec.rb
 - spec/s3_spec.rb
 - spec/sns_spec.rb
@@ -243,8 +245,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: AWS Utility methods for MovableInk