MovableInkAWS 1.0.12 → 1.0.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '092c81ea23dd01ec27099a68d8ee17510239f9d18b75992c13e7c17600fa3c13'
-  data.tar.gz: 9c1696ebf069bb8db288509014599a62dc4d4c23b9c569e88dae3a858addf054
+  metadata.gz: 4563a27230a7711f44768ef561751a2366e03195de236dfb18a29bbff559c78b
+  data.tar.gz: b42568ac0c14f98d09420d4c2819b58eb047be5c31ca9a2e576c61685f69445b
 SHA512:
-  metadata.gz: e42ac78ca0f620398222aa285f1792e257229327c7d8595ac3a4d1548b335c97dec2a3682d5d828aecae956ede68ed5336f790d91beda46a85d29b90b4d97138
-  data.tar.gz: 4f25f6de7c875a960ae306c8c337f8f9b943d132ede56b5135761449228b745c15a8a06415c48061d64197e98d8a272627b3fe775ff179897ce6f5f46e5f31a8
+  metadata.gz: e9d9d52abbe8c75ce57a6cbd4165ea5188341106c4cf79cdc49fb96b9eda8db2b1e2216b948c750867c6333a9d29c682ef12084fd75186619b29d435606a99b7
+  data.tar.gz: c956e15226b703d33fdf7d74b367dac8259106cea459f5ca00ddfbe7fd05b17ccca18bdee22bde39c697370c7fb33ddd1ddf6210141efeea329f35b23f0a95c8

data/.travis.yml

@@ -1,7 +1,8 @@
 language: ruby
 
 rvm:
-  - 2.2
+  - 2.5
+  - 2.6
 
 bundler_args: "--retry=3"
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    MovableInkAWS (1.0.12)
+    MovableInkAWS (1.0.17)
       aws-sdk-athena (~> 1)
       aws-sdk-autoscaling (~> 1)
       aws-sdk-cloudwatch (~> 1)
@@ -19,48 +19,48 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    aws-eventstream (1.0.3)
-    aws-partitions (1.280.0)
-    aws-sdk-athena (1.22.0)
+    aws-eventstream (1.1.0)
+    aws-partitions (1.322.0)
+    aws-sdk-athena (1.26.0)
       aws-sdk-core (~> 3, >= 3.71.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-autoscaling (1.32.0)
+    aws-sdk-autoscaling (1.37.0)
       aws-sdk-core (~> 3, >= 3.71.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-cloudwatch (1.33.0)
+    aws-sdk-cloudwatch (1.38.0)
       aws-sdk-core (~> 3, >= 3.71.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-core (3.90.1)
-      aws-eventstream (~> 1.0, >= 1.0.2)
+    aws-sdk-core (3.97.0)
+      aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-ec2 (1.147.0)
+    aws-sdk-ec2 (1.164.0)
       aws-sdk-core (~> 3, >= 3.71.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-elasticache (1.29.0)
+    aws-sdk-elasticache (1.35.0)
       aws-sdk-core (~> 3, >= 3.71.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-kms (1.29.0)
+    aws-sdk-kms (1.32.0)
       aws-sdk-core (~> 3, >= 3.71.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-rds (1.78.0)
+    aws-sdk-rds (1.85.0)
       aws-sdk-core (~> 3, >= 3.71.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-route53 (1.30.0)
+    aws-sdk-route53 (1.35.0)
       aws-sdk-core (~> 3, >= 3.71.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.60.2)
-      aws-sdk-core (~> 3, >= 3.83.0)
+    aws-sdk-s3 (1.67.0)
+      aws-sdk-core (~> 3, >= 3.96.1)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.1)
-    aws-sdk-sns (1.21.0)
+    aws-sdk-sns (1.24.0)
       aws-sdk-core (~> 3, >= 3.71.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-ssm (1.71.0)
+    aws-sdk-ssm (1.79.0)
       aws-sdk-core (~> 3, >= 3.71.0)
       aws-sigv4 (~> 1.1)
-    aws-sigv4 (1.1.1)
+    aws-sigv4 (1.1.4)
       aws-eventstream (~> 1.0, >= 1.0.2)
     diff-lcs (1.3)
     httparty (0.16.3)
@@ -69,7 +69,7 @@ GEM
     jmespath (1.4.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2019.1009)
+    mime-types-data (3.2020.0512)
     multi_xml (0.6.0)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
@@ -93,4 +93,4 @@ DEPENDENCIES
   rspec (~> 3.6)
 
 BUNDLED WITH
-   1.17.1
+   1.17.3

data/lib/movable_ink/aws.rb

@@ -59,6 +59,7 @@ module MovableInk
           return yield
         rescue Aws::EC2::Errors::RequestLimitExceeded,
                Aws::EC2::Errors::ResourceAlreadyAssociated,
+               Aws::EC2::Errors::Unavailable,
                Aws::SNS::Errors::ThrottledException,
                Aws::AutoScaling::Errors::ThrottledException,
                Aws::S3::Errors::SlowDown,
@@ -76,12 +77,15 @@ module MovableInk
             notify_and_sleep(sleep_time, $!.class)
           end
         rescue Aws::Errors::ServiceError => e
-          message = "#{e.class}: #{e.message}\nFrom `#{e.backtrace.last.gsub("`","'")}`"
+          message = "#{e.class}: #{e.message}\nFrom #{$0}\n```\n#{e.backtrace.first(3).join("\n").gsub("`","'")}\n```"
           notify_slack(subject: 'Unhandled AWS API Error', message: message)
           puts message
           raise MovableInk::AWS::Errors::ServiceError
         end
       end
+      message = "From: #{$0}\n```\n#{Thread.current.backtrace.first(3).join("\n").gsub("`","'")}\n```"
+      notify_slack(subject: "AWS API failed after #{tries} attempts", message: message)
+      puts message
       raise MovableInk::AWS::Errors::FailedWithBackoff
     end
 

data/lib/movable_ink/aws/ec2.rb

@@ -14,7 +14,8 @@ module MovableInk
 
       def mi_env
         @mi_env ||= if File.exist?(mi_env_cache_file_path)
-          File.read(mi_env_cache_file_path)
+          value = File.read(mi_env_cache_file_path)
+          value.empty? ? load_mi_env : value
         else
           load_mi_env
         end
@@ -28,23 +29,12 @@ module MovableInk
         raise MovableInk::AWS::Errors::NoEnvironmentTagError
       end
 
-      def thopter_filter
-        [{
-          name: 'tag:mi:roles',
-          values: ['*thopter*']
-        },
-        {
-          name: 'tag:mi:env',
-          values: [mi_env]
-        },
-        {
-          name: 'instance-state-name',
-          values: ['running']
-        }]
-      end
-
       def thopter_instance
-        @thopter_instance ||= load_all_instances('us-east-1', filter: thopter_filter)
+        @thopter_instance ||= all_instances(region: 'us-east-1').select do |instance|
+          instance.tags.select{ |tag| tag.key == 'mi:roles' }.detect do |tag|
+            tag.value.include?('thopter')
+          end
+        end
       end
 
       def all_instances(region: my_region, no_filter: false)
@@ -101,18 +91,27 @@ module MovableInk
         @me ||= all_instances.select{|instance| instance.instance_id == instance_id}.first rescue nil
       end
 
-      def instances(role:, exclude_roles: [], region: my_region, availability_zone: nil, exact_match: false)
-        instances = all_instances(region: region).select { |instance|
-          instance.tags.select{ |tag| tag.key == 'mi:roles' }.detect { |tag|
-            roles = tag.value.split(/\s*,\s*/)
-            if exact_match
-              roles == [role]
-            else
-              exclude_roles.push('decommissioned')
-              roles.include?(role) && !roles.any? { |role| exclude_roles.include?(role) }
-            end
+      def instances(role:, exclude_roles: [], region: my_region, availability_zone: nil, exact_match: false, use_cache: true)
+        roles = role.split(/\s*,\s*/)
+        if use_cache == false
+          filter = default_filter.push({
+            name: 'tag:mi:roles',
+            values: roles
+          })
+          instances = load_all_instances(my_region, filter: filter)
+        else
+          instances = all_instances(region: region).select { |instance|
+            instance.tags.select{ |tag| tag.key == 'mi:roles' }.detect { |tag|
+              tag_roles = tag.value.split(/\s*,\s*/)
+              if exact_match
+                tag_roles == roles
+              else
+                exclude_roles.push('decommissioned')
+                tag_roles.any? { |tag_role| roles.include?(tag_role) } && !tag_roles.any? { |role| exclude_roles.include?(role) }
+              end
+            }
           }
-        }
+        end
 
         if availability_zone
           instances.select { |instance|

data/lib/movable_ink/version.rb

@@ -1,5 +1,5 @@
 module MovableInk
   class AWS
-    VERSION = '1.0.12'
+    VERSION = '1.0.17'
   end
 end

data/spec/aws_spec.rb

@@ -42,9 +42,9 @@ describe MovableInk::AWS do
         ec2 = Aws::EC2::Client.new(stub_responses: true)
         ec2.stub_responses(:describe_instances, 'RequestLimitExceeded')
 
-        expect(aws).to receive(:notify_slack).exactly(9).times
+        expect(aws).to receive(:notify_slack).exactly(10).times
         expect(aws).to receive(:sleep).exactly(9).times.and_return(true)
-        expect(STDOUT).to receive(:puts).exactly(9).times
+        expect(STDOUT).to receive(:puts).exactly(10).times
 
         aws.run_with_backoff { ec2.describe_instances } rescue nil
       end
@@ -64,9 +64,9 @@ describe MovableInk::AWS do
         ec2 = Aws::EC2::Client.new(stub_responses: true)
         ec2.stub_responses(:describe_instances, 'RequestLimitExceeded')
 
-        expect(aws).to receive(:notify_slack).exactly(1).times
+        expect(aws).to receive(:notify_slack).exactly(2).times
         expect(aws).to receive(:sleep).exactly(9).times.and_return(true)
-        expect(STDOUT).to receive(:puts).exactly(1).times
+        expect(STDOUT).to receive(:puts).exactly(2).times
 
         aws.run_with_backoff(quiet: true) { ec2.describe_instances } rescue nil
       end
@@ -77,6 +77,8 @@ describe MovableInk::AWS do
         ec2.stub_responses(:describe_instances, 'RequestLimitExceeded')
 
         expect(aws).to receive(:notify_and_sleep).exactly(9).times
+        expect(aws).to receive(:notify_slack).exactly(1).times
+        expect(STDOUT).to receive(:puts).exactly(1).times
         expect{ aws.run_with_backoff { ec2.describe_instances } }.to raise_error(MovableInk::AWS::Errors::FailedWithBackoff)
       end
     end

data/spec/ec2_spec.rb

@@ -45,6 +45,19 @@ describe MovableInk::AWS::EC2 do
       expect(aws.mi_env).to eq('staging')
     end
 
+    it 'will read tag data if the cache file is empty' do
+      ec2.stub_responses(:describe_tags, tag_data)
+      allow(aws).to receive(:my_region).and_return('us-east-1')
+      allow(aws).to receive(:instance_id).and_return('i-12345')
+      allow(aws).to receive(:ec2).and_return(ec2)
+
+      f = Tempfile.new('cache')
+      f.write('')
+      f.close
+      allow(aws).to receive(:mi_env_cache_file_path).and_return(f.path)
+      expect(aws.mi_env).to eq('test')
+    end
+
     it "should find the environment from the current instance's tags" do
       ec2.stub_responses(:describe_tags, tag_data)
       allow(aws).to receive(:my_region).and_return('us-east-1')
@@ -170,8 +183,8 @@ describe MovableInk::AWS::EC2 do
     context "instances" do
       let(:my_availability_zone) { 'us-east-1a' }
       let(:other_availability_zone) { 'us-east-1b' }
-      let(:instance_data) { ec2.stub_data(:describe_instances, reservations: [
-        instances: [
+      let(:instances) {
+        [
           {
             tags: [
               {
@@ -239,12 +252,54 @@ describe MovableInk::AWS::EC2 do
             placement: {
               availability_zone: other_availability_zone
             }
+          },
+          {
+            tags: [
+              {
+                key: 'mi:name',
+                value: 'instance5'
+              },
+              {
+                key: 'mi:roles',
+                value: 'app_db'
+              }
+            ],
+            instance_id: 'i-321cba',
+            private_ip_address: '10.0.0.5',
+            placement: {
+              availability_zone: other_availability_zone
+            }
           }
-        ]])
+        ]
+      }
+      let(:single_role_instance_data) { ec2.stub_data(:describe_instances, reservations: [
+        instances: instances.select { |instance|
+          instance[:tags].detect { |tag| tag[:key] == 'mi:roles' && tag[:value] == 'app_db_replica' }
+        }
+      ])}
+      let(:multi_role_instance_data) { ec2.stub_data(:describe_instances, reservations: [
+        instances: instances.select { |instance|
+          instance[:tags].detect { |tag|
+            tag[:key] == 'mi:roles' && ['app_db', 'app_db_replica'].any?(tag[:value])
+          }
+        }
+      ])}
+      let(:all_roles_instance_data) { ec2.stub_data(:describe_instances, reservations: [ instances: instances ])
       }
 
       before(:each) do
-        ec2.stub_responses(:describe_instances, instance_data)
+        ec2.stub_responses(:describe_instances, -> (context) {
+          if (context.params[:filters].length == 2)
+            all_roles_instance_data
+          else
+            role_filter = context.params[:filters].detect { |filter| filter[:name] == 'tag:mi:roles' }
+            if role_filter[:values].length == 1
+              single_role_instance_data
+            else
+              multi_role_instance_data
+            end
+          end
+        })
         allow(aws).to receive(:mi_env).and_return('test')
         allow(aws).to receive(:availability_zone).and_return(my_availability_zone)
         allow(aws).to receive(:my_region).and_return('us-east-1')
@@ -270,6 +325,16 @@ describe MovableInk::AWS::EC2 do
         expect(instances.map{|i| i.tags.first.value }).to eq(['instance4'])
       end
 
+      it "returns roles with exactly the specified role, letting the API to the filtering" do
+        instances = aws.instances(role: 'app_db_replica', use_cache: false)
+        expect(instances.map{|i| i.tags.first.value }).to eq(['instance4'])
+      end
+
+      it "returns roles with any of the specified roles, letting the API to the filtering" do
+        instances = aws.instances(role: 'app_db,app_db_replica', use_cache: false)
+        expect(instances.map{|i| i.tags.first.value }).to eq(['instance4', 'instance5'])
+      end
+
       it "excludes requested roles" do
         instances = aws.instances(role: 'app_db_replica', exclude_roles: ['db'])
         expect(instances.map{|i| i.tags.first.value }).to eq(['instance1', 'instance4'])

data/spec/ssm_spec.rb

@@ -82,6 +82,8 @@ describe MovableInk::AWS::SSM do
       allow(aws).to receive(:ssm_client).and_return(1)
       allow(aws).to receive(:ssm_client_failover).and_return(2)
       allow(aws).to receive(:notify_and_sleep).and_return(nil)
+      allow(aws).to receive(:notify_slack).and_return(nil)
+      allow(STDOUT).to receive(:puts).and_return(nil)
 
       results = []
       calls = 0

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: MovableInkAWS
 version: !ruby/object:Gem::Version
-  version: 1.0.12
+  version: 1.0.17
 platform: ruby
 authors:
 - Matt Chesler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-08 00:00:00.000000000 Z
+date: 2020-05-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-core