MovableInkAWS 2.0.1 → 2.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9748092eb913e843641b0ac9d68106cb8ae3dc141c5f3ef0cf2794c7b52a41f5
-  data.tar.gz: 1bcbc2de4d41898654cf8df5e09fb1a43465d338f73b64a36fcf785db882fb15
+  metadata.gz: eb083b13e12b669f0843ea32ea2b8b4280ccce9b5ac6055964d9ca1214d36f60
+  data.tar.gz: 967f30f88ebef0135620f12930075da3bdeb05d4b449c2e5612a267f3197ff73
 SHA512:
-  metadata.gz: ed19423e404a705e9caf74762f6bd1cd08b5f552af5b24d6c355364d59c422d2cfba5f225e10cd021a79e77ae5d65ee8773c86eda5340e32d69d088cfb4789c4
-  data.tar.gz: 6c5edc3a00e2d798140c30d2cce8279c03099e9b30aa1c1630e14a10ae075756328d9327ebfab94171bea47394503d4f9a592c9a8d5e706be37fe6c1dfbbd7f6
+  metadata.gz: 687824a79299c74f15b6c9d1f3fded4663020670ae533619045e2097b99a38782348c3572d51bcb0d130d6b0e4e84b0ee78b778795792ba375b45118a94aaba9
+  data.tar.gz: 85feb71050191506f20429f851fca622597090365fe6817b54091daf539f7c84918466b20ba8566d2c948c1326188f4a33bdc8b3989d2f80e9b82cb6ffeebedf

data/.github/workflows/test.yaml

@@ -0,0 +1,39 @@
+name: test
+
+on:
+  push:
+    branches: ['**']
+    tags: ['v*']
+  pull_request:
+    branches: [$default-branch]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-18.04
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Install ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 2.6
+
+    - name: Install dependencies
+      run: bundle install --retry=3
+
+    - name: Run tests
+      run: bundle exec rspec spec/
+
+    - name: Publish to RubyGems
+      if: startsWith(github.ref, 'refs/tags/')
+      run: |
+        mkdir -p $HOME/.gem
+        touch $HOME/.gem/credentials
+        chmod 0600 $HOME/.gem/credentials
+        printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+        gem build *.gemspec
+        gem push *.gem
+      env:
+        GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    MovableInkAWS (2.0.1)
+    MovableInkAWS (2.2.1)
       aws-sdk-athena (~> 1)
       aws-sdk-autoscaling (~> 1)
       aws-sdk-cloudwatch (~> 1)
@@ -23,51 +23,51 @@ GEM
   specs:
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
-    aws-eventstream (1.1.0)
-    aws-partitions (1.390.0)
-    aws-sdk-athena (1.33.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-eventstream (1.1.1)
+    aws-partitions (1.433.0)
+    aws-sdk-athena (1.37.0)
+      aws-sdk-core (~> 3, >= 3.112.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-autoscaling (1.48.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-sdk-autoscaling (1.58.0)
+      aws-sdk-core (~> 3, >= 3.112.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-cloudwatch (1.46.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-sdk-cloudwatch (1.50.0)
+      aws-sdk-core (~> 3, >= 3.112.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-core (3.109.2)
+    aws-sdk-core (3.113.0)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-ec2 (1.207.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-sdk-ec2 (1.227.0)
+      aws-sdk-core (~> 3, >= 3.112.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-eks (1.45.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-sdk-eks (1.51.0)
+      aws-sdk-core (~> 3, >= 3.112.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-elasticache (1.45.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-sdk-elasticache (1.54.0)
+      aws-sdk-core (~> 3, >= 3.112.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-kms (1.39.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-sdk-kms (1.43.0)
+      aws-sdk-core (~> 3, >= 3.112.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-rds (1.105.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-sdk-rds (1.117.0)
+      aws-sdk-core (~> 3, >= 3.112.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-route53 (1.44.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-sdk-route53 (1.47.0)
+      aws-sdk-core (~> 3, >= 3.112.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.83.1)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-sdk-s3 (1.91.0)
+      aws-sdk-core (~> 3, >= 3.112.0)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.1)
-    aws-sdk-sns (1.35.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-sdk-sns (1.39.0)
+      aws-sdk-core (~> 3, >= 3.112.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-ssm (1.95.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-sdk-ssm (1.106.0)
+      aws-sdk-core (~> 3, >= 3.112.0)
       aws-sigv4 (~> 1.1)
-    aws-sigv4 (1.2.2)
+    aws-sigv4 (1.2.3)
       aws-eventstream (~> 1, >= 1.0.2)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
@@ -85,7 +85,7 @@ GEM
     jmespath (1.4.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2020.1104)
+    mime-types-data (3.2021.0225)
     multi_xml (0.6.0)
     multipart-post (2.1.1)
     public_suffix (4.0.5)

data/lib/movable_ink/aws.rb

@@ -10,6 +10,7 @@ require_relative 'aws/s3'
 require_relative 'aws/eks'
 require_relative 'aws/elasticache'
 require_relative 'aws/api_gateway'
+require_relative 'consul/consul'
 require 'aws-sdk-cloudwatch'
 
 

data/lib/movable_ink/aws/ec2.rb

@@ -112,50 +112,72 @@ module MovableInk
           raise MovableInk::AWS::Errors::RoleNameRequiredError
         end
 
-        if role.include?('_')
-          raise MovableInk::AWS::Errors::RoleNameInvalidError
-        end
-
-        Diplomat.configure do |config|
-          config.url = "https://localhost:8501"
-          config.options = { ssl: { verify: false } }
-        end
-
-        consul_service_options = Hash.new
-
-        consul_service_options[:dc] = datacenter(region: region)
-        consul_service_options[:stale] = true
-        consul_service_options[:cached] = true
-        consul_service_options[:passing] = true
-
+        consul_service_options = {
+          dc: datacenter(region: region),
+          stale: true,
+          cached: true,
+          passing: true,
+        }
         consul_service_options[:node_meta] = "availability_zone:#{availability_zone}" unless availability_zone.nil?
 
-        Diplomat::Health.service(role, consul_service_options).map { |endpoint|
-          OpenStruct.new (
-          {
-            private_ip_address: endpoint.Node['Address'],
-            instance_id: endpoint.Node['Meta']['instance_id'],
-            tags: [
-              {
-                key: 'Name',
-                value: endpoint.Node['Node']
-              },
-              {
-                key: 'mi:roles',
-                value: endpoint.Node['Meta']['mi_roles']
-              },
-              {
-                key: 'mi:monitoring_roles',
-                value: endpoint.Node['Meta']['mi_monitoring_roles']
-              }
-            ],
-            placement: {
-              availability_zone: endpoint.Node['Meta']['availability_zone']
-            }
-          })
+        # We replace underscores with dashes in the role name in order to comply with
+        # consul service naming conventions while still retaining the role name we use
+        # within MI configuration
+        Diplomat::Health.service(role.gsub('_', '-'), consul_service_options).map { |endpoint|
+          if endpoint.Node.dig('Meta', 'external-source') == 'kubernetes'
+            map_k8s_consul_endpoint(endpoint, role)
+          else
+            map_ec2_consul_endpoint(endpoint)
+          end
         }
       end
 
+      def map_k8s_consul_endpoint(endpoint, role)
+        OpenStruct.new ({
+          private_ip_address: endpoint.Service.dig('Address'),
+          instance_id: nil,
+          tags: [
+            {
+              key: 'Name',
+              value: endpoint.Service.dig('ID')
+            },
+            {
+              key: 'mi:roles',
+              value: role
+            },
+            {
+              key: 'mi:monitoring_roles',
+              value: role
+            }
+          ],
+          placement: { availability_zone: nil }
+        })
+      end
+
+      def map_ec2_consul_endpoint(endpoint)
+        OpenStruct.new ({
+          private_ip_address: endpoint.Node['Address'],
+          instance_id: endpoint.Node['Meta']['instance_id'],
+          tags: [
+            {
+              key: 'Name',
+              value: endpoint.Node['Node']
+            },
+            {
+              key: 'mi:roles',
+              value: endpoint.Node['Meta']['mi_roles']
+            },
+            {
+              key: 'mi:monitoring_roles',
+              value: endpoint.Node['Meta']['mi_monitoring_roles']
+            }
+          ],
+          placement: {
+            availability_zone: endpoint.Node['Meta']['availability_zone']
+          }
+        })
+      end
+
       def instances(role:, exclude_roles: [], region: my_region, availability_zone: nil, exact_match: false, use_cache: true, discovery_type: 'ec2')
         if discovery_type == 'ec2'
           instances_with_ec2_discovery(role: role, exclude_roles: exclude_roles, region: region, availability_zone: availability_zone, exact_match: exact_match, use_cache: use_cache)

data/lib/movable_ink/aws/sns.rb

@@ -29,42 +29,27 @@ module MovableInk
         sleep seconds
       end
 
-      def notify_nsq_can_not_be_drained
-        notify_slack(subject: 'NSQ not drained',
-                     message: "Unable to drain NSQ for instance <https://#{my_region}.console.aws.amazon.com/ec2/v2/home?region=#{my_region}#Instances:search=#{instance_id};sort=instanceId|#{instance_id}>")
-        notify_pagerduty(region: my_region, instance_id: instance_id)
-      end
-
-      def notify_pagerduty(region:, instance_id:)
-        summary = "Unable to drain NSQ for instance #{instance_id} in region #{region}"
-
-        # the PagerDuty integration key is added to the payload in the AWS integration
-        json_message = {
-          pagerduty: {
-            event_action: 'trigger',
-            payload: {
-              source: 'MovableInkAWS',
-              summary: summary,
-              timestamp: Time.now.utc.iso8601,
-              severity: 'error',
-              component: 'nsq',
-              group: 'nsq',
-              custom_details: {
-                InstanceId: instance_id,
-              },
-            },
-            dedup_key: "nsq-not-draining-#{instance_id}",
-            links: [{
-              href: "https://#{region}.console.aws.amazon.com/ec2/v2/home?region=#{region}#Instances:search=#{instance_id};sort=instanceId",
-              text: 'View Instance'
-            }],
-          }
-        }.to_json
-
+      def send_alert(
+        source: instance_id,
+        links: [],
+        custom_details: {},
+        summary:,
+        dedup_key:
+      )
         run_with_backoff do
-          sns.publish(topic_arn: sns_pagerduty_topic_arn,
-                      subject: "Unable to drain NSQ",
-                      message: json_message)
+          message_json = pd_message_json({
+            source: source,
+            summary: summary,
+            links: links,
+            custom_details: custom_details,
+            dedup_key: dedup_key,
+          })
+
+          sns.publish({
+            topic_arn: sns_pagerduty_topic_arn,
+            subject: summary,
+            message: message_json
+          })
         end
       end
 
@@ -78,6 +63,33 @@ module MovableInk
                       message: "#{required_info}\n#{message}")
         end
       end
+
+      private
+
+      def pd_message_json(
+        source:,
+        summary:,
+        links:,
+        custom_details:,
+        dedup_key:
+      )
+        {
+          pagerduty: {
+            event_action: 'trigger',
+            payload: {
+              source: source,
+              summary: summary,
+              timestamp: Time.now.utc.iso8601,
+              severity: 'error',
+              component: source,
+              group: source,
+              custom_details: custom_details,
+            },
+            dedup_key: dedup_key,
+            links: links,
+          }
+        }.to_json
+      end
     end
   end
 end

data/lib/movable_ink/consul/consul.rb

@@ -0,0 +1,21 @@
+require 'diplomat'
+
+Diplomat.configure do |config|
+  config.url = "https://localhost:8501"
+  config.options = { ssl: { verify: false } }
+end
+
+module MovableInk
+  module Consul
+    module Kv
+      def self.get(*args)
+        begin
+          result = Diplomat::Kv.get(*args)
+          JSON.parse(result) if !result.nil?
+        rescue JSON::ParserError
+          result
+        end
+      end
+    end
+  end
+end

data/lib/movable_ink/version.rb

@@ -1,5 +1,5 @@
 module MovableInk
   class AWS
-    VERSION = '2.0.1'
+    VERSION = '2.2.1'
   end
 end

data/spec/consul_spec.rb

@@ -0,0 +1,22 @@
+require_relative '../lib/movable_ink/aws'
+require 'webmock/rspec'
+
+describe MovableInk::Consul do
+  describe MovableInk::Consul::Kv do
+    it 'get returns a hash of json content' do
+      allow(Diplomat::Kv).to receive(:get).and_return({ hello: 'world' }.to_json)
+      expect(MovableInk::Consul::Kv.get('ham')).to eq({ 'hello' => 'world' })
+    end
+
+    it 'returns a string if the stored value is not json' do
+      value = 'some-database-password'
+      allow(Diplomat::Kv).to receive(:get).and_return(value)
+      expect(MovableInk::Consul::Kv.get('db_password')).to eq(value)
+    end
+
+    it 'forwards all arguments to Diplomat' do
+      expect(Diplomat::Kv).to receive(:get).with('foo/', recurse: true)
+      MovableInk::Consul::Kv.get('foo/', recurse: true)
+    end
+  end
+end

data/spec/ec2_spec.rb

@@ -386,6 +386,36 @@ describe MovableInk::AWS::EC2 do
         }]
       }
 
+      let(:consul_kubernetes_service_instances) do
+        [
+          {
+            "Node": {
+              "ID": "",
+              "Node": "k8s-sync",
+              "Address": "127.0.0.1",
+              "Datacenter": "my_datacenter",
+              "TaggedAddresses": nil,
+              "Meta": {
+                "external-source": "kubernetes",
+                "CreateIndex": 987654321,
+                "ModifyIndex": 123456789,
+              }
+            },
+            "Service": {
+              "ID": "pod-name-ac8f920827af",
+              "Service": "kubernetes-service-name",
+              "Tags": ["k8s"],
+              "Address": "10.0.0.1",
+              "Port": 8080,
+              "Meta": {
+                "external-source": "kubernetes",
+                "external-k8s-ns": "default",
+              }
+            }
+          }
+        ]
+      end
+
       before(:each) do
         allow(aws).to receive(:mi_env).and_return('test')
         allow(aws).to receive(:availability_zone).and_return(my_availability_zone)
@@ -397,10 +427,6 @@ describe MovableInk::AWS::EC2 do
         expect{ aws.instances(role: nil, discovery_type: 'consul') }.to raise_error(MovableInk::AWS::Errors::RoleNameRequiredError)
       end
 
-      it "returns an error if an invalid role is passed" do
-        expect{ aws.instances(role: 'asset_proxy', discovery_type: 'consul') }.to raise_error(MovableInk::AWS::Errors::RoleNameInvalidError)
-      end
-
       it "returns all instances matching a consul service" do
         miaws = double(MovableInk::AWS)
         allow(miaws).to receive(:my_region).and_return('us-east-1')
@@ -449,6 +475,33 @@ describe MovableInk::AWS::EC2 do
         ojos_instances = aws.instances(role: 'ojos', availability_zone: other_availability_zone, discovery_type: 'consul')
         expect(ojos_instances.map{|i| i.tags.first[:value]}).to eq(['ojos_instance2', 'ojos_instance3'])
       end
+
+      it "returns backends that are synced from consul-k8s" do
+        miaws = double(MovableInk::AWS)
+        allow(miaws).to receive(:my_region).and_return('us-east-1')
+
+        json = JSON.generate(consul_kubernetes_service_instances)
+        stub_request(:get, "https://localhost:8501/v1/health/service/kubernetes-service-name?cached&dc=#{my_datacenter}&passing&stale")
+          .with({
+            headers: {
+              'Accept'=>'*/*',
+              'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
+              'User-Agent'=>'Faraday v1.0.1'
+            }
+          })
+          .to_return(status: 200, body: json, headers: {})
+
+         resp_instances = aws.instances(role: 'kubernetes-service-name', discovery_type: 'consul')
+         backend = resp_instances.first
+         name_tag = backend.tags.find {|t| t[:key] == 'Name' }
+         roles_tag = backend.tags.find {|t| t[:key] == 'mi:roles' }
+
+         expect(name_tag[:value]).to eq('pod-name-ac8f920827af')
+         expect(roles_tag[:value]).to eq('kubernetes-service-name')
+         expect(backend.instance_id).to eq(nil)
+         expect(backend.private_ip_address).to eq('10.0.0.1')
+         expect(backend.private_ip_address).to eq('10.0.0.1')
+      end
     end
 
     context "ordered roles" do

data/spec/sns_spec.rb

@@ -47,7 +47,7 @@ describe MovableInk::AWS::SNS do
     allow(aws).to receive(:sns).and_return(sns)
     allow(aws).to receive(:private_ipv4).and_return('10.0.0.1')
 
-    expect(aws.notify_pagerduty(region: 'us-east-1', instance_id: 'i-987654321').message_id).to eq('messageId')
+    expect(aws.send_alert(summary: 'Something bad happened', dedup_key: 'i-987654321').message_id).to eq('messageId')
   end
 
   it "should truncate subjects longer than 100 characters" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: MovableInkAWS
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 2.2.1
 platform: ruby
 authors:
 - Matt Chesler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-06 00:00:00.000000000 Z
+date: 2021-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-core
@@ -227,7 +227,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".github/workflows/pr-lint.yaml"
-- ".travis.yml"
+- ".github/workflows/test.yaml"
 - Gemfile
 - Gemfile.lock
 - MovableInkAWS.gemspec
@@ -246,9 +246,11 @@ files:
 - lib/movable_ink/aws/s3.rb
 - lib/movable_ink/aws/sns.rb
 - lib/movable_ink/aws/ssm.rb
+- lib/movable_ink/consul/consul.rb
 - lib/movable_ink/version.rb
 - spec/autoscaling_spec.rb
 - spec/aws_spec.rb
+- spec/consul_spec.rb
 - spec/ec2_spec.rb
 - spec/elasticache_spec.rb
 - spec/metadata_spec.rb
@@ -274,7 +276,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.8
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: AWS Utility methods for MovableInk

data/.travis.yml

@@ -1,18 +0,0 @@
-language: ruby
-
-rvm:
-  - 2.6
-
-bundler_args: "--retry=3"
-
-script:
-  - rspec spec/
-
-deploy:
-  provider: rubygems
-  api_key:
-    secure: NAHVFo9MwOnAYhq1NzZz22/2p34Iw/L71ZNt2WHSbWuMVnA/xyvVu7Ja/S25aQmaV/btNkxTYtbJbf83QZTO9/IwkashWFcWJtLiptZkBRdpkiunMPWX9Gw3Ap5fODUIPxLFyLYWd53huNBJppBCVQTUPwkLEcWJcQZP4MU7dkhW5QKdQgmyNTlMy8epxyMqCQe7V0n98LCwreUsHxHMToe6/V0FemysjYjpnyNvcFpGFa9kf28VQw8ZpDR0iobw9hg6qs61Xj69Y3ns9fWI3sdnk8VjY29M+V415/ksyn7kzCjYiQsGGCFerzjMffIt/4PXqX81GRvcymxmpXWwtikJYP95kiHYbLGFUNS5GJngbyVgO5+amjx7vSZfaBGSXcUCSkxLV1Ki4QfMYnA4ihogwTdQhIALSEV9GzOfFt147lUok2LgZDzNfn9nCPL+qKMBNaBRe7/YoWGuvLQG/smYjunPYhis0ynAQ1Sytc7lgtbdEtLFzfET87ITVS4Y2F9TQ+z1fxROAn/YwMmdSPyiaXVLEOIUc/gx0BjJneTl43M4l6ZId8Q6maJTNG2Fd0yK0hwLJcsN/257HbnPEiTTgkXBcg58E6/ABfZqwDnJrP3sQPem85M4BxLJIPUvfBkYzyeAEV+ALYDJx+KLaqYCCgavF9M+41VbkkNZyKU=
-  gem: MovableInkAWS
-  on:
-    tags: true
-    repo: movableink/awslib