MovableInkAWS 0.2.8 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 88c1b6ff56ab368ab4d97a3f539b6a822e7af3c0447f8373a3601907c4e90393
-  data.tar.gz: 31146689d3eab6d8514f88cb85c013b1d4314ff5ad06376fc6e2f7b577dc94a2
+  metadata.gz: d362679456a51366b015bb76f2c7f8e578acc5d6f6d21d8e05bc2f4794842205
+  data.tar.gz: 879834f53d480a7e9fe5ca0f11daff419828525a78fb98bb7a6c1de8a744078d
 SHA512:
-  metadata.gz: 18352f2392290945415bf01b8be8daf41dfa4e5d146174f9d2bebe7154e74681eaced174a1a44cd072190521cb03c2045436d6c9eed8b246b885c267ded92370
-  data.tar.gz: f76fdc11f3109407596a82005400701345d9bea7803f14b3ebf2be283c3ca8ad1072c313370590b86974759dbd5e38cc98631c314ddb0d0b825b7672796734b4
+  metadata.gz: 05d9112da634b7ddec10dfaae68d70875f3832f4768dfa6bb279e315d63b50ff34b3f1bcd15643cff04eaca097b2c28da4c9a4321add29dccee00b90dd73572d
+  data.tar.gz: 3393a133714eb6bce8dce30195a0e6fdf20128129a32bf0bc1c67d9477e911f06eabe61765a8cea269bcea20e985957e46d8048ce2b80f7335f37b85e77e25a5

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    MovableInkAWS (0.2.8)
+    MovableInkAWS (0.3.0)
       aws-sdk (= 2.11.240)
 
 GEM

data/lib/movable_ink/aws/sns.rb

@@ -1,3 +1,6 @@
+require 'time'
+require 'json'
+
 module MovableInk
   class AWS
     module SNS
@@ -16,6 +19,16 @@ module MovableInk
         end
       end
 
+      def sns_pagerduty_topic_arn
+        run_with_backoff do
+          sns.list_topics.each do |resp|
+            resp.topics.each do |topic|
+              return topic.topic_arn if topic.topic_arn.include? "pagerduty-custom-alerts"
+            end
+          end
+        end
+      end
+
       def notify_and_sleep(seconds, error_class)
         message = "Throttled by AWS. Sleeping #{seconds} seconds, (#{error_class})"
         notify_slack(subject: 'API Throttled',
@@ -26,15 +39,52 @@ module MovableInk
 
       def notify_nsq_can_not_be_drained
         notify_slack(subject: 'NSQ not drained',
-                     message: 'Unable to drain NSQ')
+                     message: "Unable to drain NSQ for instance <https://#{my_region}.console.aws.amazon.com/ec2/v2/home?region=#{my_region}#Instances:search=#{instance_id};sort=instanceId|#{instance_id}>")
+        notify_pagerduty(region: my_region, instance_id: instance_id)
       end
 
-      def notify_slack(subject:, message:)
+      def notify_pagerduty(region:, instance_id:)
+        summary = "Unable to drain NSQ for instance #{instance_id} in region #{region}"
+
+        # the PagerDuty integration key is added to the payload in the AWS integration
+        json_message = {
+          pagerduty: {
+            event_action: 'trigger',
+            payload: {
+              source: 'MovableInkAWS',
+              summary: summary,
+              timestamp: Time.now.utc.iso8601,
+              severity: 'error',
+              component: 'nsq',
+              group: 'nsq',
+              custom_details: {
+                InstanceId: instance_id,
+              },
+            },
+            dedup_key: "nsq-not-draining-#{instance_id}",
+            links: [{
+              href: "https://#{region}.console.aws.amazon.com/ec2/v2/home?region=#{region}#Instances:search=#{instance_id};sort=instanceId",
+              text: 'View Instance'
+            }],
+          }
+        }.to_json
+
+        run_with_backoff do
+          sns.publish(topic_arn: sns_pagerduty_topic_arn,
+                      subject: add_subject_info(subject: "Unable to drain NSQ"),
+                      message: json_message)
+        end
+      end
+
+      def add_subject_info(subject:)
         required_info = " (#{instance_id}, #{my_region})"
-        subject = "#{subject.slice(0, 99-required_info.length)}#{required_info}"
+        "#{subject.slice(0, 99-required_info.length)}#{required_info}"
+      end
+
+      def notify_slack(subject:, message:)
         run_with_backoff do
           sns.publish(topic_arn: sns_slack_topic_arn,
-                      subject: subject,
+                      subject: add_subject_info(subject: subject),
                       message: message)
         end
       end

data/lib/movable_ink/version.rb

@@ -1,5 +1,5 @@
 module MovableInk
   class AWS
-    VERSION = '0.2.8'
+    VERSION = '0.3.0'
   end
 end

data/spec/sns_spec.rb

@@ -6,6 +6,9 @@ describe MovableInk::AWS::SNS do
   let(:topic_data) { sns.stub_data(:list_topics, topics: [
       {
         topic_arn: 'slack-aws-alerts'
+      },
+      {
+        topic_arn: 'pagerduty-custom-alerts'
       }
     ])
   }
@@ -19,6 +22,14 @@ describe MovableInk::AWS::SNS do
     expect(aws.sns_slack_topic_arn).to eq('slack-aws-alerts')
   end
 
+  it 'should find the pagerduty slack sns topic' do
+    sns.stub_responses(:list_topics, topic_data)
+    allow(aws).to receive(:my_region).and_return('us-east-1')
+    allow(aws).to receive(:sns).and_return(sns)
+
+    expect(aws.sns_pagerduty_topic_arn).to eq('pagerduty-custom-alerts')
+  end
+
   it "should notify with the specified subject and message" do
     sns.stub_responses(:list_topics, topic_data)
     allow(aws).to receive(:my_region).and_return('us-east-1')
@@ -26,7 +37,15 @@ describe MovableInk::AWS::SNS do
     allow(aws).to receive(:sns).and_return(sns)
 
     expect(aws.notify_slack(subject: 'Test subject', message: 'Test message').message_id).to eq('messageId')
+  end
 
+  it 'should notify pagerduty with related information' do
+    sns.stub_responses(:list_topics, topic_data)
+    allow(aws).to receive(:my_region).and_return('us-east-1')
+    allow(aws).to receive(:instance_id).and_return('test instance')
+    allow(aws).to receive(:sns).and_return(sns)
+
+    expect(aws.notify_pagerduty(region: 'us-east-1', instance_id: 'i-987654321').message_id).to eq('messageId')
   end
 
   it "should truncate subjects longer than 100 characters" do
@@ -41,6 +60,5 @@ describe MovableInk::AWS::SNS do
     expect(sns).to receive(:publish).with({:topic_arn=>"slack-aws-alerts", :message=>"Test message", :subject => expected_subject})
 
     aws.notify_slack(subject: subject, message: message)
-
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: MovableInkAWS
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.3.0
 platform: ruby
 authors:
 - Matt Chesler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-18 00:00:00.000000000 Z
+date: 2019-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk