kubernetes_health_checker 0.0.0.3 → 0.0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a41c0e00613fdcadb3390cd838e58758d77dc7a4
-  data.tar.gz: 0e078558a40550481b7e7dc15a75f9a74a0bab24
+  metadata.gz: 8a44862baa23546298cbe135442791f9413b092f
+  data.tar.gz: 158d714234ec07627bd6e9fce0834aa906506ce8
 SHA512:
-  metadata.gz: 25468d36cfaf12e62c897ff6618197989a1c1ae9da83eeb0469129a7c32d35b2721a2a7e5bc8bf98c6ec5a3f8509b70d9b2434c6a146f9c910dd7b36b0e1e207
-  data.tar.gz: a3b8345de9736ad37dd8ef8bbf857c8f602eea7167dd51e394564e6d28de008267405f90f4a82e5bdb0ac1d86a79cf2b1060ee4fa9dca4b110cec62b8ea8b1c1
+  metadata.gz: 55c4048409051f6b23a8760be6640d6441673eb4b90a6df9e88cf7700b836ed5c93e5549df2d08eef30bd19639a1ef1ef564d549a4f08bec5057f52073b1da31
+  data.tar.gz: cae075720d0244dadc3e9c3a529afb09d4766f779a9d78d3792083338e01093bab0393da590c2ca70fe1c4a668013d0878b75ae2f4e6fb9ebba94ff557400be4

data/bin/kubernetes_health_checker

@@ -2,6 +2,7 @@
 
 require 'rubygems'
 require 'thor'
+require 'byebug'
 require 'json'
 
 module KubernetesHealthChecker
@@ -17,34 +18,19 @@ module KubernetesHealthChecker
 
     ALERT_STATUSES = ['CrashLoopBackOff'].freeze
 
-    def start
-      opts = options.inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo}
-      @alert_threshold = opts[:alert_threshold] || 5
-      @slack_channel = opts[:channel] || '@rohan'
-      @namespace = opts[:namespace] || 'default'
-      @test = opts[:test] || false
-
-      raise 'must provide a url to POST alert to' if !opts[:url]
-      @url = opts[:url]
-
-      if @test
-        path = File.expand_path('../mock_output.txt', __FILE__)
-        output = File.read(path)
-      else
-        output = `kubectl get pods --namespace #{options[:namespace]}`
-      end
-
-      message = ''
-      rows = output.split("\n")
-      rows.each_with_index do |row, index|
-        next if index == 0
-        row = row.split(" ")
-        response = check_pod_state(pod_name: row[0], status: row[2], restarts: row[3].to_i)
+    RUN_INTERVAL = 10
 
-        message += response[:message] if response[:should_alert]
+    def start
+      set_defaults(options)
+      run_validations if !@test
+      while true
+        puts 'starting...'
+        output = get_cli_output
+        message = construct_message(output)
+        send_alert(message) if !message.empty?
+        puts 'sleeping...'
+        sleep RUN_INTERVAL
       end
-
-      send_alert(message) if !message.empty?
     end
 
     desc 'version', 'Prints version'
@@ -53,7 +39,36 @@ module KubernetesHealthChecker
     end
 
     no_commands do
+      def set_defaults(options)
+        opts = options.inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo}
+        @alert_threshold = opts[:alert_threshold] || 5
+        @slack_channel = opts[:channel] || '@rohan'
+        @namespace = opts[:namespace] || 'default'
+        @test = opts[:test] || false
+        @url = opts[:url]
+        @pods_data = {}
+      end
+
+      def run_validations
+        raise 'must provide a url to POST alert to' if !@url
+      end
+
+      def get_cli_output
+        if @test
+          # so hacky, but pulls from second mock output if
+          # it's the second time running to test state changes
+          file_name = @pods_data.empty? ? '../mock_output.txt' : '../mock_output_two.txt'
+          path = File.expand_path(file_name, __FILE__)
+          output = File.read(path)
+        else
+          output = `kubectl get pods --namespace #{@namespace}`
+        end
+      end
+
       def send_alert(message)
+        puts "Sending message: #{message}"
+        return if @test
+
         payload = {
           channel: @slack_channel,
           username: 'kubernetes_health_check_bot',
@@ -64,19 +79,44 @@ module KubernetesHealthChecker
         `curl -X POST --data-urlencode 'payload=#{JSON.generate(payload)}' #{@url}`
       end
 
-      def check_pod_state(pod_name:, status:, restarts:)
-        should_alert = false
-        text = nil
+      def get_pod_message(pod_name:, new_status:, new_restarts:)
+        old_restarts = @pods_data[pod_name][:restarts]
+        old_status = @pods_data[pod_name][:status]
+        text = ""
+
+        if !@pods_data[pod_name].empty? && old_restarts == new_restarts && old_status == new_status
+          # we've already alerted from this state, skip
+        elsif ALERT_STATUSES.include?(new_status)
+          text = "Pod *#{pod_name}* is in status: *#{new_status}*.\n"
+        elsif new_restarts > @alert_threshold
+          text = "Pod *#{pod_name}* has restarted *#{new_restarts}* times and has status: *#{new_status}*.\n"
+        end
+
+        text
+      end
+
+      def construct_message(output)
+        message = ''
+        new_pod_store = {}
+
+        rows = output.split("\n")
+        rows.each_with_index do |row, index|
+          next if index == 0
+
+          row = row.split(" ")
+          pod_name = row[0]
+          status = row[2]
+          restarts = row[3].to_i
+
+          @pods_data[pod_name] ||= {}
+          message += get_pod_message(pod_name: pod_name, new_status: status, new_restarts: restarts)
 
-        if ALERT_STATUSES.include?(status)
-          should_alert = true
-          text = "Pod *#{pod_name}* is in status: *#{status}*.\n"
-        elsif restarts > @alert_threshold
-          should_alert = true
-          text = "Pod *#{pod_name}* has restarted *#{restarts}* times and has status: *#{status}*.\n"
+          # update our data store for the pods
+          @pods_data[pod_name][:status] = status
+          @pods_data[pod_name][:restarts] = restarts
         end
 
-        {should_alert: should_alert, message: text}
+        message
       end
     end
   end

data/bin/kubernetes_health_checkerd

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'daemons'
+
+Daemons.run('./bin/kubernetes_health_checker')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kubernetes_health_checker
 version: !ruby/object:Gem::Version
-  version: 0.0.0.3
+  version: 0.0.0.4
 platform: ruby
 authors:
 - Rohan Sahai
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: daemons
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -43,10 +57,12 @@ description: Simple script to send slack notifications for pods that are crashin
 email: rohan@sah.ai
 executables:
 - kubernetes_health_checker
+- kubernetes_health_checkerd
 extensions: []
 extra_rdoc_files: []
 files:
 - bin/kubernetes_health_checker
+- bin/kubernetes_health_checkerd
 - bin/mock_output.txt
 homepage: http://rubygems.org/gems/kubernetes_health_checker
 licenses: