watchmonkey_cli 1.4

data/lib/watchmonkey_cli/checker.rb

@@ -0,0 +1,228 @@
+module WatchmonkeyCli
+  class Checker
+    # Descendant tracking for inherited classes.
+    def self.descendants
+      @descendants ||= []
+    end
+
+    def self.inherited(descendant)
+      descendants << descendant
+    end
+
+    def self.checker_name
+      @checker_name || self.name
+    end
+
+    def self.checker_name= name
+      @checker_name = name
+    end
+
+    module AppHelper
+      def init_checkers!
+        @checkers = {}
+        WatchmonkeyCli::Checker.descendants.uniq.each do |klass|
+          debug "[SETUP] Initializing checker `#{klass.name}'"
+          @checkers[klass.checker_name] = klass.new(self)
+        end
+      end
+
+      def start_checkers!
+        @checkers.each do |key, instance|
+          debug "[SETUP] Starting checker `#{key}' (#{instance.class.name})"
+          instance.start
+        end
+      end
+
+      def stop_checkers!
+        @checkers.each do |key, instance|
+          debug "[SETUP] Stopping checker `#{key}' (#{instance.class.name})"
+          instance.stop
+        end
+      end
+    end
+
+    class Result
+      attr_reader :checker, :type, :args
+      attr_accessor :result, :command, :data
+
+      def initialize checker, *args
+        @checker = checker
+        @args = args
+        @mutex = Monitor.new
+        @type = :info
+        @spool = { error: [], info: [], debug: []}
+      end
+
+      def sync &block
+        @mutex.synchronize(&block)
+      end
+
+      def descriptor
+        "[#{@checker.class.checker_name} | #{args.join(" | ")}]"
+      end
+
+      def str_safe
+        "#{descriptor}\n\t"
+      end
+
+      def str_running
+        "Running checker #{@checker.class.checker_name} with [#{args.join(" | ")}]"
+      end
+
+      def str_descriptor
+        "#{descriptor}\n\t"
+      end
+
+      def messages
+        @spool.map(&:second).flatten
+      end
+
+      def dump!
+        sync do
+          @spool.each do |t, messages|
+            while messages.any?
+              @checker.send(t, "#{str_descriptor}#{messages.shift}", self)
+            end
+          end
+        end
+      end
+
+      [:info, :debug, :error].each do |meth|
+        define_method meth do |msg|
+          sync { @spool[meth] << msg }
+        end
+        define_method :"#{meth}!" do |msg = nil|
+          sync do
+            @spool[meth] << msg if msg
+            @type = meth
+          end
+        end
+        define_method :"#{meth}?" do
+          sync { @type == meth }
+        end
+      end
+    end
+
+    # -------------------
+
+    attr_reader :app
+
+    def initialize app
+      @app = app
+      send(:init) if respond_to?(:init)
+    end
+
+    def info msg, robj = nil
+      app.fire(:on_info, msg, robj)
+      return if app.opts[:quiet]
+      _tolog(msg, :info)
+      app.sync do
+        puts app.c(msg, :blue)
+      end
+    end
+
+    def debug msg, robj = nil
+      app.fire(:on_debug, msg, robj)
+      app.fire(:on_message, msg, robj)
+      return if app.opts[:quiet] || app.opts[:silent]
+      _tolog(msg, :debug)
+      app.sync do
+        puts app.c(msg, :black)
+      end
+    end
+
+    def error msg, robj = nil
+      app.fire(:on_error, msg, robj)
+      app.fire(:on_message, msg, robj)
+      _tolog(msg, :error)
+      app.sync { app.error(msg) }
+    end
+
+    def _tolog msg, meth = :log
+      return unless app.opts[:logfile]
+      app.logger.public_send(meth, msg)
+    end
+
+    def spawn_sub which, *args
+      return false if app.running?
+      if sec = app.checkers["ssl_expiration"]
+        sec.enqueue(*args)
+      end
+    end
+
+    # def to_s
+    #   string = "#<#{self.class.name}:#{self.object_id} "
+    #   fields = self.class.inspector_fields.map{|field| "#{field}: #{self.send(field)}"}
+    #   string << fields.join(", ") << ">"
+    # end
+
+    def local
+      @app.fetch_connection(:loopback, :local)
+    end
+
+    def safe descriptor = nil, &block
+      tries = 0
+      begin
+        tries += 1
+        block.call
+      rescue StandardError => e
+        unless tries > 3
+          app.sync do
+            error "#{descriptor}retry #{tries} reason is `#{e.class}: #{e.message}'"
+            e.backtrace.each{|l| debug "\t\t#{l}" }
+          end
+          sleep 1
+          retry
+        end
+        error "#{descriptor}retries exceeded"
+      end
+    end
+
+    def rsafe resultobj, &block
+      tries = 0
+      begin
+        tries += 1
+        block.call
+      rescue StandardError => e
+        unless tries > 3
+          resultobj.sync do
+            resultobj.error! "retry #{tries} reason is `#{e.class}: #{e.message}'"
+            e.backtrace.each{|l| resultobj.debug "\t\t#{l}" }
+            resultobj.dump!
+          end
+          sleep 1
+          retry
+        end
+        resultobj.error! "#{descriptor}retries exceeded"
+        resultobj.dump!
+      end
+    end
+
+
+    # =================
+    # =      API      =
+    # =================
+
+    def init
+      # hook method (called when checker is being initialized)
+    end
+
+    def start
+      # hook method (called after all checkers were initialized and configs + hosts are loaded)
+      # can/should be used for starting connections, etc.
+    end
+
+    def stop
+      # hook method (called on application shutdown)
+      # connections should be closed here
+
+      # DO NOT CLOSE CONNECTIONS HANDLED BY THE APP!
+      # Keep in mind that the checkers run concurrently
+      # and therefore shared resources might still be in use
+    end
+
+    def check! *a
+      raise NotImplementedError, "a checker (#{self.class.name}) must implement `#check!' method!"
+    end
+  end
+end

data/lib/watchmonkey_cli/checkers/ftp_availability.rb

@@ -0,0 +1,22 @@
+module WatchmonkeyCli
+  module Checkers
+    class FtpAvailability < Checker
+      self.checker_name = "ftp_availability"
+
+      def enqueue host, opts = {}
+        opts = { threshold: 1.months }.merge(opts)
+        app.enqueue(self, host, opts)
+      end
+
+      def check! result, host, opts = {}
+        Net::FTP.open(host) do |ftp|
+          ftp.login(opts[:user], opts[:password])
+        end
+      rescue Net::FTPPermError
+        result.error "Invalid credentials!"
+      rescue SocketError => e
+        result.error "#{e.class}: #{e.message}"
+      end
+    end
+  end
+end

data/lib/watchmonkey_cli/checkers/mysql_replication.rb

@@ -0,0 +1,57 @@
+module WatchmonkeyCli
+  module Checkers
+    class MysqlReplication < Checker
+      self.checker_name = "mysql_replication"
+
+      def enqueue host, opts = {}
+        opts = { host: "127.0.0.1", user: "root", sbm_threshold: 60 }.merge(opts)
+        host = app.fetch_connection(:loopback, :local) if !host || host == :local
+        host = app.fetch_connection(:ssh, host) if host.is_a?(Symbol)
+        app.enqueue(self, host, opts)
+      end
+
+      def check! result, host, opts = {}
+        cmd = ["mysql"]
+        cmd << "-u#{opts[:user]}" if opts[:user]
+        cmd << "-p#{opts[:password]}" if opts[:password]
+        cmd << "-h#{opts[:host]}" if opts[:host]
+        cmd << "-P#{opts[:port]}" if opts[:port]
+        cmd << %{-e "SHOW SLAVE STATUS\\G"}
+        result.command = cmd.join(" ")
+        result.result = host.exec(result.command)
+        result.data = _parse_response(result.result)
+
+        io  = result.data["Slave_IO_Running"]
+        sql = result.data["Slave_SQL_Running"]
+        sbm = result.data["Seconds_Behind_Master"]
+        pres = io.nil? && sql.nil? ? "\n\t#{res}" : ""
+
+        if !io && !sql
+          result.error! "MySQL replication is offline (IO=#{io},SQL=#{sql})#{pres}"
+        elsif !io || !sql
+          result.error! "MySQL replication is BROKEN (IO=#{io},SQL=#{sql})#{pres}"
+        elsif sbm > opts[:sbm_threshold]
+          result.error! "MySQL replication is #{sbm} SECONDS BEHIND master (IO=#{io},SQL=#{sql})#{pres}"
+        end
+      end
+
+      def _parse_response res
+        {}.tap do |r|
+          res.split("\n").map(&:strip).reject(&:blank?).each do |line|
+            next if line.start_with?("***")
+            chunks = line.split(":")
+            key = chunks.shift.strip
+            val = chunks.join(":").strip
+
+            # value cast
+            val = false if %w[no No].include?(val)
+            val = true if %w[yes Yes].include?(val)
+            val = val.to_i if val =~ /^\d+$/
+
+            r[key] = val
+          end
+        end
+      end
+    end
+  end
+end

data/lib/watchmonkey_cli/checkers/ssl_expiration.rb

@@ -0,0 +1,83 @@
+module WatchmonkeyCli
+  module Checkers
+    class SslExpiration < Checker
+      self.checker_name = "ssl_expiration"
+
+      def enqueue page, opts = {}
+        opts = { threshold: 1.months }.merge(opts)
+        app.enqueue(self, page, opts)
+      end
+
+      def check! result, page, opts = {}
+        uri = URI.parse(page)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        cert = nil
+        http.start do |h|
+          cert = h.peer_cert
+        end
+
+        if cert.not_before > Time.current
+          result.error! "Certificate is not yet valid (will in #{fseconds(cert.not_before - Time.current)}, #{cert.not_before})!"
+          return
+        end
+
+        if cert.not_after <= Time.current
+          result.error! "Certificate is EXPIRED (since #{fseconds(cert.not_after - Time.current)}, #{cert.not_after})!"
+          return
+        end
+
+        if cert.not_after <= Time.current + opts[:threshold]
+          result.error! "Certificate is about to expire within threshold (in #{fseconds(cert.not_after - Time.current)}, #{cert.not_after})!"
+          return
+        else
+          result.info! "Certificate for `#{page}' expires in #{fseconds(cert.not_after - Time.current)} (#{cert.not_after})!"
+        end
+      end
+
+      def fseconds secs
+        secs = secs.to_i
+        t_minute = 60
+        t_hour = t_minute * 60
+        t_day = t_hour * 24
+        t_week = t_day * 7
+        t_month = t_day * 30
+        t_year = t_month * 12
+        "".tap do |r|
+          if secs >= t_year
+            r << "#{secs / t_year}y "
+            secs = secs % t_year
+          end
+
+          if secs >= t_month
+            r << "#{secs / t_month}m "
+            secs = secs % t_month
+          end
+
+          if secs >= t_week
+            r << "#{secs / t_week}w "
+            secs = secs % t_week
+          end
+
+          if secs >= t_day || !r.blank?
+            r << "#{secs / t_day}d "
+            secs = secs % t_day
+          end
+
+          if secs >= t_hour || !r.blank?
+            r << "#{secs / t_hour}h "
+            secs = secs % t_hour
+          end
+
+          if secs >= t_minute || !r.blank?
+            r << "#{secs / t_minute}m "
+            secs = secs % t_minute
+          end
+
+          r << "#{secs}s" unless r.include?("d")
+        end.strip
+      end
+    end
+  end
+end

data/lib/watchmonkey_cli/checkers/unix_defaults.rb

@@ -0,0 +1,40 @@
+module WatchmonkeyCli
+  module Checkers
+    class UnixDefaults < Checker
+      self.checker_name = "unix_defaults"
+
+      def enqueue host, opts = {}
+        return if app.running? # we only ever want to run this once to spawn sub checkers
+        opts = { unix_load: {}, unix_memory: {}, unix_df: {}, unix_mdadm: {} }.merge(opts)
+
+        host = app.fetch_connection(:loopback, :local) if !host || host == :local
+        host = app.fetch_connection(:ssh, host) if host.is_a?(Symbol)
+
+        # option shortcuts
+        opts[:unix_load][:limits] = opts[:load] if opts[:load]
+        opts[:unix_load] = false if opts[:load] == false
+        opts[:unix_memory][:min_percent] = opts[:memory_min] if opts[:memory_min]
+        opts[:unix_memory] = false if opts[:memory_min] == false
+        opts[:unix_df][:min_percent] = opts[:df_min] if opts[:df_min]
+        opts[:unix_df] = false if opts[:df_min] == false
+        opts[:unix_mdadm] = false if opts[:mdadm] == false
+        opts.delete(:load)
+        opts.delete(:memory_min)
+        opts.delete(:df_min)
+        opts.delete(:mdadm)
+
+        app.enqueue(self, host, opts)
+      end
+
+      def check! result, host, opts = {}
+        [:unix_load, :unix_memory, :unix_df, :unix_mdadm].each do |which|
+          # if opts[which] && sec = app.checkers[which.to_s]
+          #   sec.check!(result, host, opts[which])
+          # end
+          # app.enqueue_sub(self, which, host, opts[which]) if opts[which]
+          spawn_sub(which, host, opts[which].is_a?(Hash) ? opts[which] : {})
+        end
+      end
+    end
+  end
+end

data/lib/watchmonkey_cli/checkers/unix_df.rb

@@ -0,0 +1,41 @@
+module WatchmonkeyCli
+  module Checkers
+    class UnixDf < Checker
+      self.checker_name = "unix_df"
+
+      def enqueue host, opts = {}
+        opts = { min_percent: 25 }.merge(opts)
+        host = app.fetch_connection(:loopback, :local) if !host || host == :local
+        host = app.fetch_connection(:ssh, host) if host.is_a?(Symbol)
+        app.enqueue(self, host, opts)
+      end
+
+      def check! result, host, opts = {}
+        result.command = "df"
+        result.result = host.exec(result.command)
+        result.data = _parse_response(result.result)
+
+        result.data.each do |fs|
+          result.error! "disk space on `#{fs[:mountpoint] || "unmounted"}' (#{fs[:filesystem]}) is low (limit is min. #{opts[:min_percent]}%, got #{fs[:free]}%)" if fs[:free] < opts[:min_percent]
+        end if opts[:min_percent]
+      end
+
+      def _parse_response res
+        [].tap do |r|
+          res.strip.split("\n")[1..-1].each do |device|
+            chunks = device.split(" ")
+            r << {
+              filesystem: chunks[0],
+              size: chunks[1].to_i,
+              used: chunks[2].to_i,
+              available: chunks[3].to_i,
+              use: chunks[4].to_i,
+              free: 100-chunks[4].to_i,
+              mountpoint: chunks[5],
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/watchmonkey_cli/checkers/unix_file_exists.rb

@@ -0,0 +1,25 @@
+module WatchmonkeyCli
+  module Checkers
+    class FileExists < Checker
+      self.checker_name = "unix_file_exists"
+
+      def enqueue host, file, opts = {}
+        opts = { message: "File #{file} does not exist!" }.merge(opts)
+        host = app.fetch_connection(:loopback, :local) if !host || host == :local
+        host = app.fetch_connection(:ssh, host) if host.is_a?(Symbol)
+        app.enqueue(self, host, file, opts)
+      end
+
+      def check! result, host, file, opts = {}
+        descriptor = "[#{self.class.checker_name} | #{host} | #{file} | #{opts}]\n\t"
+        if host.is_a?(WatchmonkeyCli::LoopbackConnection)
+          result.error! "#{descriptor}#{opts[:message]} (ENOENT)" if !File.exist?(file)
+        else
+          result.command = "test -f #{Shellwords.escape(file)} && echo exists"
+          result.result = host.exec(result.command)
+          result.error! "#{opts[:message]} (#{result.result.presence || "ENOENT"})" if result.result != "exists"
+        end
+      end
+    end
+  end
+end

data/lib/watchmonkey_cli/checkers/unix_load.rb

@@ -0,0 +1,30 @@
+module WatchmonkeyCli
+  module Checkers
+    class UnixLoad < Checker
+      self.checker_name = "unix_load"
+
+      def enqueue host, opts = {}
+        opts = { limits: [4, 2, 1.5] }.merge(opts)
+        host = app.fetch_connection(:loopback, :local) if !host || host == :local
+        host = app.fetch_connection(:ssh, host) if host.is_a?(Symbol)
+        app.enqueue(self, host, opts)
+      end
+
+      def check! result, host, opts = {}
+        result.command = "uptime"
+        result.result = host.exec(result.command)
+        ld = result.data = _parse_response(result.result)
+
+        emsg = []
+        emsg << "load1 is to high (limit1 is #{opts[:limits][0]}, load1 is #{ld[0]})" if ld[0] > opts[:limits][0]
+        emsg << "load5 is to high (limit5 is #{opts[:limits][1]}, load5 is #{ld[1]})" if ld[1] > opts[:limits][1]
+        emsg << "load15 is to high (limit15 is #{opts[:limits][2]}, load15 is #{ld[2]})" if ld[2] > opts[:limits][2]
+        result.error!(emsg.join("\n\t")) if emsg.any?
+      end
+
+      def _parse_response res
+        res.match(/load average(?:s)?: (?:([\d\.]+), ([\d\.]+), ([\d\.]+))|(?:([\d,]+) ([\d\,]+) ([\d\,]+))/i)[1..-1].reject(&:blank?).map{|v| v.gsub(",", ".").to_f }
+      end
+    end
+  end
+end

data/lib/watchmonkey_cli/checkers/unix_mdadm.rb

@@ -0,0 +1,60 @@
+module WatchmonkeyCli
+  module Checkers
+    class UnixMdadm < Checker
+      self.checker_name = "unix_mdadm"
+
+      def enqueue host, opts = {}
+        opts = { log_checking: true }.merge(opts)
+        host = app.fetch_connection(:loopback, :local) if !host || host == :local
+        host = app.fetch_connection(:ssh, host) if host.is_a?(Symbol)
+        app.enqueue(self, host, opts)
+      end
+
+      def check! result, host, opts = {}
+        result.command = "cat /proc/mdstat"
+        result.result = host.exec(result.command)
+        result.data = _parse_response(result.result)
+
+        if !result.data
+          result.error!(result.result)
+        else
+          result.data[:devices].each do |rdev|
+            dev = rdev[0].split(" ").first
+            status = rdev[1].split(" ").last
+            progress = rdev[2].to_s
+            return if status == "chunks"
+            result.error! "#{dev} seems broken (expected U+, got `#{status}')" if status !~ /\[U+\]/
+            if opts[:log_checking] && progress && m = progress.match(/\[[=>\.]+\]\s+([^\s]+)\s+=\s+([^\s]+)\s+\(([^\/]+)\/([^\)]+)\)\s+finish=([^\s]+)\s+speed=([^\s]+)/i)
+              info "#{dev} on is checking (status:#{m[1]}|done:#{m[2]}|eta:#{m[5]}|speed:#{m[6]}|blocks_done:#{m[3]}/#{m[4]})"
+            end
+          end
+        end
+      end
+
+      def _parse_response res
+        return false if res.downcase["no such file"]
+
+        { devices: [] }.tap do |r|
+          res = res.strip
+          chunks = res.split("\n").map(&:strip)
+          chunks.reject!{|el| el =~ /\Awarning:/i }
+
+          # personalities
+          personalities = chunks.delete_at(chunks.index{|c| c =~ /^personalities/i })
+          r[:personalities] = personalities.match(/^personalities(?:\s?): (.*)$/i)[1].split(" ").map{|s| s[1..-2] }
+
+          # unusued devices
+          unused_devices = chunks.delete_at(chunks.index{|c| c =~ /^unused devices/i })
+          r[:unused_devices] = unused_devices.match(/^unused devices\s?: (.*)$/i)[1]
+
+          # device output
+          chunks.join("\n").split("\n\n").map{|sp| sp.split("\n") }.each do |rdev|
+            r[:devices] << rdev
+          end
+        end
+      rescue StandardError => e
+        return "failed to parse mdadm output - #{e.class}: #{e.message}"
+      end
+    end
+  end
+end

data/lib/watchmonkey_cli/checkers/unix_memory.rb

@@ -0,0 +1,37 @@
+module WatchmonkeyCli
+  module Checkers
+    class UnixMemory < Checker
+      self.checker_name = "unix_memory"
+
+      def enqueue host, opts = {}
+        opts = { min_percent: 25 }.merge(opts)
+        host = app.fetch_connection(:loopback, :local) if !host || host == :local
+        host = app.fetch_connection(:ssh, host) if host.is_a?(Symbol)
+        app.enqueue(self, host, opts)
+      end
+
+      def check! result, host, opts = {}
+        result.command = "cat /proc/meminfo"
+        result.result = host.exec(result.command)
+        result.data = _parse_response(result.result)
+
+        if !result.data
+          result.error! result.result
+        else
+          result.error! "memory is low (limit is min. #{opts[:min_percent]}%, got #{result.data["free"]}%)" if opts[:min_percent] && result.data["free"] < opts[:min_percent]
+        end
+      end
+
+      def _parse_response res
+        return false if res.downcase["no such file"]
+        {}.tap do |r|
+          res.strip.split("\n").each do |line|
+            chunks = line.split(":").map(&:strip)
+            r[chunks[0]] = chunks[1].to_i
+          end
+          r["free"] = ((r["MemFree"].to_i + r["Buffers"].to_i + r["Cached"].to_i) / r["MemTotal"].to_f * 100).round(2)
+        end
+      end
+    end
+  end
+end

data/lib/watchmonkey_cli/checkers/www_availability.rb

@@ -0,0 +1,54 @@
+module WatchmonkeyCli
+  module Checkers
+    class WwwAvailability < Checker
+      self.checker_name = "www_availability"
+
+      def enqueue page, opts = {}
+        app.enqueue(self, page, opts.except(:ssl_expiration))
+
+        # if available enable ssl_expiration support
+        if page.start_with?("https://") && opts[:ssl_expiration] != false
+          spawn_sub("ssl_expiration", page, opts[:ssl_expiration].is_a?(Hash) ? opts[:ssl_expiration] : {})
+        end
+      end
+
+      def check! result, page, opts = {}
+        begin
+          resp = HTTParty.get(page, no_follow: true, verify: false)
+          result.result = resp
+        rescue HTTParty::RedirectionTooDeep => e
+          result.result = e.response
+          original_response = true
+        rescue Errno::ECONNREFUSED => e
+          result.error! "Failed to fetch #{page} (#{e.class}: #{e.message})"
+          return
+        end
+
+        # status
+        if opts[:status]
+          stati = [*opts[:status]]
+          result.error! "#{result.result.code} is not in #{stati}!" if !stati.include?(result.result.code.to_i)
+        end
+
+        # body
+        if rx = opts[:body]
+          if rx.is_a?(String)
+            result.error! "body does not include #{rx}!" if !result.result.body.include?(rx)
+          elsif rx.is_a?(Regexp)
+            result.error! "body does not match #{rx}!" if !result.result.body.match(rx)
+          end
+        end
+
+        # headers
+        if opts[:headers]
+          hdata = original_response ? result.result : result.result.headers
+          opts[:headers].each do |k, v|
+            if !(v.is_a?(Regexp) ? hdata[k].match(v) : hdata[k] == v)
+              result.error! "header #{k} mismatches (expected `#{v}' got `#{hdata[k] || "nil"}')"
+            end
+          end
+        end
+      end
+    end
+  end
+end