sys_watchdog 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5ee1c0e291ad96ace550d5e8538db4e41cf5f9ff
+  data.tar.gz: dc135065fc06c720b8d3ee976abee42a00f941fa
+SHA512:
+  metadata.gz: a138bc331c7063af2bac5c17f5db75cd7205872a389a945cca9ef4f5792e678504b19d025f50741b22df8e47fbc863b8ce2be26a5a828b612b0bbcfaf9a0be0e
+  data.tar.gz: 885df78c3268ba5c59e5997996e98ba9711646ff8744b525c26b29e90d6b39dab702524d392693a608ee2f7f749e749acc021f1dcae95f5142849730192fc655

data/.gitignore

@@ -0,0 +1,50 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+# Used by dotenv library to load environment variables.
+# .env
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+#
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/README.md

@@ -0,0 +1,89 @@
+
+Sys Watchdog
+=================
+
+** perform all steps as root
+
+## Installation
+
+```
+gem install sys_watchdog
+```
+
+If using Rbenv, run ```rbenv rehash``` to make sys_watchdog binary available.
+
+## Setup
+
+If on Linux with systemd available (eg: Ubuntu 16+, RedHat 7+. Generally distro versions released from 2015):  
+
+```
+sys_watchdog setup_with_systemd
+```
+
+Otherwise:  
+
+```
+sys_watchdog setup_with_cron
+```
+
+### Configuration
+
+After install, edit ```/etc/sys_watchdog.yml```.
+You can see some examples in this file and in [/test/sys_watchdog_test.yml](https://github.com/tomlobato/sys_watchdog/blob/master/test/sys_watchdog_test.yml).  
+
+Now you can test from the command line. 
+
+```
+sys_watchdog test
+``` 
+
+### Start
+
+Finally, after installed and configured, enable periodic run...
+
+for systemd:  
+```systemctl start sys_watchdog```
+
+or uncommenting the added cron line:  
+```vim /etc/crontab```
+
+
+### Config Settings
+
+setting      | description
+-------------|-------------------------------------------------------------------------------------------------
+name         | -
+server_name  | -
+slack_token  | -
+slack_channel| -
+smtp_server  | -
+smtp_domain  | -
+mail_from    | -
+mail_to      | -
+
+#### Sys Test Settings
+
+setting           | description
+------------------|-------------------------------------------------------------------------------------------
+test_cmd          | -
+test_url          | -
+notify_on_change  | -
+restore_cmd       | -
+expected_regex    | -
+expected_string   | -
+expected_max      | -
+expected_min      | -
+
+## Create a Slack Token
+
+From https://github.com/slack-ruby/slack-ruby-client ...  
+
+This is something done in Slack, under [integrations](https://my.slack.com/services). Create a [new bot](https://my.slack.com/services/new/bot), and note its API token.
+
+![](screenshots/register-bot.png)
+
+## Copyright and License
+
+Copyright (c) 2017-2016, [Tom Lobato](https://github.com/tomlobato).
+
+This project is licensed under the MIT License.

data/Rakefile

@@ -0,0 +1,8 @@
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+desc "Run tests"
+task :default => :test

data/bin/sys_watchdog

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+
+require 'sys_watchdog'
+
+case ARGV[0]
+
+when 'setup_with_systemd'
+  Setup.new.setup_with_systemd
+
+when 'setup_with_cron'
+  Setup.new.setup_with_cron
+
+when 'test'
+  sw = SysWatchdog.new log_file: STDOUT
+  sw.run once: true
+
+when 'once'
+  SysWatchdog.new.run once: true
+
+when nil
+  SysWatchdog.new.run
+end
+

data/lib/sys_watchdog/core_extensions.rb

@@ -0,0 +1,9 @@
+class Exception
+  def desc
+    "#{self.message} #{self.backtrace.join "\n" if self.backtrace}"
+  end
+end
+
+def log_ex e
+  STDERR.puts e.desc
+end

data/lib/sys_watchdog/main.rb

@@ -0,0 +1,83 @@
+class SysWatchdog
+    DEFAULT_CONF_FILE = '/etc/sys_watchdog.yml'
+    DEFAULT_LOG_FILE  = '/var/log/sys_watchdog.log'
+
+    def initialize conf_file: nil, log_file: nil
+        @logger = WdLogger.new (log_file || DEFAULT_LOG_FILE)
+        @trackers = {}
+        parse_conf (conf_file || DEFAULT_CONF_FILE)
+        setup
+    end
+
+    def run once: false
+        loop do
+            @tests.each{|test| run_test test}
+            return if once
+            sleep 60
+        end
+    end
+
+    private
+
+    def setup
+        if @conf.slack_token
+            Slack.configure do |config|
+              config.token = @conf.slack_token
+            end
+        end
+        if @conf.smtp_server
+            Mail.defaults do
+              delivery_method :smtp, address: @conf.smtp_server, port: 587, :domain => @conf.smtp_domain, 
+                              :enable_starttls_auto => true, :openssl_verify_mode => 'none'
+            end
+        end
+    end
+
+    def parse_conf conf_file
+        raise "Conf file #{conf_file} not found." unless File.exist? conf_file
+
+        conf = YAML.load_file conf_file
+        conf.deep_symbolize_keys!
+
+        @conf = OpenStruct.new conf[:config]
+
+        @tests = conf[:tests].keys.map { |name|
+            WdTest.new(name, conf[:tests][name], @logger)
+        }
+    end
+
+    def run_test test, after_restore: false
+        success, exitstatus, output = test.run
+
+        if test.notify_on_change
+            if @trackers[test.name] != output
+                notify "#{test.name} changed", "old: #{@trackers[test.name]}\nnew: #{output}"
+            end
+            @trackers[test.name] = output
+        end
+
+        if success
+            if test.fail
+                test.fail = false
+                notify "#{test.name} ok"
+            end
+        else
+            unless test.fail
+                if test.restore_cmd and not after_restore
+                    test.restore
+                    run_test test, after_restore: true
+                else
+                    fail test, exitstatus, output
+                end
+            end
+        end
+    rescue => e
+        @logger.error e.desc
+    end
+
+    def fail test, exitstatus, output
+        test.fail = true
+        body = "output: #{output}" if body and not body.empty?
+        notify "#{test.name} fail", body
+    end
+end

data/lib/sys_watchdog/notify.rb

@@ -0,0 +1,55 @@
+class SysWatchdog
+    private
+
+    def notify sub, body = ""
+        begin
+            send_mail sub, body if @conf.smtp_server
+        rescue => e
+            @logger.error e.desc
+        end
+        begin
+            send_slack_msg "#{sub}\n#{body}" if @conf.slack_token
+        rescue => e
+            @logger.error e.desc
+        end
+    end
+
+    def send_slack_msg msg
+        slack_client = Slack::Web::Client.new
+        slack_client.chat_postMessage(channel: @conf[:slack_channel], text: "[#{server_name}] #{msg}", as_user: true)
+    end
+
+    def send_mail sub, _body
+        @logger.info "Sending email: #{ sub }"
+
+        body = _body || ""
+        body += append_sys_info
+        
+        mail = Mail.new do
+          from     @conf.mail_from
+          to       @conf.mail_to
+          subject  "Watchdog #{@conf.name} [#{server_name}]: #{ sub }"
+          body     body
+        end
+
+        mail.deliver!
+    end
+
+    def server_name
+        @conf.server_name || `hostname`
+    end
+
+    def append_sys_info
+        ret = "\n\n--------------- sys info ---------------"
+        %Q(
+            ps aux
+            df -h
+            uptime
+
+        ).split("\n").map(&:strip).compact.reject(&:empty?).each do |cmd|
+            cmd_result = `#{ cmd }`
+            ret += "\n\n#{ cmd }:\n#{ cmd_result }"
+        end
+        ret
+    end
+end

data/lib/sys_watchdog/setup.rb

@@ -0,0 +1,69 @@
+
+class Setup
+  def initialize
+    @thisdir = File.join File.dirname(__FILE__)
+  end
+
+  def with_systemd
+    copy_sample_conf
+    install_systemd_service
+
+    puts "Installed."
+
+    puts "\nEdit #{SysWatchdog::DEFAULT_CONF_FILE} and start:"
+    puts "systemctl start sys_watchdog"
+
+    puts "\nTo check daemon status:"
+    puts "systemctl status sys_watchdog"
+  end
+
+  def with_cron
+    copy_sample_conf
+    add_cron_line
+
+    puts "Installed."
+
+    puts "\nEdit #{SysWatchdog::DEFAULT_CONF_FILE} and uncomment the cron line added."
+  end
+  
+  private
+
+  def add_cron_line
+    run "echo '#* *   * * * root sys_watchdog' >> /etc/crontab"
+  end
+
+  def install_systemd_service
+    services_dir = "/lib/systemd/system/"
+
+    if `which systemctl`.empty?
+      STDERR.puts "SysWatchdog install requires systemctl. Aborting."
+      exit 1
+    end
+
+    unless File.exist? services_dir
+      STDERR.puts "SysWatchdog install requires dir #{services_dir}. Aborting."
+      exit 1
+    end
+    
+    copy "#{@thisdir}/../../util/sys_watchdog.service", 
+         services_dir
+
+    run 'systemctl enable sys_watchdog'
+  end
+
+  def copy_sample_conf
+    copy "#{@thisdir}/../../util/sys_watchdog_sample.yml", 
+          SysWatchdog::DEFAULT_CONF_FILE
+  end
+
+  def copy from, to
+    puts "Copying #{from} to #{to}..."
+    FileUtils.cp from, to
+  end
+
+  def run cmd
+    puts "Running #{cmd}..."
+    system cmd
+  end
+
+end

data/lib/sys_watchdog/wd_logger.rb

@@ -0,0 +1,34 @@
+
+require 'syslog'
+require 'syslog/logger'
+
+class WdLogger < Logger
+  SYSLOG_NAME = 'sys_watchdog'
+
+  def initialize *args
+    @syslog = Syslog::Logger.new SYSLOG_NAME
+    super
+  end
+
+  def add(severity, message = nil, progname = nil, &block)
+    super
+
+    if message.nil?
+      if block_given?
+        message = yield
+      else
+        message = progname
+      end
+    end
+
+    added severity, message
+  end
+
+  def added severity, message
+    if severity >= Logger::WARN
+      @syslog.send Logger::Severity::constants[severity].downcase, message
+    end
+  end
+
+end
+

data/lib/sys_watchdog/wd_test.rb

@@ -0,0 +1,91 @@
+class WdTest
+    attr_accessor :name, 
+                  :test_cmd, :test_url, :notify_on_change, :restore_cmd, 
+                  :expected_regex, :expected_string, :expected_max, :expected_min, 
+                  :fail
+
+    def initialize name, params, logger
+        @logger = logger
+
+        @name = name
+
+        @test_cmd = params[:test_cmd]
+        @test_url = params[:test_url]
+        @notify_on_change = params[:notify_on_change]
+        @restore_cmd = params[:restore_cmd]
+
+        @expected_regex = params[:expected_regex]
+        @expected_string = params[:expected_string]
+        @expected_max = params[:expected_max]
+        @expected_min = params[:expected_min]
+
+        @fail = false
+
+        setup
+    end
+
+    def restore
+        exitstatus, output = run_cmd @restore_cmd
+        unless exitstatus == 0
+            @logger.error "restore exited with non-zero: #{exitstatus} #{output}"
+        end
+    end
+
+    def run after_restore: false
+        @logger.info "========== testing #{@name}"
+
+        unless @test_cmd
+            @logger.error "test cmd or url required"
+            return
+        end
+
+        exitstatus, output = run_cmd @test_cmd
+
+        success = check_result exitstatus, output
+        @logger.info "success: #{success.to_s}"
+
+        [success, exitstatus, output]
+    end
+
+    private
+
+    def run_cmd cmd, allow_raise = false
+        @logger.info "run: #{ cmd }"
+
+        output = IO.popen(cmd, "r") {|pipe| pipe.read}
+        exitstatus = $?.exitstatus
+
+        if exitstatus != 0
+            @logger.error "#{cmd} -> #{output}"
+        end
+
+        [exitstatus, output]
+    end
+
+    def setup
+        if @test_url
+            @test_cmd = "wget -O - '#{ @test_url }' > /dev/null 2>&1"
+        end
+        if @expected_regex
+            @expected_regex = Regexp.new @expected_regex
+        end
+        if @notify_on_change
+            @test_cmd = @notify_on_change
+        end
+    end
+
+    def check_result exitstatus, output
+        success = if @expected_regex
+            output =~ @expected_regex
+        elsif @expected_string
+            output.index @expected_string
+        elsif @expected_max
+            output.to_f <= @expected_max
+        elsif @expected_min
+            output.to_f <= @expected_min
+        else
+            exitstatus == 0
+        end
+        !!success
+    end
+end

data/lib/sys_watchdog.rb

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+
+require "net/https"  
+require 'net/smtp'
+require "uri"
+require 'fileutils'
+require 'logger'
+require 'ostruct'
+
+require 'slack-ruby-client'
+require 'mail'
+
+require 'sys_watchdog/core_extensions'
+require 'sys_watchdog/wd_logger'
+require 'sys_watchdog/wd_test'
+require 'sys_watchdog/install'
+
+require 'sys_watchdog/main'
+require 'sys_watchdog/notify'

data/sys_watchdog.gemspec

@@ -0,0 +1,21 @@
+Gem::Specification.new do |s|
+  s.name          = "sys_watchdog"
+  s.version       = "0.0.1"
+  s.authors       = ["Tom Lobato"]
+  s.email         = "lobato@bettercall.io"
+  s.homepage      = "http://rubygems.org/gems/sys_watchdog"
+  s.summary       = "SysWatchdog keeps your UNIX servers green by performing periodic checks and opitionaly actions and notifications"
+  s.description   = "SysWatchdog keeps your UNIX servers green by performing periodic checks and opitionaly actions and notifications."
+  s.licenses      = ["MIT"]
+  s.platform      = Gem::Platform::RUBY
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- test/*`.split("\n")
+  s.require_paths = ["lib"]
+  s.executables   = %w(sys_watchdog)
+  s.required_ruby_version = '>= 2.1.0'
+
+  s.add_dependency("mail", "~> 2.5")
+  s.add_dependency("slack-ruby-client", "~> 0.8.1")
+end
+

data/test/sys_watchdog_test.yml

@@ -0,0 +1,26 @@
+
+config:
+  name: My Website Server
+  server_name: main website
+
+tests:
+  ls_success:
+    test_cmd: ls /
+
+  ls_fail:
+    test_cmd: ls /erfe/4ytwrg/qwewqf/34ter
+
+  site_status_success:
+    test_url: https://www.google.com
+
+  site_status_fail:
+    test_url: https://www.6wybwudybsuadbysiudkd.com
+
+  number_output_success:
+    test_cmd: echo 1
+    expected_max: 2
+
+  number_output_fail:
+    test_cmd: echo 3
+    expected_max: 2
+

data/test/test_sys_watchdog.rb

@@ -0,0 +1,10 @@
+require 'test/unit'
+require 'sys_watchdog'
+
+class SysWatchdogTest < Test::Unit::TestCase
+  def test_run
+    conf_file = File.join File.dirname(__FILE__), 'sys_watchdog_test.yml'
+    sw = SysWatchdog.new conf_file: conf_file, log_file: STDOUT
+    sw.run once: true
+  end
+end

data/util/sys_watchdog.service

@@ -0,0 +1,11 @@
+
+[Unit]
+Description=SysWatchdog supervisor
+
+[Service]
+User=root
+Group=root
+WorkingDirectory=/var/local/sys_watchdog
+Restart=on-failure
+ExecStart=/usr/local/sys_watchdog/sys_watchdog
+

data/util/sys_watchdog_sample.yml

@@ -0,0 +1,56 @@
+
+config:
+  name: My Website Server
+  server_name: main website # if not present, `hostname` will be used
+
+  # Send alerts to Slack (create your slack token in https://github.com/slack-ruby/slack-ruby-client#usage)
+  # slack_token: xoxb-...
+  # slack_channel: '#alerts'
+
+  # Send email alerts (via any smtp server: https://sendgrid.com, https://postmarkapp.com/ or your own)
+  # smtp_server: smtp.mysite.com
+  # smtp_domain: mysite.com
+  # mail_from: system@mysite.com
+  # mail_to: 
+  #   - myself@mysite.com
+
+tests:
+  ## General
+  boot_time:
+    notify_on_change: uptime -s
+
+  ## URLs
+  site_status:
+    test_url: https://www.mysite.com
+
+  # helpdesk_status:
+  #   test_url: https://helpdesk_status.mysite.com
+
+  ## Disks
+  disk_root:
+    test_cmd: "df / | grep -v Filesystem | awk '{print $5}' | sed 's/%//'"
+    expected_max: 90
+
+  # disk_shared_storage:
+  #   test_cmd: "df /var/www/site/shared | grep -v Filesystem | awk '{print $5}' | sed 's/%//'"
+  #   expected_max: 90
+
+  ## Mounts
+  # shared_storage:
+  #   test_cmd: mount | grep /var/www/site/shared
+
+  ## Services
+  # dns_server:
+  #   test_cmd: host -t A www.mysite.com ns1.mydns.com | tail -1
+  #   expected_regex: ^www.mysite.com has address
+  #   restore_cmd: systemctl restart bind9
+
+  # redis_server:
+  #   test_cmd: systemctl status redis-server
+  #   expected_string: Active: active (running)
+  #   restore_cmd: systemctl restart redis-server
+
+  ## Processes
+  # worker_status:
+  #   test_cmd: ps aux | grep 'sidekiq 4.2.10 site'
+

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification
+name: sys_watchdog
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Tom Lobato
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-06-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mail
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+- !ruby/object:Gem::Dependency
+  name: slack-ruby-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+description: SysWatchdog keeps your UNIX servers green by performing periodic checks
+  and opitionaly actions and notifications.
+email: lobato@bettercall.io
+executables:
+- sys_watchdog
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- README.md
+- Rakefile
+- bin/sys_watchdog
+- lib/sys_watchdog.rb
+- lib/sys_watchdog/core_extensions.rb
+- lib/sys_watchdog/main.rb
+- lib/sys_watchdog/notify.rb
+- lib/sys_watchdog/setup.rb
+- lib/sys_watchdog/wd_logger.rb
+- lib/sys_watchdog/wd_test.rb
+- screenshots/register-bot.png
+- sys_watchdog.gemspec
+- test/sys_watchdog_test.yml
+- test/test_sys_watchdog.rb
+- util/sys_watchdog.service
+- util/sys_watchdog_sample.yml
+homepage: http://rubygems.org/gems/sys_watchdog
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: SysWatchdog keeps your UNIX servers green by performing periodic checks and
+  opitionaly actions and notifications
+test_files:
+- test/sys_watchdog_test.yml
+- test/test_sys_watchdog.rb