hostmon 0.0.1

data/bin/hostmon

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 'hostmon'
+
+Hostmon::App.run!

data/examples/config.yml

@@ -0,0 +1,14 @@
+---
+:main:
+  :report_dir: "./reports" # where read from and store reports, relative to # this file
+  # :saved_reports: 1
+
+:bugzilla:
+  :bugzilla_url: "https://bugzilla.mozilla.org/enter_bug.cgi?alias=&assigned_to=nobody%40mozilla.org&blocked=&bug_file_loc=http%3A%2F%2F&bug_severity=normal&bug_status=NEW&cf_blocking_191=---&cf_blocking_192=---&cf_blocking_20=---&cf_blocking_fennec=---&cf_crash_signature=&cf_status_191=---&cf_status_192=---&cf_status_20=---&cf_status_firefox5=---&cf_status_firefox6=---&cf_status_firefox7=---&cf_status_firefox8=---&cf_tracking_firefox5=---&cf_tracking_firefox6=---&cf_tracking_firefox7=---&cf_tracking_firefox8=---&component=Operations%3A%20Metrics%2FMonitoring&contenttypeentry=&contenttypemethod=autodetect&contenttypeselection=text%2Fplain&data=&dependson=&description=&flag_type-325=X&flag_type-37=X&flag_type-4=X&flag_type-607=X&flag_type-708=X&flag_type-720=X&flag_type-721=X&flag_type-737=X&form_name=enter_bug&groups=services-infra&maketemplate=Remember%20values%20as%20bookmarkable%20template&op_sys=Linux&priority=--&product=Mozilla%20Services&qa_contact=operations%40services.bugs&rep_platform=x86_64&requestee_type-325=&requestee_type-4=&requestee_type-607=&status_whiteboard=&target_milestone=---&version=unspecified"
+  :bugzilla_search_url: "https://bugzilla.mozilla.org/buglist.cgi?list_id=1127320&resolution=---&resolution=DUPLICATE&query_format=advanced&short_desc_type=allwords&component=Operations%3A%20Metrics%2FMonitoring&product=Mozilla%20Services"
+  :hostlint_url: "localhost:9998"
+
+# :amqp:
+#   :host: localhost
+#   :vhost: /
+#   :exchange: hostlint

data/examples/reports/bad_exit_status/cluster1/mach1/report_2012-06-28-16:49:32.yaml

@@ -0,0 +1,11 @@
+--- 
+:status: 
+- :CHECK_OK
+- :FAILED_BAD_EXIT_STATUS
+:stdout: ""
+:stderr: ""
+:exit_status: 1
+:timestamp: Thu Jun 28 16:49:32 -0700 2012
+:cluster: cluster1
+:host: mach1
+:name: bad_exit_status

data/examples/reports/bad_exit_status/cluster2/mach1/report_2012-06-14-15:06:12.yaml

@@ -0,0 +1,11 @@
+---
+:status:
+- :CHECK_OK
+- :FAILED_BAD_EXIT_STATUS
+:stdout: ''
+:stderr: ''
+:cluster: cluster2
+:exit_status: 1
+:name: bad_exit_status
+:timestamp: Thu Jun 14 15:06:12 -0700 2012
+:host: mach1

data/examples/reports/bad_exit_status/cluster2/mach2/report_2012-06-14-15:06:12.yaml

@@ -0,0 +1,11 @@
+---
+:status:
+- :CHECK_OK
+- :FAILED_BAD_EXIT_STATUS
+:stdout: ''
+:stderr: ''
+:cluster: cluster2
+:exit_status: 1
+:name: bad_exit_status
+:timestamp: Thu Jun 14 15:06:12 -0700 2012
+:host: mach2

data/examples/reports/foo/cluster1/mach1/report_2012-06-28-16:49:32.yaml

@@ -0,0 +1,10 @@
+--- 
+:status: 
+- :CHECK_OK
+:stdout: ""
+:stderr: ""
+:exit_status: 0
+:timestamp: Thu Jun 28 16:49:32 -0700 2012
+:cluster: cluster1
+:host: mach1
+:name: foo

data/examples/reports/foo/cluster2/mach1/report_2012-06-14-15:06:12.yaml

@@ -0,0 +1,10 @@
+---
+:status:
+- :CHECK_OK
+:stdout: ''
+:stderr: ''
+:cluster: cluster2
+:exit_status: 0
+:name: foo
+:timestamp: Thu Jun 14 15:06:12 -0700 2012
+:host: mach1

data/examples/reports/foo/cluster2/mach2/report_2012-06-14-15:06:12.yaml

@@ -0,0 +1,10 @@
+---
+:status:
+- :CHECK_OK
+:stdout: ''
+:stderr: ''
+:cluster: cluster2
+:exit_status: 0
+:name: foo
+:timestamp: Thu Jun 14 15:06:12 -0700 2012
+:host: mach2

data/examples/reports/missing_prelude/cluster1/mach1/report_2012-06-28-16:49:32.yaml

@@ -0,0 +1,12 @@
+--- 
+:status: 
+- :FAILED_MISSING_PRELUDE
+:stdout: |
+  this should fail because the prelude is missing
+
+:stderr: ""
+:exit_status: 0
+:timestamp: Thu Jun 28 16:49:32 -0700 2012
+:cluster: cluster1
+:host: mach1
+:name: missing_prelude

data/examples/reports/missing_prelude/cluster2/mach1/report_2012-06-14-15:06:12.yaml

@@ -0,0 +1,12 @@
+---
+:status:
+- :FAILED_MISSING_PRELUDE
+:stdout: ! 'this should fail because the prelude is missing
+
+'
+:stderr: ''
+:cluster: cluster2
+:exit_status: 0
+:name: missing_prelude
+:timestamp: Thu Jun 14 15:06:12 -0700 2012
+:host: mach1

data/examples/reports/missing_prelude/cluster2/mach2/report_2012-06-14-15:06:12.yaml

@@ -0,0 +1,12 @@
+---
+:status:
+- :FAILED_MISSING_PRELUDE
+:stdout: ! 'this should fail because the prelude is missing
+
+'
+:stderr: ''
+:cluster: cluster2
+:exit_status: 0
+:name: missing_prelude
+:timestamp: Thu Jun 14 15:06:12 -0700 2012
+:host: mach2

data/examples/reports/network/config/bootproto_dhcp/cluster1/mach1/report_2012-06-28-16:49:32.yaml

@@ -0,0 +1,10 @@
+--- 
+:status: 
+- :CHECK_OK
+:stdout: ""
+:stderr: ""
+:exit_status: 0
+:timestamp: Thu Jun 28 16:49:32 -0700 2012
+:cluster: cluster1
+:host: mach1
+:name: network.config.bootproto_dhcp

data/examples/reports/network/config/bootproto_dhcp/cluster2/mach1/report_2012-06-14-15:06:12.yaml

@@ -0,0 +1,10 @@
+---
+:status:
+- :CHECK_OK
+:stdout: ''
+:stderr: ''
+:cluster: cluster2
+:exit_status: 0
+:name: network.config.bootproto_dhcp
+:timestamp: Thu Jun 14 15:06:12 -0700 2012
+:host: mach1

data/examples/reports/network/config/bootproto_dhcp/cluster2/mach2/report_2012-06-14-15:06:12.yaml

@@ -0,0 +1,10 @@
+---
+:status:
+- :CHECK_OK
+:stdout: ''
+:stderr: ''
+:cluster: cluster2
+:exit_status: 0
+:name: network.config.bootproto_dhcp
+:timestamp: Thu Jun 14 15:06:12 -0700 2012
+:host: mach2

data/examples/reports/network/ethernet_bonded/cluster1/mach1/report_2012-06-28-16:49:32.yaml

@@ -0,0 +1,31 @@
+--- 
+:status: 
+- :CHECK_FAILED
+:stdout: |
+  output of ifconfig:
+  lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 16436
+          inet 127.0.0.1  netmask 255.0.0.0
+          inet6 ::1  prefixlen 128  scopeid 0x10<host>
+   \       loop  txqueuelen 0  (Local Loopback)
+          RX packets 2592342  bytes 411323230 (392.2 MiB)
+          RX errors 0  dropped 0  overruns 0  frame 0
+          TX packets 2592342  bytes 411323230 (392.2 MiB)
+          TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
+  
+  p128p1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> \ mtu 1500
+          inet 10.250.3.163  netmask 255.255.248.0  broadcast 10.250.7.255
+   \       inet6 2620:101:8003:200:a6ba:dbff:fefd:fe71  prefixlen 64  scopeid 0x0<global>
+   \       inet6 fe80::a6ba:dbff:fefd:fe71  prefixlen 64  scopeid 0x20<link>
+          ether a4:ba:db:fd:fe:71  txqueuelen 1000  (Ethernet)
+          RX packets 15272170  bytes 2840450940 (2.6 GiB)
+          RX errors 0  dropped 7  overruns 0  frame 0
+          TX packets 312785  bytes 39125389 (37.3 MiB)
+          TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
+          device interrupt 16  
+
+:stderr: ""
+:exit_status: 0
+:timestamp: Thu Jun 28 16:49:32 -0700 2012
+:cluster: cluster1
+:host: mach1
+:name: network.ethernet_bonded

data/examples/reports/network/ethernet_bonded/cluster2/mach1/report_2012-06-14-15:06:12.yaml

@@ -0,0 +1,22 @@
+---
+:status:
+- :CHECK_FAILED
+:stdout: ! "output of ifconfig:\nlo: flags=73<UP,LOOPBACK,RUNNING>  mtu 16436\n        inet
+  127.0.0.1  netmask 255.0.0.0\n        inet6 ::1  prefixlen 128  scopeid 0x10<host>\n
+  \       loop  txqueuelen 0  (Local Loopback)\n        RX packets 2651014  bytes
+  410810736 (391.7 MiB)\n        RX errors 0  dropped 0  overruns 0  frame 0\n        TX
+  packets 2651014  bytes 410810736 (391.7 MiB)\n        TX errors 0  dropped 0 overruns
+  0  carrier 0  collisions 0\n\np128p1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>
+  \ mtu 1500\n        inet 10.250.3.163  netmask 255.255.248.0  broadcast 10.250.7.255\n
+  \       inet6 2620:101:8003:200:a6ba:dbff:fefd:fe71  prefixlen 64  scopeid 0x0<global>\n
+  \       inet6 fe80::a6ba:dbff:fefd:fe71  prefixlen 64  scopeid 0x20<link>\n        ether
+  a4:ba:db:fd:fe:71  txqueuelen 1000  (Ethernet)\n        RX packets 16682970  bytes
+  3053667650 (2.8 GiB)\n        RX errors 0  dropped 1  overruns 0  frame 0\n        TX
+  packets 306643  bytes 40247466 (38.3 MiB)\n        TX errors 0  dropped 0 overruns
+  0  carrier 0  collisions 0\n        device interrupt 16  \n"
+:stderr: ''
+:cluster: cluster2
+:exit_status: 0
+:name: network.ethernet_bonded
+:timestamp: Thu Jun 14 15:06:12 -0700 2012
+:host: mach1

data/examples/reports/network/ethernet_bonded/cluster2/mach2/report_2012-06-14-15:06:12.yaml

@@ -0,0 +1,22 @@
+---
+:status:
+- :CHECK_OK
+:stdout: ! "output of ifconfig:\nlo: flags=73<UP,LOOPBACK,RUNNING>  mtu 16436\n        inet
+  127.0.0.1  netmask 255.0.0.0\n        inet6 ::1  prefixlen 128  scopeid 0x10<host>\n
+  \       loop  txqueuelen 0  (Local Loopback)\n        RX packets 2651014  bytes
+  410810736 (391.7 MiB)\n        RX errors 0  dropped 0  overruns 0  frame 0\n        TX
+  packets 2651014  bytes 410810736 (391.7 MiB)\n        TX errors 0  dropped 0 overruns
+  0  carrier 0  collisions 0\n\np128p1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>
+  \ mtu 1500\n        inet 10.250.3.163  netmask 255.255.248.0  broadcast 10.250.7.255\n
+  \       inet6 2620:101:8003:200:a6ba:dbff:fefd:fe71  prefixlen 64  scopeid 0x0<global>\n
+  \       inet6 fe80::a6ba:dbff:fefd:fe71  prefixlen 64  scopeid 0x20<link>\n        ether
+  a4:ba:db:fd:fe:71  txqueuelen 1000  (Ethernet)\n        RX packets 16682970  bytes
+  3053667650 (2.8 GiB)\n        RX errors 0  dropped 1  overruns 0  frame 0\n        TX
+  packets 306643  bytes 40247466 (38.3 MiB)\n        TX errors 0  dropped 0 overruns
+  0  carrier 0  collisions 0\n        device interrupt 16  \n"
+:stderr: ''
+:cluster: cluster2
+:exit_status: 0
+:name: network.ethernet_bonded
+:timestamp: Thu Jun 14 15:06:12 -0700 2012
+:host: mach2

data/examples/reports/nonexec/cluster1/mach1/report_2012-06-28-16:49:32.yaml

@@ -0,0 +1,10 @@
+--- 
+:status: 
+- :FAILED_SKIPPED
+:stdout: ""
+:stderr: ""
+:exit_status: 
+:timestamp: Thu Jun 28 16:49:32 -0700 2012
+:cluster: cluster1
+:host: mach1
+:name: nonexec

data/examples/reports/nonexec/cluster2/mach1/report_2012-06-14-15:06:12.yaml

@@ -0,0 +1,10 @@
+---
+:status:
+- :FAILED_SKIPPED
+:stdout: ''
+:stderr: ''
+:cluster: cluster2
+:exit_status: 
+:name: nonexec
+:timestamp: Thu Jun 14 15:06:12 -0700 2012
+:host: mach1

data/examples/reports/nonexec/cluster2/mach2/report_2012-06-14-15:06:12.yaml

@@ -0,0 +1,10 @@
+---
+:status:
+- :FAILED_SKIPPED
+:stdout: ''
+:stderr: ''
+:cluster: cluster2
+:exit_status: 
+:name: nonexec
+:timestamp: Thu Jun 14 15:06:12 -0700 2012
+:host: mach2

data/examples/reports/yum/badcheck/cluster1/mach1/report_2012-06-28-16:49:32.yaml

@@ -0,0 +1,13 @@
+--- 
+:status: 
+- :CHECK_OK
+- :FAILED_STDERR_NONEMPTY
+:stdout: ""
+:stderr: |
+  standard error
+
+:exit_status: 0
+:timestamp: Thu Jun 28 16:49:32 -0700 2012
+:cluster: cluster1
+:host: mach1
+:name: yum.badcheck

data/examples/reports/yum/badcheck/cluster2/mach1/report_2012-06-14-15:06:12.yaml

@@ -0,0 +1,13 @@
+---
+:status:
+- :CHECK_OK
+- :FAILED_STDERR_NONEMPTY
+:stdout: ''
+:stderr: ! 'standard error
+
+'
+:cluster: cluster2
+:exit_status: 0
+:name: yum.badcheck
+:timestamp: Thu Jun 14 15:06:12 -0700 2012
+:host: mach1

data/examples/reports/yum/badcheck/cluster2/mach2/report_2012-06-14-15:06:12.yaml

@@ -0,0 +1,13 @@
+---
+:status:
+- :CHECK_OK
+- :FAILED_STDERR_NONEMPTY
+:stdout: ''
+:stderr: ! 'standard error
+
+'
+:cluster: cluster2
+:exit_status: 0
+:name: yum.badcheck
+:timestamp: Thu Jun 14 15:06:12 -0700 2012
+:host: mach2

data/lib/hostmon.rb

@@ -0,0 +1,208 @@
+require "erb"
+require "rack"
+require "sinatra/base"
+require "open-uri"
+require "yaml"
+require "thread"
+require "set"
+require "fileutils"
+require "digest/sha1"
+require "bunny"
+require "pathname"
+require "pp"
+
+require "hostmon/version"
+require "hostmon/models"
+require "hostmon/config"
+require "hostmon/helpers"
+require "hostmon/models"
+require "hostmon/rubyfixes"
+require "bunny"
+
+# should really be using queues (thread-safe according to pickaxe)
+
+module Hostmon
+  class App < Sinatra::Base
+    include Hostmon::Models
+    helpers Hostmon::Helpers
+    set :config, Hostmon::Config.new
+    set :port, config.global_config[:port]
+    set :run, true
+    set :root, File.expand_path('../hostmon', __FILE__)
+    set :static, true
+    set :logging, true
+
+    confd = Pathname.new(config.conf_file).dirname.realpath
+    reportd = Pathname.new(config.report_dir)
+    if reportd.relative?
+      reportd = File.join(confd, reportd)
+    end
+
+    Host.saved_reports = config.global_config[:saved_reports]
+    Report.report_base = reportd
+
+    # timestamp order is relevant when loading reports from disk (but not amqp)
+    Dir.glob(File.join(reportd, "/**/*.yaml")).sort do |a, b|
+      Time.parse(a) <=> Time.parse(b)
+    end.each do |f|
+      puts "load #{f}"
+      # YAML::load_documents doesn't work in 1.8.7
+      stream = YAML::load_stream(File.read(f))
+      stream.documents.each do |check|
+        Report.new(check)
+      end
+    end
+
+    # spin off the amqp thread
+    # rescue errors and bail "correctly"
+    amqp = Bunny.new(config.amqp_config)
+    amqp.start
+    queue = amqp.queue("hostlint-#{Process.pid}", :auto_delete => true)
+    exchange = amqp.exchange(config.amqp_config[:exchange]||"hostlint",
+                             :type => :topic,
+                             :durable => true)
+    queue.bind(exchange)
+    Thread.new do
+      begin
+        queue.subscribe do |msg|
+          puts "processing amqp report"
+          Report.new(YAML.load(msg[:payload]))
+        end
+      rescue Bunny::ServerDownError, Bunny::ConnectionError
+        # TODO: implement reconnect
+        throw
+      end # begin
+    end # Thread.new
+
+    before do
+      @request_time = Time.now
+    end
+
+    get '/' do
+      @title = "Overview"
+      erb :view
+    end
+
+    get '/check/:check/?' do
+      unless check = Check.find(params[:check])
+        # could probably do better than raise an exception
+        raise "check not found #{params[:check]}"
+      end
+      @title = "check :: #{params[:check]}"
+      @check = check
+
+      @statuses = [[Host::OK, "passing"],
+                   [Host::FAIL, "failing"],
+                   [Host::EXEC_FAIL, "failing to run"],
+                   [nil, "all"]]
+
+      erb :check
+    end
+
+    get '/check/:check/:status/?' do
+      unless check = Check.find(params[:check])
+        # could probably do better than raise an exception
+        raise "check not found #{params[:check]}"
+      end
+      @title1 = "check :: #{params[:check]} :: "
+      @check = check
+
+      @status = params[:status]
+      unless Host.statuses.member?(@status)
+        raise "unknown status #{@status}"
+      end
+
+      @statuses = [[Host::OK, "passing"],
+                   [Host::FAIL, "failing"],
+                   [Host::EXEC_FAIL, "failing to run"],
+                   ["", "all"]]
+      case @status
+      when Host::OK
+        @subtitle = "passing"
+        @method = :hosts_ok
+      when Host::FAIL
+        @subtitle = "failing"
+        @method = :hosts_failing
+      when Host::EXEC_FAIL
+        @subtitle = "failing to run"
+        @method = :hosts_check_failing
+      end
+      @title = @title1 + @subtitle
+      erb :check_status
+    end
+
+    # fixme dup
+    get '/host/:cluster/:host/?' do
+      @title = "#{params[:cluster]} :: host :: #{params[:host]}"
+      @host = Host.find_by_name_and_cluster(params[:host], params[:cluster])
+
+      @statuses = [[Host::FAIL, @host.failing, "failing", "checks failing"],
+                   [Host::OK, @host.succeeding, "succeeding", "checks passing"],
+                   [Host::EXEC_FAIL, @host.checks_failing, "failing to run", "checks failing to run"],
+                   [nil, [], "all", "all"]]
+      erb :host
+    end
+
+    get '/host/:cluster/:host/:status?/?' do
+      @title = "#{params[:cluster]} :: host :: #{params[:host]} :: FIXME"
+      @host = Host.find_by_name_and_cluster(params[:host], params[:cluster])
+      raise "unknown host #{params[:host]}" unless @host
+      @status = params[:status]
+      raise "unknown status #{@status}" unless Host.statuses.member?(@status)
+
+
+      @statuses = [[Host::FAIL, @host.failing, "failing", "checks failing"],
+                   [Host::OK, @host.succeeding, "succeeding", "checks passing"],
+                   [Host::EXEC_FAIL, @host.checks_failing, "failing to run", "checks failing to run"],
+                   [nil, [], "all", "all"]]
+
+      if @status
+        @statuses2 = @statuses.select {|s| s.first == @status}
+      end
+
+      erb :host_status
+    end
+
+    # get '/host/:cluster/:host/:check/?' do
+    #   "fixme"
+    # end
+
+    get '/cluster/:cluster/?' do
+      @title = "cluster :: #{params[:cluster]}"
+      erb :cluster
+    end
+
+    get '/status/:status/?' do
+      @status = params[:status]
+      unless Host.statuses.member?(@status)
+        raise "unknown status #{@status}"
+      end
+
+      case @status
+      when Host::OK
+        @title = "checks passing"
+        @method = :hosts_ok
+      when Host::FAIL
+        @title = "checks failing"
+        @method = :hosts_failing
+      when Host::EXEC_FAIL
+        @title = "checks failing to run"
+        @method = :hosts_check_failing
+      end
+      erb :status
+    end
+
+    get '/search' do
+      regexp = Regexp.new(params["keyword"], Regexp::IGNORECASE)
+      case params["cat"]
+      when "0" # all
+      when "1" # hosts
+      when "2" # checks
+      when "3" # failing
+      when "4" # succeeding
+      end
+      regexp.to_s
+    end
+
+  end
+end