remon 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +10 -0
- data/.rspec +3 -0
- data/FEATURES.md +39 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +21 -0
- data/README.md +41 -0
- data/Rakefile +27 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/dev_exe/remon +4 -0
- data/exe/remon +101 -0
- data/lib/remon.rb +11 -0
- data/lib/remon/check.rb +145 -0
- data/lib/remon/check_dsl.rb +92 -0
- data/lib/remon/check_runner.rb +53 -0
- data/lib/remon/checks/consul.rb +41 -0
- data/lib/remon/checks/disk.rb +36 -0
- data/lib/remon/checks/http.rb +53 -0
- data/lib/remon/checks/oom.rb +26 -0
- data/lib/remon/checks/redis.rb +23 -0
- data/lib/remon/checks/salt.rb +27 -0
- data/lib/remon/checks/system.rb +96 -0
- data/lib/remon/checks/yum.rb +30 -0
- data/lib/remon/config.rb +101 -0
- data/lib/remon/custom_logger.rb +6 -0
- data/lib/remon/deduped_queue.rb +38 -0
- data/lib/remon/error.rb +4 -0
- data/lib/remon/event_processor.rb +33 -0
- data/lib/remon/ext/num_ext.rb +23 -0
- data/lib/remon/helper.rb +41 -0
- data/lib/remon/logger.rb +17 -0
- data/lib/remon/metrics/consul.rb +32 -0
- data/lib/remon/metrics/disk.rb +24 -0
- data/lib/remon/metrics/http.rb +40 -0
- data/lib/remon/metrics/oom.rb +32 -0
- data/lib/remon/metrics/salt.rb +18 -0
- data/lib/remon/metrics/system.rb +63 -0
- data/lib/remon/metrics/yum.rb +20 -0
- data/lib/remon/proc_check.rb +26 -0
- data/lib/remon/scheduler.rb +106 -0
- data/lib/remon/scripts/salt-status +24 -0
- data/lib/remon/scripts/yum-status +12 -0
- data/lib/remon/sysinfo.rb +69 -0
- data/lib/remon/version.rb +3 -0
- data/remon.gemspec +26 -0
- data/test_config.rb +44 -0
- metadata +146 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 7dfc750910ee08bb5b65c02e00d54e88a255f4c2c4de853fdcf7ba9f86e49bca
|
4
|
+
data.tar.gz: 1362364de36a0545397e2e4c7d154cbcdb484eab1c452f64cca9f72359462932
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 92b80c7d8f8e68ece7560df7f1fd94cd044738fcaf19ab55a52ac28b7b3c693f144be0170c976aaf5763b2f1ca337e7e8d3db551eb70d764f6250b6fcae52daf
|
7
|
+
data.tar.gz: 775f49e88b719c765f8aaa5372d4e6344087cd7e7339ccfa67a6ce833b5d4446d3bb6bedca7ac717c8e0d49587da262636646a20969d11f23e549005ed98f0b5
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/FEATURES.md
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
# Features
|
2
|
+
- [x] namespace checks system/cpu or system:cpu => system.rb or system/cpu.rb
|
3
|
+
- [x] defcheck:
|
4
|
+
- [x] proc check semantics:
|
5
|
+
- [x] proc_check { }
|
6
|
+
- [x] runnable without .new
|
7
|
+
- [x] has access to :event/:sys method and also access to lexical scope
|
8
|
+
- [x] anonymous defcheck but still needs to be initialized using new
|
9
|
+
- [x] check.rb:
|
10
|
+
- [x] allow only predefined options
|
11
|
+
- [x] randomized scheduling with offset(for yum/salt) per "every" block and not per task/check
|
12
|
+
- [x] scheduler offset
|
13
|
+
- [x] check_name, to_s for debugging
|
14
|
+
- [x] better synchronize which returns nil and does not wait
|
15
|
+
- [x] num_workers in config
|
16
|
+
|
17
|
+
# Checks Library
|
18
|
+
- [x] system: load/cpu/memory/iowait/uptime
|
19
|
+
- [x] disk: disk usage
|
20
|
+
- [x] yum: check updates(with timeout)
|
21
|
+
- [x] salt: check highstate drift ( with timeout)
|
22
|
+
- [x] http: check http/https url status
|
23
|
+
- [x] consul: check for failed nodes
|
24
|
+
- [x] oom: oom count daily/weekly
|
25
|
+
- [ ] postgres master/slave
|
26
|
+
- [ ] elasticsearch
|
27
|
+
- [ ] redis stats
|
28
|
+
- [ ] memcached stats
|
29
|
+
|
30
|
+
# TODO
|
31
|
+
- [ ] http check: modify service name
|
32
|
+
- [ ] rpm spec
|
33
|
+
- [ ] list checks and opts --list-checks <check1> <check2>
|
34
|
+
- [ ] self health check: tainted/ok
|
35
|
+
- [ ] packaging user gems
|
36
|
+
- [ ] RUBY_GC_HEAP_GROWTH_FACTOR=1.05
|
37
|
+
- [ ] better code coverage
|
38
|
+
- [ ] use libpcp metrics if possible
|
39
|
+
- [ ] jemalloc MALLOC_CONF="narenas:1,lg_chunk:18"
|
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2016 Neeraj
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
# Remon
|
2
|
+
|
3
|
+
Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/remon`. To experiment with that code, run `bin/console` for an interactive prompt.
|
4
|
+
|
5
|
+
TODO: Delete this and the text above, and describe your gem
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
Add this line to your application's Gemfile:
|
10
|
+
|
11
|
+
```ruby
|
12
|
+
gem 'remon'
|
13
|
+
```
|
14
|
+
|
15
|
+
And then execute:
|
16
|
+
|
17
|
+
$ bundle
|
18
|
+
|
19
|
+
Or install it yourself as:
|
20
|
+
|
21
|
+
$ gem install remon
|
22
|
+
|
23
|
+
## Usage
|
24
|
+
|
25
|
+
TODO: Write usage instructions here
|
26
|
+
|
27
|
+
## Development
|
28
|
+
|
29
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
30
|
+
|
31
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
32
|
+
|
33
|
+
## Contributing
|
34
|
+
|
35
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/remon.
|
36
|
+
|
37
|
+
|
38
|
+
## License
|
39
|
+
|
40
|
+
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
41
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
2
|
+
require "rspec/core/rake_task"
|
3
|
+
|
4
|
+
RSpec::Core::RakeTask.new(:spec)
|
5
|
+
|
6
|
+
task :default => :spec
|
7
|
+
task :console do
|
8
|
+
require 'pry'
|
9
|
+
require 'remon/check_dsl'
|
10
|
+
l = Remon::CheckDsl.new(["#{__dir__}/lib/Remon/checks"])
|
11
|
+
l.check "system"
|
12
|
+
binding.pry
|
13
|
+
end
|
14
|
+
|
15
|
+
require_relative "lib/remon/version"
|
16
|
+
task :build do
|
17
|
+
name = "remon"
|
18
|
+
version = Remon::VERSION
|
19
|
+
file_name = "#{name}-#{version}.tar.gz"
|
20
|
+
puts "building archive at #{file_name}"
|
21
|
+
ok = system "git archive --prefix=#{name}-#{version}/ -o #{file_name} HEAD"
|
22
|
+
if ok
|
23
|
+
puts "build successful"
|
24
|
+
else
|
25
|
+
abort "error in build"
|
26
|
+
end
|
27
|
+
end
|
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "remon"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start
|
data/bin/setup
ADDED
data/dev_exe/remon
ADDED
data/exe/remon
ADDED
@@ -0,0 +1,101 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
Signal.trap("INT") { exit 1 }
|
3
|
+
|
4
|
+
require 'remon'
|
5
|
+
require 'remon/logger'
|
6
|
+
require 'set'
|
7
|
+
require 'optparse'
|
8
|
+
|
9
|
+
logger = Remon::Logger.logger
|
10
|
+
logger.level = Logger::INFO
|
11
|
+
|
12
|
+
options = {}
|
13
|
+
check_load_paths = Set.new
|
14
|
+
|
15
|
+
opts_parser = OptionParser.new do |opts|
|
16
|
+
|
17
|
+
banner = []
|
18
|
+
banner << "Usage: remon [options] /path/to/config.rb"
|
19
|
+
|
20
|
+
banner << "Options: "
|
21
|
+
opts.banner = banner.join("\n")
|
22
|
+
|
23
|
+
opts.on("-d", "--config-dir [config dir]" , "set config dir") do |d|
|
24
|
+
options[:config_dir] = d
|
25
|
+
end
|
26
|
+
|
27
|
+
opts.on("--checks-dir [checks dir]" , "set checks dir") do |d|
|
28
|
+
check_load_paths << d
|
29
|
+
end
|
30
|
+
|
31
|
+
opts.on("-v", "--version", "Show version") do |v|
|
32
|
+
puts ::Remon::VERSION
|
33
|
+
exit
|
34
|
+
end
|
35
|
+
|
36
|
+
opts.on("--debug", "Show debug messages") do
|
37
|
+
options[:debug] = true
|
38
|
+
logger.level = ::Logger::DEBUG
|
39
|
+
end
|
40
|
+
|
41
|
+
opts.on("--trace", "Show debug messages and exception stack trace") do
|
42
|
+
options[:debug] = true
|
43
|
+
options[:trace] = true
|
44
|
+
logger.level = ::Logger::DEBUG
|
45
|
+
logger.trace_mode = true
|
46
|
+
end
|
47
|
+
|
48
|
+
opts.on_tail("-h", "--help", "Show this message") do
|
49
|
+
puts opts
|
50
|
+
exit
|
51
|
+
end
|
52
|
+
end
|
53
|
+
opts_parser.order!(ARGV)
|
54
|
+
|
55
|
+
require 'remon/config'
|
56
|
+
require 'remon/scheduler'
|
57
|
+
require 'remon/deduped_queue'
|
58
|
+
require 'remon/check_runner'
|
59
|
+
require 'remon/event_processor'
|
60
|
+
require 'remon/sysinfo'
|
61
|
+
|
62
|
+
begin
|
63
|
+
Remon::Sysinfo.init
|
64
|
+
Thread.abort_on_exception = true
|
65
|
+
|
66
|
+
config_path = ARGV.shift
|
67
|
+
config = Remon::Config.new(
|
68
|
+
config_file: config_path,
|
69
|
+
config_dir: options[:config_dir],
|
70
|
+
load_paths: check_load_paths
|
71
|
+
).config
|
72
|
+
|
73
|
+
queue = Remon::DedupedQueue.new
|
74
|
+
|
75
|
+
scheduler = Remon::Scheduler.new(config[:schedule], queue: queue, scheduler_offset: config[:scheduler_offset])
|
76
|
+
|
77
|
+
processor = Remon::EventProcessor.new(config[:process_proc])
|
78
|
+
processor.start
|
79
|
+
|
80
|
+
check_runner = Remon::CheckRunner.new(
|
81
|
+
task_queue: queue,
|
82
|
+
result_queue: processor.queue,
|
83
|
+
num_workers: config[:workers]
|
84
|
+
)
|
85
|
+
check_runner.start
|
86
|
+
|
87
|
+
i = ENV["SCHEDULER_ITERATIONS"] || :inf
|
88
|
+
i = i.is_a?(String) ? i.to_i : i
|
89
|
+
scheduler.run(i)
|
90
|
+
|
91
|
+
rescue Remon::Error => e
|
92
|
+
cause = e.cause
|
93
|
+
if true
|
94
|
+
puts cause
|
95
|
+
cause ? (raise cause) : (raise e)
|
96
|
+
else
|
97
|
+
logger.debug "#{cause.message}" if cause
|
98
|
+
logger.error "#{e.message}"
|
99
|
+
abort
|
100
|
+
end
|
101
|
+
end
|
data/lib/remon.rb
ADDED
data/lib/remon/check.rb
ADDED
@@ -0,0 +1,145 @@
|
|
1
|
+
require_relative 'logger'
|
2
|
+
module Remon
|
3
|
+
class Check
|
4
|
+
|
5
|
+
def self.name
|
6
|
+
@name || "undefined"
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.name=(name)
|
10
|
+
@name = name
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.opts(h)
|
14
|
+
@opts = h
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.get_opts
|
18
|
+
@opts ||= {}
|
19
|
+
end
|
20
|
+
|
21
|
+
WARNING = "warning".freeze
|
22
|
+
CRITICAL = "critical".freeze
|
23
|
+
OK = "ok".freeze
|
24
|
+
|
25
|
+
attr_reader :mutex, :opts
|
26
|
+
|
27
|
+
include Logger
|
28
|
+
|
29
|
+
def initialize(*args, **kwargs, &block)
|
30
|
+
@tags = kwargs[:tags] || []
|
31
|
+
@ttl = kwargs[:ttl] || 10
|
32
|
+
@host = kwargs[:host] || Remon.host
|
33
|
+
|
34
|
+
opts = kwargs[:opts] || {}
|
35
|
+
verfiy_opts(opts)
|
36
|
+
default_opts = self.class.get_opts
|
37
|
+
@opts = default_opts.merge opts
|
38
|
+
|
39
|
+
@mutex = Mutex.new
|
40
|
+
return if not respond_to? :init
|
41
|
+
|
42
|
+
# propagate only those kwargs which are defined in "init" method definition
|
43
|
+
filtered_kwargs = filtered_kwargs(kwargs)
|
44
|
+
|
45
|
+
#workaround a bug in ruby for methods that take 0 args
|
46
|
+
if filtered_kwargs.empty?
|
47
|
+
init(*args, &block) if respond_to? :init
|
48
|
+
else
|
49
|
+
init(*args, **filtered_kwargs, &block) if respond_to? :init
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def run
|
54
|
+
raise NotImplementedError.new "run method not implemented"
|
55
|
+
end
|
56
|
+
|
57
|
+
def run_mutex
|
58
|
+
synchronize { run }
|
59
|
+
end
|
60
|
+
|
61
|
+
def check_name
|
62
|
+
self.class.name
|
63
|
+
end
|
64
|
+
|
65
|
+
def to_s
|
66
|
+
"<check:#{check_name}>"
|
67
|
+
end
|
68
|
+
|
69
|
+
private
|
70
|
+
|
71
|
+
def event(**kwargs)
|
72
|
+
kwargs[:time] = Time.now.to_i
|
73
|
+
kwargs[:tags] = @tags
|
74
|
+
kwargs[:ttl] = @ttl
|
75
|
+
kwargs[:host] = @host if @host
|
76
|
+
kwargs
|
77
|
+
end
|
78
|
+
|
79
|
+
def warning_event(service)
|
80
|
+
event({
|
81
|
+
service: service,
|
82
|
+
description: "failing to execute check",
|
83
|
+
state: "warning",
|
84
|
+
metric: 0.9
|
85
|
+
})
|
86
|
+
end
|
87
|
+
|
88
|
+
def critical_event(service)
|
89
|
+
event({
|
90
|
+
service: service,
|
91
|
+
description: "failing to execute check",
|
92
|
+
state: "critical",
|
93
|
+
metric: 1
|
94
|
+
})
|
95
|
+
end
|
96
|
+
|
97
|
+
def state(metric, warning:, critical:)
|
98
|
+
if metric < warning
|
99
|
+
OK
|
100
|
+
elsif metric >= warning && metric < critical
|
101
|
+
WARNING
|
102
|
+
else
|
103
|
+
CRITICAL
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def service_state(service = nil, metric)
|
108
|
+
wkey = service ? "#{service}_warning".to_sym : :warning
|
109
|
+
ckey = service ? "#{service}_critical".to_sym : :critical
|
110
|
+
warning = opts.fetch wkey
|
111
|
+
critical = opts.fetch ckey
|
112
|
+
state(metric, warning: warning, critical: critical)
|
113
|
+
end
|
114
|
+
|
115
|
+
alias_method :o, :opts
|
116
|
+
|
117
|
+
def filtered_kwargs(kwargs)
|
118
|
+
params = method(:init).parameters
|
119
|
+
init_kwargs = params.select { |i| i[0] == :key }.map { |i| i[1] }
|
120
|
+
kwargs.select { |k,v| init_kwargs.include? k }
|
121
|
+
end
|
122
|
+
|
123
|
+
def verfiy_opts(opts)
|
124
|
+
default_opts = self.class.get_opts
|
125
|
+
valid_opts = default_opts.keys
|
126
|
+
opts.keys.each do |k|
|
127
|
+
raise Error, "invalid opt #{k}" if not valid_opts.include? k
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
def synchronize
|
132
|
+
locked = false
|
133
|
+
locked = @mutex.try_lock
|
134
|
+
if locked
|
135
|
+
yield
|
136
|
+
else
|
137
|
+
logger.error "#{self} already running in another thread"
|
138
|
+
return false
|
139
|
+
end
|
140
|
+
ensure
|
141
|
+
@mutex.unlock if locked
|
142
|
+
end
|
143
|
+
|
144
|
+
end
|
145
|
+
end
|