heroku_mongo_watcher 0.2.5.beta → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +22 -8
- data/example/.watcher +2 -3
- data/lib/heroku_mongo_watcher/autoscaler.rb +86 -0
- data/lib/heroku_mongo_watcher/cli.rb +63 -87
- data/lib/heroku_mongo_watcher/configuration.rb +5 -1
- data/lib/heroku_mongo_watcher/data_row.rb +65 -52
- data/lib/heroku_mongo_watcher/mailer.rb +63 -0
- data/lib/heroku_mongo_watcher/version.rb +1 -1
- data/lib/heroku_mongo_watcher/watcher.rb +43 -0
- metadata +16 -13
data/README.md
CHANGED
@@ -17,14 +17,14 @@ It needed to accomplish the following:
|
|
17
17
|
|
18
18
|
The output looks like the following ...
|
19
19
|
|
20
|
-
|<---- heroku stats
|
21
|
-
dyno reqs
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
20
|
+
|<---- heroku stats ------------------------------------------------------------------->|<----mongo stats ------------------------------------------------>|
|
21
|
+
| dyno reqs art max r_err w_err %err wait queue slowest |insrt query updt flt lck lck:mrq qr|qw netI/O time |
|
22
|
+
6 3096 57 870 0 2 0.06% 0 0 /assets/companions/50104c| 41 0 79 0 2.2% 0.71 0|0 36k/30k 15:59:29
|
23
|
+
6 2705 80 3314 0 0 0.0% 0 0 /assets/companions/50104b| 34 0 67 0 2% 0.74 0|0 30k/25k 16:00:29
|
24
|
+
6 2469 122 5708 0 0 0.0% 0 0 /ads/gw/50074348451823003| 30 0 57 0 1.7% 0.69 0|0 26k/22k 16:01:29
|
25
|
+
6 2465 89 1347 0 0 0.0% 0 0 /assets/videos/501050b991| 30 0 59 0 1.8% 0.73 0|0 27k/22k 16:02:29
|
26
|
+
6 2301 83 1912 0 4 0.17% 0 0 /assets/companions/501050| 28 0 57 0 1.7% 0.74 0|0 25k/21k 16:03:29
|
27
|
+
6 1951 64 830 0 0 0.0% 0 0 /ads/gw/50074348451823003| 24 0 45 0 1.4% 0.72 0|0 21k/18k 16:04:29
|
28
28
|
|
29
29
|
### Legend
|
30
30
|
<table>
|
@@ -42,6 +42,8 @@ The output looks like the following ...
|
|
42
42
|
<tr><td>query</td><td>number of mongo queries</td></tr>
|
43
43
|
<tr><td>update</td><td>number of mongo updates</td></tr>
|
44
44
|
<tr><td>faults</td><td>number of mongo page faults</td></tr>
|
45
|
+
<tr><td>lck</td><td>mongo lock percentage</td></tr>
|
46
|
+
<tr><td>lck:mrq</td><td>ratio of lock% to 1000 requests</td></tr>
|
45
47
|
<tr><td>qr|qw</td><td>number of mongo's queued read and writes</td></tr>
|
46
48
|
<tr><td>netIO</td><td>size on mongo net in/ net out</td></tr>
|
47
49
|
<tr><td>time</td><td>the time sampled</td></tr>
|
@@ -74,4 +76,16 @@ locally
|
|
74
76
|
* `--print-errors` to print a summary of errors during each sample
|
75
77
|
* `--print-requests` to print a summary of requests during each sample
|
76
78
|
|
79
|
+
##Experimental
|
80
|
+
|
81
|
+
( you probably
|
82
|
+
|
83
|
+
* `--autoscale` to autoscale dynos
|
84
|
+
* `--min-dynos=n` to set the min default 6
|
85
|
+
* `--max-dynos=n` to set the min default 50
|
86
|
+
|
77
87
|
note: you can set these defaults in your .watcher file
|
88
|
+
|
89
|
+
## Contact
|
90
|
+
|
91
|
+
jtushman@pipewave.com
|
data/example/.watcher
CHANGED
@@ -21,9 +21,8 @@ mongo_password: 'skret'
|
|
21
21
|
# note you already need to have the heroku gem installed
|
22
22
|
# with creditails already set up for this directory
|
23
23
|
heroku_appname: 'my-heroku-app-name'
|
24
|
-
|
25
|
-
|
26
|
-
heroku_account: 'my-heroku-account'
|
24
|
+
heroku_username: 'ninja'
|
25
|
+
heroku_password: 'skret'
|
27
26
|
|
28
27
|
# If you want to display the list of errors in the logs
|
29
28
|
print_errors: true
|
@@ -0,0 +1,86 @@
|
|
1
|
+
require 'heroku_mongo_watcher'
|
2
|
+
require 'heroku_mongo_watcher/data_row'
|
3
|
+
require 'heroku_mongo_watcher/configuration'
|
4
|
+
require 'heroku'
|
5
|
+
class HerokuMongoWatcher::Autoscaler
|
6
|
+
include Singleton
|
7
|
+
|
8
|
+
attr_reader :last_scaled, :options
|
9
|
+
|
10
|
+
def initialize()
|
11
|
+
@last_scaled = Time.now - 60
|
12
|
+
@options = default_options.merge(config)
|
13
|
+
end
|
14
|
+
|
15
|
+
def default_options
|
16
|
+
{
|
17
|
+
min_dynos: 6,
|
18
|
+
max_dynos: 50,
|
19
|
+
step: 5,
|
20
|
+
requests_per_dyno: 1000,
|
21
|
+
min_frequency: 60 # seconds
|
22
|
+
}
|
23
|
+
end
|
24
|
+
|
25
|
+
def config
|
26
|
+
HerokuMongoWatcher::Configuration.instance.config
|
27
|
+
end
|
28
|
+
|
29
|
+
# should play it safe and add config[:step] to the calculated result
|
30
|
+
# if rpm > 10,000 scale to 15
|
31
|
+
# if rpm > 15,000 scale to 20
|
32
|
+
# if rpm > 20,000 scale to 25
|
33
|
+
# if rpm then drops to 14,000 drop to 15
|
34
|
+
# if rpm then drops to 300 drop to 6 (minimum)
|
35
|
+
#
|
36
|
+
# also do not scale down for 5 minutes
|
37
|
+
# always allow to scale up
|
38
|
+
def scale(data_row)
|
39
|
+
|
40
|
+
rpm = data_row.total_requests
|
41
|
+
current_dynos = data_row.dynos
|
42
|
+
|
43
|
+
# 32,012 rpm => 32 dynos
|
44
|
+
ideal_dynos = (rpm / options[:requests_per_dyno]).round
|
45
|
+
|
46
|
+
# return if the delta is less than 5 dynos | don't think I need to do this ...
|
47
|
+
#return if (ideal_dynos - current_dynos).abs < options[:step]
|
48
|
+
|
49
|
+
#this turns 32 into 30
|
50
|
+
stepped_dynos = (ideal_dynos/options[:step]).round * options[:step]
|
51
|
+
|
52
|
+
#this turns 30 into 35
|
53
|
+
stepped_dynos += options[:step]
|
54
|
+
|
55
|
+
#this makes sure that it stays within the min max bounds
|
56
|
+
stepped_dynos = options[:min_dynos] if stepped_dynos < options[:min_dynos]
|
57
|
+
stepped_dynos = options[:max_dynos] if stepped_dynos > options[:max_dynos]
|
58
|
+
|
59
|
+
# Don't allow downscaling until 5 minutes
|
60
|
+
if stepped_dynos < current_dynos && (Time.now - last_scaled) < (60 * 60) # 1 hour
|
61
|
+
#puts ">> Current: [#{current_dynos}], Ideal: [#{stepped_dynos}] | will not downscale within an hour"
|
62
|
+
nil
|
63
|
+
elsif stepped_dynos != current_dynos
|
64
|
+
set_dynos(stepped_dynos)
|
65
|
+
stepped_dynos
|
66
|
+
else
|
67
|
+
nil
|
68
|
+
end
|
69
|
+
|
70
|
+
end
|
71
|
+
|
72
|
+
def heroku
|
73
|
+
@heroku ||= Heroku::Client.new(config[:heroku_username], config[:heroku_password])
|
74
|
+
end
|
75
|
+
|
76
|
+
def set_dynos(count)
|
77
|
+
@last_scaled = Time.now
|
78
|
+
t = Thread.new('Setting Dynos') do
|
79
|
+
i = heroku.ps_scale(config[:heroku_appname], :type => 'web', :qty => count)
|
80
|
+
end
|
81
|
+
t.join
|
82
|
+
|
83
|
+
end
|
84
|
+
|
85
|
+
|
86
|
+
end
|
@@ -1,20 +1,9 @@
|
|
1
|
+
require 'heroku'
|
1
2
|
require 'heroku_mongo_watcher'
|
2
3
|
require 'heroku_mongo_watcher/configuration'
|
4
|
+
require 'heroku_mongo_watcher/autoscaler'
|
5
|
+
require 'heroku_mongo_watcher/mailer'
|
3
6
|
require 'heroku_mongo_watcher/data_row'
|
4
|
-
require 'trollop'
|
5
|
-
|
6
|
-
#http://stackoverflow.com/a/9117903/192791
|
7
|
-
require 'net/smtp'
|
8
|
-
Net.instance_eval {remove_const :SMTPSession} if defined?(Net::SMTPSession)
|
9
|
-
|
10
|
-
require 'net/pop'
|
11
|
-
Net::POP.instance_eval {remove_const :Revision} if defined?(Net::POP::Revision)
|
12
|
-
Net.instance_eval {remove_const :POP} if defined?(Net::POP)
|
13
|
-
Net.instance_eval {remove_const :POPSession} if defined?(Net::POPSession)
|
14
|
-
Net.instance_eval {remove_const :POP3Session} if defined?(Net::POP3Session)
|
15
|
-
Net.instance_eval {remove_const :APOPSession} if defined?(Net::APOPSession)
|
16
|
-
|
17
|
-
require 'tlsmail'
|
18
7
|
|
19
8
|
class HerokuMongoWatcher::CLI
|
20
9
|
|
@@ -22,9 +11,20 @@ class HerokuMongoWatcher::CLI
|
|
22
11
|
HerokuMongoWatcher::Configuration.instance.config
|
23
12
|
end
|
24
13
|
|
14
|
+
def self.mailer
|
15
|
+
HerokuMongoWatcher::Mailer.instance
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.autoscaler
|
19
|
+
HerokuMongoWatcher::Autoscaler.instance
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.heroku
|
23
|
+
@heroku || Heroku::Client.new(config[:heroku_username], config[:heroku_password])
|
24
|
+
end
|
25
|
+
|
25
26
|
def self.watch
|
26
27
|
Thread.abort_on_exception = true
|
27
|
-
notify("Mongo Watcher enabled!")
|
28
28
|
|
29
29
|
# lock warnings flags
|
30
30
|
@lock_critical_notified = false
|
@@ -39,37 +39,14 @@ class HerokuMongoWatcher::CLI
|
|
39
39
|
|
40
40
|
@mutex = Mutex.new
|
41
41
|
|
42
|
-
#
|
43
|
-
|
42
|
+
# Let people know that its on, and confirm emails are being received
|
43
|
+
mailer.notify(@current_row, "Mongo Watcher enabled!")
|
44
44
|
|
45
|
-
|
46
|
-
|
47
|
-
IO.popen(cmd_string) do |f|
|
48
|
-
while line = f.gets
|
49
|
-
@mutex.synchronize do
|
50
|
-
@current_row.process_heroku_router_line(line) if line.include? 'heroku[router]'
|
51
|
-
@current_row.process_heroku_web_line(line) if line.include? 'app[web'
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
55
|
-
end
|
45
|
+
# Call Tails heroku logs updates counts
|
46
|
+
heroku_watcher = Thread.new('heroku_logs') { tail_and_process_heroku }
|
56
47
|
|
57
48
|
# Call Heroku PS to check the number of up dynos
|
58
|
-
heroku_ps = Thread.new('heroku_ps')
|
59
|
-
while true do
|
60
|
-
dynos = 0
|
61
|
-
cmd_string = "heroku ps --app #{config[:heroku_appname]}"
|
62
|
-
cmd_string = cmd_string + " --account #{config[:heroku_account]}" if config[:heroku_account] && config[:heroku_account].length > 0
|
63
|
-
IO.popen(cmd_string) do |p|
|
64
|
-
while line = p.gets
|
65
|
-
dynos += 1 if line =~ /^web/ && line.split(' ')[1] == 'up'
|
66
|
-
end
|
67
|
-
end
|
68
|
-
@mutex.synchronize { @current_row.dynos = dynos }
|
69
|
-
sleep(30)
|
70
|
-
end
|
71
|
-
end
|
72
|
-
|
49
|
+
heroku_ps = Thread.new('heroku_ps') { periodicly_capture_heroku_ps }
|
73
50
|
|
74
51
|
HerokuMongoWatcher::DataRow.print_header
|
75
52
|
|
@@ -83,6 +60,8 @@ class HerokuMongoWatcher::CLI
|
|
83
60
|
|
84
61
|
check_and_notify
|
85
62
|
|
63
|
+
autoscale if config[:autoscale]
|
64
|
+
|
86
65
|
@last_row = @current_row
|
87
66
|
@current_row = HerokuMongoWatcher::DataRow.new
|
88
67
|
@current_row.dynos = @last_row.dynos
|
@@ -94,6 +73,40 @@ class HerokuMongoWatcher::CLI
|
|
94
73
|
heroku_watcher.join
|
95
74
|
heroku_ps.join
|
96
75
|
|
76
|
+
rescue Interrupt
|
77
|
+
puts "\nexiting ..."
|
78
|
+
end
|
79
|
+
|
80
|
+
def self.autoscale
|
81
|
+
new_dyno_level = autoscaler.scale(@current_row)
|
82
|
+
if new_dyno_level
|
83
|
+
puts "! Scaling -> #{new_dyno_level}"
|
84
|
+
mailer.notify(@current_row, "Scaling to #{new_dyno_level}")
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
def self.periodicly_capture_heroku_ps
|
89
|
+
while true do
|
90
|
+
results = heroku.ps(config[:heroku_appname])
|
91
|
+
dynos = results.select { |ps| ps['process'] =~ /^web./ && ps['state'] == 'up' }.count
|
92
|
+
|
93
|
+
@mutex.synchronize { @current_row.dynos = dynos }
|
94
|
+
sleep(30)
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
def self.tail_and_process_heroku
|
99
|
+
heroku.read_logs(config[:heroku_appname], ['tail=1']) do |chunk|
|
100
|
+
unless chunk.empty?
|
101
|
+
chunk.split("\n").each do |line|
|
102
|
+
@mutex.synchronize do
|
103
|
+
@current_row.process_heroku_router_line(line) if line.include? 'heroku[router]'
|
104
|
+
@current_row.process_heroku_web_line(line) if line.include? 'app[web'
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
end
|
97
110
|
end
|
98
111
|
|
99
112
|
def self.check_and_notify
|
@@ -104,14 +117,14 @@ class HerokuMongoWatcher::CLI
|
|
104
117
|
protected
|
105
118
|
|
106
119
|
def self.check_and_notify_locks
|
107
|
-
l = Float(@current_row.
|
120
|
+
l = Float(@current_row.locked)
|
108
121
|
if l > 90
|
109
|
-
notify '[CRITICAL] Locks above 90%' unless @lock_critical_notified
|
122
|
+
mailer.notify(@current_row, '[CRITICAL] Locks above 90%') unless @lock_critical_notified
|
110
123
|
elsif l > 70
|
111
|
-
notify '[WARNING] Locks above 70%' unless @lock_warning_notified
|
124
|
+
mailer.notify(@current_row, '[WARNING] Locks above 70%') unless @lock_warning_notified
|
112
125
|
elsif l < 50
|
113
126
|
if @lock_warning_notified || @lock_critical_notified
|
114
|
-
notify '[Resolved] locks below 50%'
|
127
|
+
mailer.notify(@current_row, '[Resolved] locks below 50%')
|
115
128
|
@lock_warning_notified = false
|
116
129
|
@lock_critical_notified = false
|
117
130
|
end
|
@@ -122,55 +135,18 @@ class HerokuMongoWatcher::CLI
|
|
122
135
|
def self.check_and_notify_response_time
|
123
136
|
return unless @current_row.total_requests > 200
|
124
137
|
if @current_row.average_response_time > 10_000 || @current_row.error_rate > 4
|
125
|
-
notify "[SEVERE WARNING] Application not healthy | [#{@current_row.total_requests} rpm,#{@current_row.average_response_time} art]" unless @art_critical_notified
|
138
|
+
mailer.notify @current_row, "[SEVERE WARNING] Application not healthy | [#{@current_row.total_requests} rpm,#{@current_row.average_response_time} art]" unless @art_critical_notified
|
126
139
|
@art_critical_notified = true
|
127
140
|
elsif @current_row.average_response_time > 500 || @current_row.error_rate > 1 || @current_row.total_requests > 30_000
|
128
|
-
notify "[WARNING] Application heating up | [#{@current_row.total_requests} rpm,#{@current_row.average_response_time} art]" unless @art_warning_notified
|
141
|
+
mailer.notify @current_row, "[WARNING] Application heating up | [#{@current_row.total_requests} rpm,#{@current_row.average_response_time} art]" unless @art_warning_notified
|
129
142
|
@art_warning_notified = true
|
130
143
|
elsif @current_row.average_response_time < 300 && @current_row.total_requests < 25_000
|
131
144
|
if @art_warning_notified || @art_critical_notified
|
132
|
-
notify "[RESOLVED] | [#{@current_row.total_requests} rpm,#{@current_row.average_response_time} art]"
|
145
|
+
mailer.notify @current_row, "[RESOLVED] | [#{@current_row.total_requests} rpm,#{@current_row.average_response_time} art]"
|
133
146
|
@art_warning_notified = false
|
134
147
|
@art_critical_notified = false
|
135
148
|
end
|
136
149
|
end
|
137
150
|
end
|
138
151
|
|
139
|
-
def self.notify(msg)
|
140
|
-
Thread.new('notify_admins') do
|
141
|
-
subscribers = config[:notify] || []
|
142
|
-
subscribers.each { |user_email| send_email(user_email, msg) }
|
143
|
-
end
|
144
|
-
end
|
145
|
-
|
146
|
-
def self.send_email(to, msg)
|
147
|
-
return unless config[:gmail_username] && config[:gmail_password]
|
148
|
-
content = [
|
149
|
-
"From: Mongo Watcher <#{config[:gmail_username]}>",
|
150
|
-
"To: #{to}",
|
151
|
-
"Subject: #{msg}",
|
152
|
-
"",
|
153
|
-
"RPM: #{@last_row.total_requests}",
|
154
|
-
"Average Reponse Time: #{@last_row.average_response_time}",
|
155
|
-
"Application Errors: #{@last_row.total_web_errors}",
|
156
|
-
"Router Errors (timeouts): #{@last_row.total_router_errors}",
|
157
|
-
"Error Rate: #{@last_row.error_rate}%",
|
158
|
-
"Dynos: #{@last_row.dynos}",
|
159
|
-
"",
|
160
|
-
"Locks: #{@last_row.lock}",
|
161
|
-
"Queries: #{@last_row.queries}",
|
162
|
-
"Inserts: #{@last_row.inserts}",
|
163
|
-
"Updates: #{@last_row.updates}",
|
164
|
-
"Faults: #{@last_row.faults}",
|
165
|
-
"NetI/O: #{@last_row.net_in}/#{@last_row.net_out}",
|
166
|
-
]
|
167
|
-
content = content + @last_row.error_content_for_email + @last_row.request_content_for_email
|
168
|
-
|
169
|
-
content = content.join("\r\n")
|
170
|
-
Net::SMTP.enable_tls(OpenSSL::SSL::VERIFY_NONE)
|
171
|
-
Net::SMTP.start('smtp.gmail.com', 587, 'gmail.com', config[:gmail_username], config[:gmail_password], :login) do |smtp|
|
172
|
-
smtp.send_message(content, config[:gmail_username], to)
|
173
|
-
end
|
174
|
-
end
|
175
|
-
|
176
152
|
end
|
@@ -14,8 +14,9 @@ class HerokuMongoWatcher::Configuration
|
|
14
14
|
mongo_host: '',
|
15
15
|
mongo_username: '',
|
16
16
|
mongo_password: '',
|
17
|
+
heroku_username: '',
|
18
|
+
heroku_password: '',
|
17
19
|
heroku_appname: '',
|
18
|
-
heroku_account: '',
|
19
20
|
print_errors: false,
|
20
21
|
print_requests: false
|
21
22
|
}
|
@@ -30,6 +31,9 @@ class HerokuMongoWatcher::Configuration
|
|
30
31
|
opt :print_errors, "show aggregate error summaries", default: false
|
31
32
|
opt :print_requests, "show aggregate requests summaries", default: false
|
32
33
|
opt :interval, "frequency in seconds to poll mongostat and print out results", default: 60
|
34
|
+
opt :autoscale, "autoscale dynos -- WARNING ONLY DO THIS IF YOU KNOW WHAT YOUR ARE DOING", default: false
|
35
|
+
opt :min_dynos, "For autoscaling, the minimum dynos to use", default: 6
|
36
|
+
opt :max_dynos, "For autoscaling, the max dynos to use", default: 50
|
33
37
|
end
|
34
38
|
|
35
39
|
@@config.merge!(opts)
|
@@ -10,7 +10,7 @@ class HerokuMongoWatcher::DataRow
|
|
10
10
|
|
11
11
|
# Mongo Attributes
|
12
12
|
@@attributes.concat [:inserts, :queries, :ops, :updates, :deletes,
|
13
|
-
:faults, :
|
13
|
+
:faults, :locked, :qrw, :net_in, :net_out]
|
14
14
|
|
15
15
|
@@attributes.each { |attr| attr_accessor attr }
|
16
16
|
|
@@ -41,13 +41,53 @@ class HerokuMongoWatcher::DataRow
|
|
41
41
|
total_requests > 0 ? ((((total_web_errors + total_router_errors)*(1.0)) / total_requests)* 100).round(2) : 'N/A'
|
42
42
|
end
|
43
43
|
|
44
|
+
def lock_request_ratio
|
45
|
+
total_requests > 0 ? ((((Float(locked) * 1.0)/ total_requests)) * 1_000).round(2) : 'N/A'
|
46
|
+
end
|
47
|
+
|
48
|
+
def self.print_header
|
49
|
+
puts
|
50
|
+
puts "|<---- heroku stats ------------------------------------------------------------------->|<----mongo stats ------------------------------------------------>|"
|
51
|
+
puts "| dyno reqs art max r_err w_err %err wait queue slowest |insrt query updt flt lck lck:mrq qr|qw netI/O time |"
|
52
|
+
end
|
53
|
+
|
54
|
+
def error_content_for_email
|
55
|
+
content = []
|
56
|
+
if @errors && @errors.keys && @errors.keys.length > 0
|
57
|
+
content << ""
|
58
|
+
content << "Errors"
|
59
|
+
@errors.each do |error,count|
|
60
|
+
content << "\t\t[#{count}] #{error}"
|
61
|
+
end
|
62
|
+
end
|
63
|
+
content
|
64
|
+
end
|
65
|
+
|
66
|
+
def request_content_for_email
|
67
|
+
content = []
|
68
|
+
if @requests && @requests.keys && @requests.keys.length > 0
|
69
|
+
content << ""
|
70
|
+
content << "Requests"
|
71
|
+
@requests.sort_by{|req,count| -count}.first(10).each do |row|
|
72
|
+
content << "\t\t[#{row.last}] #{row.first}"
|
73
|
+
end
|
74
|
+
end
|
75
|
+
content
|
76
|
+
end
|
77
|
+
|
44
78
|
def process_heroku_router_line(line)
|
45
79
|
items = line.split
|
46
80
|
|
81
|
+
self.total_requests +=1
|
82
|
+
|
47
83
|
if line =~ /Error/
|
48
84
|
# Note: The lion share of these are timeouts
|
49
85
|
#Full list here: https://devcenter.heroku.com/articles/error-codes
|
50
86
|
self.total_router_errors += 1
|
87
|
+
time = items[0]
|
88
|
+
process = items[1]
|
89
|
+
clean_line = line.sub(time, '').sub(process, '').strip
|
90
|
+
add_error(clean_line)
|
51
91
|
else
|
52
92
|
time = items[0]
|
53
93
|
process = items[1]
|
@@ -63,7 +103,6 @@ class HerokuMongoWatcher::DataRow
|
|
63
103
|
path = extract_path(url)
|
64
104
|
|
65
105
|
if is_number?(service) && is_number?(wait) && is_number?(queue)
|
66
|
-
self.total_requests +=1
|
67
106
|
self.total_service += Integer(service) if service
|
68
107
|
self.total_wait += Integer(wait) if wait
|
69
108
|
self.total_queue += Integer(queue) if queue
|
@@ -83,12 +122,6 @@ class HerokuMongoWatcher::DataRow
|
|
83
122
|
end
|
84
123
|
end
|
85
124
|
|
86
|
-
def extract_path(url)
|
87
|
-
URI('http://' + url).path
|
88
|
-
rescue
|
89
|
-
return url
|
90
|
-
end
|
91
|
-
|
92
125
|
def process_heroku_web_line(line)
|
93
126
|
# Only care about errors
|
94
127
|
error_messages = config[:error_messages]
|
@@ -100,15 +133,19 @@ class HerokuMongoWatcher::DataRow
|
|
100
133
|
clean_line = line.sub(time, '').sub(process, '').strip
|
101
134
|
|
102
135
|
self.total_web_errors += 1
|
103
|
-
|
104
|
-
self.errors[clean_line] = self.errors[clean_line] + 1
|
105
|
-
else
|
106
|
-
self.errors[clean_line] = 1
|
107
|
-
end
|
136
|
+
add_error(clean_line)
|
108
137
|
end
|
109
138
|
|
110
139
|
end
|
111
140
|
|
141
|
+
def add_error(error_line)
|
142
|
+
if self.errors.has_key? error_line
|
143
|
+
self.errors[error_line] = self.errors[error_line] + 1
|
144
|
+
else
|
145
|
+
self.errors[error_line] = 1
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
112
149
|
def process_mongo_line(line)
|
113
150
|
items = line.split
|
114
151
|
|
@@ -136,12 +173,6 @@ class HerokuMongoWatcher::DataRow
|
|
136
173
|
|
137
174
|
end
|
138
175
|
|
139
|
-
def self.print_header
|
140
|
-
puts
|
141
|
-
puts "|<---- heroku stats ------------------------------------------------------------------->|<----mongo stats ------------------------------------------------>|"
|
142
|
-
puts "| dyno reqs art max r_err w_err %err wait queue slowest | insert query update faults locked qr|qw netI/O time |"
|
143
|
-
end
|
144
|
-
|
145
176
|
def print_row
|
146
177
|
print_hash(@errors) if config[:print_errors]
|
147
178
|
print_hash(@requests) if config[:print_requests]
|
@@ -153,21 +184,24 @@ class HerokuMongoWatcher::DataRow
|
|
153
184
|
color_print @total_router_errors, warning: 1
|
154
185
|
color_print @total_web_errors, warning: 1
|
155
186
|
color_print error_rate, warning: 1, critical: 3, percent: true
|
156
|
-
color_print
|
157
|
-
color_print
|
187
|
+
color_print @total_wait #, warning: 10, critical: 100
|
188
|
+
color_print @total_queue #, warning: 10, critical: 100
|
158
189
|
color_print @slowest_request, length: 28, slice: 25
|
159
190
|
print '|'
|
160
|
-
color_print @inserts
|
161
|
-
color_print @queries
|
162
|
-
color_print @updates
|
163
|
-
color_print @faults
|
164
|
-
color_print @locked, bold: true, warning: 40, critical: 70
|
191
|
+
color_print @inserts, length: 5
|
192
|
+
color_print @queries, length: 6
|
193
|
+
color_print @updates, length: 5
|
194
|
+
color_print @faults, length: 5
|
195
|
+
color_print @locked, bold: true, warning: 40, critical: 70, length: 6, percent: true
|
196
|
+
color_print lock_request_ratio
|
165
197
|
color_print @qrw
|
166
198
|
color_print "#{@net_in}/#{@net_out}", length: 10
|
167
|
-
color_print @mongo_time, length:
|
199
|
+
color_print @mongo_time, length: 13
|
168
200
|
printf "\n"
|
169
201
|
end
|
170
202
|
|
203
|
+
private
|
204
|
+
|
171
205
|
def print_hash(hash)
|
172
206
|
if hash && hash.keys && hash.keys.length > 0
|
173
207
|
hash.sort_by{|key,count| -count}.first(10).each do |row|
|
@@ -176,32 +210,12 @@ class HerokuMongoWatcher::DataRow
|
|
176
210
|
end
|
177
211
|
end
|
178
212
|
|
179
|
-
def
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
content << "Errors"
|
184
|
-
@errors.each do |error,count|
|
185
|
-
content << "\t\t[#{count}] #{error}"
|
186
|
-
end
|
187
|
-
end
|
188
|
-
content
|
189
|
-
end
|
190
|
-
|
191
|
-
def request_content_for_email
|
192
|
-
content = []
|
193
|
-
if @requests && @requests.keys && @requests.keys.length > 0
|
194
|
-
content << ""
|
195
|
-
content << "Requests"
|
196
|
-
@requests.sort_by{|req,count| -count}.first(10).each do |row|
|
197
|
-
content << "\t\t[#{row.last}] #{row.first}"
|
198
|
-
end
|
199
|
-
end
|
200
|
-
content
|
213
|
+
def extract_path(url)
|
214
|
+
URI('http://' + url).path
|
215
|
+
rescue
|
216
|
+
return url
|
201
217
|
end
|
202
218
|
|
203
|
-
private
|
204
|
-
|
205
219
|
def is_number?(string)
|
206
220
|
_is_number = true
|
207
221
|
begin
|
@@ -235,5 +249,4 @@ class HerokuMongoWatcher::DataRow
|
|
235
249
|
print "\a"
|
236
250
|
end
|
237
251
|
|
238
|
-
|
239
252
|
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
require 'heroku_mongo_watcher'
|
2
|
+
require 'heroku_mongo_watcher/configuration'
|
3
|
+
|
4
|
+
#http://stackoverflow.com/a/9117903/192791
|
5
|
+
require 'net/smtp'
|
6
|
+
Net.instance_eval {remove_const :SMTPSession} if defined?(Net::SMTPSession)
|
7
|
+
|
8
|
+
require 'net/pop'
|
9
|
+
Net::POP.instance_eval {remove_const :Revision} if defined?(Net::POP::Revision)
|
10
|
+
Net.instance_eval {remove_const :POP} if defined?(Net::POP)
|
11
|
+
Net.instance_eval {remove_const :POPSession} if defined?(Net::POPSession)
|
12
|
+
Net.instance_eval {remove_const :POP3Session} if defined?(Net::POP3Session)
|
13
|
+
Net.instance_eval {remove_const :APOPSession} if defined?(Net::APOPSession)
|
14
|
+
|
15
|
+
require 'tlsmail'
|
16
|
+
|
17
|
+
class HerokuMongoWatcher::Mailer
|
18
|
+
include Singleton
|
19
|
+
|
20
|
+
def config
|
21
|
+
HerokuMongoWatcher::Configuration.instance.config
|
22
|
+
end
|
23
|
+
|
24
|
+
def notify(data_row,msg)
|
25
|
+
Thread.new('notify_admins') do
|
26
|
+
subscribers = config[:notify] || []
|
27
|
+
subscribers.each { |user_email| send_email(user_email, msg, data_row) }
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
protected
|
32
|
+
|
33
|
+
def send_email(to, msg, data_row)
|
34
|
+
return unless config[:gmail_username] && config[:gmail_password]
|
35
|
+
content = [
|
36
|
+
"From: Mongo Watcher <#{config[:gmail_username]}>",
|
37
|
+
"To: #{to}",
|
38
|
+
"Subject: #{msg}",
|
39
|
+
"",
|
40
|
+
"RPM: #{data_row.total_requests}",
|
41
|
+
"Average Reponse Time: #{data_row.average_response_time}",
|
42
|
+
"Application Errors: #{data_row.total_web_errors}",
|
43
|
+
"Router Errors (timeouts): #{data_row.total_router_errors}",
|
44
|
+
"Error Rate: #{data_row.error_rate}%",
|
45
|
+
"Dynos: #{data_row.dynos}",
|
46
|
+
"",
|
47
|
+
"Locks: #{data_row.locked}",
|
48
|
+
"Queries: #{data_row.queries}",
|
49
|
+
"Inserts: #{data_row.inserts}",
|
50
|
+
"Updates: #{data_row.updates}",
|
51
|
+
"Faults: #{data_row.faults}",
|
52
|
+
"NetI/O: #{data_row.net_in}/#{data_row.net_out}",
|
53
|
+
]
|
54
|
+
content = content + data_row.error_content_for_email + data_row.request_content_for_email
|
55
|
+
|
56
|
+
content = content.join("\r\n")
|
57
|
+
Net::SMTP.enable_tls(OpenSSL::SSL::VERIFY_NONE)
|
58
|
+
Net::SMTP.start('smtp.gmail.com', 587, 'gmail.com', config[:gmail_username], config[:gmail_password], :login) do |smtp|
|
59
|
+
smtp.send_message(content, config[:gmail_username], to)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
require 'eventmachine'
|
2
|
+
require 'heroku_mongo_watcher'
|
3
|
+
require 'heroku_mongo_watcher/configuration'
|
4
|
+
|
5
|
+
module MongoStatProcessor
|
6
|
+
def post_init
|
7
|
+
puts "started to read mongostat"
|
8
|
+
end
|
9
|
+
|
10
|
+
def receive_data data
|
11
|
+
puts "ruby sent me: #{data}"
|
12
|
+
end
|
13
|
+
|
14
|
+
def unbind
|
15
|
+
puts "ruby died with exit status: #{get_status.exitstatus}"
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
|
20
|
+
class HerokuMongoWatcher::Watcher
|
21
|
+
|
22
|
+
def self.config
|
23
|
+
HerokuMongoWatcher::Configuration.instance.config
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.watch
|
27
|
+
|
28
|
+
EM::run {
|
29
|
+
|
30
|
+
|
31
|
+
cmd = "mongostat --rowcount 0 10 --host #{config[:mongo_host]} \
|
32
|
+
--username #{config[:mongo_username]} --password #{config[:mongo_password]} --noheaders"
|
33
|
+
|
34
|
+
EM.popen(cmd, MongoStatProcessor)
|
35
|
+
|
36
|
+
|
37
|
+
}
|
38
|
+
|
39
|
+
|
40
|
+
end
|
41
|
+
|
42
|
+
|
43
|
+
end
|
metadata
CHANGED
@@ -1,19 +1,19 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: heroku_mongo_watcher
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
5
|
-
prerelease:
|
4
|
+
version: 0.3.0
|
5
|
+
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- Jonathan Tushman
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-08-01 00:00:00.000000000Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: term-ansicolor
|
16
|
-
requirement: &
|
16
|
+
requirement: &2157847200 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ! '>='
|
@@ -21,10 +21,10 @@ dependencies:
|
|
21
21
|
version: '0'
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *2157847200
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: tlsmail
|
27
|
-
requirement: &
|
27
|
+
requirement: &2157846540 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - ! '>='
|
@@ -32,10 +32,10 @@ dependencies:
|
|
32
32
|
version: '0'
|
33
33
|
type: :runtime
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
35
|
+
version_requirements: *2157846540
|
36
36
|
- !ruby/object:Gem::Dependency
|
37
37
|
name: heroku
|
38
|
-
requirement: &
|
38
|
+
requirement: &2157845780 !ruby/object:Gem::Requirement
|
39
39
|
none: false
|
40
40
|
requirements:
|
41
41
|
- - ! '>='
|
@@ -43,10 +43,10 @@ dependencies:
|
|
43
43
|
version: '0'
|
44
44
|
type: :runtime
|
45
45
|
prerelease: false
|
46
|
-
version_requirements: *
|
46
|
+
version_requirements: *2157845780
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: trollop
|
49
|
-
requirement: &
|
49
|
+
requirement: &2157845020 !ruby/object:Gem::Requirement
|
50
50
|
none: false
|
51
51
|
requirements:
|
52
52
|
- - ! '>='
|
@@ -54,7 +54,7 @@ dependencies:
|
|
54
54
|
version: '0'
|
55
55
|
type: :runtime
|
56
56
|
prerelease: false
|
57
|
-
version_requirements: *
|
57
|
+
version_requirements: *2157845020
|
58
58
|
description: Also notifies you when certain thresholds are hit. I have found this
|
59
59
|
much more accurate than New Relic
|
60
60
|
email:
|
@@ -72,10 +72,13 @@ files:
|
|
72
72
|
- example/.watcher
|
73
73
|
- heroku_mongo_watcher.gemspec
|
74
74
|
- lib/heroku_mongo_watcher.rb
|
75
|
+
- lib/heroku_mongo_watcher/autoscaler.rb
|
75
76
|
- lib/heroku_mongo_watcher/cli.rb
|
76
77
|
- lib/heroku_mongo_watcher/configuration.rb
|
77
78
|
- lib/heroku_mongo_watcher/data_row.rb
|
79
|
+
- lib/heroku_mongo_watcher/mailer.rb
|
78
80
|
- lib/heroku_mongo_watcher/version.rb
|
81
|
+
- lib/heroku_mongo_watcher/watcher.rb
|
79
82
|
homepage: https://github.com/jtushman/heroku_mongo_watcher
|
80
83
|
licenses: []
|
81
84
|
post_install_message:
|
@@ -91,9 +94,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
91
94
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
92
95
|
none: false
|
93
96
|
requirements:
|
94
|
-
- - ! '
|
97
|
+
- - ! '>='
|
95
98
|
- !ruby/object:Gem::Version
|
96
|
-
version:
|
99
|
+
version: '0'
|
97
100
|
requirements: []
|
98
101
|
rubyforge_project: heroku_mongo_watcher
|
99
102
|
rubygems_version: 1.8.17
|