droid19 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Rakefile +41 -0
- data/VERSION +1 -0
- data/bin/bleedq +19 -0
- data/droid.gemspec +97 -0
- data/examples/async_reply.rb +25 -0
- data/examples/heroku_async_reply.rb +22 -0
- data/examples/sync.rb +32 -0
- data/examples/worker.rb +58 -0
- data/lib/droid.rb +140 -0
- data/lib/droid/em.rb +55 -0
- data/lib/droid/heroku.rb +102 -0
- data/lib/droid/heroku/local_stats.rb +145 -0
- data/lib/droid/heroku/logger_client.rb +218 -0
- data/lib/droid/heroku/memcache_cluster.rb +152 -0
- data/lib/droid/heroku/stats.rb +14 -0
- data/lib/droid/json_server.rb +109 -0
- data/lib/droid/monkey.rb +8 -0
- data/lib/droid/publish.rb +31 -0
- data/lib/droid/queue.rb +196 -0
- data/lib/droid/request.rb +110 -0
- data/lib/droid/sync.rb +87 -0
- data/lib/droid/utilization.rb +113 -0
- data/lib/droid/utils.rb +113 -0
- data/lib/heroku_droid.rb +1 -0
- data/lib/local_stats.rb +1 -0
- data/lib/memcache_cluster.rb +1 -0
- data/lib/stats.rb +1 -0
- data/spec/publish_spec.rb +27 -0
- data/spec/response_spec.rb +47 -0
- data/spec/spec_helper.rb +12 -0
- data/spec/utils_spec.rb +43 -0
- data/spec/wait_for_port_spec.rb +16 -0
- metadata +190 -0
data/lib/droid/heroku.rb
ADDED
@@ -0,0 +1,102 @@
|
|
1
|
+
require 'droid'
|
2
|
+
require 'droid/heroku/local_stats'
|
3
|
+
require 'droid/heroku/logger_client'
|
4
|
+
|
5
|
+
Log.configure do |c|
|
6
|
+
c.component = LocalStats.slot
|
7
|
+
c.instance_name = LocalStats.instance_name
|
8
|
+
end
|
9
|
+
|
10
|
+
Droid.log = Log
|
11
|
+
|
12
|
+
class Droid
|
13
|
+
module Utils
|
14
|
+
def self.generate_name_for_instance(name)
|
15
|
+
"#{name}.#{LocalStats.slot}.#{LocalStats.ion_instance_id}"
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
begin
|
21
|
+
require 'droid/json_server'
|
22
|
+
rescue LoadError => e
|
23
|
+
Droid.log.error "Could not load JSONServer", :exception => e
|
24
|
+
end
|
25
|
+
|
26
|
+
class HerokuDroid < Droid
|
27
|
+
attr_reader :extended_stats
|
28
|
+
|
29
|
+
def initialize(name, opts={}, &blk)
|
30
|
+
@extended_stats = !(opts[:extended_stats] == false)
|
31
|
+
|
32
|
+
super(name, opts) do |droid|
|
33
|
+
setup_standard_topics(self) unless opts[:standard_topics] == false
|
34
|
+
LocalStats.attach
|
35
|
+
blk.call(self)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def stats(&blk)
|
40
|
+
@stats = blk
|
41
|
+
out = call_stats
|
42
|
+
out = out.inspect unless out.kind_of?(String)
|
43
|
+
Log.notice out
|
44
|
+
end
|
45
|
+
|
46
|
+
def call_stats
|
47
|
+
@stats ? @stats.call : nil
|
48
|
+
end
|
49
|
+
|
50
|
+
def name
|
51
|
+
Droid.name
|
52
|
+
end
|
53
|
+
|
54
|
+
def ruby_path
|
55
|
+
if File.exists?("/usr/ruby1.8.6/bin/ruby")
|
56
|
+
"/usr/ruby1.8.6/bin"
|
57
|
+
else
|
58
|
+
"/usr/local/bin"
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def setup_standard_topics(droid)
|
63
|
+
setup_ping_topic(droid)
|
64
|
+
end
|
65
|
+
|
66
|
+
def setup_ping_topic(droid)
|
67
|
+
require 'time'
|
68
|
+
|
69
|
+
droid.listener('ping').subscribe do |req|
|
70
|
+
Droid::Utilization.latency = (Time.now.to_f - req['departed_at']).abs
|
71
|
+
end
|
72
|
+
|
73
|
+
blk = Proc.new do |d|
|
74
|
+
begin
|
75
|
+
t1 = Time.now
|
76
|
+
response = {}.merge(LocalStats.stats)
|
77
|
+
|
78
|
+
estats = nil
|
79
|
+
if self.extended_stats
|
80
|
+
estats = droid.call_stats
|
81
|
+
estats = { :notes => estats } unless estats.kind_of?(Hash)
|
82
|
+
estats[:notes] ||= estats.map { |k, v| "#{v} #{k}" }.join(", ")
|
83
|
+
estats.merge!(LocalStats.extended_stats)
|
84
|
+
end
|
85
|
+
|
86
|
+
response.merge!({
|
87
|
+
:extended_stats => estats,
|
88
|
+
:droid_name => Droid.name,
|
89
|
+
:latency => Droid::Utilization.latency,
|
90
|
+
})
|
91
|
+
|
92
|
+
d.publish('pong', response.merge(:stat_collection => (Time.now - t1)))
|
93
|
+
rescue Object => e
|
94
|
+
log.error "Ping Block Error: #{e.class} -> #{e.message}", :exception => e
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
EM.add_timer(2) { blk.call(droid) }
|
99
|
+
EM.add_periodic_timer(50 + (rand*15).to_i) { blk.call(droid) }
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
@@ -0,0 +1,145 @@
|
|
1
|
+
require 'rest_client'
|
2
|
+
|
3
|
+
module LocalStats
|
4
|
+
extend self
|
5
|
+
|
6
|
+
def load_avg
|
7
|
+
File.read('/proc/loadavg').split(' ', 2).first.to_f
|
8
|
+
end
|
9
|
+
|
10
|
+
def memory_info
|
11
|
+
info = {}
|
12
|
+
File.read('/proc/meminfo').split("\n").each do |line|
|
13
|
+
name, value = line.split(/:\s+/, 2)
|
14
|
+
info[name] = value.to_i
|
15
|
+
end
|
16
|
+
info
|
17
|
+
end
|
18
|
+
|
19
|
+
def memory_use
|
20
|
+
info = memory_info
|
21
|
+
used = info['MemTotal'] - info['MemFree'] - info['Cached'] - info['Buffers']
|
22
|
+
(100 * used / info['MemTotal']).to_i
|
23
|
+
end
|
24
|
+
|
25
|
+
def swap_use
|
26
|
+
_, total, use, _ = `free | grep Swap:`.strip.split
|
27
|
+
return 0 if total.to_i == 0
|
28
|
+
(100 * use.to_i / total.to_i).to_i
|
29
|
+
end
|
30
|
+
|
31
|
+
def local_ip
|
32
|
+
@local_ip ||= fetch_local_ip
|
33
|
+
end
|
34
|
+
|
35
|
+
def fetch_local_ip
|
36
|
+
`/sbin/ifconfig eth0 | grep inet | awk '{print $2}'`.gsub('addr:', '').strip
|
37
|
+
end
|
38
|
+
|
39
|
+
def public_ip
|
40
|
+
retries = 0
|
41
|
+
@public_ip ||= begin
|
42
|
+
RestClient.get("http://169.254.169.254/latest/meta-data/public-ipv4").to_s.strip
|
43
|
+
rescue RestClient::RequestTimeout => e
|
44
|
+
retries += 1
|
45
|
+
raise if retries > 5
|
46
|
+
sleep 1
|
47
|
+
retry
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def disk_stats
|
52
|
+
raw = `df -P | grep -v tmpfs | grep -v udev | grep -v slugs | awk '{print $1 " ## " $6 " ## " $5}'`.split("\n").collect { |d| d }
|
53
|
+
raw.shift
|
54
|
+
raw.collect do |d|
|
55
|
+
tokens = d.split("##").collect { |t| t.strip }
|
56
|
+
{
|
57
|
+
:device => tokens[0],
|
58
|
+
:path => tokens[1],
|
59
|
+
:usage => tokens[2].gsub('%','').to_i
|
60
|
+
}
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def ion_instance_id
|
65
|
+
@ion_instance_id ||= File.read('/etc/heroku/ion_instance_id').strip rescue nil
|
66
|
+
end
|
67
|
+
|
68
|
+
def slot
|
69
|
+
@slot ||= File.read('/etc/heroku/slot').strip.gsub(/64$/,'').split('-').first rescue nil
|
70
|
+
end
|
71
|
+
|
72
|
+
def instance_name
|
73
|
+
return nil if slot.nil? or ion_instance_id.nil?
|
74
|
+
"#{slot}.#{ion_instance_id}"
|
75
|
+
end
|
76
|
+
|
77
|
+
alias :this_instance_name :instance_name
|
78
|
+
|
79
|
+
def stats
|
80
|
+
{
|
81
|
+
:slot => slot,
|
82
|
+
:ion_id => ion_instance_id,
|
83
|
+
:load_avg => load_avg,
|
84
|
+
:memory_use => memory_use,
|
85
|
+
:swap_use => swap_use,
|
86
|
+
:local_ip => local_ip,
|
87
|
+
:instance_name => this_instance_name,
|
88
|
+
:public_ip => public_ip,
|
89
|
+
:net_in_rate => receive_rate,
|
90
|
+
:net_out_rate => transmit_rate
|
91
|
+
}
|
92
|
+
end
|
93
|
+
|
94
|
+
def extended_stats
|
95
|
+
{
|
96
|
+
:disk_stats => disk_stats
|
97
|
+
}
|
98
|
+
end
|
99
|
+
|
100
|
+
# sample ifstat every 30 seconds
|
101
|
+
IFSTAT_INTERVAL = 30
|
102
|
+
|
103
|
+
# bytes received per second and bytes transmitted per
|
104
|
+
# second as a two tuple
|
105
|
+
def transfer_rates
|
106
|
+
[receive_rate, transmit_rate]
|
107
|
+
end
|
108
|
+
|
109
|
+
# bytes received per second
|
110
|
+
def receive_rate
|
111
|
+
@rxrate || 0
|
112
|
+
end
|
113
|
+
|
114
|
+
# bytes transmitted per second
|
115
|
+
def transmit_rate
|
116
|
+
@txrate || 0
|
117
|
+
end
|
118
|
+
|
119
|
+
# sample the current RX and TX bytes from ifconfig and
|
120
|
+
# return as a two-tuple.
|
121
|
+
def sample
|
122
|
+
data = ifconfig.match(/RX bytes:(\d+).*TX bytes:(\d+)/)
|
123
|
+
[data[1].to_i, data[2].to_i]
|
124
|
+
rescue => boom
|
125
|
+
Log.notice "error sampling network rate: #{boom.class} #{boom.message}"
|
126
|
+
end
|
127
|
+
|
128
|
+
def update_counters
|
129
|
+
rx, tx = sample
|
130
|
+
@rxrate = (rx - @rx) / IFSTAT_INTERVAL if @rx
|
131
|
+
@txrate = (tx - @tx) / IFSTAT_INTERVAL if @tx
|
132
|
+
@rx, @tx = rx, tx
|
133
|
+
end
|
134
|
+
|
135
|
+
# called when a droid starts up - setup timers and whatnot
|
136
|
+
def attach
|
137
|
+
@rx, @tx, @rxrate, @txrate = nil
|
138
|
+
update_counters
|
139
|
+
EM.add_periodic_timer(IFSTAT_INTERVAL) { update_counters }
|
140
|
+
end
|
141
|
+
|
142
|
+
def ifconfig
|
143
|
+
`/sbin/ifconfig eth0`
|
144
|
+
end
|
145
|
+
end
|
@@ -0,0 +1,218 @@
|
|
1
|
+
# Logger Client
|
2
|
+
# Use as a gem or Rails plugin. Example configuration:
|
3
|
+
#
|
4
|
+
# Log.configure do |config|
|
5
|
+
# config.component = 'core'
|
6
|
+
# config.instance = 'userapps.123'
|
7
|
+
# config.failsafe = :file
|
8
|
+
# end
|
9
|
+
|
10
|
+
require 'time'
|
11
|
+
require 'syslog'
|
12
|
+
|
13
|
+
module Log; end
|
14
|
+
|
15
|
+
class Log::InvalidConfiguration < RuntimeError
|
16
|
+
def message
|
17
|
+
"Invalid component. Configure with Log.configure { |c| c.component = 'myComponent' }"
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
class Log::Config
|
22
|
+
def initialize
|
23
|
+
@contents = { }
|
24
|
+
end
|
25
|
+
|
26
|
+
def method_missing(method, value)
|
27
|
+
@contents[method.to_s.gsub(/=$/, '').to_sym] = value
|
28
|
+
end
|
29
|
+
|
30
|
+
def to_hash
|
31
|
+
@contents
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
module Log
|
36
|
+
extend self
|
37
|
+
|
38
|
+
unless defined? SyslogConvertion
|
39
|
+
SyslogConvertion = {
|
40
|
+
'error' => 3,
|
41
|
+
'warning' => 4,
|
42
|
+
'notice' => 5,
|
43
|
+
'debug' => 7,
|
44
|
+
}
|
45
|
+
end
|
46
|
+
|
47
|
+
def debug(msg, options={})
|
48
|
+
log msg, options.merge(:level => 'debug')
|
49
|
+
end
|
50
|
+
|
51
|
+
def notice(msg, options={})
|
52
|
+
log msg, options.merge(:level => 'notice')
|
53
|
+
end
|
54
|
+
|
55
|
+
alias :info :notice
|
56
|
+
|
57
|
+
def warning(msg, options={})
|
58
|
+
log msg, options.merge(:level => 'warning')
|
59
|
+
end
|
60
|
+
|
61
|
+
def error(msg, options={})
|
62
|
+
# Fake an exception for error messages with no exception object
|
63
|
+
# so that we get a backtrace.
|
64
|
+
if options[:exception].nil?
|
65
|
+
begin
|
66
|
+
raise StandardError, msg
|
67
|
+
rescue => error
|
68
|
+
error.backtrace.shift
|
69
|
+
options[:exception] = error
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
log msg, options.merge(:level => 'error')
|
74
|
+
end
|
75
|
+
|
76
|
+
##########
|
77
|
+
|
78
|
+
def log(msg, options={})
|
79
|
+
console_log(msg)
|
80
|
+
syslog(msg, options)
|
81
|
+
rescue InvalidConfiguration => e
|
82
|
+
raise e
|
83
|
+
end
|
84
|
+
|
85
|
+
def default_error(e)
|
86
|
+
# avoid backtrace in /usr or vendor if possible
|
87
|
+
system, app = e.backtrace.partition { |b| b =~ /(^\/usr\/|vendor)/ }
|
88
|
+
reordered_backtrace = app + system
|
89
|
+
|
90
|
+
# avoid "/" as the method name (we want the controller action)
|
91
|
+
row = 0
|
92
|
+
row = 1 if reordered_backtrace[row].match(/in `\/'$/)
|
93
|
+
|
94
|
+
# get file and method name
|
95
|
+
begin
|
96
|
+
file, method = reordered_backtrace[row].match(/(.*):in `(.*)'$/)[1..2]
|
97
|
+
file.gsub!(/.*\//, '')
|
98
|
+
self.log "#{e.class} in #{file} #{method}: #{e.message}", :exception => e, :level => 'error'
|
99
|
+
rescue
|
100
|
+
self.log "#{e.class} in #{e.backtrace.first}: #{e.message}", :exception => e, :level => 'error'
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
def web_error(args)
|
105
|
+
e = args[:exception]
|
106
|
+
summary = "#{e.class} processing #{args[:url]}"
|
107
|
+
|
108
|
+
body = []
|
109
|
+
body << "\tURL: #{args[:url]}"
|
110
|
+
body << "\tParams: #{args[:params].inspect}"
|
111
|
+
body << "\tUser: #{args[:user]}" if args[:user]
|
112
|
+
body << "\tBacktrace:"
|
113
|
+
body << "\t\t#{e.class} (#{e.message}):"
|
114
|
+
body += e.backtrace.map { |l| "\t\t#{l}" }
|
115
|
+
body = body.join("\n")
|
116
|
+
|
117
|
+
log "#{summary}\n#{body}", :level => 'error'
|
118
|
+
end
|
119
|
+
|
120
|
+
def console_puts(*args)
|
121
|
+
$stderr.puts(*args)
|
122
|
+
end
|
123
|
+
|
124
|
+
def exception(e)
|
125
|
+
msg = "Exception #{e.class} -> #{e.message}\n"
|
126
|
+
msg += filtered_backtrace(e.backtrace)
|
127
|
+
Log.error e.message, :exception => e
|
128
|
+
end
|
129
|
+
|
130
|
+
def filtered_backtrace(backtrace)
|
131
|
+
backtrace.select do |line|
|
132
|
+
!line.match(/^\/usr/)
|
133
|
+
end.map do |line|
|
134
|
+
" #{line}\n"
|
135
|
+
end.join
|
136
|
+
end
|
137
|
+
|
138
|
+
def event(name, options = {})
|
139
|
+
console_log("EVENT: #{name} begins")
|
140
|
+
result = yield
|
141
|
+
console_log("EVENT: #{name} complete")
|
142
|
+
result
|
143
|
+
end
|
144
|
+
|
145
|
+
def context(options)
|
146
|
+
prev_options = default_options.dup
|
147
|
+
default_options.merge!(options)
|
148
|
+
yield
|
149
|
+
@@default_options = prev_options
|
150
|
+
end
|
151
|
+
|
152
|
+
def configure
|
153
|
+
config = Config.new
|
154
|
+
yield(config)
|
155
|
+
set_default_options(config.to_hash)
|
156
|
+
end
|
157
|
+
|
158
|
+
def set_default_options(options)
|
159
|
+
default_options.merge!(options)
|
160
|
+
end
|
161
|
+
|
162
|
+
def default_options
|
163
|
+
@@default_options ||= { :console_log => true }
|
164
|
+
end
|
165
|
+
|
166
|
+
def failsafe(params)
|
167
|
+
case default_options[:failsafe]
|
168
|
+
when :file
|
169
|
+
dir = defined?(RAILS_ROOT) ? RAILS_ROOT : '.'
|
170
|
+
File.open("#{dir}/failsafe.log", "a") do |f|
|
171
|
+
f.puts "#{Time.now} #{params[:log][:level]} : #{params[:log][:message]}"
|
172
|
+
end
|
173
|
+
when :console, nil
|
174
|
+
console_log(params[:log][:message])
|
175
|
+
end
|
176
|
+
end
|
177
|
+
|
178
|
+
def console_log(msg)
|
179
|
+
console_puts "#{Time.now.iso8601} #{msg}" if default_options[:console_log]
|
180
|
+
end
|
181
|
+
|
182
|
+
def syslog(msg, opts = {})
|
183
|
+
@retried = false
|
184
|
+
begin
|
185
|
+
level = SyslogConvertion[opts[:level]]
|
186
|
+
if opts[:exception]
|
187
|
+
msg += "\n" + format_syslog_exception(opts[:exception])
|
188
|
+
end
|
189
|
+
syslog_resource.log(level, '%s', trim_syslog_msg(msg))
|
190
|
+
rescue Exception => e
|
191
|
+
failsafe(:log => { :level => 'error', :message => "could not log to syslog: #{e.class.name} #{e.message}"})
|
192
|
+
unless @retried
|
193
|
+
@retried = true
|
194
|
+
@@syslog.close rescue nil
|
195
|
+
@@syslog = Syslog.open(default_options[:component]) rescue nil
|
196
|
+
retry
|
197
|
+
end
|
198
|
+
end
|
199
|
+
end
|
200
|
+
|
201
|
+
def format_syslog_exception(e)
|
202
|
+
if e.respond_to?(:backtrace)
|
203
|
+
"\t#{e.class}: #{e.message}\n" + e.backtrace.map { |t| "\t#{t}" }.join("\n")
|
204
|
+
else
|
205
|
+
"\t#{e.class}: #{e.message}"
|
206
|
+
end
|
207
|
+
end
|
208
|
+
|
209
|
+
def trim_syslog_msg(msg)
|
210
|
+
return msg if msg.size < 800
|
211
|
+
msg[0, 796] + " ..."
|
212
|
+
end
|
213
|
+
|
214
|
+
def syslog_resource
|
215
|
+
@@syslog ||= Syslog.open(default_options[:component].to_s, Syslog::LOG_PID | Syslog::LOG_CONS, default_options[:syslog_facility] || Syslog::LOG_USER)
|
216
|
+
end
|
217
|
+
end
|
218
|
+
|