droid 0.9.5 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +2 -0
- data/Rakefile +38 -0
- data/VERSION +1 -0
- data/bin/bleedq +19 -0
- data/droid.gemspec +100 -0
- data/examples/async_reply.rb +25 -0
- data/examples/heroku_async_reply.rb +22 -0
- data/examples/sync.rb +32 -0
- data/examples/worker.rb +58 -0
- data/lib/droid.rb +88 -476
- data/lib/droid/em.rb +55 -0
- data/lib/droid/heroku.rb +102 -0
- data/lib/droid/heroku/local_stats.rb +145 -0
- data/{vendor/logger_client/lib → lib/droid/heroku}/logger_client.rb +9 -5
- data/lib/droid/heroku/memcache_cluster.rb +129 -0
- data/lib/droid/heroku/stats.rb +30 -0
- data/lib/droid/json_server.rb +109 -0
- data/lib/droid/monkey.rb +8 -0
- data/lib/droid/publish.rb +24 -0
- data/lib/droid/queue.rb +196 -0
- data/lib/droid/request.rb +110 -0
- data/lib/droid/sync.rb +88 -0
- data/lib/droid/utilization.rb +113 -0
- data/lib/droid/utils.rb +113 -0
- data/lib/heroku_droid.rb +1 -86
- data/lib/local_stats.rb +1 -143
- data/lib/memcache_cluster.rb +1 -129
- data/lib/stats.rb +1 -30
- data/spec/publish_spec.rb +27 -0
- data/spec/response_spec.rb +47 -0
- data/spec/spec_helper.rb +11 -0
- data/spec/utils_spec.rb +43 -0
- data/{test/wait_for_port_test.rb → spec/wait_for_port_spec.rb} +2 -9
- metadata +109 -35
- data/README.md +0 -34
- data/lib/utilization.rb +0 -90
- data/test/base.rb +0 -43
- data/test/droid_test.rb +0 -53
- data/test/heroku_droid_test.rb +0 -42
- data/vendor/logger_client/Rakefile +0 -53
- data/vendor/logger_client/init.rb +0 -1
- data/vendor/logger_client/test.rb +0 -18
data/lib/droid/em.rb
ADDED
@@ -0,0 +1,55 @@
|
|
1
|
+
class Droid
|
2
|
+
module EMTimerUtils
|
3
|
+
def self.included(base)
|
4
|
+
base.extend(ClassMethods)
|
5
|
+
end
|
6
|
+
|
7
|
+
module ClassMethods
|
8
|
+
# Trap exceptions leaving the block and log them. Do not re-raise
|
9
|
+
def trap_exceptions
|
10
|
+
yield
|
11
|
+
rescue => e
|
12
|
+
em_exception(e)
|
13
|
+
end
|
14
|
+
|
15
|
+
def em_exception(e)
|
16
|
+
msg = format_em_exception(e)
|
17
|
+
log.error "[EM.timer] #{msg}", :exception => e
|
18
|
+
end
|
19
|
+
|
20
|
+
def format_em_exception(e)
|
21
|
+
# avoid backtrace in /usr or vendor if possible
|
22
|
+
system, app = e.backtrace.partition { |b| b =~ /(^\/usr\/|vendor)/ }
|
23
|
+
reordered_backtrace = app + system
|
24
|
+
|
25
|
+
# avoid "/" as the method name (we want the controller action)
|
26
|
+
row = 0
|
27
|
+
row = 1 if reordered_backtrace[row].match(/in `\/'$/)
|
28
|
+
|
29
|
+
# get file and method name
|
30
|
+
begin
|
31
|
+
file, method = reordered_backtrace[row].match(/(.*):in `(.*)'$/)[1..2]
|
32
|
+
file.gsub!(/.*\//, '')
|
33
|
+
"#{e.class} in #{file} #{method}: #{e.message}"
|
34
|
+
rescue
|
35
|
+
"#{e.class} in #{e.backtrace.first}: #{e.message}"
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
# One-shot timer
|
40
|
+
def timer(duration, &blk)
|
41
|
+
EM.add_timer(duration) { trap_exceptions(&blk) }
|
42
|
+
end
|
43
|
+
|
44
|
+
# Add a periodic timer. If the now argument is true, run the block
|
45
|
+
# immediately in addition to scheduling the periodic timer.
|
46
|
+
def periodic_timer(duration, now=false, &blk)
|
47
|
+
timer(1, &blk) if now
|
48
|
+
EM.add_periodic_timer(duration) { trap_exceptions(&blk) }
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def timer(*args, &blk); self.class.timer(*args, &blk); end
|
53
|
+
def periodic_timer(*args, &blk); self.class.periodic_timer(*args, &blk); end
|
54
|
+
end
|
55
|
+
end
|
data/lib/droid/heroku.rb
ADDED
@@ -0,0 +1,102 @@
|
|
1
|
+
require 'droid'
|
2
|
+
require 'droid/heroku/local_stats'
|
3
|
+
require 'droid/heroku/logger_client'
|
4
|
+
|
5
|
+
Log.configure do |c|
|
6
|
+
c.component = LocalStats.slot
|
7
|
+
c.instance_name = LocalStats.instance_name
|
8
|
+
end
|
9
|
+
|
10
|
+
Droid.log = Log
|
11
|
+
|
12
|
+
class Droid
|
13
|
+
module Utils
|
14
|
+
def self.generate_name_for_instance(name)
|
15
|
+
"#{name}.#{LocalStats.slot}.#{LocalStats.ion_instance_id}"
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
begin
|
21
|
+
require 'droid/json_server'
|
22
|
+
rescue LoadError => e
|
23
|
+
Droid.log.error "Could not load JSONServer", :exception => e
|
24
|
+
end
|
25
|
+
|
26
|
+
class HerokuDroid < Droid
|
27
|
+
attr_reader :extended_stats
|
28
|
+
|
29
|
+
def initialize(name, opts={}, &blk)
|
30
|
+
@extended_stats = !(opts[:extended_stats] == false)
|
31
|
+
|
32
|
+
super(name, opts) do |droid|
|
33
|
+
setup_standard_topics(self) unless opts[:standard_topics] == false
|
34
|
+
LocalStats.attach
|
35
|
+
blk.call(self)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def stats(&blk)
|
40
|
+
@stats = blk
|
41
|
+
out = call_stats
|
42
|
+
out = out.inspect unless out.kind_of?(String)
|
43
|
+
Log.notice out
|
44
|
+
end
|
45
|
+
|
46
|
+
def call_stats
|
47
|
+
@stats ? @stats.call : nil
|
48
|
+
end
|
49
|
+
|
50
|
+
def name
|
51
|
+
Droid.name
|
52
|
+
end
|
53
|
+
|
54
|
+
def ruby_path
|
55
|
+
if File.exists?("/usr/ruby1.8.6/bin/ruby")
|
56
|
+
"/usr/ruby1.8.6/bin"
|
57
|
+
else
|
58
|
+
"/usr/local/bin"
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def setup_standard_topics(droid)
|
63
|
+
setup_ping_topic(droid)
|
64
|
+
end
|
65
|
+
|
66
|
+
def setup_ping_topic(droid)
|
67
|
+
require 'time'
|
68
|
+
|
69
|
+
droid.listener('ping').subscribe do |req|
|
70
|
+
Droid::Utilization.latency = (Time.now.to_f - req['departed_at']).abs
|
71
|
+
end
|
72
|
+
|
73
|
+
blk = Proc.new do |d|
|
74
|
+
begin
|
75
|
+
t1 = Time.now
|
76
|
+
response = {}.merge(LocalStats.stats)
|
77
|
+
|
78
|
+
estats = nil
|
79
|
+
if self.extended_stats
|
80
|
+
estats = droid.call_stats
|
81
|
+
estats = { :notes => estats } unless estats.kind_of?(Hash)
|
82
|
+
estats[:notes] ||= estats.map { |k, v| "#{v} #{k}" }.join(", ")
|
83
|
+
estats.merge!(LocalStats.extended_stats)
|
84
|
+
end
|
85
|
+
|
86
|
+
response.merge!({
|
87
|
+
:extended_stats => estats,
|
88
|
+
:droid_name => Droid.name,
|
89
|
+
:latency => Droid::Utilization.latency,
|
90
|
+
})
|
91
|
+
|
92
|
+
d.publish('pong', response.merge(:stat_collection => (Time.now - t1)))
|
93
|
+
rescue Object => e
|
94
|
+
log.error "Ping Block Error: #{e.class} -> #{e.message}", :exception => e
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
EM.add_timer(2) { blk.call(droid) }
|
99
|
+
EM.add_periodic_timer(50 + (rand*15).to_i) { blk.call(droid) }
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
@@ -0,0 +1,145 @@
|
|
1
|
+
require 'rest_client'
|
2
|
+
|
3
|
+
module LocalStats
|
4
|
+
extend self
|
5
|
+
|
6
|
+
def load_avg
|
7
|
+
File.read('/proc/loadavg').split(' ', 2).first.to_f
|
8
|
+
end
|
9
|
+
|
10
|
+
def memory_info
|
11
|
+
info = {}
|
12
|
+
File.read('/proc/meminfo').split("\n").each do |line|
|
13
|
+
name, value = line.split(/:\s+/, 2)
|
14
|
+
info[name] = value.to_i
|
15
|
+
end
|
16
|
+
info
|
17
|
+
end
|
18
|
+
|
19
|
+
def memory_use
|
20
|
+
info = memory_info
|
21
|
+
used = info['MemTotal'] - info['MemFree'] - info['Cached'] - info['Buffers']
|
22
|
+
(100 * used / info['MemTotal']).to_i
|
23
|
+
end
|
24
|
+
|
25
|
+
def swap_use
|
26
|
+
_, total, use, _ = `free | grep Swap:`.strip.split
|
27
|
+
return 0 if total.to_i == 0
|
28
|
+
(100 * use.to_i / total.to_i).to_i
|
29
|
+
end
|
30
|
+
|
31
|
+
def local_ip
|
32
|
+
@local_ip ||= fetch_local_ip
|
33
|
+
end
|
34
|
+
|
35
|
+
def fetch_local_ip
|
36
|
+
`/sbin/ifconfig eth0 | grep inet | awk '{print $2}'`.gsub('addr:', '').strip
|
37
|
+
end
|
38
|
+
|
39
|
+
def public_ip
|
40
|
+
retries = 0
|
41
|
+
@public_ip ||= begin
|
42
|
+
RestClient.get("http://169.254.169.254/latest/meta-data/public-ipv4").to_s.strip
|
43
|
+
rescue RestClient::RequestTimeout => e
|
44
|
+
retries += 1
|
45
|
+
raise if retries > 5
|
46
|
+
sleep 1
|
47
|
+
retry
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def disk_stats
|
52
|
+
raw = `df -P | grep -v tmpfs | grep -v udev | grep -v slugs | awk '{print $1 " ## " $6 " ## " $5}'`.split("\n").collect { |d| d }
|
53
|
+
raw.shift
|
54
|
+
raw.collect do |d|
|
55
|
+
tokens = d.split("##").collect { |t| t.strip }
|
56
|
+
{
|
57
|
+
:device => tokens[0],
|
58
|
+
:path => tokens[1],
|
59
|
+
:usage => tokens[2].gsub('%','').to_i
|
60
|
+
}
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def ion_instance_id
|
65
|
+
@ion_instance_id ||= File.read('/etc/heroku/ion_instance_id').strip rescue nil
|
66
|
+
end
|
67
|
+
|
68
|
+
def slot
|
69
|
+
@slot ||= File.read('/etc/heroku/slot').strip.gsub(/64$/,'').split('-').first rescue nil
|
70
|
+
end
|
71
|
+
|
72
|
+
def instance_name
|
73
|
+
return nil if slot.nil? or ion_instance_id.nil?
|
74
|
+
"#{slot}.#{ion_instance_id}"
|
75
|
+
end
|
76
|
+
|
77
|
+
alias :this_instance_name :instance_name
|
78
|
+
|
79
|
+
def stats
|
80
|
+
{
|
81
|
+
:slot => slot,
|
82
|
+
:ion_id => ion_instance_id,
|
83
|
+
:load_avg => load_avg,
|
84
|
+
:memory_use => memory_use,
|
85
|
+
:swap_use => swap_use,
|
86
|
+
:local_ip => local_ip,
|
87
|
+
:instance_name => this_instance_name,
|
88
|
+
:public_ip => public_ip,
|
89
|
+
:net_in_rate => receive_rate,
|
90
|
+
:net_out_rate => transmit_rate
|
91
|
+
}
|
92
|
+
end
|
93
|
+
|
94
|
+
def extended_stats
|
95
|
+
{
|
96
|
+
:disk_stats => disk_stats
|
97
|
+
}
|
98
|
+
end
|
99
|
+
|
100
|
+
# sample ifstat every 30 seconds
|
101
|
+
IFSTAT_INTERVAL = 30
|
102
|
+
|
103
|
+
# bytes received per second and bytes transmitted per
|
104
|
+
# second as a two tuple
|
105
|
+
def transfer_rates
|
106
|
+
[receive_rate, transmit_rate]
|
107
|
+
end
|
108
|
+
|
109
|
+
# bytes received per second
|
110
|
+
def receive_rate
|
111
|
+
@rxrate || 0
|
112
|
+
end
|
113
|
+
|
114
|
+
# bytes transmitted per second
|
115
|
+
def transmit_rate
|
116
|
+
@txrate || 0
|
117
|
+
end
|
118
|
+
|
119
|
+
# sample the current RX and TX bytes from ifconfig and
|
120
|
+
# return as a two-tuple.
|
121
|
+
def sample
|
122
|
+
data = ifconfig.match(/RX bytes:(\d+).*TX bytes:(\d+)/)
|
123
|
+
[data[1].to_i, data[2].to_i]
|
124
|
+
rescue => boom
|
125
|
+
Log.notice "error sampling network rate: #{boom.class} #{boom.message}"
|
126
|
+
end
|
127
|
+
|
128
|
+
def update_counters
|
129
|
+
rx, tx = sample
|
130
|
+
@rxrate = (rx - @rx) / IFSTAT_INTERVAL if @rx
|
131
|
+
@txrate = (tx - @tx) / IFSTAT_INTERVAL if @tx
|
132
|
+
@rx, @tx = rx, tx
|
133
|
+
end
|
134
|
+
|
135
|
+
# called when a droid starts up - setup timers and whatnot
|
136
|
+
def attach
|
137
|
+
@rx, @tx, @rxrate, @txrate = nil
|
138
|
+
update_counters
|
139
|
+
EM.add_periodic_timer(IFSTAT_INTERVAL) { update_counters }
|
140
|
+
end
|
141
|
+
|
142
|
+
def ifconfig
|
143
|
+
`/sbin/ifconfig eth0`
|
144
|
+
end
|
145
|
+
end
|
@@ -7,7 +7,6 @@
|
|
7
7
|
# config.failsafe = :file
|
8
8
|
# end
|
9
9
|
|
10
|
-
require 'rubygems'
|
11
10
|
require 'time'
|
12
11
|
require 'syslog'
|
13
12
|
|
@@ -21,7 +20,7 @@ end
|
|
21
20
|
|
22
21
|
class Log::Config
|
23
22
|
def initialize
|
24
|
-
@contents = {
|
23
|
+
@contents = { }
|
25
24
|
end
|
26
25
|
|
27
26
|
def method_missing(method, value)
|
@@ -53,6 +52,8 @@ module Log
|
|
53
52
|
log msg, options.merge(:level => 'notice')
|
54
53
|
end
|
55
54
|
|
55
|
+
alias :info :notice
|
56
|
+
|
56
57
|
def warning(msg, options={})
|
57
58
|
log msg, options.merge(:level => 'warning')
|
58
59
|
end
|
@@ -116,10 +117,13 @@ module Log
|
|
116
117
|
log "#{summary}\n#{body}", :level => 'error'
|
117
118
|
end
|
118
119
|
|
120
|
+
def console_puts(*args)
|
121
|
+
$stderr.puts(*args)
|
122
|
+
end
|
123
|
+
|
119
124
|
def exception(e)
|
120
125
|
msg = "Exception #{e.class} -> #{e.message}\n"
|
121
126
|
msg += filtered_backtrace(e.backtrace)
|
122
|
-
STDERR.puts msg
|
123
127
|
Log.error e.message, :exception => e
|
124
128
|
end
|
125
129
|
|
@@ -156,7 +160,7 @@ module Log
|
|
156
160
|
end
|
157
161
|
|
158
162
|
def default_options
|
159
|
-
@@default_options ||= {}
|
163
|
+
@@default_options ||= { :console_log => true }
|
160
164
|
end
|
161
165
|
|
162
166
|
def failsafe(params)
|
@@ -172,7 +176,7 @@ module Log
|
|
172
176
|
end
|
173
177
|
|
174
178
|
def console_log(msg)
|
175
|
-
|
179
|
+
console_puts "#{Time.now.iso8601} #{msg}" if default_options[:console_log]
|
176
180
|
end
|
177
181
|
|
178
182
|
def syslog(msg, opts = {})
|
@@ -0,0 +1,129 @@
|
|
1
|
+
require 'memcache'
|
2
|
+
require 'yaml'
|
3
|
+
require 'digest/sha1'
|
4
|
+
|
5
|
+
# Manages a pool of memcache servers. This class should not be called
|
6
|
+
# outside of the reactor - it does not account for asynchronous access
|
7
|
+
# to the server list.
|
8
|
+
module MemcacheCluster
|
9
|
+
extend self
|
10
|
+
|
11
|
+
HEROKU_NAMESPACE = '0Xfa15837Z' # heroku's internal memcache namespace
|
12
|
+
|
13
|
+
# A MemCache object configured with heroku's internal memcache namespace.
|
14
|
+
def heroku
|
15
|
+
cache(HEROKU_NAMESPACE)
|
16
|
+
end
|
17
|
+
|
18
|
+
def cache_retry(prefix, opts={})
|
19
|
+
opts[:retries] ||= 5
|
20
|
+
opts[:delay] ||= 0.5
|
21
|
+
|
22
|
+
retried = 0
|
23
|
+
begin
|
24
|
+
c = cache(prefix)
|
25
|
+
yield c if block_given?
|
26
|
+
rescue MemCache::MemCacheError => e
|
27
|
+
Log.error "#{e.class} -> #{e.message}", :exception => e
|
28
|
+
raise if retried > opts[:retries]
|
29
|
+
retried += 1
|
30
|
+
sleep opts[:delay]
|
31
|
+
@caches = { }
|
32
|
+
retry
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def set(prefix, *args)
|
37
|
+
res = nil
|
38
|
+
cache_retry(prefix) do |c|
|
39
|
+
res = c.set(*args)
|
40
|
+
end
|
41
|
+
res
|
42
|
+
end
|
43
|
+
|
44
|
+
def get(prefix, *args)
|
45
|
+
res = nil
|
46
|
+
cache_retry(prefix) do |c|
|
47
|
+
res = c.get(*args)
|
48
|
+
end
|
49
|
+
res
|
50
|
+
end
|
51
|
+
|
52
|
+
# Create listeners for standard memcache cluster related topics.
|
53
|
+
def attach(droid, file='memcached.yml')
|
54
|
+
load_from_file(file)
|
55
|
+
|
56
|
+
droid.listen4('memcache.up', :queue => "memcache.up.#{LocalStats.this_instance_name}.#$$") { |msg| add(msg['address'], msg['port']) }
|
57
|
+
droid.listen4('instance.down', :queue => "instance.down.#{LocalStats.this_instance_name}.#$$") { |msg| remove(msg['local_ip']) if msg['slot'] == 'memcache' }
|
58
|
+
EM.add_timer(1) { droid.publish('memcache.needed', {}) }
|
59
|
+
end
|
60
|
+
|
61
|
+
# A MemCache object configured with the given prefix.
|
62
|
+
def cache(prefix, options={})
|
63
|
+
caches[prefix] ||=
|
64
|
+
MemCache.new(servers, options.merge(:namespace => prefix))
|
65
|
+
end
|
66
|
+
|
67
|
+
alias_method :[], :cache
|
68
|
+
|
69
|
+
def caches
|
70
|
+
reload_if_stale
|
71
|
+
@caches ||= {}
|
72
|
+
end
|
73
|
+
|
74
|
+
def servers
|
75
|
+
reload_if_stale
|
76
|
+
@servers ||= []
|
77
|
+
end
|
78
|
+
|
79
|
+
def add(ip, port)
|
80
|
+
host = [ip, port].join(':')
|
81
|
+
return if servers.include?(host)
|
82
|
+
|
83
|
+
log { "#{host} added" }
|
84
|
+
@servers.push host
|
85
|
+
@servers.sort!
|
86
|
+
@caches = {}
|
87
|
+
write_to_file
|
88
|
+
@last_read = Time.now
|
89
|
+
end
|
90
|
+
|
91
|
+
def remove(host)
|
92
|
+
if servers.reject!{ |s| s =~ /^#{host}/ }
|
93
|
+
log { "#{host} removed" }
|
94
|
+
caches.clear
|
95
|
+
write_to_file
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
def reload_if_stale
|
100
|
+
if @last_read &&
|
101
|
+
(Time.now - @last_read) > 5 &&
|
102
|
+
File.mtime(@file) > @last_read
|
103
|
+
log { "server list modified. reloading." }
|
104
|
+
load_from_file(@file)
|
105
|
+
end
|
106
|
+
rescue => boom
|
107
|
+
# ignore errors accessing/reading file.
|
108
|
+
end
|
109
|
+
|
110
|
+
def load_from_file(file)
|
111
|
+
@file = file
|
112
|
+
@last_read = Time.now
|
113
|
+
@servers = YAML.load(File.read(file)) rescue []
|
114
|
+
@caches = {}
|
115
|
+
end
|
116
|
+
|
117
|
+
def write_to_file
|
118
|
+
log { "writing server list: #{@file}" }
|
119
|
+
File.open(@file, 'w') do |f|
|
120
|
+
f.flock(File::LOCK_EX)
|
121
|
+
f.write YAML.dump(@servers)
|
122
|
+
f.flock(File::LOCK_UN)
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
def log(type=:debug)
|
127
|
+
Log.send(type, "memcached: #{yield}")
|
128
|
+
end
|
129
|
+
end
|