droid 1.0.0 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. data/VERSION +1 -1
  2. data/droid.gemspec +2 -2
  3. data/lib/droid/heroku/local_stats.rb +141 -141
  4. data/lib/droid/heroku/logger_client.rb +193 -193
  5. data/lib/droid/heroku/memcache_cluster.rb +143 -120
  6. data/lib/droid/heroku/stats.rb +25 -25
  7. data/lib/droid/utilization.rb +6 -6
  8. metadata +3 -3
data/VERSION CHANGED
@@ -1 +1 @@
1
- 1.0.0
1
+ 1.0.1
data/droid.gemspec CHANGED
@@ -5,11 +5,11 @@
5
5
 
6
6
  Gem::Specification.new do |s|
7
7
  s.name = %q{droid}
8
- s.version = "1.0.0"
8
+ s.version = "1.0.1"
9
9
 
10
10
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
11
  s.authors = ["Ricardo Chimal, Jr."]
12
- s.date = %q{2010-10-21}
12
+ s.date = %q{2010-11-12}
13
13
  s.default_executable = %q{bleedq}
14
14
  s.description = %q{Easy to use AMQP Library with constructs for typical usage patterns}
15
15
  s.email = %q{ricardo@heroku.com}
data/lib/droid/heroku/local_stats.rb CHANGED
@@ -1,145 +1,145 @@
1
1
  require 'rest_client'
2
2
 
3
3
  module LocalStats
4
- extend self
5
-
6
- def load_avg
7
- File.read('/proc/loadavg').split(' ', 2).first.to_f
8
- end
9
-
10
- def memory_info
11
- info = {}
12
- File.read('/proc/meminfo').split("\n").each do |line|
13
- name, value = line.split(/:\s+/, 2)
14
- info[name] = value.to_i
15
- end
16
- info
17
- end
18
-
19
- def memory_use
20
- info = memory_info
21
- used = info['MemTotal'] - info['MemFree'] - info['Cached'] - info['Buffers']
22
- (100 * used / info['MemTotal']).to_i
23
- end
24
-
25
- def swap_use
26
- _, total, use, _ = `free | grep Swap:`.strip.split
27
- return 0 if total.to_i == 0
28
- (100 * use.to_i / total.to_i).to_i
29
- end
30
-
31
- def local_ip
32
- @local_ip ||= fetch_local_ip
33
- end
34
-
35
- def fetch_local_ip
36
- `/sbin/ifconfig eth0 | grep inet | awk '{print $2}'`.gsub('addr:', '').strip
37
- end
38
-
39
- def public_ip
40
- retries = 0
41
- @public_ip ||= begin
42
- RestClient.get("http://169.254.169.254/latest/meta-data/public-ipv4").to_s.strip
43
- rescue RestClient::RequestTimeout => e
44
- retries += 1
45
- raise if retries > 5
46
- sleep 1
47
- retry
48
- end
49
- end
50
-
51
- def disk_stats
52
- raw = `df -P | grep -v tmpfs | grep -v udev | grep -v slugs | awk '{print $1 " ## " $6 " ## " $5}'`.split("\n").collect { |d| d }
53
- raw.shift
54
- raw.collect do |d|
55
- tokens = d.split("##").collect { |t| t.strip }
56
- {
57
- :device => tokens[0],
58
- :path => tokens[1],
59
- :usage => tokens[2].gsub('%','').to_i
60
- }
61
- end
62
- end
63
-
64
- def ion_instance_id
65
- @ion_instance_id ||= File.read('/etc/heroku/ion_instance_id').strip rescue nil
66
- end
67
-
68
- def slot
69
- @slot ||= File.read('/etc/heroku/slot').strip.gsub(/64$/,'').split('-').first rescue nil
70
- end
71
-
72
- def instance_name
73
- return nil if slot.nil? or ion_instance_id.nil?
74
- "#{slot}.#{ion_instance_id}"
75
- end
76
-
77
- alias :this_instance_name :instance_name
78
-
79
- def stats
80
- {
81
- :slot => slot,
82
- :ion_id => ion_instance_id,
83
- :load_avg => load_avg,
84
- :memory_use => memory_use,
85
- :swap_use => swap_use,
86
- :local_ip => local_ip,
87
- :instance_name => this_instance_name,
88
- :public_ip => public_ip,
89
- :net_in_rate => receive_rate,
90
- :net_out_rate => transmit_rate
91
- }
92
- end
93
-
94
- def extended_stats
95
- {
96
- :disk_stats => disk_stats
97
- }
98
- end
99
-
100
- # sample ifstat every 30 seconds
101
- IFSTAT_INTERVAL = 30
102
-
103
- # bytes received per second and bytes transmitted per
104
- # second as a two tuple
105
- def transfer_rates
106
- [receive_rate, transmit_rate]
107
- end
108
-
109
- # bytes received per second
110
- def receive_rate
111
- @rxrate || 0
112
- end
113
-
114
- # bytes transmitted per second
115
- def transmit_rate
116
- @txrate || 0
117
- end
118
-
119
- # sample the current RX and TX bytes from ifconfig and
120
- # return as a two-tuple.
121
- def sample
122
- data = ifconfig.match(/RX bytes:(\d+).*TX bytes:(\d+)/)
123
- [data[1].to_i, data[2].to_i]
124
- rescue => boom
125
- Log.notice "error sampling network rate: #{boom.class} #{boom.message}"
126
- end
127
-
128
- def update_counters
129
- rx, tx = sample
130
- @rxrate = (rx - @rx) / IFSTAT_INTERVAL if @rx
131
- @txrate = (tx - @tx) / IFSTAT_INTERVAL if @tx
132
- @rx, @tx = rx, tx
133
- end
134
-
135
- # called when a droid starts up - setup timers and whatnot
136
- def attach
137
- @rx, @tx, @rxrate, @txrate = nil
138
- update_counters
139
- EM.add_periodic_timer(IFSTAT_INTERVAL) { update_counters }
140
- end
141
-
142
- def ifconfig
143
- `/sbin/ifconfig eth0`
144
- end
4
+ extend self
5
+
6
+ def load_avg
7
+ File.read('/proc/loadavg').split(' ', 2).first.to_f
8
+ end
9
+
10
+ def memory_info
11
+ info = {}
12
+ File.read('/proc/meminfo').split("\n").each do |line|
13
+ name, value = line.split(/:\s+/, 2)
14
+ info[name] = value.to_i
15
+ end
16
+ info
17
+ end
18
+
19
+ def memory_use
20
+ info = memory_info
21
+ used = info['MemTotal'] - info['MemFree'] - info['Cached'] - info['Buffers']
22
+ (100 * used / info['MemTotal']).to_i
23
+ end
24
+
25
+ def swap_use
26
+ _, total, use, _ = `free | grep Swap:`.strip.split
27
+ return 0 if total.to_i == 0
28
+ (100 * use.to_i / total.to_i).to_i
29
+ end
30
+
31
+ def local_ip
32
+ @local_ip ||= fetch_local_ip
33
+ end
34
+
35
+ def fetch_local_ip
36
+ `/sbin/ifconfig eth0 | grep inet | awk '{print $2}'`.gsub('addr:', '').strip
37
+ end
38
+
39
+ def public_ip
40
+ retries = 0
41
+ @public_ip ||= begin
42
+ RestClient.get("http://169.254.169.254/latest/meta-data/public-ipv4").to_s.strip
43
+ rescue RestClient::RequestTimeout => e
44
+ retries += 1
45
+ raise if retries > 5
46
+ sleep 1
47
+ retry
48
+ end
49
+ end
50
+
51
+ def disk_stats
52
+ raw = `df -P | grep -v tmpfs | grep -v udev | grep -v slugs | awk '{print $1 " ## " $6 " ## " $5}'`.split("\n").collect { |d| d }
53
+ raw.shift
54
+ raw.collect do |d|
55
+ tokens = d.split("##").collect { |t| t.strip }
56
+ {
57
+ :device => tokens[0],
58
+ :path => tokens[1],
59
+ :usage => tokens[2].gsub('%','').to_i
60
+ }
61
+ end
62
+ end
63
+
64
+ def ion_instance_id
65
+ @ion_instance_id ||= File.read('/etc/heroku/ion_instance_id').strip rescue nil
66
+ end
67
+
68
+ def slot
69
+ @slot ||= File.read('/etc/heroku/slot').strip.gsub(/64$/,'').split('-').first rescue nil
70
+ end
71
+
72
+ def instance_name
73
+ return nil if slot.nil? or ion_instance_id.nil?
74
+ "#{slot}.#{ion_instance_id}"
75
+ end
76
+
77
+ alias :this_instance_name :instance_name
78
+
79
+ def stats
80
+ {
81
+ :slot => slot,
82
+ :ion_id => ion_instance_id,
83
+ :load_avg => load_avg,
84
+ :memory_use => memory_use,
85
+ :swap_use => swap_use,
86
+ :local_ip => local_ip,
87
+ :instance_name => this_instance_name,
88
+ :public_ip => public_ip,
89
+ :net_in_rate => receive_rate,
90
+ :net_out_rate => transmit_rate
91
+ }
92
+ end
93
+
94
+ def extended_stats
95
+ {
96
+ :disk_stats => disk_stats
97
+ }
98
+ end
99
+
100
+ # sample ifstat every 30 seconds
101
+ IFSTAT_INTERVAL = 30
102
+
103
+ # bytes received per second and bytes transmitted per
104
+ # second as a two tuple
105
+ def transfer_rates
106
+ [receive_rate, transmit_rate]
107
+ end
108
+
109
+ # bytes received per second
110
+ def receive_rate
111
+ @rxrate || 0
112
+ end
113
+
114
+ # bytes transmitted per second
115
+ def transmit_rate
116
+ @txrate || 0
117
+ end
118
+
119
+ # sample the current RX and TX bytes from ifconfig and
120
+ # return as a two-tuple.
121
+ def sample
122
+ data = ifconfig.match(/RX bytes:(\d+).*TX bytes:(\d+)/)
123
+ [data[1].to_i, data[2].to_i]
124
+ rescue => boom
125
+ Log.notice "error sampling network rate: #{boom.class} #{boom.message}"
126
+ end
127
+
128
+ def update_counters
129
+ rx, tx = sample
130
+ @rxrate = (rx - @rx) / IFSTAT_INTERVAL if @rx
131
+ @txrate = (tx - @tx) / IFSTAT_INTERVAL if @tx
132
+ @rx, @tx = rx, tx
133
+ end
134
+
135
+ # called when a droid starts up - setup timers and whatnot
136
+ def attach
137
+ @rx, @tx, @rxrate, @txrate = nil
138
+ update_counters
139
+ EM.add_periodic_timer(IFSTAT_INTERVAL) { update_counters }
140
+ end
141
+
142
+ def ifconfig
143
+ `/sbin/ifconfig eth0`
144
+ end
145
145
  end
data/lib/droid/heroku/logger_client.rb CHANGED
@@ -13,206 +13,206 @@ require 'syslog'
13
13
  module Log; end
14
14
 
15
15
  class Log::InvalidConfiguration < RuntimeError
16
- def message
17
- "Invalid component. Configure with Log.configure { |c| c.component = 'myComponent' }"
18
- end
16
+ def message
17
+ "Invalid component. Configure with Log.configure { |c| c.component = 'myComponent' }"
18
+ end
19
19
  end
20
20
 
21
21
  class Log::Config
22
- def initialize
23
- @contents = { }
24
- end
22
+ def initialize
23
+ @contents = { }
24
+ end
25
25
 
26
- def method_missing(method, value)
27
- @contents[method.to_s.gsub(/=$/, '').to_sym] = value
28
- end
26
+ def method_missing(method, value)
27
+ @contents[method.to_s.gsub(/=$/, '').to_sym] = value
28
+ end
29
29
 
30
- def to_hash
31
- @contents
32
- end
30
+ def to_hash
31
+ @contents
32
+ end
33
33
  end
34
34
 
35
35
  module Log
36
- extend self
37
-
38
- unless defined? SyslogConvertion
39
- SyslogConvertion = {
40
- 'error' => 3,
41
- 'warning' => 4,
42
- 'notice' => 5,
43
- 'debug' => 7,
44
- }
45
- end
46
-
47
- def debug(msg, options={})
48
- log msg, options.merge(:level => 'debug')
49
- end
50
-
51
- def notice(msg, options={})
52
- log msg, options.merge(:level => 'notice')
53
- end
54
-
55
- alias :info :notice
56
-
57
- def warning(msg, options={})
58
- log msg, options.merge(:level => 'warning')
59
- end
60
-
61
- def error(msg, options={})
62
- # Fake an exception for error messages with no exception object
63
- # so that we get a backtrace.
64
- if options[:exception].nil?
65
- begin
66
- raise StandardError, msg
67
- rescue => error
68
- error.backtrace.shift
69
- options[:exception] = error
70
- end
71
- end
72
-
73
- log msg, options.merge(:level => 'error')
74
- end
75
-
76
- ##########
77
-
78
- def log(msg, options={})
79
- console_log(msg)
80
- syslog(msg, options)
81
- rescue InvalidConfiguration => e
82
- raise e
83
- end
84
-
85
- def default_error(e)
86
- # avoid backtrace in /usr or vendor if possible
87
- system, app = e.backtrace.partition { |b| b =~ /(^\/usr\/|vendor)/ }
88
- reordered_backtrace = app + system
89
-
90
- # avoid "/" as the method name (we want the controller action)
91
- row = 0
92
- row = 1 if reordered_backtrace[row].match(/in `\/'$/)
93
-
94
- # get file and method name
95
- begin
96
- file, method = reordered_backtrace[row].match(/(.*):in `(.*)'$/)[1..2]
97
- file.gsub!(/.*\//, '')
98
- self.log "#{e.class} in #{file} #{method}: #{e.message}", :exception => e, :level => 'error'
99
- rescue
100
- self.log "#{e.class} in #{e.backtrace.first}: #{e.message}", :exception => e, :level => 'error'
101
- end
102
- end
103
-
104
- def web_error(args)
105
- e = args[:exception]
106
- summary = "#{e.class} processing #{args[:url]}"
107
-
108
- body = []
109
- body << "\tURL: #{args[:url]}"
110
- body << "\tParams: #{args[:params].inspect}"
111
- body << "\tUser: #{args[:user]}" if args[:user]
112
- body << "\tBacktrace:"
113
- body << "\t\t#{e.class} (#{e.message}):"
114
- body += e.backtrace.map { |l| "\t\t#{l}" }
115
- body = body.join("\n")
116
-
117
- log "#{summary}\n#{body}", :level => 'error'
118
- end
119
-
120
- def console_puts(*args)
121
- $stderr.puts(*args)
122
- end
123
-
124
- def exception(e)
125
- msg = "Exception #{e.class} -> #{e.message}\n"
126
- msg += filtered_backtrace(e.backtrace)
127
- Log.error e.message, :exception => e
128
- end
129
-
130
- def filtered_backtrace(backtrace)
131
- backtrace.select do |line|
132
- !line.match(/^\/usr/)
133
- end.map do |line|
134
- " #{line}\n"
135
- end.join
136
- end
137
-
138
- def event(name, options = {})
139
- console_log("EVENT: #{name} begins")
140
- result = yield
141
- console_log("EVENT: #{name} complete")
142
- result
143
- end
144
-
145
- def context(options)
146
- prev_options = default_options.dup
147
- default_options.merge!(options)
148
- yield
149
- @@default_options = prev_options
150
- end
151
-
152
- def configure
153
- config = Config.new
154
- yield(config)
155
- set_default_options(config.to_hash)
156
- end
157
-
158
- def set_default_options(options)
159
- default_options.merge!(options)
160
- end
161
-
162
- def default_options
163
- @@default_options ||= { :console_log => true }
164
- end
165
-
166
- def failsafe(params)
167
- case default_options[:failsafe]
168
- when :file
169
- dir = defined?(RAILS_ROOT) ? RAILS_ROOT : '.'
170
- File.open("#{dir}/failsafe.log", "a") do |f|
171
- f.puts "#{Time.now} #{params[:log][:level]} : #{params[:log][:message]}"
172
- end
173
- when :console, nil
174
- console_log(params[:log][:message])
175
- end
176
- end
177
-
178
- def console_log(msg)
179
- console_puts "#{Time.now.iso8601} #{msg}" if default_options[:console_log]
180
- end
181
-
182
- def syslog(msg, opts = {})
183
- @retried = false
184
- begin
185
- level = SyslogConvertion[opts[:level]]
186
- if opts[:exception]
187
- msg += "\n" + format_syslog_exception(opts[:exception])
188
- end
189
- syslog_resource.log(level, '%s', trim_syslog_msg(msg))
190
- rescue Exception => e
191
- failsafe(:log => { :level => 'error', :message => "could not log to syslog: #{e.class.name} #{e.message}"})
192
- unless @retried
193
- @retried = true
194
- @@syslog.close rescue nil
195
- @@syslog = Syslog.open(default_options[:component]) rescue nil
196
- retry
197
- end
198
- end
199
- end
200
-
201
- def format_syslog_exception(e)
202
- if e.respond_to?(:backtrace)
203
- "\t#{e.class}: #{e.message}\n" + e.backtrace.map { |t| "\t#{t}" }.join("\n")
204
- else
205
- "\t#{e.class}: #{e.message}"
206
- end
207
- end
208
-
209
- def trim_syslog_msg(msg)
210
- return msg if msg.size < 800
211
- msg[0, 796] + " ..."
212
- end
213
-
214
- def syslog_resource
215
- @@syslog ||= Syslog.open(default_options[:component].to_s, Syslog::LOG_PID | Syslog::LOG_CONS, default_options[:syslog_facility] || Syslog::LOG_USER)
216
- end
36
+ extend self
37
+
38
+ unless defined? SyslogConvertion
39
+ SyslogConvertion = {
40
+ 'error' => 3,
41
+ 'warning' => 4,
42
+ 'notice' => 5,
43
+ 'debug' => 7,
44
+ }
45
+ end
46
+
47
+ def debug(msg, options={})
48
+ log msg, options.merge(:level => 'debug')
49
+ end
50
+
51
+ def notice(msg, options={})
52
+ log msg, options.merge(:level => 'notice')
53
+ end
54
+
55
+ alias :info :notice
56
+
57
+ def warning(msg, options={})
58
+ log msg, options.merge(:level => 'warning')
59
+ end
60
+
61
+ def error(msg, options={})
62
+ # Fake an exception for error messages with no exception object
63
+ # so that we get a backtrace.
64
+ if options[:exception].nil?
65
+ begin
66
+ raise StandardError, msg
67
+ rescue => error
68
+ error.backtrace.shift
69
+ options[:exception] = error
70
+ end
71
+ end
72
+
73
+ log msg, options.merge(:level => 'error')
74
+ end
75
+
76
+ ##########
77
+
78
+ def log(msg, options={})
79
+ console_log(msg)
80
+ syslog(msg, options)
81
+ rescue InvalidConfiguration => e
82
+ raise e
83
+ end
84
+
85
+ def default_error(e)
86
+ # avoid backtrace in /usr or vendor if possible
87
+ system, app = e.backtrace.partition { |b| b =~ /(^\/usr\/|vendor)/ }
88
+ reordered_backtrace = app + system
89
+
90
+ # avoid "/" as the method name (we want the controller action)
91
+ row = 0
92
+ row = 1 if reordered_backtrace[row].match(/in `\/'$/)
93
+
94
+ # get file and method name
95
+ begin
96
+ file, method = reordered_backtrace[row].match(/(.*):in `(.*)'$/)[1..2]
97
+ file.gsub!(/.*\//, '')
98
+ self.log "#{e.class} in #{file} #{method}: #{e.message}", :exception => e, :level => 'error'
99
+ rescue
100
+ self.log "#{e.class} in #{e.backtrace.first}: #{e.message}", :exception => e, :level => 'error'
101
+ end
102
+ end
103
+
104
+ def web_error(args)
105
+ e = args[:exception]
106
+ summary = "#{e.class} processing #{args[:url]}"
107
+
108
+ body = []
109
+ body << "\tURL: #{args[:url]}"
110
+ body << "\tParams: #{args[:params].inspect}"
111
+ body << "\tUser: #{args[:user]}" if args[:user]
112
+ body << "\tBacktrace:"
113
+ body << "\t\t#{e.class} (#{e.message}):"
114
+ body += e.backtrace.map { |l| "\t\t#{l}" }
115
+ body = body.join("\n")
116
+
117
+ log "#{summary}\n#{body}", :level => 'error'
118
+ end
119
+
120
+ def console_puts(*args)
121
+ $stderr.puts(*args)
122
+ end
123
+
124
+ def exception(e)
125
+ msg = "Exception #{e.class} -> #{e.message}\n"
126
+ msg += filtered_backtrace(e.backtrace)
127
+ Log.error e.message, :exception => e
128
+ end
129
+
130
+ def filtered_backtrace(backtrace)
131
+ backtrace.select do |line|
132
+ !line.match(/^\/usr/)
133
+ end.map do |line|
134
+ " #{line}\n"
135
+ end.join
136
+ end
137
+
138
+ def event(name, options = {})
139
+ console_log("EVENT: #{name} begins")
140
+ result = yield
141
+ console_log("EVENT: #{name} complete")
142
+ result
143
+ end
144
+
145
+ def context(options)
146
+ prev_options = default_options.dup
147
+ default_options.merge!(options)
148
+ yield
149
+ @@default_options = prev_options
150
+ end
151
+
152
+ def configure
153
+ config = Config.new
154
+ yield(config)
155
+ set_default_options(config.to_hash)
156
+ end
157
+
158
+ def set_default_options(options)
159
+ default_options.merge!(options)
160
+ end
161
+
162
+ def default_options
163
+ @@default_options ||= { :console_log => true }
164
+ end
165
+
166
+ def failsafe(params)
167
+ case default_options[:failsafe]
168
+ when :file
169
+ dir = defined?(RAILS_ROOT) ? RAILS_ROOT : '.'
170
+ File.open("#{dir}/failsafe.log", "a") do |f|
171
+ f.puts "#{Time.now} #{params[:log][:level]} : #{params[:log][:message]}"
172
+ end
173
+ when :console, nil
174
+ console_log(params[:log][:message])
175
+ end
176
+ end
177
+
178
+ def console_log(msg)
179
+ console_puts "#{Time.now.iso8601} #{msg}" if default_options[:console_log]
180
+ end
181
+
182
+ def syslog(msg, opts = {})
183
+ @retried = false
184
+ begin
185
+ level = SyslogConvertion[opts[:level]]
186
+ if opts[:exception]
187
+ msg += "\n" + format_syslog_exception(opts[:exception])
188
+ end
189
+ syslog_resource.log(level, '%s', trim_syslog_msg(msg))
190
+ rescue Exception => e
191
+ failsafe(:log => { :level => 'error', :message => "could not log to syslog: #{e.class.name} #{e.message}"})
192
+ unless @retried
193
+ @retried = true
194
+ @@syslog.close rescue nil
195
+ @@syslog = Syslog.open(default_options[:component]) rescue nil
196
+ retry
197
+ end
198
+ end
199
+ end
200
+
201
+ def format_syslog_exception(e)
202
+ if e.respond_to?(:backtrace)
203
+ "\t#{e.class}: #{e.message}\n" + e.backtrace.map { |t| "\t#{t}" }.join("\n")
204
+ else
205
+ "\t#{e.class}: #{e.message}"
206
+ end
207
+ end
208
+
209
+ def trim_syslog_msg(msg)
210
+ return msg if msg.size < 800
211
+ msg[0, 796] + " ..."
212
+ end
213
+
214
+ def syslog_resource
215
+ @@syslog ||= Syslog.open(default_options[:component].to_s, Syslog::LOG_PID | Syslog::LOG_CONS, default_options[:syslog_facility] || Syslog::LOG_USER)
216
+ end
217
217
  end
218
218
 
data/lib/droid/heroku/memcache_cluster.rb CHANGED
@@ -6,124 +6,147 @@ require 'digest/sha1'
6
6
  # outside of the reactor - it does not account for asynchronous access
7
7
  # to the server list.
8
8
  module MemcacheCluster
9
- extend self
10
-
11
- HEROKU_NAMESPACE = '0Xfa15837Z' # heroku's internal memcache namespace
12
-
13
- # A MemCache object configured with heroku's internal memcache namespace.
14
- def heroku
15
- cache(HEROKU_NAMESPACE)
16
- end
17
-
18
- def cache_retry(prefix, opts={})
19
- opts[:retries] ||= 5
20
- opts[:delay] ||= 0.5
21
-
22
- retried = 0
23
- begin
24
- c = cache(prefix)
25
- yield c if block_given?
26
- rescue MemCache::MemCacheError => e
27
- Log.error "#{e.class} -> #{e.message}", :exception => e
28
- raise if retried > opts[:retries]
29
- retried += 1
30
- sleep opts[:delay]
31
- @caches = { }
32
- retry
33
- end
34
- end
35
-
36
- def set(prefix, *args)
37
- res = nil
38
- cache_retry(prefix) do |c|
39
- res = c.set(*args)
40
- end
41
- res
42
- end
43
-
44
- def get(prefix, *args)
45
- res = nil
46
- cache_retry(prefix) do |c|
47
- res = c.get(*args)
48
- end
49
- res
50
- end
51
-
52
- # Create listeners for standard memcache cluster related topics.
53
- def attach(droid, file='memcached.yml')
54
- load_from_file(file)
55
-
56
- droid.listen4('memcache.up', :queue => "memcache.up.#{LocalStats.this_instance_name}.#$$") { |msg| add(msg['address'], msg['port']) }
57
- droid.listen4('instance.down', :queue => "instance.down.#{LocalStats.this_instance_name}.#$$") { |msg| remove(msg['local_ip']) if msg['slot'] == 'memcache' }
58
- EM.add_timer(1) { droid.publish('memcache.needed', {}) }
59
- end
60
-
61
- # A MemCache object configured with the given prefix.
62
- def cache(prefix, options={})
63
- caches[prefix] ||=
64
- MemCache.new(servers, options.merge(:namespace => prefix))
65
- end
66
-
67
- alias_method :[], :cache
68
-
69
- def caches
70
- reload_if_stale
71
- @caches ||= {}
72
- end
73
-
74
- def servers
75
- reload_if_stale
76
- @servers ||= []
77
- end
78
-
79
- def add(ip, port)
80
- host = [ip, port].join(':')
81
- return if servers.include?(host)
82
-
83
- log { "#{host} added" }
84
- @servers.push host
85
- @servers.sort!
86
- @caches = {}
87
- write_to_file
88
- @last_read = Time.now
89
- end
90
-
91
- def remove(host)
92
- if servers.reject!{ |s| s =~ /^#{host}/ }
93
- log { "#{host} removed" }
94
- caches.clear
95
- write_to_file
96
- end
97
- end
98
-
99
- def reload_if_stale
100
- if @last_read &&
101
- (Time.now - @last_read) > 5 &&
102
- File.mtime(@file) > @last_read
103
- log { "server list modified. reloading." }
104
- load_from_file(@file)
105
- end
106
- rescue => boom
107
- # ignore errors accessing/reading file.
108
- end
109
-
110
- def load_from_file(file)
111
- @file = file
112
- @last_read = Time.now
113
- @servers = YAML.load(File.read(file)) rescue []
114
- @caches = {}
115
- end
116
-
117
- def write_to_file
118
- log { "writing server list: #{@file}" }
119
- File.open(@file, 'w') do |f|
120
- f.flock(File::LOCK_EX)
121
- f.write YAML.dump(@servers)
122
- f.flock(File::LOCK_UN)
123
- end
124
- end
125
-
126
- def log(type=:debug)
127
- Log.send(type, "memcached: #{yield}")
128
- end
9
+ extend self
10
+
11
+ HEROKU_NAMESPACE = '0Xfa15837Z' # heroku's internal memcache namespace
12
+
13
+ class Proxy
14
+ def initialize(name, options={})
15
+ @name = name
16
+ @options = options
17
+ end
18
+
19
+ def set(*args)
20
+ MemcacheCluster.cache_internal(@name, @options).set(*args)
21
+ end
22
+
23
+ def get(*args)
24
+ MemcacheCluster.cache_internal(@name, @options).get(*args)
25
+ end
26
+
27
+ def method_missing(method_name, *args)
28
+ MemcacheCluster.cache_internal(@name, @options).send(method_name, *args)
29
+ end
30
+ end
31
+
32
+ # A MemCache object configured with heroku's internal memcache namespace.
33
+ def heroku
34
+ cache(HEROKU_NAMESPACE)
35
+ end
36
+
37
+ def cache_retry(prefix, opts={})
38
+ opts[:retries] ||= 5
39
+ opts[:delay] ||= 0.5
40
+
41
+ retried = 0
42
+ begin
43
+ c = cache_internal(prefix)
44
+ yield c if block_given?
45
+ rescue MemCache::MemCacheError => e
46
+ Log.error "#{e.class} -> #{e.message}", :exception => e
47
+ raise if retried > opts[:retries]
48
+ retried += 1
49
+ sleep opts[:delay]
50
+ @caches = { }
51
+ retry
52
+ end
53
+ end
54
+
55
+ def set(prefix, *args)
56
+ res = nil
57
+ cache_retry(prefix) do |c|
58
+ res = c.set(*args)
59
+ end
60
+ res
61
+ end
62
+
63
+ def get(prefix, *args)
64
+ res = nil
65
+ cache_retry(prefix) do |c|
66
+ res = c.get(*args)
67
+ end
68
+ res
69
+ end
70
+
71
+ # Create listeners for standard memcache cluster related topics.
72
+ def attach(droid, file='memcached.yml')
73
+ load_from_file(file)
74
+
75
+ droid.listen4('memcache.up', :queue => "memcache.up.#{LocalStats.this_instance_name}.#$$") { |msg| add(msg['address'], msg['port']) }
76
+ droid.listen4('instance.down', :queue => "instance.down.#{LocalStats.this_instance_name}.#$$") { |msg| remove(msg['local_ip']) if msg['slot'] == 'memcache' }
77
+ EM.add_timer(1) { droid.publish('memcache.needed', {}) }
78
+ end
79
+
80
+ # A MemCache object configured with the given prefix.
81
+ def cache_internal(prefix, options={})
82
+ caches[prefix] ||=
83
+ MemCache.new(servers, options.merge(:namespace => prefix))
84
+ end
85
+
86
+ def cache(prefix, options={})
87
+ Proxy.new(prefix, options)
88
+ end
89
+
90
+ alias_method :[], :cache
91
+
92
+ def caches
93
+ reload_if_stale
94
+ @caches ||= {}
95
+ end
96
+
97
+ def servers
98
+ reload_if_stale
99
+ @servers ||= []
100
+ end
101
+
102
+ def add(ip, port)
103
+ host = [ip, port].join(':')
104
+ return if servers.include?(host)
105
+
106
+ log { "action=added server=#{host}" }
107
+ @servers.push host
108
+ @servers.sort!
109
+ @caches = {}
110
+ write_to_file
111
+ @last_read = Time.now.to_i
112
+ end
113
+
114
+ def remove(host)
115
+ if servers.reject!{ |s| s =~ /^#{host}/ }
116
+ log { "action=remove server=#{host}" }
117
+ caches.clear
118
+ write_to_file
119
+ end
120
+ end
121
+
122
+ def reload_if_stale
123
+ if @last_read &&
124
+ (Time.now.to_i - @last_read) > 20 &&
125
+ File.mtime(@file).to_i > @last_read
126
+ load_from_file(@file)
127
+ end
128
+ rescue => e
129
+ log { "action=error file=#{@file} error_class='#{e.class}' error_message='#{e.message}'" }
130
+ end
131
+
132
+ def load_from_file(file)
133
+ @file = file
134
+ @last_read = Time.now.to_i
135
+ @servers = YAML.load(File.read(file)) rescue []
136
+ @caches = {}
137
+ log { "action=load file=#{@file} servers=#{@servers.join(',')}" }
138
+ end
139
+
140
+ def write_to_file
141
+ log { "action=write file=#{@file} servers=#{@servers.join(',')}" }
142
+ File.open(@file, 'w') do |f|
143
+ f.flock(File::LOCK_EX)
144
+ f.write YAML.dump(@servers)
145
+ f.flock(File::LOCK_UN)
146
+ end
147
+ end
148
+
149
+ def log(type=:notice)
150
+ Log.send(type, "memcache_cluster #{yield}")
151
+ end
129
152
  end
data/lib/droid/heroku/stats.rb CHANGED
@@ -1,30 +1,30 @@
1
1
  module Stats
2
- # The MemCache instance used to manipulate stats.
3
- def cache
4
- MemcacheCluster.cache("heroku:stats")
5
- end
2
+ # The MemCache instance used to manipulate stats.
3
+ def cache
4
+ @@cache ||= MemcacheCluster.cache("heroku:stats")
5
+ end
6
6
 
7
- # Increment a stat counter. If the counter does not exist,
8
- # yield to the block and use the result as the current counter
9
- # value. With no block, the counter will be started at zero.
10
- def increment(key, amount=1)
11
- if (value = cache.incr(key, amount)).nil?
12
- value = yield if block_given?
13
- value = (value || 0) + amount
14
- cache.add(key, value.to_s, 0, true)
15
- end
16
- rescue => boom
17
- Log.default_error(boom)
18
- nil
19
- end
7
+ # Increment a stat counter. If the counter does not exist,
8
+ # yield to the block and use the result as the current counter
9
+ # value. With no block, the counter will be started at zero.
10
+ def increment(key, amount=1)
11
+ if (value = cache.incr(key, amount)).nil?
12
+ value = yield if block_given?
13
+ value = (value || 0) + amount
14
+ cache.add(key, value.to_s, 0, true)
15
+ end
16
+ rescue => boom
17
+ Log.default_error(boom)
18
+ nil
19
+ end
20
20
 
21
- # Set the stat counter to a specific value.
22
- def sample(key, value)
23
- cache.set(key, value.to_s, 0, true)
24
- rescue => boom
25
- Log.default_error(boom)
26
- nil
27
- end
21
+ # Set the stat counter to a specific value.
22
+ def sample(key, value)
23
+ cache.set(key, value.to_s, 0, true)
24
+ rescue => boom
25
+ Log.default_error(boom)
26
+ nil
27
+ end
28
28
 
29
- extend self
29
+ extend self
30
30
  end
data/lib/droid/utilization.rb CHANGED
@@ -6,7 +6,7 @@ class Droid
6
6
  def latency=(val); @latency = val; end
7
7
  def latency; @latency; end
8
8
 
9
- @@start = Time.now
9
+ @@start = Time.now.getutc
10
10
  @@data = { }
11
11
 
12
12
  def start
@@ -14,7 +14,7 @@ class Droid
14
14
  end
15
15
 
16
16
  def reinit
17
- @@start = Time.now
17
+ @@start = Time.now.getutc
18
18
  @@data = { }
19
19
  end
20
20
 
@@ -39,7 +39,7 @@ class Droid
39
39
  def calc_utilization(topic, t2=nil)
40
40
  d = data(topic)
41
41
  t1 = @@start
42
- t2 ||= Time.now
42
+ t2 ||= Time.now.getutc.to_i
43
43
  secs = (t2 - t1)
44
44
  secs = 1 if secs <= 0.0
45
45
  if d['msgs'] == 0
@@ -60,18 +60,18 @@ class Droid
60
60
  def monitor(topic, opts={})
61
61
  topic = 'temporary' if opts[:temp]
62
62
 
63
- t1 = Time.now
63
+ t1 = Time.now.getutc
64
64
  begin
65
65
  yield if block_given?
66
66
  ensure
67
- t2 = Time.now
67
+ t2 = Time.now.getutc
68
68
  record(topic, t2 - t1)
69
69
  end
70
70
  end
71
71
 
72
72
  def report_data
73
73
  data = {}
74
- t2 = Time.now
74
+ t2 = Time.now.getutc
75
75
  topics.each do |topic|
76
76
  data[topic] = calc_utilization(topic, t2)
77
77
  end
metadata CHANGED
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
5
5
  segments:
6
6
  - 1
7
7
  - 0
8
- - 0
9
- version: 1.0.0
8
+ - 1
9
+ version: 1.0.1
10
10
  platform: ruby
11
11
  authors:
12
12
  - Ricardo Chimal, Jr.
@@ -14,7 +14,7 @@ autorequire:
14
14
  bindir: bin
15
15
  cert_chain: []
16
16
 
17
- date: 2010-10-21 00:00:00 -07:00
17
+ date: 2010-11-12 00:00:00 -08:00
18
18
  default_executable: bleedq
19
19
  dependencies:
20
20
  - !ruby/object:Gem::Dependency