droid19 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,152 @@
1
+ require 'memcache'
2
+ require 'yaml'
3
+ require 'digest/sha1'
4
+
5
+ # Manages a pool of memcache servers. This class should not be called
6
+ # outside of the reactor - it does not account for asynchronous access
7
+ # to the server list.
8
+ module MemcacheCluster
9
+ extend self
10
+
11
+ HEROKU_NAMESPACE = '0Xfa15837Z' # heroku's internal memcache namespace
12
+
13
+ class Proxy
14
+ def initialize(name, options={})
15
+ @name = name
16
+ @options = options
17
+ end
18
+
19
+ def set(*args)
20
+ MemcacheCluster.cache_internal(@name, @options).set(*args)
21
+ end
22
+
23
+ def get(*args)
24
+ MemcacheCluster.cache_internal(@name, @options).get(*args)
25
+ end
26
+
27
+ def method_missing(method_name, *args)
28
+ MemcacheCluster.cache_internal(@name, @options).send(method_name, *args)
29
+ end
30
+ end
31
+
32
+ # A MemCache object configured with heroku's internal memcache namespace.
33
+ def heroku
34
+ cache(HEROKU_NAMESPACE)
35
+ end
36
+
37
+ def cache_retry(prefix, opts={})
38
+ opts[:retries] ||= 5
39
+ opts[:delay] ||= 0.5
40
+
41
+ retried = 0
42
+ begin
43
+ c = cache_internal(prefix)
44
+ yield c if block_given?
45
+ rescue MemCache::MemCacheError => e
46
+ Log.error "#{e.class} -> #{e.message}", :exception => e
47
+ raise if retried > opts[:retries]
48
+ retried += 1
49
+ sleep opts[:delay]
50
+ @caches = { }
51
+ retry
52
+ end
53
+ end
54
+
55
+ def set(prefix, *args)
56
+ res = nil
57
+ cache_retry(prefix) do |c|
58
+ res = c.set(*args)
59
+ end
60
+ res
61
+ end
62
+
63
+ def get(prefix, *args)
64
+ res = nil
65
+ cache_retry(prefix) do |c|
66
+ res = c.get(*args)
67
+ end
68
+ res
69
+ end
70
+
71
+ # Create listeners for standard memcache cluster related topics.
72
+ def attach(droid, file='memcached.yml')
73
+ load_from_file(file)
74
+
75
+ droid.listen4('memcache.up', :queue => "memcache.up.#{LocalStats.this_instance_name}.#$$") { |msg| add(msg['address'], msg['port']) }
76
+ droid.listen4('instance.down', :queue => "instance.down.#{LocalStats.this_instance_name}.#$$") { |msg| remove(msg['local_ip']) if msg['slot'] == 'memcache' }
77
+ EM.add_timer(1) { droid.publish('memcache.needed', {}) }
78
+ end
79
+
80
+ # A MemCache object configured with the given prefix.
81
+ def cache_internal(prefix, options={})
82
+ caches[prefix] ||=
83
+ MemCache.new(servers, options.merge(:namespace => prefix))
84
+ end
85
+
86
+ def cache(prefix, options={})
87
+ Proxy.new(prefix, options)
88
+ end
89
+
90
+ alias_method :[], :cache
91
+
92
+ def caches
93
+ reload_if_stale
94
+ @caches ||= {}
95
+ end
96
+
97
+ def servers
98
+ reload_if_stale
99
+ @servers ||= []
100
+ end
101
+
102
+ def add(ip, port)
103
+ host = [ip, port].join(':')
104
+ return if servers.include?(host)
105
+
106
+ log { "action=added server=#{host}" }
107
+ @servers.push host
108
+ @servers.sort!
109
+ @caches = {}
110
+ write_to_file
111
+ @last_read = Time.now.to_i
112
+ end
113
+
114
+ def remove(host)
115
+ if servers.reject!{ |s| s =~ /^#{host}/ }
116
+ log { "action=remove server=#{host}" }
117
+ caches.clear
118
+ write_to_file
119
+ end
120
+ end
121
+
122
+ def reload_if_stale
123
+ if @last_read &&
124
+ (Time.now.to_i - @last_read) > 20 &&
125
+ File.mtime(@file).to_i > @last_read
126
+ load_from_file(@file)
127
+ end
128
+ rescue => e
129
+ log { "action=error file=#{@file} error_class='#{e.class}' error_message='#{e.message}'" }
130
+ end
131
+
132
+ def load_from_file(file)
133
+ @file = file
134
+ @last_read = Time.now.to_i
135
+ @servers = YAML.load(File.read(file)) rescue []
136
+ @caches = {}
137
+ log { "action=load file=#{@file} servers=#{@servers.join(',')}" }
138
+ end
139
+
140
+ def write_to_file
141
+ log { "action=write file=#{@file} servers=#{@servers.join(',')}" }
142
+ File.open(@file, 'w') do |f|
143
+ f.flock(File::LOCK_EX)
144
+ f.write YAML.dump(@servers)
145
+ f.flock(File::LOCK_UN)
146
+ end
147
+ end
148
+
149
+ def log(type=:notice)
150
+ Log.send(type, "memcache_cluster #{yield}")
151
+ end
152
+ end
@@ -0,0 +1,14 @@
1
+ # Deprecated
2
+ module Stats
3
+ def cache
4
+ @@cache ||= MemcacheCluster.cache("heroku:stats")
5
+ end
6
+
7
+ def increment(key, amount=1)
8
+ end
9
+
10
+ def sample(key, value)
11
+ end
12
+
13
+ extend self
14
+ end
@@ -0,0 +1,109 @@
1
+ require 'eventmachine'
2
+ require 'evma_httpserver'
3
+
4
+ class Droid
5
+ class JSONServer < ::EM::Connection
6
+ include ::EM::HttpServer
7
+
8
+ def post_init
9
+ super
10
+ no_environment_strings
11
+ end
12
+
13
+ def process_http_request
14
+ return not_found_response if @http_request_method != "GET"
15
+
16
+ if @http_request_uri == "/"
17
+ default_response
18
+ else
19
+ method_name = "get_#{@http_request_uri.split("/")[1]}".gsub(/[^\d\w_]/,'').downcase
20
+ if public_methods.include?(method_name)
21
+ generate_response(method_name)
22
+ else
23
+ not_found_response
24
+ end
25
+ end
26
+ end
27
+
28
+ def generate_response(method_name)
29
+ status, data, content_type = self.send(method_name)
30
+
31
+ response = ::EM::DelegatedHttpResponse.new(self)
32
+ response.status = status
33
+ response.content_type content_type
34
+ response.content = data
35
+ response.send_response
36
+ end
37
+
38
+ def default_response
39
+ response = ::EM::DelegatedHttpResponse.new(self)
40
+ response.status = 200
41
+ response.content_type 'application/json'
42
+ response.content = {"status" => "OK"}.to_json
43
+ response.send_response
44
+ end
45
+
46
+ def not_found_response
47
+ response = ::EM::DelegatedHttpResponse.new(self)
48
+ response.status = 404
49
+ response.content_type 'text/plain'
50
+ response.content = "Not Found"
51
+ response.send_response
52
+ end
53
+
54
+ def get_droid
55
+ report_data = Droid::Utilization.report_data
56
+
57
+ metrics = {}
58
+ report_data.each do |topic, data|
59
+ metrics[topic] = data['msgs']
60
+ end
61
+ metrics['latency'] = Droid::Utilization.latency
62
+
63
+ summary = Droid::Utilization.report_summary(report_data)
64
+
65
+ metrics['total_msgs'] = summary['msgs']
66
+
67
+ status = "AMQP: #{summary['msgs']} msgs processed since #{Droid::Utilization.start.utc}"
68
+
69
+ # reset metrics data
70
+ Droid::Utilization.reinit
71
+
72
+ data = {
73
+ 'status' => status,
74
+ 'state' => 'ok',
75
+ 'metrics' => hash_to_metrics(metrics)
76
+ }
77
+ [200, data.to_json, "application/json"]
78
+ end
79
+
80
+ def hash_to_metrics(hash); self.class.hash_to_metrics(hash); end
81
+
82
+ # utility method to convert a ruby hash to a metrics format
83
+ # that can be consumed by cloudkick
84
+ def self.hash_to_metrics(hash)
85
+ hash.collect do |k,v|
86
+ name = k.to_s
87
+ value = v
88
+ type = if v.kind_of?(Integer)
89
+ 'int'
90
+ elsif v.kind_of?(Float)
91
+ 'float'
92
+ else
93
+ 'string'
94
+ end
95
+
96
+ # bool -> int conversion
97
+ if [TrueClass, FalseClass].include?(v.class)
98
+ value = v ? 1 : 0
99
+ type = 'int'
100
+ end
101
+
102
+ # if type is really string then it should respond to .to_s
103
+ value = value.to_s if type == 'string'
104
+
105
+ { 'name' => name, 'type' => type, 'value' => value }
106
+ end
107
+ end
108
+ end
109
+ end
@@ -0,0 +1,8 @@
1
+ module AMQP
2
+ module Client
3
+ def reconnect(*args)
4
+ sleep 10
5
+ exit 1
6
+ end
7
+ end
8
+ end
@@ -0,0 +1,31 @@
1
+ require 'droid/utils'
2
+
3
+ class Droid
4
+ # publish to queue directly
5
+ def self.publish_to_q(queue_name, data, opts={}, popts={})
6
+ q = ::MQ.queue(queue_name)
7
+ json, popts = Droid::Utils.format_publish(data, opts, popts)
8
+ q.publish(json, popts)
9
+ log.info "amqp_publish queue=#{queue_name} #{Droid::Utils.format_data_summary(data, popts[:headers])}" unless opts[:log] == false
10
+ end
11
+
12
+ def self.reply_to_q(queue_name, data, opts={}, popts={})
13
+ q = ::MQ.queue(queue_name, :auto_delete => true)
14
+ json, popts = Droid::Utils.format_publish(data, opts, popts)
15
+ q.publish(json, popts)
16
+ log.info "amqp_reply queue=#{queue_name} #{Droid::Utils.format_data_summary(data, popts[:headers])}" unless opts[:log] == false
17
+ end
18
+
19
+ # publish to exchange directly
20
+ def self.publish_to_ex(ex_name, data, opts={}, popts={})
21
+ ex = ::MQ.direct(ex_name)
22
+ json, popts = Droid::Utils.format_publish(data, opts, popts)
23
+ ex.publish(json, popts)
24
+ log.info "amqp_publish exchange=#{ex_name} #{Droid::Utils.format_data_summary(data, popts[:headers])}" unless opts[:log] == false
25
+ end
26
+
27
+ # default is publish to exchange
28
+ def self.publish(ex_name, data, opts={}, popts={})
29
+ publish_to_ex(ex_name, data, opts, popts)
30
+ end
31
+ end
@@ -0,0 +1,196 @@
1
+ class Droid
2
+ class ExpiredMessage < RuntimeError; end
3
+
4
+ class BaseQueue
5
+ attr_reader :queue_name, :opts
6
+ attr_reader :q, :ex, :mq
7
+
8
+ def initialize(queue_name, opts={})
9
+ opts[:auto_delete] = true unless opts.has_key?(:auto_delete) and opts[:auto_delete] === false
10
+
11
+ @queue_name, @opts = queue_name, opts
12
+ end
13
+
14
+ def setup
15
+ @mq = MQ.new
16
+ @q = @mq.queue(queue_name, opts)
17
+ # if we don't specify an exchange name it defaults to the queue_name
18
+ @ex = @mq.direct(opts[:exchange_name] || queue_name)
19
+ end
20
+
21
+ def temp?
22
+ false
23
+ end
24
+
25
+ def log
26
+ Droid.log
27
+ end
28
+
29
+ def tag
30
+ s = "queue=#{q.name}"
31
+ s += " exchange=#{ex.name}" if ex
32
+ end
33
+
34
+ def subscribe(amqp_opts={}, opts={})
35
+ setup
36
+
37
+ q.bind(ex) if ex
38
+ q.subscribe(amqp_opts) do |header, message|
39
+ Droid::Utilization.monitor(q.name, :temp => temp?) do
40
+ request = Droid::Request.new(self, header, message)
41
+ log.info "amqp_message #{tag} action=received ttl=#{request.ttl} age=#{request.age} #{request.data_summary}"
42
+ begin
43
+ raise Droid::ExpiredMessage if request.expired?
44
+ yield request if block_given?
45
+ finished = Time.now.getgm.to_i
46
+ log.info "amqp_message action=processed #{tag} elapsed=#{finished-request.start} ttl=#{request.ttl} age=#{request.age} #{request.data_summary}"
47
+ rescue Droid::ExpiredMessage
48
+ log.info "amqp_message action=timeout #{tag} ttl=#{request.ttl} age=#{request.age} #{request.data_summary}"
49
+ request.ack if amqp_opts[:ack]
50
+ rescue => e
51
+ request.ack if amqp_opts[:ack]
52
+ Droid.handle_error(e)
53
+ end
54
+ end
55
+ end
56
+ log.info "amqp_subscribe #{tag}"
57
+ self
58
+ end
59
+
60
+ def teardown
61
+ @q.unsubscribe
62
+ @mq.close
63
+ log.info "amqp_unsubscribe #{tag}"
64
+ end
65
+
66
+ def destroy
67
+ teardown
68
+ end
69
+ end
70
+
71
+ class WorkerQueue < BaseQueue
72
+ attr_reader :prefetch
73
+
74
+ def initialize(queue_name, opts={})
75
+ @prefetch = opts.delete(:prefetch) || 1
76
+ opts[:auto_delete] = false
77
+
78
+ super(queue_name, opts)
79
+ end
80
+
81
+ def setup
82
+ super
83
+ @mq.prefetch(self.prefetch)
84
+ end
85
+
86
+ def subscribe(amqp_opts={}, opts={})
87
+ amqp_opts[:ack] = true
88
+ super(amqp_opts, opts) do |request|
89
+ begin
90
+ yield request if block_given?
91
+ ensure
92
+ request.ack unless amqp_opts[:auto_ack] == false
93
+ end
94
+ end
95
+ end
96
+ end
97
+
98
+ class ListenQueue < BaseQueue
99
+ def initialize(exchange_name, opts={})
100
+ opts[:exchange_name] = exchange_name
101
+ queue_name = opts.delete(:queue) || Droid::Utils.generate_queue(exchange_name)
102
+ super(queue_name, opts)
103
+ end
104
+ end
105
+
106
+ class ReplyQueue < BaseQueue
107
+ def initialize(queue_name, opts={})
108
+ opts[:auto_delete] = true
109
+ super
110
+ end
111
+
112
+ def setup
113
+ @mq = MQ.new
114
+ @q = @mq.queue(queue_name, opts)
115
+ @ex = nil
116
+ end
117
+
118
+ def temp?
119
+ true
120
+ end
121
+
122
+ def subscribe(amqp_opts={}, opts={})
123
+ super(amqp_opts, opts) do |request|
124
+ yield request if block_given?
125
+ self.destroy
126
+ end
127
+ end
128
+
129
+ def teardown
130
+ @q.delete
131
+ super
132
+ end
133
+ end
134
+
135
+ module QueueMethods
136
+ def worker(queue_name, opts={})
137
+ WorkerQueue.new(queue_name, opts)
138
+ end
139
+
140
+ def listener(exchange_name, opts={})
141
+ ListenQueue.new(exchange_name, opts)
142
+ end
143
+ end
144
+
145
+ class BackwardsCompatibleQueue < BaseQueue
146
+ def initialize(exchange_name, opts={})
147
+ opts[:auto_delete] = true unless opts.has_key?(:auto_delete) and opts[:auto_delete] === false
148
+ opts[:exchange_name] = exchange_name
149
+ queue_name = opts.delete(:queue) || Droid::Utils.generate_queue(exchange_name)
150
+ @queue_name, @opts = queue_name, opts
151
+ end
152
+
153
+ def setup
154
+ @mq = MQ.new
155
+ @q = @mq.queue(queue_name, opts)
156
+ @ex = @mq.direct(opts[:exchange_name])
157
+
158
+ @mq.prefetch(opts[:prefetch]) if opts[:prefetch]
159
+ end
160
+
161
+ def subscribe(amqp_opts={}, opts={})
162
+ super(amqp_opts, opts) do |request|
163
+ if block_given?
164
+ if opts[:detail]
165
+ yield request, request.header, request.raw_message if block_given?
166
+ else
167
+ yield request if block_given?
168
+ end
169
+ end
170
+ end
171
+
172
+ self
173
+ end
174
+ end
175
+
176
+ module BackwardsCompatibleMethods
177
+ def listen4(key, orig_opts={}, &block)
178
+ opts = {}
179
+ amqp_opts = {}
180
+ subscribe_opts = {}
181
+
182
+ if orig_opts[:prefetch] || orig_opts[:ack]
183
+ opts[:prefetch] = orig_opts[:prefetch] || 1
184
+ opts[:ack] = true
185
+ end
186
+ if orig_opts[:queue]
187
+ opts[:queue] = orig_opts[:queue]
188
+ end
189
+ if orig_opts[:detail]
190
+ subscribe_opts[:detail] = true
191
+ end
192
+
193
+ BackwardsCompatibleQueue.new(key, opts).subscribe(amqp_opts, subscribe_opts, &block)
194
+ end
195
+ end
196
+ end