rightscale-nanite 0.4.1.1 → 0.4.1.2
Sign up to get free protection for your applications and to get access to all the features.
- data/README.rdoc +18 -19
- data/Rakefile +2 -0
- data/lib/nanite/actor_registry.rb +3 -1
- data/lib/nanite/admin.rb +1 -16
- data/lib/nanite/agent.rb +13 -10
- data/lib/nanite/cluster.rb +66 -37
- data/lib/nanite/dispatcher.rb +1 -0
- data/lib/nanite/job.rb +2 -2
- data/lib/nanite/log.rb +5 -4
- data/lib/nanite/mapper.rb +24 -12
- data/lib/nanite/mapper_proxy.rb +1 -0
- data/lib/nanite/packets.rb +188 -49
- data/lib/nanite/state.rb +2 -2
- data/spec/actor_registry_spec.rb +2 -2
- data/spec/cluster_spec.rb +176 -5
- data/spec/job_spec.rb +38 -6
- data/spec/spec_helper.rb +11 -1
- metadata +3 -2
data/README.rdoc
CHANGED
@@ -296,26 +296,25 @@ Mongrel on the other hand does not use EventMachine and therefore requires to wr
|
|
296
296
|
|
297
297
|
Using nanite with Passenger:
|
298
298
|
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
# catch these, stop AMQP, stop eventmachine, and re-throw to
|
308
|
-
Mongrel/Passenger's signal traps
|
309
|
-
EM.run do
|
310
|
-
['INT', 'TERM'].each do |sig|
|
311
|
-
old = trap(sig) do
|
312
|
-
AMQP.stop do
|
313
|
-
EM.stop
|
314
|
-
old.call
|
315
|
-
end
|
316
|
-
end
|
317
|
-
end
|
299
|
+
if defined?(PhusionPassenger)
|
300
|
+
PhusionPassenger.on_event(:starting_worker_process) do |forked|
|
301
|
+
if forked
|
302
|
+
if EM.reactor_running?
|
303
|
+
EM.stop_event_loop
|
304
|
+
EM.release_machine
|
305
|
+
EM.instance_variable_set( '@reactor_running', false )
|
318
306
|
end
|
307
|
+
Thread.current[:mq] = nil
|
308
|
+
AMQP.instance_variable_set('@conn', nil)
|
309
|
+
end
|
310
|
+
|
311
|
+
th = Thread.current
|
312
|
+
Thread.new{
|
313
|
+
Nanite.start_mapper(:host => 'localhost', :user => 'mapper', :pass => 'testing', :vhost => '/nanite', :log_level => 'info')
|
314
|
+
}
|
315
|
+
Thread.stop
|
316
|
+
end
|
317
|
+
end
|
319
318
|
|
320
319
|
=======
|
321
320
|
Where to put the mapper initialization code depends on the framework and our preference.
|
data/Rakefile
CHANGED
@@ -8,7 +8,9 @@ module Nanite
|
|
8
8
|
|
9
9
|
def register(actor, prefix)
|
10
10
|
raise ArgumentError, "#{actor.inspect} is not a Nanite::Actor subclass instance" unless Nanite::Actor === actor
|
11
|
-
|
11
|
+
log_msg = "[actor] #{actor.class.to_s}"
|
12
|
+
log_msg += ", prefix #{prefix}" if prefix && !prefix.empty?
|
13
|
+
Nanite::Log.info(log_msg)
|
12
14
|
prefix ||= actor.class.default_prefix
|
13
15
|
actors[prefix.to_s] = actor
|
14
16
|
end
|
data/lib/nanite/admin.rb
CHANGED
@@ -66,21 +66,6 @@ module Nanite
|
|
66
66
|
<meta content='Engineyard' name='author' />
|
67
67
|
<title>Nanite Control Tower</title>
|
68
68
|
|
69
|
-
<!-- Google AJAX Libraries API -->
|
70
|
-
<script src="http://www.google.com/jsapi"></script>
|
71
|
-
<script type="text/javascript">
|
72
|
-
google.load("jquery", "1");
|
73
|
-
</script>
|
74
|
-
|
75
|
-
<script type="text/javascript">
|
76
|
-
$(document).ready(function(){
|
77
|
-
|
78
|
-
// set the focus to the payload field
|
79
|
-
$("#payload").focus();
|
80
|
-
|
81
|
-
});
|
82
|
-
</script>
|
83
|
-
|
84
69
|
<style>
|
85
70
|
body {margin: 0; font-family: verdana; background-color: #fcfcfc;}
|
86
71
|
ul {margin: 0; padding: 0; margin-left: 10px}
|
@@ -137,7 +122,7 @@ module Nanite
|
|
137
122
|
<div class="section">
|
138
123
|
#{"No nanites online." if @mapper.cluster.nanites.size == 0}
|
139
124
|
<ul>
|
140
|
-
#{@mapper.cluster.nanites.map {|k,v| "<li>identity : #{k}<br />load : #{v[:status]}<br />services : #{v[:services].to_a.inspect}</li>" }.join}
|
125
|
+
#{@mapper.cluster.nanites.map {|k,v| "<li>identity : #{k}<br />load : #{v[:status]}<br />services : #{v[:services].to_a.inspect}<br />tags: #{v[:tags].to_a.inspect}</li>" }.join}
|
141
126
|
</ul>
|
142
127
|
</div>
|
143
128
|
<div id="footer">
|
data/lib/nanite/agent.rb
CHANGED
@@ -158,7 +158,7 @@ module Nanite
|
|
158
158
|
actors = @options[:actors]
|
159
159
|
Dir["#{actors_dir}/*.rb"].each do |actor|
|
160
160
|
next if actors && !actors.include?(File.basename(actor, ".rb"))
|
161
|
-
Nanite::Log.info("loading
|
161
|
+
Nanite::Log.info("[setup] loading #{actor}")
|
162
162
|
require actor
|
163
163
|
end
|
164
164
|
init_path = @options[:initrb] || File.join(options[:root], 'init.rb')
|
@@ -166,25 +166,27 @@ module Nanite
|
|
166
166
|
end
|
167
167
|
|
168
168
|
def receive(packet)
|
169
|
+
Nanite::Log.debug("RECV #{packet.to_s}")
|
169
170
|
case packet
|
170
171
|
when Advertise
|
171
|
-
Nanite::Log.
|
172
|
+
Nanite::Log.info("RECV #{packet.to_s}") unless Nanite::Log.level == Logger::DEBUG
|
172
173
|
advertise_services
|
173
174
|
when Request, Push
|
174
|
-
Nanite::Log.debug("handling Request: #{packet.inspect}")
|
175
175
|
if @security && !@security.authorize(packet)
|
176
|
+
Nanite::Log.warn("RECV NOT AUTHORIZED #{packet.to_s}")
|
176
177
|
if packet.kind_of?(Request)
|
177
178
|
r = Result.new(packet.token, packet.reply_to, @deny_token, identity)
|
178
179
|
amq.queue(packet.reply_to, :no_declare => options[:secure]).publish(serializer.dump(r))
|
179
180
|
end
|
180
181
|
else
|
182
|
+
Nanite::Log.info("RECV #{packet.to_s([:from, :tags])}") unless Nanite::Log.level == Logger::DEBUG
|
181
183
|
dispatcher.dispatch(packet)
|
182
184
|
end
|
183
185
|
when Result
|
184
|
-
Nanite::Log.
|
186
|
+
Nanite::Log.info("RECV #{packet.to_s([])}") unless Nanite::Log.level == Logger::DEBUG
|
185
187
|
@mapper_proxy.handle_result(packet)
|
186
188
|
when IntermediateMessage
|
187
|
-
Nanite::Log.
|
189
|
+
Nanite::Log.info("RECV #{packet.to_s([])}") unless Nanite::Log.level == Logger::DEBUG
|
188
190
|
@mapper_proxy.handle_intermediate_result(packet)
|
189
191
|
end
|
190
192
|
end
|
@@ -198,10 +200,9 @@ module Nanite
|
|
198
200
|
amq.queue(identity, :durable => true).subscribe(:ack => true) do |info, msg|
|
199
201
|
begin
|
200
202
|
info.ack
|
201
|
-
|
202
|
-
receive(packet)
|
203
|
+
receive(serializer.load(msg))
|
203
204
|
rescue Exception => e
|
204
|
-
Nanite::Log.error("
|
205
|
+
Nanite::Log.error("RECV #{e.message}")
|
205
206
|
end
|
206
207
|
end
|
207
208
|
end
|
@@ -231,13 +232,15 @@ module Nanite
|
|
231
232
|
def un_register
|
232
233
|
unless @unregistered
|
233
234
|
@unregistered = true
|
235
|
+
Nanite::Log.info("SEND [un_register]")
|
234
236
|
amq.fanout('registration', :no_declare => options[:secure]).publish(serializer.dump(UnRegister.new(identity)))
|
235
237
|
end
|
236
238
|
end
|
237
239
|
|
238
240
|
def advertise_services
|
239
|
-
|
240
|
-
|
241
|
+
reg = Register.new(identity, registry.services, status_proc.call, self.tags)
|
242
|
+
Nanite::Log.info("SEND #{reg.to_s}")
|
243
|
+
amq.fanout('registration', :no_declare => options[:secure]).publish(serializer.dump(reg))
|
241
244
|
end
|
242
245
|
|
243
246
|
def parse_uptime(up)
|
data/lib/nanite/cluster.rb
CHANGED
@@ -1,23 +1,17 @@
|
|
1
1
|
module Nanite
|
2
2
|
class Cluster
|
3
|
-
attr_reader :agent_timeout, :nanites, :reaper, :serializer, :identity, :amq, :redis, :mapper
|
3
|
+
attr_reader :agent_timeout, :nanites, :reaper, :serializer, :identity, :amq, :redis, :mapper, :callbacks
|
4
4
|
|
5
|
-
def initialize(amq, agent_timeout, identity, serializer, mapper,
|
5
|
+
def initialize(amq, agent_timeout, identity, serializer, mapper, state_configuration=nil, callbacks = {})
|
6
6
|
@amq = amq
|
7
7
|
@agent_timeout = agent_timeout
|
8
8
|
@identity = identity
|
9
9
|
@serializer = serializer
|
10
10
|
@mapper = mapper
|
11
|
-
@
|
11
|
+
@state = state_configuration
|
12
12
|
@security = SecurityProvider.get
|
13
|
-
|
14
|
-
|
15
|
-
require 'nanite/state'
|
16
|
-
@nanites = ::Nanite::State.new(redis)
|
17
|
-
else
|
18
|
-
require 'nanite/local_state'
|
19
|
-
@nanites = Nanite::LocalState.new
|
20
|
-
end
|
13
|
+
@callbacks = callbacks
|
14
|
+
setup_state
|
21
15
|
@reaper = Reaper.new(agent_timeout)
|
22
16
|
setup_queues
|
23
17
|
end
|
@@ -35,18 +29,27 @@ module Nanite
|
|
35
29
|
case reg
|
36
30
|
when Register
|
37
31
|
if @security.authorize_registration(reg)
|
32
|
+
Nanite::Log.info("RECV #{reg.to_s}")
|
38
33
|
nanites[reg.identity] = { :services => reg.services, :status => reg.status, :tags => reg.tags }
|
39
|
-
reaper.timeout(reg.identity, agent_timeout + 1) {
|
40
|
-
|
34
|
+
reaper.timeout(reg.identity, agent_timeout + 1) { nanite_timed_out(reg.identity) }
|
35
|
+
callbacks[:register].call(reg.identity, mapper) if callbacks[:register]
|
41
36
|
else
|
42
|
-
Nanite::Log.
|
37
|
+
Nanite::Log.warn("RECV NOT AUTHORIZED #{reg.to_s}")
|
43
38
|
end
|
44
39
|
when UnRegister
|
40
|
+
Nanite::Log.info("RECV #{reg.to_s}")
|
45
41
|
nanites.delete(reg.identity)
|
46
|
-
|
42
|
+
callbacks[:unregister].call(reg.identity, mapper) if callbacks[:unregister]
|
43
|
+
else
|
44
|
+
Nanite::Log.warn("RECV [register] Invalid packet type: #{reg.class}")
|
47
45
|
end
|
48
46
|
end
|
49
47
|
|
48
|
+
def nanite_timed_out(token)
|
49
|
+
nanite = nanites.delete(token)
|
50
|
+
callbacks[:timeout].call(token, mapper) if callbacks[:timeout]
|
51
|
+
end
|
52
|
+
|
50
53
|
def route(request, targets)
|
51
54
|
EM.next_tick { targets.map { |target| publish(request, target) } }
|
52
55
|
end
|
@@ -57,6 +60,7 @@ module Nanite
|
|
57
60
|
begin
|
58
61
|
old_target = request.target
|
59
62
|
request.target = target unless target == 'mapper-offline'
|
63
|
+
Nanite::Log.info("SEND #{request.to_s([:from, :tags, :target])}")
|
60
64
|
amq.queue(target).publish(serializer.dump(request), :persistent => request.persistent)
|
61
65
|
ensure
|
62
66
|
request.target = old_target
|
@@ -68,36 +72,46 @@ module Nanite
|
|
68
72
|
# updates nanite information (last ping timestamps, status)
|
69
73
|
# when heartbeat message is received
|
70
74
|
def handle_ping(ping)
|
71
|
-
|
72
|
-
nanite
|
73
|
-
|
74
|
-
|
75
|
-
|
75
|
+
begin
|
76
|
+
if nanite = nanites[ping.identity]
|
77
|
+
nanite[:status] = ping.status
|
78
|
+
reaper.reset_with_autoregister_hack(ping.identity, agent_timeout + 1) { nanite_timed_out(ping.identity) }
|
79
|
+
else
|
80
|
+
packet = Advertise.new
|
81
|
+
Nanite::Log.info("SEND #{packet.to_s} to #{ping.identity}")
|
82
|
+
amq.queue(ping.identity).publish(serializer.dump(packet))
|
83
|
+
end
|
76
84
|
end
|
77
85
|
end
|
78
86
|
|
79
87
|
# forward request coming from agent
|
80
88
|
def handle_request(request)
|
81
89
|
if @security.authorize_request(request)
|
82
|
-
|
83
|
-
|
84
|
-
|
90
|
+
Nanite::Log.info("RECV #{request.to_s([:from, :target, :tags])}") unless Nanite::Log.level == Logger::DEBUG
|
91
|
+
Nanite::Log.debug("RECV #{request.to_s}")
|
92
|
+
|
93
|
+
intm_handler = lambda do |result, job|
|
94
|
+
result = IntermediateMessage.new(request.token, job.request.from, mapper.identity, nil, result)
|
85
95
|
forward_response(result, request.persistent)
|
86
96
|
end
|
97
|
+
|
98
|
+
result = Result.new(request.token, request.from, nil, mapper.identity)
|
87
99
|
ok = mapper.send_request(request, :intermediate_handler => intm_handler) do |res|
|
88
100
|
result.results = res
|
89
101
|
forward_response(result, request.persistent)
|
90
102
|
end
|
103
|
+
|
91
104
|
if ok == false
|
92
105
|
forward_response(result, request.persistent)
|
93
106
|
end
|
94
107
|
else
|
95
|
-
Nanite::Log.
|
108
|
+
Nanite::Log.warn("RECV NOT AUTHORIZED #{request.to_s}")
|
96
109
|
end
|
97
110
|
end
|
98
111
|
|
99
112
|
# forward response back to agent that originally made the request
|
100
113
|
def forward_response(res, persistent)
|
114
|
+
Nanite::Log.info("SEND #{res.to_s([:to])}")
|
101
115
|
amq.queue(res.to).publish(serializer.dump(res), :persistent => persistent)
|
102
116
|
end
|
103
117
|
|
@@ -150,14 +164,14 @@ module Nanite
|
|
150
164
|
handler = lambda do |ping|
|
151
165
|
begin
|
152
166
|
ping = serializer.load(ping)
|
153
|
-
Nanite::Log.debug("
|
167
|
+
Nanite::Log.debug("RECV #{ping.to_s}") if ping.respond_to?(:to_s)
|
154
168
|
handle_ping(ping)
|
155
169
|
rescue Exception => e
|
156
|
-
Nanite::Log.error("
|
170
|
+
Nanite::Log.error("RECV [ping] #{e.message}")
|
157
171
|
end
|
158
172
|
end
|
159
173
|
hb_fanout = amq.fanout('heartbeat', :durable => true)
|
160
|
-
if
|
174
|
+
if shared_state?
|
161
175
|
amq.queue("heartbeat").bind(hb_fanout).subscribe &handler
|
162
176
|
else
|
163
177
|
amq.queue("heartbeat-#{identity}", :exclusive => true).bind(hb_fanout).subscribe &handler
|
@@ -167,15 +181,13 @@ module Nanite
|
|
167
181
|
def setup_registration_queue
|
168
182
|
handler = lambda do |msg|
|
169
183
|
begin
|
170
|
-
|
171
|
-
Nanite::Log.debug("got registration from #{msg.identity}")
|
172
|
-
register(msg)
|
184
|
+
register(serializer.load(msg))
|
173
185
|
rescue Exception => e
|
174
|
-
Nanite::Log.error("
|
186
|
+
Nanite::Log.error("RECV [register] #{e.message}")
|
175
187
|
end
|
176
188
|
end
|
177
189
|
reg_fanout = amq.fanout('registration', :durable => true)
|
178
|
-
if
|
190
|
+
if shared_state?
|
179
191
|
amq.queue("registration").bind(reg_fanout).subscribe &handler
|
180
192
|
else
|
181
193
|
amq.queue("registration-#{identity}", :exclusive => true).bind(reg_fanout).subscribe &handler
|
@@ -185,19 +197,36 @@ module Nanite
|
|
185
197
|
def setup_request_queue
|
186
198
|
handler = lambda do |msg|
|
187
199
|
begin
|
188
|
-
|
189
|
-
Nanite::Log.debug("got request from #{msg.from} of type #{msg.type}")
|
190
|
-
handle_request(msg)
|
200
|
+
handle_request(serializer.load(msg))
|
191
201
|
rescue Exception => e
|
192
|
-
Nanite::Log.error("
|
202
|
+
Nanite::Log.error("RECV [request] #{e.message}")
|
193
203
|
end
|
194
204
|
end
|
195
205
|
req_fanout = amq.fanout('request', :durable => true)
|
196
|
-
if
|
206
|
+
if shared_state?
|
197
207
|
amq.queue("request").bind(req_fanout).subscribe &handler
|
198
208
|
else
|
199
209
|
amq.queue("request-#{identity}", :exclusive => true).bind(req_fanout).subscribe &handler
|
200
210
|
end
|
201
211
|
end
|
212
|
+
|
213
|
+
def setup_state
|
214
|
+
case @state
|
215
|
+
when String
|
216
|
+
# backwards compatibility, we assume redis if the configuration option
|
217
|
+
# was a string
|
218
|
+
Nanite::Log.info("[setup] using redis for state storage")
|
219
|
+
require 'nanite/state'
|
220
|
+
@nanites = Nanite::State.new(@state)
|
221
|
+
when Hash
|
222
|
+
else
|
223
|
+
require 'nanite/local_state'
|
224
|
+
@nanites = Nanite::LocalState.new
|
225
|
+
end
|
226
|
+
end
|
227
|
+
|
228
|
+
def shared_state?
|
229
|
+
!@state.nil?
|
230
|
+
end
|
202
231
|
end
|
203
232
|
end
|
data/lib/nanite/dispatcher.rb
CHANGED
@@ -31,6 +31,7 @@ module Nanite
|
|
31
31
|
callback = lambda do |r|
|
32
32
|
if deliverable.kind_of?(Request)
|
33
33
|
r = Result.new(deliverable.token, deliverable.reply_to, r, identity)
|
34
|
+
Nanite::Log.info("SEND #{r.to_s([])}")
|
34
35
|
amq.queue(deliverable.reply_to, :no_declare => options[:secure]).publish(serializer.dump(r))
|
35
36
|
end
|
36
37
|
r # For unit tests
|
data/lib/nanite/job.rb
CHANGED
@@ -14,8 +14,6 @@ module Nanite
|
|
14
14
|
end
|
15
15
|
|
16
16
|
def process(msg)
|
17
|
-
Nanite::Log.debug("processing message: #{msg.inspect}")
|
18
|
-
|
19
17
|
if job = jobs[msg.token]
|
20
18
|
job.process(msg)
|
21
19
|
|
@@ -29,6 +27,8 @@ module Nanite
|
|
29
27
|
handler = job.intermediate_handler_for_key(key)
|
30
28
|
if handler
|
31
29
|
case handler.arity
|
30
|
+
when 2
|
31
|
+
handler.call(job.intermediate_state[msg.from][key].last, job)
|
32
32
|
when 3
|
33
33
|
handler.call(key, msg.from, job.intermediate_state[msg.from][key].last)
|
34
34
|
when 4
|
data/lib/nanite/log.rb
CHANGED
@@ -13,7 +13,7 @@ module Nanite
|
|
13
13
|
# Use Nanite::Logger.init when you want to set up the logger manually.
|
14
14
|
# If this method is called with no arguments, it will log to STDOUT at the :info level.
|
15
15
|
# It also configures the Logger instance it creates to use the custom Nanite::Log::Formatter class.
|
16
|
-
def init(identity, path = false)
|
16
|
+
def init(identity = nil, path = false)
|
17
17
|
if path
|
18
18
|
@file = File.join(path, "nanite.#{identity}.log")
|
19
19
|
else
|
@@ -30,7 +30,8 @@ module Nanite
|
|
30
30
|
def level=(loglevel)
|
31
31
|
init() unless @logger
|
32
32
|
loglevel = loglevel.intern if loglevel.is_a?(String)
|
33
|
-
@logger.info("
|
33
|
+
@logger.info("[setup] setting log level to #{loglevel.to_s.upcase}")
|
34
|
+
@level = loglevel
|
34
35
|
case loglevel
|
35
36
|
when :debug
|
36
37
|
@logger.level = Logger::DEBUG
|
@@ -46,12 +47,12 @@ module Nanite
|
|
46
47
|
raise ArgumentError, "Log level must be one of :debug, :info, :warn, :error, or :fatal"
|
47
48
|
end
|
48
49
|
end
|
49
|
-
|
50
|
+
|
50
51
|
# Passes any other method calls on directly to the underlying Logger object created with init. If
|
51
52
|
# this method gets hit before a call to Nanite::Logger.init has been made, it will call
|
52
53
|
# Nanite::Logger.init() with no arguments.
|
53
54
|
def method_missing(method_symbol, *args)
|
54
|
-
init
|
55
|
+
init unless @logger
|
55
56
|
if args.length > 0
|
56
57
|
@logger.send(method_symbol, *args)
|
57
58
|
else
|
data/lib/nanite/mapper.rb
CHANGED
@@ -24,7 +24,8 @@ module Nanite
|
|
24
24
|
attr_reader :cluster, :identity, :job_warden, :options, :serializer, :amq
|
25
25
|
|
26
26
|
DEFAULT_OPTIONS = COMMON_DEFAULT_OPTIONS.merge({:user => 'mapper', :identity => Identity.generate, :agent_timeout => 15,
|
27
|
-
:offline_redelivery_frequency => 10, :persistent => false, :offline_failsafe => false
|
27
|
+
:offline_redelivery_frequency => 10, :persistent => false, :offline_failsafe => false,
|
28
|
+
:callbacks => {} }) unless defined?(DEFAULT_OPTIONS)
|
28
29
|
|
29
30
|
# Initializes a new mapper and establishes
|
30
31
|
# AMQP connection. This must be used inside EM.run block or if EventMachine reactor
|
@@ -107,12 +108,7 @@ module Nanite
|
|
107
108
|
end
|
108
109
|
|
109
110
|
def run
|
110
|
-
|
111
|
-
if @options[:daemonize]
|
112
|
-
log_path = (@options[:log_dir] || @options[:root] || Dir.pwd)
|
113
|
-
end
|
114
|
-
Nanite::Log.init(@identity, log_path)
|
115
|
-
Nanite::Log.level = @options[:log_level] if @options[:log_level]
|
111
|
+
setup_logging
|
116
112
|
@serializer = Serializer.new(@options[:format])
|
117
113
|
pid_file = PidFile.new(@identity, @options)
|
118
114
|
pid_file.check
|
@@ -120,11 +116,13 @@ module Nanite
|
|
120
116
|
daemonize
|
121
117
|
pid_file.write
|
122
118
|
at_exit { pid_file.remove }
|
119
|
+
else
|
120
|
+
trap("INT") {exit}
|
123
121
|
end
|
124
122
|
@amq = start_amqp(@options)
|
125
123
|
@job_warden = JobWarden.new(@serializer)
|
126
|
-
|
127
|
-
Nanite::Log.info('starting mapper')
|
124
|
+
setup_cluster
|
125
|
+
Nanite::Log.info('[setup] starting mapper')
|
128
126
|
setup_queues
|
129
127
|
start_console if @options[:console] && !@options[:daemonize]
|
130
128
|
end
|
@@ -264,14 +262,28 @@ module Nanite
|
|
264
262
|
def setup_message_queue
|
265
263
|
amq.queue(identity, :exclusive => true).bind(amq.fanout(identity)).subscribe do |msg|
|
266
264
|
begin
|
267
|
-
msg = serializer.load(msg)
|
268
|
-
Nanite::Log.debug("
|
265
|
+
msg = serializer.load(msg)
|
266
|
+
Nanite::Log.debug("RECV #{msg.to_s}")
|
267
|
+
Nanite::Log.info("RECV #{msg.to_s([:from])}") unless Nanite::Log.level == Logger::DEBUG
|
269
268
|
job_warden.process(msg)
|
270
269
|
rescue Exception => e
|
271
|
-
Nanite::Log.error("
|
270
|
+
Nanite::Log.error("RECV [result] #{e.message}")
|
272
271
|
end
|
273
272
|
end
|
274
273
|
end
|
274
|
+
|
275
|
+
def setup_logging
|
276
|
+
log_path = false
|
277
|
+
if @options[:daemonize]
|
278
|
+
log_path = (@options[:log_dir] || @options[:root] || Dir.pwd)
|
279
|
+
end
|
280
|
+
Nanite::Log.init(@identity, log_path)
|
281
|
+
Nanite::Log.level = @options[:log_level] if @options[:log_level]
|
282
|
+
end
|
283
|
+
|
284
|
+
def setup_cluster
|
285
|
+
@cluster = Cluster.new(@amq, @options[:agent_timeout], @options[:identity], @serializer, self, @options[:redis], @options[:callbacks])
|
286
|
+
end
|
275
287
|
end
|
276
288
|
end
|
277
289
|
|
data/lib/nanite/mapper_proxy.rb
CHANGED
@@ -37,6 +37,7 @@ module Nanite
|
|
37
37
|
request.persistent = opts.key?(:persistent) ? opts[:persistent] : options[:persistent]
|
38
38
|
pending_requests[request.token] =
|
39
39
|
{ :intermediate_handler => opts[:intermediate_handler], :result_handler => blk }
|
40
|
+
Nanite::Log.info("SEND #{request.to_s([:tags, :target])}")
|
40
41
|
amqp.fanout('request', :no_declare => options[:secure]).publish(serializer.dump(request))
|
41
42
|
end
|
42
43
|
|
data/lib/nanite/packets.rb
CHANGED
@@ -2,58 +2,111 @@ module Nanite
|
|
2
2
|
# Base class for all Nanite packets,
|
3
3
|
# knows how to dump itself to JSON
|
4
4
|
class Packet
|
5
|
+
|
6
|
+
attr_accessor :size
|
7
|
+
|
5
8
|
def initialize
|
6
9
|
raise NotImplementedError.new("#{self.class.name} is an abstract class.")
|
7
10
|
end
|
11
|
+
|
8
12
|
def to_json(*a)
|
9
|
-
{
|
13
|
+
js = {
|
10
14
|
'json_class' => self.class.name,
|
11
15
|
'data' => instance_variables.inject({}) {|m,ivar| m[ivar.sub(/@/,'')] = instance_variable_get(ivar); m }
|
12
16
|
}.to_json(*a)
|
17
|
+
js = js.chop + ",\"size\":#{js.size}}"
|
18
|
+
js
|
19
|
+
end
|
20
|
+
|
21
|
+
# Log representation
|
22
|
+
def to_s(filter=nil)
|
23
|
+
res = "[#{ self.class.to_s.split('::').last.
|
24
|
+
gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
|
25
|
+
gsub(/([a-z\d])([A-Z])/,'\1_\2').
|
26
|
+
downcase }]"
|
27
|
+
res += " (#{size.to_s.gsub(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1,")} bytes)" if size && !size.to_s.empty?
|
28
|
+
res
|
29
|
+
end
|
30
|
+
|
31
|
+
# Log friendly name for given agent id
|
32
|
+
def id_to_s(id)
|
33
|
+
case id
|
34
|
+
when /^mapper-/ then 'mapper'
|
35
|
+
when /^nanite-(.*)/ then Regexp.last_match(1)
|
36
|
+
else id
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
# Wrap given string to given maximum number of characters per line
|
41
|
+
def wrap(txt, col=120)
|
42
|
+
txt.gsub(/(.{1,#{col}})( +|$\n?)|(.{1,#{col}})/, "\\1\\3\n").chomp
|
13
43
|
end
|
44
|
+
|
14
45
|
end
|
15
46
|
|
16
47
|
# packet that means start of a file transfer
|
17
48
|
# operation
|
18
49
|
class FileStart < Packet
|
50
|
+
|
19
51
|
attr_accessor :filename, :token, :dest
|
20
|
-
|
52
|
+
|
53
|
+
def initialize(filename, dest, token, size=nil)
|
21
54
|
@filename = filename
|
22
55
|
@dest = dest
|
23
56
|
@token = token
|
57
|
+
@size = size
|
24
58
|
end
|
25
59
|
|
26
60
|
def self.json_create(o)
|
27
61
|
i = o['data']
|
28
|
-
new(i['filename'], i['dest'], i['token'])
|
62
|
+
new(i['filename'], i['dest'], i['token'], o['size'])
|
63
|
+
end
|
64
|
+
|
65
|
+
def to_s
|
66
|
+
wrap("#{super} <#{token}> #{filename} to #{dest}")
|
29
67
|
end
|
30
68
|
end
|
31
69
|
|
32
70
|
# packet that means end of a file transfer
|
33
71
|
# operation
|
34
72
|
class FileEnd < Packet
|
73
|
+
|
35
74
|
attr_accessor :token, :meta
|
36
|
-
|
75
|
+
|
76
|
+
def initialize(token, meta, size=nil)
|
37
77
|
@token = token
|
38
78
|
@meta = meta
|
79
|
+
@size = size
|
39
80
|
end
|
40
81
|
|
41
82
|
def self.json_create(o)
|
42
83
|
i = o['data']
|
43
|
-
new(i['token'], i['meta'])
|
84
|
+
new(i['token'], i['meta'], o['size'])
|
85
|
+
end
|
86
|
+
|
87
|
+
def to_s
|
88
|
+
wrap("#{super} <#{token}> meta #{meta}")
|
44
89
|
end
|
45
90
|
end
|
46
91
|
|
47
92
|
# packet that carries data chunks during a file transfer
|
48
93
|
class FileChunk < Packet
|
94
|
+
|
49
95
|
attr_accessor :chunk, :token
|
50
|
-
|
96
|
+
|
97
|
+
def initialize(token, size=nil, chunk=nil)
|
51
98
|
@chunk = chunk
|
52
99
|
@token = token
|
100
|
+
@size = size
|
53
101
|
end
|
102
|
+
|
54
103
|
def self.json_create(o)
|
55
104
|
i = o['data']
|
56
|
-
new(i['token'], i['chunk'])
|
105
|
+
new(i['token'], o['size'], i['chunk'])
|
106
|
+
end
|
107
|
+
|
108
|
+
def to_s
|
109
|
+
"#{super} <#{token}>"
|
57
110
|
end
|
58
111
|
end
|
59
112
|
|
@@ -71,25 +124,43 @@ module Nanite
|
|
71
124
|
# target is the target nanite for the request
|
72
125
|
# persistent signifies if this request should be saved to persistent storage by the AMQP broker
|
73
126
|
class Request < Packet
|
127
|
+
|
74
128
|
attr_accessor :from, :payload, :type, :token, :reply_to, :selector, :target, :persistent, :tags
|
129
|
+
|
75
130
|
DEFAULT_OPTIONS = {:selector => :least_loaded}
|
76
|
-
|
131
|
+
|
132
|
+
def initialize(type, payload, size=nil, opts={})
|
77
133
|
opts = DEFAULT_OPTIONS.merge(opts)
|
78
|
-
@type
|
79
|
-
@payload
|
80
|
-
@
|
81
|
-
@
|
82
|
-
@
|
83
|
-
@
|
84
|
-
@
|
85
|
-
@
|
86
|
-
@
|
134
|
+
@type = type
|
135
|
+
@payload = payload
|
136
|
+
@size = size
|
137
|
+
@from = opts[:from]
|
138
|
+
@token = opts[:token]
|
139
|
+
@reply_to = opts[:reply_to]
|
140
|
+
@selector = opts[:selector]
|
141
|
+
@target = opts[:target]
|
142
|
+
@persistent = opts[:persistent]
|
143
|
+
@tags = opts[:tags] || []
|
87
144
|
end
|
145
|
+
|
88
146
|
def self.json_create(o)
|
89
147
|
i = o['data']
|
90
|
-
new(i['type'], i['payload'],
|
91
|
-
|
148
|
+
new(i['type'], i['payload'], o['size'], { :from => i['from'], :token => i['token'],
|
149
|
+
:reply_to => i['reply_to'], :selector => i['selector'],
|
150
|
+
:target => i['target'], :persistent => i['persistent'],
|
151
|
+
:tags => i['tags'] })
|
152
|
+
end
|
153
|
+
|
154
|
+
def to_s(filter=nil)
|
155
|
+
log_msg = "#{super} <#{token}> #{type}"
|
156
|
+
log_msg += " from #{id_to_s(from)}" if filter.nil? || filter.include?(:from)
|
157
|
+
log_msg += " to #{id_to_s(target)}" if target && (filter.nil? || filter.include?(:target))
|
158
|
+
log_msg += ", reply_to #{id_to_s(reply_to)}" if reply_to && (filter.nil? || filter.include?(:reply_to))
|
159
|
+
log_msg += ", tags #{tags.inspect}" if tags && !tags.empty? && (filter.nil? || filter.include?(:tags))
|
160
|
+
log_msg += ", payload #{payload.inspect}" if filter.nil? || filter.include?(:payload)
|
161
|
+
wrap(log_msg)
|
92
162
|
end
|
163
|
+
|
93
164
|
end
|
94
165
|
|
95
166
|
# packet that means a work push from mapper
|
@@ -105,23 +176,38 @@ module Nanite
|
|
105
176
|
# target is the target nanite for the request
|
106
177
|
# persistent signifies if this request should be saved to persistent storage by the AMQP broker
|
107
178
|
class Push < Packet
|
179
|
+
|
108
180
|
attr_accessor :from, :payload, :type, :token, :selector, :target, :persistent, :tags
|
181
|
+
|
109
182
|
DEFAULT_OPTIONS = {:selector => :least_loaded}
|
110
|
-
|
183
|
+
|
184
|
+
def initialize(type, payload, size=nil, opts={})
|
111
185
|
opts = DEFAULT_OPTIONS.merge(opts)
|
112
|
-
@type
|
113
|
-
@payload
|
114
|
-
@
|
115
|
-
@
|
116
|
-
@
|
117
|
-
@
|
118
|
-
@
|
119
|
-
@
|
186
|
+
@type = type
|
187
|
+
@payload = payload
|
188
|
+
@size = size
|
189
|
+
@from = opts[:from]
|
190
|
+
@token = opts[:token]
|
191
|
+
@selector = opts[:selector]
|
192
|
+
@target = opts[:target]
|
193
|
+
@persistent = opts[:persistent]
|
194
|
+
@tags = opts[:tags] || []
|
120
195
|
end
|
196
|
+
|
121
197
|
def self.json_create(o)
|
122
198
|
i = o['data']
|
123
|
-
new(i['type'], i['payload'],
|
124
|
-
|
199
|
+
new(i['type'], i['payload'], o['size'], { :from => i['from'], :token => i['token'],
|
200
|
+
:selector => i['selector'], :target => i['target'],
|
201
|
+
:persistent => i['persistent'], :tags => i['tags'] })
|
202
|
+
end
|
203
|
+
|
204
|
+
def to_s(filter=nil)
|
205
|
+
log_msg = "#{super} <#{token}> #{type}"
|
206
|
+
log_msg += " from #{id_to_s(from)}" if filter.nil? || filter.include?(:from)
|
207
|
+
log_msg += ", target #{id_to_s(target)}" if target && (filter.nil? || filter.include?(:target))
|
208
|
+
log_msg += ", tags #{tags.inspect}" if tags && !tags.empty? && (filter.nil? || filter.include?(:tags))
|
209
|
+
log_msg += ", payload #{payload.inspect}" if filter.nil? || filter.include?(:payload)
|
210
|
+
wrap(log_msg)
|
125
211
|
end
|
126
212
|
end
|
127
213
|
|
@@ -132,16 +218,28 @@ module Nanite
|
|
132
218
|
# token is a generated request id that mapper uses to identify replies
|
133
219
|
# to is identity of the node result should be delivered to
|
134
220
|
class Result < Packet
|
221
|
+
|
135
222
|
attr_accessor :token, :results, :to, :from
|
136
|
-
|
223
|
+
|
224
|
+
def initialize(token, to, results, from, size=nil)
|
137
225
|
@token = token
|
138
226
|
@to = to
|
139
227
|
@from = from
|
140
228
|
@results = results
|
229
|
+
@size = size
|
141
230
|
end
|
231
|
+
|
142
232
|
def self.json_create(o)
|
143
233
|
i = o['data']
|
144
|
-
new(i['token'], i['to'], i['results'], i['from'])
|
234
|
+
new(i['token'], i['to'], i['results'], i['from'], o['size'])
|
235
|
+
end
|
236
|
+
|
237
|
+
def to_s(filter=nil)
|
238
|
+
log_msg = "#{super} <#{token}>"
|
239
|
+
log_msg += " from #{id_to_s(from)}" if filter.nil? || filter.include?(:from)
|
240
|
+
log_msg += " to #{id_to_s(to)}" if filter.nil? || filter.include?(:to)
|
241
|
+
log_msg += " results: #{results.inspect}" if filter.nil? || filter.include?(:results)
|
242
|
+
wrap(log_msg)
|
145
243
|
end
|
146
244
|
end
|
147
245
|
|
@@ -153,17 +251,25 @@ module Nanite
|
|
153
251
|
# token is a generated request id that mapper uses to identify replies
|
154
252
|
# to is identity of the node result should be delivered to
|
155
253
|
class IntermediateMessage < Packet
|
254
|
+
|
156
255
|
attr_accessor :token, :messagekey, :message, :to, :from
|
157
|
-
|
158
|
-
|
159
|
-
@
|
160
|
-
@
|
256
|
+
|
257
|
+
def initialize(token, to, from, messagekey, message, size=nil)
|
258
|
+
@token = token
|
259
|
+
@to = to
|
260
|
+
@from = from
|
161
261
|
@messagekey = messagekey
|
162
|
-
@message
|
262
|
+
@message = message
|
263
|
+
@size = size
|
163
264
|
end
|
265
|
+
|
164
266
|
def self.json_create(o)
|
165
267
|
i = o['data']
|
166
|
-
new(i['token'], i['to'], i['from'], i['messagekey'], i['message'])
|
268
|
+
new(i['token'], i['to'], i['from'], i['messagekey'], i['message'], o['size'])
|
269
|
+
end
|
270
|
+
|
271
|
+
def to_s
|
272
|
+
wrap("#{super} <#{token}> from #{id_to_s(from)}, key #{messagekey}")
|
167
273
|
end
|
168
274
|
end
|
169
275
|
|
@@ -174,16 +280,27 @@ module Nanite
|
|
174
280
|
# status is a load of the node by default, but may be any criteria
|
175
281
|
# agent may use to report it's availability, load, etc
|
176
282
|
class Register < Packet
|
283
|
+
|
177
284
|
attr_accessor :identity, :services, :status, :tags
|
178
|
-
|
179
|
-
|
180
|
-
@
|
285
|
+
|
286
|
+
def initialize(identity, services, status, tags, size=nil)
|
287
|
+
@status = status
|
288
|
+
@tags = tags
|
181
289
|
@identity = identity
|
182
290
|
@services = services
|
291
|
+
@size = size
|
183
292
|
end
|
293
|
+
|
184
294
|
def self.json_create(o)
|
185
295
|
i = o['data']
|
186
|
-
new(i['identity'], i['services'], i['status'], i['tags'])
|
296
|
+
new(i['identity'], i['services'], i['status'], i['tags'], o['size'])
|
297
|
+
end
|
298
|
+
|
299
|
+
def to_s
|
300
|
+
log_msg = "#{super} #{id_to_s(identity)}"
|
301
|
+
log_msg += ", services: #{services.join(', ')}" if services && !services.empty?
|
302
|
+
log_msg += ", tags: #{tags.join(', ')}" if tags && !tags.empty?
|
303
|
+
wrap(log_msg)
|
187
304
|
end
|
188
305
|
end
|
189
306
|
|
@@ -191,13 +308,21 @@ module Nanite
|
|
191
308
|
#
|
192
309
|
# from is sender identity
|
193
310
|
class UnRegister < Packet
|
311
|
+
|
194
312
|
attr_accessor :identity
|
195
|
-
|
313
|
+
|
314
|
+
def initialize(identity, size=nil)
|
196
315
|
@identity = identity
|
316
|
+
@size = size
|
197
317
|
end
|
318
|
+
|
198
319
|
def self.json_create(o)
|
199
320
|
i = o['data']
|
200
|
-
new(i['identity'])
|
321
|
+
new(i['identity'], o['size'])
|
322
|
+
end
|
323
|
+
|
324
|
+
def to_s
|
325
|
+
"#{super} #{id_to_s(identity)}"
|
201
326
|
end
|
202
327
|
end
|
203
328
|
|
@@ -206,26 +331,40 @@ module Nanite
|
|
206
331
|
# identity is sender's identity
|
207
332
|
# status is sender's status (see Register packet documentation)
|
208
333
|
class Ping < Packet
|
334
|
+
|
209
335
|
attr_accessor :identity, :status
|
210
|
-
|
211
|
-
|
336
|
+
|
337
|
+
def initialize(identity, status, size=nil)
|
338
|
+
@status = status
|
212
339
|
@identity = identity
|
340
|
+
@size = size
|
213
341
|
end
|
342
|
+
|
214
343
|
def self.json_create(o)
|
215
344
|
i = o['data']
|
216
|
-
new(i['identity'], i['status'])
|
345
|
+
new(i['identity'], i['status'], o['size'])
|
346
|
+
end
|
347
|
+
|
348
|
+
def to_s
|
349
|
+
"#{super} #{id_to_s(identity)} status #{status}"
|
217
350
|
end
|
351
|
+
|
218
352
|
end
|
219
353
|
|
220
354
|
# packet that is sent by workers to the mapper
|
221
355
|
# when worker initially comes online to advertise
|
222
356
|
# it's services
|
223
357
|
class Advertise < Packet
|
224
|
-
|
358
|
+
|
359
|
+
def initialize(size=nil)
|
360
|
+
@size = size
|
225
361
|
end
|
362
|
+
|
226
363
|
def self.json_create(o)
|
227
|
-
new
|
364
|
+
new(o['size'])
|
228
365
|
end
|
366
|
+
|
229
367
|
end
|
368
|
+
|
230
369
|
end
|
231
370
|
|
data/lib/nanite/state.rb
CHANGED
@@ -17,7 +17,7 @@ module Nanite
|
|
17
17
|
# service, so for each service the agent provides, we add the nanite to a SET
|
18
18
|
# of all the nanites that provide said service:
|
19
19
|
#
|
20
|
-
# /
|
20
|
+
# foo/bar: { nanite-foobar, nanite-nickelbag, nanite-another } # redis SET
|
21
21
|
#
|
22
22
|
# we do that same thing for tags:
|
23
23
|
# some-tag: { nanite-foobar, nanite-nickelbag, nanite-another } # redis SET
|
@@ -30,7 +30,7 @@ module Nanite
|
|
30
30
|
# of these two service tags
|
31
31
|
|
32
32
|
def initialize(redis)
|
33
|
-
Nanite::Log.info("initializing redis state: #{redis}")
|
33
|
+
Nanite::Log.info("[setup] initializing redis state: #{redis}")
|
34
34
|
host, port = redis.split(':')
|
35
35
|
host ||= '127.0.0.1'
|
36
36
|
port ||= '6379'
|
data/spec/actor_registry_spec.rb
CHANGED
@@ -49,14 +49,14 @@ describe Nanite::ActorRegistry do
|
|
49
49
|
|
50
50
|
it "should log info message that actor was registered" do
|
51
51
|
importer = WebDocumentImporter.new
|
52
|
-
Nanite::Log.should_receive(:info).with("
|
52
|
+
Nanite::Log.should_receive(:info).with("[actor] #{importer.class.to_s}")
|
53
53
|
@registry.register(importer, nil)
|
54
54
|
end
|
55
55
|
|
56
56
|
it "should handle actors registered with a custom prefix" do
|
57
57
|
importer = WebDocumentImporter.new
|
58
58
|
@registry.register(importer, 'monkey')
|
59
|
-
@registry.
|
59
|
+
@registry.actor_for('monkey').should == importer
|
60
60
|
end
|
61
61
|
|
62
62
|
end # Nanite::ActorRegistry
|
data/spec/cluster_spec.rb
CHANGED
@@ -2,6 +2,8 @@ require File.join(File.dirname(__FILE__), 'spec_helper')
|
|
2
2
|
|
3
3
|
describe Nanite::Cluster do
|
4
4
|
|
5
|
+
include SpecHelpers
|
6
|
+
|
5
7
|
describe "Intialization" do
|
6
8
|
|
7
9
|
before(:each) do
|
@@ -91,6 +93,20 @@ describe Nanite::Cluster do
|
|
91
93
|
|
92
94
|
end # Reaper
|
93
95
|
|
96
|
+
describe "State" do
|
97
|
+
require 'nanite/state'
|
98
|
+
it "should use a local state by default" do
|
99
|
+
cluster = Nanite::Cluster.new(@amq, 443, "the_identity", @serializer, @mapper)
|
100
|
+
cluster.nanites.instance_of?(Nanite::LocalState).should == true
|
101
|
+
end
|
102
|
+
|
103
|
+
it "should set up a redis state when requested" do
|
104
|
+
state = Nanite::State.new("")
|
105
|
+
Nanite::State.should_receive(:new).with("localhost:1234").and_return(state)
|
106
|
+
cluster = Nanite::Cluster.new(@amq, 443, "the_identity", @serializer, @mapper, "localhost:1234")
|
107
|
+
cluster.nanites.instance_of?(Nanite::State).should == true
|
108
|
+
end
|
109
|
+
end
|
94
110
|
end # Intialization
|
95
111
|
|
96
112
|
|
@@ -189,9 +205,94 @@ describe Nanite::Cluster do
|
|
189
205
|
@cluster.register(@register_packet)
|
190
206
|
end
|
191
207
|
|
208
|
+
describe "with registered callbacks" do
|
209
|
+
before(:each) do
|
210
|
+
@register_callback = lambda {|request, mapper|}
|
211
|
+
@cluster = Nanite::Cluster.new(@amq, 32, "the_identity", @serializer, @mapper, nil, :register => @register_callback)
|
212
|
+
end
|
213
|
+
|
214
|
+
it "should call the registration callback" do
|
215
|
+
@register_callback.should_receive(:call).with("nanite_id", @mapper)
|
216
|
+
@cluster.register(@register_packet)
|
217
|
+
end
|
218
|
+
end
|
219
|
+
|
220
|
+
describe "when sending an invalid packet to the registration queue" do
|
221
|
+
it "should log a message statement" do
|
222
|
+
Nanite::Log.logger.should_receive(:warn).with("RECV [register] Invalid packet type: Nanite::Ping")
|
223
|
+
@cluster.register(Nanite::Ping.new(nil, nil))
|
224
|
+
end
|
225
|
+
end
|
192
226
|
end # Nanite Registration
|
193
227
|
|
194
|
-
|
228
|
+
describe "Unregister" do
|
229
|
+
before(:each) do
|
230
|
+
@fanout = mock("fanout")
|
231
|
+
@binding = mock("binding", :subscribe => true)
|
232
|
+
@queue = mock("queue", :bind => @binding)
|
233
|
+
@amq = mock("AMQueue", :queue => @queue, :fanout => @fanout)
|
234
|
+
@serializer = mock("Serializer")
|
235
|
+
@reaper = mock("Reaper", :timeout => true)
|
236
|
+
Nanite::Log.stub!(:info)
|
237
|
+
Nanite::Reaper.stub!(:new).and_return(@reaper)
|
238
|
+
@cluster = Nanite::Cluster.new(@amq, 32, "the_identity", @serializer, @mapper)
|
239
|
+
@cluster.nanites["nanite_id"] = "nanite_id"
|
240
|
+
@unregister_packet = Nanite::UnRegister.new("nanite_id")
|
241
|
+
end
|
242
|
+
|
243
|
+
it "should delete the nanite" do
|
244
|
+
@cluster.register(@unregister_packet)
|
245
|
+
@cluster.nanites["nanite_id"].should == nil
|
246
|
+
end
|
247
|
+
|
248
|
+
describe "with registered callbacks" do
|
249
|
+
before(:each) do
|
250
|
+
@unregister_callback = lambda {|request, mapper| }
|
251
|
+
@cluster = Nanite::Cluster.new(@amq, 32, "the_identity", @serializer, @mapper, nil, :unregister => @unregister_callback)
|
252
|
+
@cluster.nanites["nanite_id"] = "nanite_id"
|
253
|
+
end
|
254
|
+
|
255
|
+
it "should call the unregister callback" do
|
256
|
+
@unregister_callback.should_receive(:call).with("nanite_id", @mapper)
|
257
|
+
@cluster.register(@unregister_packet)
|
258
|
+
end
|
259
|
+
end
|
260
|
+
end
|
261
|
+
|
262
|
+
describe "Nanite timed out" do
|
263
|
+
before(:each) do
|
264
|
+
@fanout = mock("fanout")
|
265
|
+
@binding = mock("binding", :subscribe => true)
|
266
|
+
@queue = mock("queue", :bind => @binding)
|
267
|
+
@amq = mock("AMQueue", :queue => @queue, :fanout => @fanout)
|
268
|
+
@serializer = mock("Serializer")
|
269
|
+
Nanite::Log.stub!(:info)
|
270
|
+
@register_packet = Nanite::Register.new("nanite_id", ["the_nanite_services"], "nanite_status",[])
|
271
|
+
end
|
272
|
+
|
273
|
+
it "should remove the nanite when timed out" do
|
274
|
+
EM.run do
|
275
|
+
@cluster = Nanite::Cluster.new(@amq, 0.01, "the_identity", @serializer, @mapper)
|
276
|
+
@cluster.register(@register_packet)
|
277
|
+
EM.add_timer(1.1) {
|
278
|
+
@cluster.nanites["nanite_id"].should == nil
|
279
|
+
EM.stop_event_loop
|
280
|
+
}
|
281
|
+
end
|
282
|
+
end
|
283
|
+
|
284
|
+
it "should call the timed out callback handler when registered" do
|
285
|
+
EM.run do
|
286
|
+
@cluster = Nanite::Cluster.new(@amq, 0.01, "the_identity", @serializer, @mapper)
|
287
|
+
@cluster.register(@register_packet)
|
288
|
+
EM.add_timer(1.1) {
|
289
|
+
@cluster.nanites["nanite_id"].should == nil
|
290
|
+
EM.stop_event_loop
|
291
|
+
}
|
292
|
+
end
|
293
|
+
end
|
294
|
+
end
|
295
|
+
|
195
296
|
describe "Route" do
|
196
297
|
|
197
298
|
before(:each) do
|
@@ -231,7 +332,7 @@ describe Nanite::Cluster do
|
|
231
332
|
@reaper = mock("Reaper")
|
232
333
|
Nanite::Reaper.stub!(:new).and_return(@reaper)
|
233
334
|
@cluster = Nanite::Cluster.new(@amq, 32, "the_identity", @serializer, @mapper)
|
234
|
-
@request = mock("Request", :persistent => true)
|
335
|
+
@request = mock("Request", :persistent => true, :target => nil, :target= => nil, :to_s => nil)
|
235
336
|
@target = mock("Target of Request")
|
236
337
|
end
|
237
338
|
|
@@ -274,9 +375,10 @@ describe Nanite::Cluster do
|
|
274
375
|
@reaper = mock("Reaper")
|
275
376
|
Nanite::Reaper.stub!(:new).and_return(@reaper)
|
276
377
|
@request_without_target = mock("Request", :target => nil, :token => "Token",
|
277
|
-
:reply_to => "Reply To", :from => "From", :persistent => true, :identity => "Identity"
|
378
|
+
:reply_to => "Reply To", :from => "From", :persistent => true, :identity => "Identity",
|
379
|
+
:payload => "Payload", :to_s => nil)
|
278
380
|
@request_with_target = mock("Request", :target => "Target", :token => "Token",
|
279
|
-
:reply_to => "Reply To", :from => "From", :persistent => true)
|
381
|
+
:reply_to => "Reply To", :from => "From", :persistent => true, :payload => "Payload", :to_s => nil)
|
280
382
|
@mapper_with_target = mock("Mapper", :identity => "id")
|
281
383
|
@mapper_without_target = mock("Mapper", :request => false, :identity => @request_without_target.identity)
|
282
384
|
@cluster_with_target = Nanite::Cluster.new(@amq, 32, "the_identity", @serializer, @mapper_with_target)
|
@@ -292,9 +394,78 @@ describe Nanite::Cluster do
|
|
292
394
|
it "should reply back with nil results for requests with no target when offline queue is disabled" do
|
293
395
|
@mapper_without_target.should_receive(:send_request).with(@request_without_target, anything())
|
294
396
|
Nanite::Result.should_receive(:new).with(@request_without_target.token, @request_without_target.from, nil, @request_without_target.identity)
|
295
|
-
|
397
|
+
@cluster_without_target.__send__(:handle_request, @request_without_target)
|
296
398
|
end
|
297
399
|
|
400
|
+
it "should hand in an intermediate handler" do
|
401
|
+
@mapper_with_target.should_receive(:send_request) do |request, opts|
|
402
|
+
opts[:intermediate_handler].should be_instance_of(Proc)
|
403
|
+
end
|
404
|
+
|
405
|
+
@cluster_with_target.__send__(:handle_request, @request_with_target)
|
406
|
+
end
|
407
|
+
|
408
|
+
it "should forward the message when send_request failed" do
|
409
|
+
@mapper_with_target.stub!(:send_request).and_return(false)
|
410
|
+
@cluster_with_target.should_receive(:forward_response)
|
411
|
+
@cluster_with_target.__send__(:handle_request, @request_with_target)
|
412
|
+
end
|
298
413
|
end # Agent Request Handling
|
299
414
|
|
415
|
+
describe "Heartbeat" do
|
416
|
+
before(:each) do
|
417
|
+
@fanout = mock("fanout")
|
418
|
+
@binding = mock("binding", :subscribe => true)
|
419
|
+
@queue = mock("queue", :bind => @binding, :publish => true)
|
420
|
+
@amq = mock("AMQueue", :queue => @queue, :fanout => @fanout)
|
421
|
+
@serializer = mock("Serializer", :dump => "dumped_value")
|
422
|
+
Nanite::Log.stub!(:info)
|
423
|
+
@ping = stub("ping", :status => 0.3, :identity => "nanite_id")
|
424
|
+
end
|
425
|
+
|
426
|
+
it "should update the nanite status" do
|
427
|
+
run_in_em do
|
428
|
+
@cluster = Nanite::Cluster.new(@amq, 32, "the_identity", @serializer, @mapper)
|
429
|
+
@cluster.nanites["nanite_id"] = {:status => "nanite_status"}
|
430
|
+
@cluster.send :handle_ping, @ping
|
431
|
+
@cluster.nanites["nanite_id"][:status].should == 0.3
|
432
|
+
end
|
433
|
+
end
|
434
|
+
|
435
|
+
it "should reset the agent time out" do
|
436
|
+
run_in_em do
|
437
|
+
@cluster = Nanite::Cluster.new(@amq, 32, "the_identity", @serializer, @mapper)
|
438
|
+
@cluster.reaper.should_receive(:reset_with_autoregister_hack).with("nanite_id", 33)
|
439
|
+
@cluster.nanites["nanite_id"] = {:status => "nanite_status"}
|
440
|
+
@cluster.send :handle_ping, @ping
|
441
|
+
end
|
442
|
+
end
|
443
|
+
|
444
|
+
describe "when timing out after a heartbeat" do
|
445
|
+
it "should remove the nanite" do
|
446
|
+
run_in_em(false) do
|
447
|
+
@cluster = Nanite::Cluster.new(@amq, 0.1, "the_identity", @serializer, @mapper)
|
448
|
+
@cluster.nanites["nanite_id"] = {:status => "nanite_status"}
|
449
|
+
@cluster.send :handle_ping, @ping
|
450
|
+
EM.add_timer(1.5) do
|
451
|
+
@cluster.nanites["nanite_id"].should == nil
|
452
|
+
EM.stop_event_loop
|
453
|
+
end
|
454
|
+
end
|
455
|
+
end
|
456
|
+
|
457
|
+
it "should call the timeout callback when defined" do
|
458
|
+
run_in_em(false) do
|
459
|
+
@timeout_callback = lambda {|nanite, mapper| }
|
460
|
+
@timeout_callback.should_receive(:call).with("nanite_id", @mapper)
|
461
|
+
@cluster = Nanite::Cluster.new(@amq, 0.1, "the_identity", @serializer, @mapper, nil, :timeout => @timeout_callback)
|
462
|
+
@cluster.nanites["nanite_id"] = {:status => "nanite_status"}
|
463
|
+
@cluster.send :handle_ping, @ping
|
464
|
+
EM.add_timer(1.5) do
|
465
|
+
EM.stop_event_loop
|
466
|
+
end
|
467
|
+
end
|
468
|
+
end
|
469
|
+
end
|
470
|
+
end
|
300
471
|
end # Nanite::Cluster
|
data/spec/job_spec.rb
CHANGED
@@ -33,7 +33,44 @@ describe Nanite::JobWarden do
|
|
33
33
|
|
34
34
|
end # Creating a new Job
|
35
35
|
|
36
|
-
|
36
|
+
describe "Processing an intermediate message" do
|
37
|
+
before(:each) do
|
38
|
+
@intm_handler = lambda {|arg1, arg2, arg3| puts 'ehlo'}
|
39
|
+
@message = mock("Message", :token => "3faba24fcc", :from => 'nanite-agent')
|
40
|
+
@serializer = mock("Serializer", :load => @message)
|
41
|
+
@warden = Nanite::JobWarden.new(@serializer)
|
42
|
+
@job = Nanite::Job.new(stub("request", :token => "3faba24fcc"), [], @intm_handler)
|
43
|
+
@job.instance_variable_set(:@pending_keys, ["defaultkey"])
|
44
|
+
@job.instance_variable_set(:@intermediate_state, {"nanite-agent" => {"defaultkey" => [1]}})
|
45
|
+
@warden.jobs[@job.token] = @job
|
46
|
+
Nanite::Log.stub!(:debug)
|
47
|
+
Nanite::Log.stub!(:error)
|
48
|
+
end
|
49
|
+
|
50
|
+
it "should call the intermediate handler with three parameters" do
|
51
|
+
@intm_handler.should_receive(:call).with("defaultkey", "nanite-agent", 1)
|
52
|
+
@warden.process(@message)
|
53
|
+
end
|
54
|
+
|
55
|
+
it "should call the intermediate handler with four parameters" do
|
56
|
+
@intm_handler.stub!(:arity).and_return(4)
|
57
|
+
@intm_handler.should_receive(:call).with("defaultkey", "nanite-agent", 1, @job)
|
58
|
+
@warden.process(@message)
|
59
|
+
end
|
60
|
+
|
61
|
+
it "should not call the intermediate handler when it can't be found for the specified key" do
|
62
|
+
@intm_handler.should_not_receive(:call)
|
63
|
+
@job.instance_variable_set(:@intermediate_handler, nil)
|
64
|
+
@warden.process(@message)
|
65
|
+
end
|
66
|
+
|
67
|
+
it "should call the intermediate handler with one parameter which needs to be the result" do
|
68
|
+
@intm_handler.should_receive(:call).with(1, @job)
|
69
|
+
@intm_handler.stub!(:arity).and_return(2)
|
70
|
+
@warden.process(@message)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
37
74
|
describe "Processing a Message" do
|
38
75
|
|
39
76
|
before(:each) do
|
@@ -44,11 +81,6 @@ describe Nanite::JobWarden do
|
|
44
81
|
Nanite::Log.stub!(:debug)
|
45
82
|
end
|
46
83
|
|
47
|
-
it "should log debug message about message to be processed" do
|
48
|
-
Nanite::Log.should_receive(:debug)
|
49
|
-
@warden.process(@message)
|
50
|
-
end
|
51
|
-
|
52
84
|
it "should hand over processing to job" do
|
53
85
|
Nanite::Job.stub!(:new).and_return(@job)
|
54
86
|
@job.should_receive(:process).with(@message)
|
data/spec/spec_helper.rb
CHANGED
@@ -6,7 +6,10 @@ require 'spec'
|
|
6
6
|
require 'nanite'
|
7
7
|
|
8
8
|
module SpecHelpers
|
9
|
-
|
9
|
+
|
10
|
+
# Initialize logger so it writes to file instead of STDOUT
|
11
|
+
Nanite::Log.init('test', File.join(File.dirname(__FILE__)))
|
12
|
+
|
10
13
|
# Create test certificate
|
11
14
|
def issue_cert
|
12
15
|
test_dn = { 'C' => 'US',
|
@@ -20,4 +23,11 @@ module SpecHelpers
|
|
20
23
|
[ Nanite::Certificate.new(key, dn, dn), key ]
|
21
24
|
end
|
22
25
|
|
26
|
+
def run_in_em(stop_event_loop = true)
|
27
|
+
EM.run do
|
28
|
+
yield
|
29
|
+
EM.stop_event_loop if stop_event_loop
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
23
33
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rightscale-nanite
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.1.
|
4
|
+
version: 0.4.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ezra Zygmuntowicz
|
@@ -102,6 +102,7 @@ files:
|
|
102
102
|
- spec/distinguished_name_spec.rb
|
103
103
|
has_rdoc: true
|
104
104
|
homepage: http://github.com/ezmobius/nanite
|
105
|
+
licenses:
|
105
106
|
post_install_message:
|
106
107
|
rdoc_options: []
|
107
108
|
|
@@ -122,7 +123,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
122
123
|
requirements: []
|
123
124
|
|
124
125
|
rubyforge_project:
|
125
|
-
rubygems_version: 1.
|
126
|
+
rubygems_version: 1.3.5
|
126
127
|
signing_key:
|
127
128
|
specification_version: 2
|
128
129
|
summary: self assembling fabric of ruby daemons
|