rightscale-nanite 0.4.1.4 → 0.4.1.10
Sign up to get free protection for your applications and to get access to all the features.
- data/Rakefile +3 -1
- data/lib/nanite/agent.rb +24 -8
- data/lib/nanite/amqp.rb +5 -1
- data/lib/nanite/cluster.rb +74 -54
- data/lib/nanite/daemonize.rb +4 -4
- data/lib/nanite/local_state.rb +5 -1
- data/lib/nanite/log.rb +1 -1
- data/lib/nanite/mapper.rb +17 -11
- data/lib/nanite/mapper_proxy.rb +9 -1
- data/lib/nanite/{dispatcher.rb → nanite_dispatcher.rb} +0 -0
- data/lib/nanite/packets.rb +55 -19
- data/lib/nanite/reaper.rb +15 -14
- data/lib/nanite/redis_tag_store.rb +141 -0
- data/lib/nanite/state.rb +87 -116
- data/lib/nanite/util.rb +21 -1
- data/lib/nanite.rb +2 -2
- metadata +34 -58
- data/spec/actor_registry_spec.rb +0 -60
- data/spec/actor_spec.rb +0 -77
- data/spec/agent_spec.rb +0 -240
- data/spec/cached_certificate_store_proxy_spec.rb +0 -34
- data/spec/certificate_cache_spec.rb +0 -49
- data/spec/certificate_spec.rb +0 -27
- data/spec/cluster_spec.rb +0 -485
- data/spec/dispatcher_spec.rb +0 -136
- data/spec/distinguished_name_spec.rb +0 -24
- data/spec/encrypted_document_spec.rb +0 -21
- data/spec/job_spec.rb +0 -251
- data/spec/local_state_spec.rb +0 -112
- data/spec/log_spec.rb +0 -49
- data/spec/mapper_proxy_spec.rb +0 -72
- data/spec/mapper_spec.rb +0 -70
- data/spec/nanite_spec.rb +0 -36
- data/spec/packet_spec.rb +0 -220
- data/spec/rsa_key_pair_spec.rb +0 -33
- data/spec/secure_serializer_spec.rb +0 -41
- data/spec/serializer_spec.rb +0 -107
- data/spec/signature_spec.rb +0 -30
- data/spec/spec_helper.rb +0 -30
- data/spec/static_certificate_store_spec.rb +0 -30
- data/spec/util_spec.rb +0 -63
data/Rakefile
CHANGED
@@ -10,7 +10,7 @@ end
|
|
10
10
|
require 'rake/clean'
|
11
11
|
require 'lib/nanite'
|
12
12
|
|
13
|
-
GEM = "nanite"
|
13
|
+
GEM = "rightscale-nanite"
|
14
14
|
AUTHOR = "Ezra Zygmuntowicz"
|
15
15
|
EMAIL = "ezra@engineyard.com"
|
16
16
|
HOMEPAGE = "http://github.com/ezmobius/nanite"
|
@@ -74,3 +74,5 @@ task :docs => :rdoc do
|
|
74
74
|
sh 'firefox rdoc/index.html'
|
75
75
|
end
|
76
76
|
end
|
77
|
+
|
78
|
+
require 'rightscale-nanite.rb'
|
data/lib/nanite/agent.rb
CHANGED
@@ -86,24 +86,20 @@ module Nanite
|
|
86
86
|
def initialize(opts)
|
87
87
|
set_configuration(opts)
|
88
88
|
@tags = []
|
89
|
-
@tags << opts[:tag]
|
89
|
+
@tags << opts[:tag] if opts[:tag]
|
90
90
|
@tags.flatten!
|
91
91
|
@options.freeze
|
92
92
|
end
|
93
93
|
|
94
94
|
def run
|
95
|
-
log_path
|
96
|
-
if @options[:daemonize]
|
97
|
-
log_path = (@options[:log_dir] || @options[:root] || Dir.pwd)
|
98
|
-
end
|
99
|
-
Log.init(@identity, log_path)
|
95
|
+
Log.init(@identity, @options[:log_path])
|
100
96
|
Log.level = @options[:log_level] if @options[:log_level]
|
101
97
|
@serializer = Serializer.new(@options[:format])
|
102
98
|
@status_proc = lambda { parse_uptime(`uptime 2> /dev/null`) rescue 'no status' }
|
103
99
|
pid_file = PidFile.new(@identity, @options)
|
104
100
|
pid_file.check
|
105
101
|
if @options[:daemonize]
|
106
|
-
daemonize
|
102
|
+
daemonize(@identity, @options)
|
107
103
|
pid_file.write
|
108
104
|
at_exit { pid_file.remove }
|
109
105
|
end
|
@@ -135,6 +131,16 @@ module Nanite
|
|
135
131
|
@deny_token = deny_token
|
136
132
|
end
|
137
133
|
|
134
|
+
# Update set of tags published by agent and notify mapper
|
135
|
+
# Add tags in 'new_tags' and remove tags in 'old_tags'
|
136
|
+
def update_tags(new_tags, old_tags)
|
137
|
+
@tags += (new_tags || [])
|
138
|
+
@tags -= (old_tags || [])
|
139
|
+
@tags.uniq!
|
140
|
+
tag_update = TagUpdate.new(identity, new_tags, old_tags)
|
141
|
+
amq.fanout('registration', :no_declare => options[:secure]).publish(serializer.dump(tag_update))
|
142
|
+
end
|
143
|
+
|
138
144
|
protected
|
139
145
|
|
140
146
|
def set_configuration(opts)
|
@@ -149,6 +155,11 @@ module Nanite
|
|
149
155
|
opts.delete(:identity) unless opts[:identity]
|
150
156
|
@options.update(custom_config.merge(opts))
|
151
157
|
@options[:file_root] ||= File.join(@options[:root], 'files')
|
158
|
+
@options[:log_path] = false
|
159
|
+
if @options[:daemonize]
|
160
|
+
@options[:log_path] = (@options[:log_dir] || @options[:root] || Dir.pwd)
|
161
|
+
end
|
162
|
+
|
152
163
|
return @identity = "nanite-#{@options[:identity]}" if @options[:identity]
|
153
164
|
token = Identity.generate
|
154
165
|
@identity = "nanite-#{token}"
|
@@ -160,6 +171,7 @@ module Nanite
|
|
160
171
|
def load_actors
|
161
172
|
return unless options[:root]
|
162
173
|
actors_dir = @options[:actors_dir] || "#{@options[:root]}/actors"
|
174
|
+
Nanite::Log.warn("Actors dir #{actors_dir} does not exist or is not reachable") unless File.directory?(actors_dir)
|
163
175
|
actors = @options[:actors]
|
164
176
|
Dir["#{actors_dir}/*.rb"].each do |actor|
|
165
177
|
next if actors && !actors.include?(File.basename(actor, ".rb"))
|
@@ -167,7 +179,11 @@ module Nanite
|
|
167
179
|
require actor
|
168
180
|
end
|
169
181
|
init_path = @options[:initrb] || File.join(options[:root], 'init.rb')
|
170
|
-
|
182
|
+
if File.exist?(init_path)
|
183
|
+
instance_eval(File.read(init_path), init_path)
|
184
|
+
else
|
185
|
+
Nanite::Log.warn("init.rb #{init_path} does not exist or is not reachable") unless File.exists?(init_path)
|
186
|
+
end
|
171
187
|
end
|
172
188
|
|
173
189
|
def receive(packet)
|
data/lib/nanite/amqp.rb
CHANGED
@@ -46,7 +46,11 @@ module Nanite
|
|
46
46
|
:host => options[:host],
|
47
47
|
:port => (options[:port] || ::AMQP::PORT).to_i,
|
48
48
|
:insist => options[:insist] || false,
|
49
|
-
:retry => options[:retry] || 5
|
49
|
+
:retry => options[:retry] || 5,
|
50
|
+
:connection_status => options[:connection_callback] || proc {|event|
|
51
|
+
Nanite::Log.debug("CONNECTED to MQ") if event == :connected
|
52
|
+
Nanite::Log.debug("DISCONNECTED from MQ") if event == :disconnected
|
53
|
+
}
|
50
54
|
})
|
51
55
|
MQ.new(connection)
|
52
56
|
end
|
data/lib/nanite/cluster.rb
CHANGED
@@ -2,13 +2,14 @@ module Nanite
|
|
2
2
|
class Cluster
|
3
3
|
attr_reader :agent_timeout, :nanites, :reaper, :serializer, :identity, :amq, :redis, :mapper, :callbacks
|
4
4
|
|
5
|
-
def initialize(amq, agent_timeout, identity, serializer, mapper, state_configuration=nil, callbacks = {})
|
5
|
+
def initialize(amq, agent_timeout, identity, serializer, mapper, state_configuration=nil, tag_store=nil, callbacks = {})
|
6
6
|
@amq = amq
|
7
7
|
@agent_timeout = agent_timeout
|
8
8
|
@identity = identity
|
9
9
|
@serializer = serializer
|
10
10
|
@mapper = mapper
|
11
11
|
@state = state_configuration
|
12
|
+
@tag_store = tag_store
|
12
13
|
@security = SecurityProvider.get
|
13
14
|
@callbacks = callbacks
|
14
15
|
setup_state
|
@@ -19,7 +20,7 @@ module Nanite
|
|
19
20
|
# determine which nanites should receive the given request
|
20
21
|
def targets_for(request)
|
21
22
|
return [request.target] if request.target
|
22
|
-
__send__(request.selector, request.type, request.tags).collect {|name, state| name }
|
23
|
+
__send__(request.selector, request.from, request.type, request.tags).collect {|name, state| name }
|
23
24
|
end
|
24
25
|
|
25
26
|
# adds nanite to nanites map: key is nanite's identity
|
@@ -30,38 +31,51 @@ module Nanite
|
|
30
31
|
when Register
|
31
32
|
if @security.authorize_registration(reg)
|
32
33
|
Nanite::Log.info("RECV #{reg.to_s}")
|
33
|
-
nanites[reg.identity] = { :services => reg.services, :status => reg.status, :tags => reg.tags }
|
34
|
-
reaper.
|
34
|
+
nanites[reg.identity] = { :services => reg.services, :status => reg.status, :tags => reg.tags, :timestamp => Time.now.utc.to_i }
|
35
|
+
reaper.register(reg.identity, agent_timeout + 1) { nanite_timed_out(reg.identity) }
|
35
36
|
callbacks[:register].call(reg.identity, mapper) if callbacks[:register]
|
36
37
|
else
|
37
38
|
Nanite::Log.warn("RECV NOT AUTHORIZED #{reg.to_s}")
|
38
39
|
end
|
39
40
|
when UnRegister
|
40
41
|
Nanite::Log.info("RECV #{reg.to_s}")
|
42
|
+
reaper.unregister(reg.identity)
|
41
43
|
nanites.delete(reg.identity)
|
42
44
|
callbacks[:unregister].call(reg.identity, mapper) if callbacks[:unregister]
|
45
|
+
when TagUpdate
|
46
|
+
Nanite::Log.info("RECV #{reg.to_s}")
|
47
|
+
nanites.update_tags(reg.identity, reg.new_tags, reg.obsolete_tags)
|
43
48
|
else
|
44
49
|
Nanite::Log.warn("RECV [register] Invalid packet type: #{reg.class}")
|
45
50
|
end
|
46
51
|
end
|
47
52
|
|
48
53
|
def nanite_timed_out(token)
|
49
|
-
nanite = nanites
|
50
|
-
|
54
|
+
nanite = nanites[token]
|
55
|
+
if nanite && timed_out?(nanite)
|
56
|
+
Nanite::Log.info("Nanite #{token} timed out")
|
57
|
+
nanite = nanites.delete(token)
|
58
|
+
callbacks[:timeout].call(token, mapper) if callbacks[:timeout]
|
59
|
+
true
|
60
|
+
end
|
51
61
|
end
|
52
|
-
|
62
|
+
|
53
63
|
def route(request, targets)
|
54
64
|
EM.next_tick { targets.map { |target| publish(request, target) } }
|
55
65
|
end
|
56
66
|
|
57
67
|
def publish(request, target)
|
58
|
-
# We need to initialize the 'target' field of the request object so that the serializer
|
59
|
-
# access to it.
|
68
|
+
# We need to initialize the 'target' field of the request object so that the serializer and
|
69
|
+
# the security provider have access to it.
|
60
70
|
begin
|
61
71
|
old_target = request.target
|
62
72
|
request.target = target unless target == 'mapper-offline'
|
63
|
-
|
64
|
-
|
73
|
+
if @security.authorize_request(request)
|
74
|
+
Nanite::Log.info("SEND #{request.to_s([:from, :on_behalf, :tags, :target])}")
|
75
|
+
amq.queue(target).publish(serializer.dump(request), :persistent => request.persistent)
|
76
|
+
else
|
77
|
+
Nanite::Log.warn("RECV NOT AUTHORIZED #{request.to_s}")
|
78
|
+
end
|
65
79
|
ensure
|
66
80
|
request.target = old_target
|
67
81
|
end
|
@@ -74,8 +88,8 @@ module Nanite
|
|
74
88
|
def handle_ping(ping)
|
75
89
|
begin
|
76
90
|
if nanite = nanites[ping.identity]
|
77
|
-
|
78
|
-
reaper.
|
91
|
+
nanites.update_status(ping.identity, ping.status)
|
92
|
+
reaper.update(ping.identity, agent_timeout + 1) { nanite_timed_out(ping.identity) }
|
79
93
|
else
|
80
94
|
packet = Advertise.new
|
81
95
|
Nanite::Log.info("SEND #{packet.to_s} to #{ping.identity}")
|
@@ -83,58 +97,54 @@ module Nanite
|
|
83
97
|
end
|
84
98
|
end
|
85
99
|
end
|
86
|
-
|
100
|
+
|
87
101
|
# forward request coming from agent
|
88
102
|
def handle_request(request)
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
mapper.send_push(request)
|
95
|
-
else
|
96
|
-
intm_handler = lambda do |result, job|
|
97
|
-
result = IntermediateMessage.new(request.token, job.request.from, mapper.identity, nil, result)
|
98
|
-
forward_response(result, request.persistent)
|
99
|
-
end
|
100
|
-
|
101
|
-
result = Result.new(request.token, request.from, nil, mapper.identity)
|
102
|
-
ok = mapper.send_request(request, :intermediate_handler => intm_handler) do |res|
|
103
|
-
result.results = res
|
104
|
-
forward_response(result, request.persistent)
|
105
|
-
end
|
106
|
-
|
107
|
-
if ok == false
|
108
|
-
forward_response(result, request.persistent)
|
109
|
-
end
|
110
|
-
end
|
103
|
+
Nanite::Log.info("RECV #{request.to_s([:from, :on_behalf, :target, :tags])}") unless Nanite::Log.level == :debug
|
104
|
+
Nanite::Log.debug("RECV #{request.to_s}")
|
105
|
+
case request
|
106
|
+
when Push
|
107
|
+
mapper.send_push(request)
|
111
108
|
else
|
112
|
-
|
109
|
+
intm_handler = lambda do |result, job|
|
110
|
+
result = IntermediateMessage.new(request.token, job.request.from, mapper.identity, nil, result)
|
111
|
+
forward_response(result, request.persistent)
|
112
|
+
end
|
113
|
+
|
114
|
+
result = Result.new(request.token, request.from, nil, mapper.identity)
|
115
|
+
ok = mapper.send_request(request, :intermediate_handler => intm_handler) do |res|
|
116
|
+
result.results = res
|
117
|
+
forward_response(result, request.persistent)
|
118
|
+
end
|
119
|
+
|
120
|
+
if ok == false
|
121
|
+
forward_response(result, request.persistent)
|
122
|
+
end
|
113
123
|
end
|
114
124
|
end
|
115
|
-
|
125
|
+
|
116
126
|
# forward response back to agent that originally made the request
|
117
127
|
def forward_response(res, persistent)
|
118
128
|
Nanite::Log.info("SEND #{res.to_s([:to])}")
|
119
129
|
amq.queue(res.to).publish(serializer.dump(res), :persistent => persistent)
|
120
130
|
end
|
121
|
-
|
131
|
+
|
122
132
|
# returns least loaded nanite that provides given service
|
123
|
-
def least_loaded(service, tags=[])
|
124
|
-
candidates = nanites_providing(service,tags)
|
133
|
+
def least_loaded(from, service, tags=[])
|
134
|
+
candidates = nanites_providing(from, service, tags)
|
125
135
|
return [] if candidates.empty?
|
126
136
|
|
127
137
|
[candidates.min { |a,b| a[1][:status] <=> b[1][:status] }]
|
128
138
|
end
|
129
139
|
|
130
140
|
# returns all nanites that provide given service
|
131
|
-
def all(service, tags=[])
|
132
|
-
nanites_providing(service,tags)
|
141
|
+
def all(from, service, tags=[])
|
142
|
+
nanites_providing(from, service,tags)
|
133
143
|
end
|
134
144
|
|
135
145
|
# returns a random nanite
|
136
|
-
def random(service, tags=[])
|
137
|
-
candidates = nanites_providing(service,tags)
|
146
|
+
def random(from, service, tags=[])
|
147
|
+
candidates = nanites_providing(from, service,tags)
|
138
148
|
return [] if candidates.empty?
|
139
149
|
|
140
150
|
[candidates[rand(candidates.size)]]
|
@@ -142,10 +152,10 @@ module Nanite
|
|
142
152
|
|
143
153
|
# selects next nanite that provides given service
|
144
154
|
# using round robin rotation
|
145
|
-
def rr(service, tags=[])
|
155
|
+
def rr(from, service, tags=[])
|
146
156
|
@last ||= {}
|
147
157
|
@last[service] ||= 0
|
148
|
-
candidates = nanites_providing(service,tags)
|
158
|
+
candidates = nanites_providing(from, service,tags)
|
149
159
|
return [] if candidates.empty?
|
150
160
|
@last[service] = 0 if @last[service] >= candidates.size
|
151
161
|
candidate = candidates[@last[service]]
|
@@ -153,9 +163,18 @@ module Nanite
|
|
153
163
|
[candidate]
|
154
164
|
end
|
155
165
|
|
166
|
+
def timed_out?(nanite)
|
167
|
+
nanite[:timestamp].to_i < (Time.now.utc - agent_timeout).to_i
|
168
|
+
end
|
169
|
+
|
156
170
|
# returns all nanites that provide the given service
|
157
|
-
def nanites_providing(service,
|
158
|
-
nanites.nanites_for(service,
|
171
|
+
def nanites_providing(from, service, tags)
|
172
|
+
nanites.nanites_for(from, service, tags).delete_if do |nanite|
|
173
|
+
if res = timed_out?(nanite[1])
|
174
|
+
Nanite::Log.debug("Ignoring timed out nanite #{nanite[0]} in target selection - last seen at #{nanite[1][:timestamp]}")
|
175
|
+
end
|
176
|
+
res
|
177
|
+
end
|
159
178
|
end
|
160
179
|
|
161
180
|
def setup_queues
|
@@ -172,6 +191,7 @@ module Nanite
|
|
172
191
|
handle_ping(ping)
|
173
192
|
rescue Exception => e
|
174
193
|
Nanite::Log.error("RECV [ping] #{e.message}")
|
194
|
+
callbacks[:exception].call(e, msg, mapper) rescue nil if callbacks[:exception]
|
175
195
|
end
|
176
196
|
end
|
177
197
|
hb_fanout = amq.fanout('heartbeat', :durable => true)
|
@@ -188,6 +208,7 @@ module Nanite
|
|
188
208
|
register(serializer.load(msg))
|
189
209
|
rescue Exception => e
|
190
210
|
Nanite::Log.error("RECV [register] #{e.message}")
|
211
|
+
callbacks[:exception].call(e, msg, mapper) rescue nil if callbacks[:exception]
|
191
212
|
end
|
192
213
|
end
|
193
214
|
reg_fanout = amq.fanout('registration', :durable => true)
|
@@ -197,13 +218,14 @@ module Nanite
|
|
197
218
|
amq.queue("registration-#{identity}", :exclusive => true).bind(reg_fanout).subscribe &handler
|
198
219
|
end
|
199
220
|
end
|
200
|
-
|
221
|
+
|
201
222
|
def setup_request_queue
|
202
223
|
handler = lambda do |msg|
|
203
224
|
begin
|
204
225
|
handle_request(serializer.load(msg))
|
205
226
|
rescue Exception => e
|
206
227
|
Nanite::Log.error("RECV [request] #{e.message}")
|
228
|
+
callbacks[:exception].call(e, msg, mapper) rescue nil if callbacks[:exception]
|
207
229
|
end
|
208
230
|
end
|
209
231
|
req_fanout = amq.fanout('request', :durable => true)
|
@@ -219,16 +241,14 @@ module Nanite
|
|
219
241
|
when String
|
220
242
|
# backwards compatibility, we assume redis if the configuration option
|
221
243
|
# was a string
|
222
|
-
Nanite::Log.info("[setup] using redis for state storage")
|
223
244
|
require 'nanite/state'
|
224
|
-
@nanites = Nanite::State.new(@state)
|
225
|
-
when Hash
|
245
|
+
@nanites = Nanite::State.new(@state, @tag_store)
|
226
246
|
else
|
227
247
|
require 'nanite/local_state'
|
228
248
|
@nanites = Nanite::LocalState.new
|
229
249
|
end
|
230
250
|
end
|
231
|
-
|
251
|
+
|
232
252
|
def shared_state?
|
233
253
|
!@state.nil?
|
234
254
|
end
|
data/lib/nanite/daemonize.rb
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
module Nanite
|
2
2
|
module DaemonizeHelper
|
3
|
-
def daemonize
|
3
|
+
def daemonize(identity, options = {})
|
4
4
|
exit if fork
|
5
5
|
Process.setsid
|
6
6
|
exit if fork
|
7
|
-
File.umask 0000
|
8
7
|
STDIN.reopen "/dev/null"
|
9
|
-
STDOUT.reopen "/
|
10
|
-
STDERR.reopen
|
8
|
+
STDOUT.reopen "#{options[:log_path]}/nanite.#{identity}.out", "a"
|
9
|
+
STDERR.reopen "#{options[:log_path]}/nanite.#{identity}.err", "a"
|
10
|
+
File.umask 0000
|
11
11
|
end
|
12
12
|
end
|
13
13
|
end
|
data/lib/nanite/local_state.rb
CHANGED
@@ -14,7 +14,7 @@ module Nanite
|
|
14
14
|
all(:tags)
|
15
15
|
end
|
16
16
|
|
17
|
-
def nanites_for(service,
|
17
|
+
def nanites_for(from, service, tags)
|
18
18
|
tags = tags.dup.flatten
|
19
19
|
nanites = select { |name, state| state[:services].include?(service) }
|
20
20
|
unless tags.empty?
|
@@ -24,6 +24,10 @@ module Nanite
|
|
24
24
|
end.to_a
|
25
25
|
end
|
26
26
|
|
27
|
+
def update_status(name, status)
|
28
|
+
self[name].update(:status => status, :timestamp => Time.now.utc.to_i)
|
29
|
+
end
|
30
|
+
|
27
31
|
private
|
28
32
|
|
29
33
|
def all(key)
|
data/lib/nanite/log.rb
CHANGED
@@ -35,7 +35,7 @@ module Nanite
|
|
35
35
|
# Throws an ArgumentError if you feed it a bogus log level (that is not
|
36
36
|
# one of :debug, :info, :warn, :error, :fatal or the corresponding strings or a valid Logger level)
|
37
37
|
def level=(loglevel)
|
38
|
-
init
|
38
|
+
init unless @logger
|
39
39
|
lvl = case loglevel
|
40
40
|
when String then loglevel.intern
|
41
41
|
when Integer then LEVELS.invert[loglevel]
|
data/lib/nanite/mapper.rb
CHANGED
@@ -76,10 +76,15 @@ module Nanite
|
|
76
76
|
# secure : use Security features of rabbitmq to restrict nanites to themselves
|
77
77
|
#
|
78
78
|
# prefetch : Sets prefetch (only supported in RabbitMQ >= 1.6)
|
79
|
-
#
|
80
|
-
#
|
81
|
-
#
|
82
|
-
#
|
79
|
+
#
|
80
|
+
# callbacks : A set of callbacks to be executed on specific events. Supported events are :register,
|
81
|
+
# :unregister, :timeout and :exception. This option must be a hash with event names as
|
82
|
+
# as keys and Procs as values. The Proc's arity (number of parameters) depends on the
|
83
|
+
# type of callback:
|
84
|
+
# exception -- the exception, the message being processed, a reference to the mapper
|
85
|
+
# all others -- the corresponding nanite's identity, a reference to the mapper
|
86
|
+
#
|
87
|
+
# tag_store : Name of class which implements tag store backend interface, RedisTagStore by default
|
83
88
|
#
|
84
89
|
# Connection options:
|
85
90
|
#
|
@@ -116,6 +121,10 @@ module Nanite
|
|
116
121
|
@options.update(custom_config.merge(options))
|
117
122
|
@identity = "mapper-#{@options[:identity]}"
|
118
123
|
@options[:file_root] ||= File.join(@options[:root], 'files')
|
124
|
+
@options[:log_path] = false
|
125
|
+
if @options[:daemonize]
|
126
|
+
@options[:log_path] = (@options[:log_dir] || @options[:root] || Dir.pwd)
|
127
|
+
end
|
119
128
|
@options.freeze
|
120
129
|
end
|
121
130
|
|
@@ -125,7 +134,7 @@ module Nanite
|
|
125
134
|
pid_file = PidFile.new(@identity, @options)
|
126
135
|
pid_file.check
|
127
136
|
if @options[:daemonize]
|
128
|
-
daemonize
|
137
|
+
daemonize(@identity, @options)
|
129
138
|
pid_file.write
|
130
139
|
at_exit { pid_file.remove }
|
131
140
|
else
|
@@ -288,21 +297,18 @@ module Nanite
|
|
288
297
|
job_warden.process(msg)
|
289
298
|
rescue Exception => e
|
290
299
|
Nanite::Log.error("RECV [result] #{e.message}")
|
300
|
+
callbacks[:exception].call(e, msg, mapper) rescue nil if callbacks[:exception]
|
291
301
|
end
|
292
302
|
end
|
293
303
|
end
|
294
304
|
|
295
305
|
def setup_logging
|
296
|
-
log_path
|
297
|
-
if @options[:daemonize]
|
298
|
-
log_path = (@options[:log_dir] || @options[:root] || Dir.pwd)
|
299
|
-
end
|
300
|
-
Nanite::Log.init(@identity, log_path)
|
306
|
+
Nanite::Log.init(@identity, @options[:log_path])
|
301
307
|
Nanite::Log.level = @options[:log_level] if @options[:log_level]
|
302
308
|
end
|
303
309
|
|
304
310
|
def setup_cluster
|
305
|
-
@cluster = Cluster.new(@amq, @options[:agent_timeout], @options[:identity], @serializer, self, @options[:redis], @options[:callbacks])
|
311
|
+
@cluster = Cluster.new(@amq, @options[:agent_timeout], @options[:identity], @serializer, self, @options[:redis], @options[:tag_store], @options[:callbacks])
|
306
312
|
end
|
307
313
|
end
|
308
314
|
end
|
data/lib/nanite/mapper_proxy.rb
CHANGED
@@ -49,7 +49,15 @@ module Nanite
|
|
49
49
|
push.persistent = opts.key?(:persistent) ? opts[:persistent] : options[:persistent]
|
50
50
|
Nanite::Log.info("SEND #{push.to_s([:tags, :target])}")
|
51
51
|
amqp.fanout('request', :no_declare => options[:secure]).publish(serializer.dump(push))
|
52
|
-
end
|
52
|
+
end
|
53
|
+
|
54
|
+
# Update tags registered by mapper for agent
|
55
|
+
def update_tags(new_tags, obsolete_tags)
|
56
|
+
raise "Mapper proxy not initialized" unless identity && options
|
57
|
+
update = TagUpdate.new(identity, new_tags, obsolete_tags)
|
58
|
+
Nanite::Log.info("SEND #{update.to_s}")
|
59
|
+
amqp.fanout('registration', :no_declare => options[:secure]).publish(serializer.dump(update))
|
60
|
+
end
|
53
61
|
|
54
62
|
# Handle intermediary result
|
55
63
|
def handle_intermediate_result(res)
|
File without changes
|
data/lib/nanite/packets.rb
CHANGED
@@ -112,15 +112,16 @@ module Nanite
|
|
112
112
|
# payload is arbitrary data that is transferred from mapper to actor
|
113
113
|
#
|
114
114
|
# Options:
|
115
|
-
# from
|
116
|
-
#
|
117
|
-
#
|
118
|
-
#
|
119
|
-
#
|
115
|
+
# from is sender identity
|
116
|
+
# on_behalf is agent identity that should be used to authorize request
|
117
|
+
# token is a generated request id that mapper uses to identify replies
|
118
|
+
# reply_to is identity of the node actor replies to, usually a mapper itself
|
119
|
+
# selector is the selector used to route the request
|
120
|
+
# target is the target nanite for the request
|
120
121
|
# persistent signifies if this request should be saved to persistent storage by the AMQP broker
|
121
122
|
class Request < Packet
|
122
123
|
|
123
|
-
attr_accessor :from, :payload, :type, :token, :reply_to, :selector, :target, :persistent, :tags
|
124
|
+
attr_accessor :from, :on_behalf, :payload, :type, :token, :reply_to, :selector, :target, :persistent, :tags
|
124
125
|
|
125
126
|
DEFAULT_OPTIONS = {:selector => :least_loaded}
|
126
127
|
|
@@ -130,6 +131,7 @@ module Nanite
|
|
130
131
|
@payload = payload
|
131
132
|
@size = size
|
132
133
|
@from = opts[:from]
|
134
|
+
@on_behalf = opts[:on_behalf]
|
133
135
|
@token = opts[:token]
|
134
136
|
@reply_to = opts[:reply_to]
|
135
137
|
@selector = opts[:selector]
|
@@ -140,15 +142,17 @@ module Nanite
|
|
140
142
|
|
141
143
|
def self.json_create(o)
|
142
144
|
i = o['data']
|
143
|
-
new(i['type'], i['payload'], { :from => i['from'],
|
144
|
-
:
|
145
|
-
:
|
146
|
-
:tags => i['tags'] },
|
145
|
+
new(i['type'], i['payload'], { :from => i['from'], :on_behalf => i['on_behalf'],
|
146
|
+
:token => i['token'], :reply_to => i['reply_to'],
|
147
|
+
:selector => i['selector'], :target => i['target'],
|
148
|
+
:persistent => i['persistent'], :tags => i['tags'] },
|
149
|
+
o['size'])
|
147
150
|
end
|
148
151
|
|
149
152
|
def to_s(filter=nil)
|
150
153
|
log_msg = "#{super} <#{token}> #{type}"
|
151
154
|
log_msg += " from #{id_to_s(from)}" if filter.nil? || filter.include?(:from)
|
155
|
+
log_msg += " on behalf of #{id_to_s(on_behalf)}" if on_behalf && (filter.nil? || filter.include?(:on_behalf))
|
152
156
|
log_msg += " to #{id_to_s(target)}" if target && (filter.nil? || filter.include?(:target))
|
153
157
|
log_msg += ", reply_to #{id_to_s(reply_to)}" if reply_to && (filter.nil? || filter.include?(:reply_to))
|
154
158
|
log_msg += ", tags #{tags.inspect}" if tags && !tags.empty? && (filter.nil? || filter.include?(:tags))
|
@@ -165,14 +169,15 @@ module Nanite
|
|
165
169
|
# payload is arbitrary data that is transferred from mapper to actor
|
166
170
|
#
|
167
171
|
# Options:
|
168
|
-
# from
|
169
|
-
#
|
170
|
-
#
|
171
|
-
#
|
172
|
+
# from is sender identity
|
173
|
+
# on_behalf is agent identity that should be used to authorize request
|
174
|
+
# token is a generated request id that mapper uses to identify replies
|
175
|
+
# selector is the selector used to route the request
|
176
|
+
# target is the target nanite for the request
|
172
177
|
# persistent signifies if this request should be saved to persistent storage by the AMQP broker
|
173
178
|
class Push < Packet
|
174
179
|
|
175
|
-
attr_accessor :from, :payload, :type, :token, :selector, :target, :persistent, :tags
|
180
|
+
attr_accessor :from, :on_behalf, :payload, :type, :token, :selector, :target, :persistent, :tags
|
176
181
|
|
177
182
|
DEFAULT_OPTIONS = {:selector => :least_loaded}
|
178
183
|
|
@@ -182,6 +187,7 @@ module Nanite
|
|
182
187
|
@payload = payload
|
183
188
|
@size = size
|
184
189
|
@from = opts[:from]
|
190
|
+
@on_behalf = opts[:on_behalf]
|
185
191
|
@token = opts[:token]
|
186
192
|
@selector = opts[:selector]
|
187
193
|
@target = opts[:target]
|
@@ -191,14 +197,17 @@ module Nanite
|
|
191
197
|
|
192
198
|
def self.json_create(o)
|
193
199
|
i = o['data']
|
194
|
-
new(i['type'], i['payload'], { :from
|
195
|
-
:
|
196
|
-
:
|
200
|
+
new(i['type'], i['payload'], { :from => i['from'], :on_behalf => i['on_behalf'],
|
201
|
+
:token => i['token'], :selector => i['selector'],
|
202
|
+
:target => i['target'], :persistent => i['persistent'],
|
203
|
+
:tags => i['tags'] },
|
204
|
+
o['size'])
|
197
205
|
end
|
198
206
|
|
199
207
|
def to_s(filter=nil)
|
200
208
|
log_msg = "#{super} <#{token}> #{type}"
|
201
209
|
log_msg += " from #{id_to_s(from)}" if filter.nil? || filter.include?(:from)
|
210
|
+
log_msg += " on behalf of #{id_to_s(on_behalf)}" if on_behalf && (filter.nil? || filter.include?(:on_behalf))
|
202
211
|
log_msg += ", target #{id_to_s(target)}" if target && (filter.nil? || filter.include?(:target))
|
203
212
|
log_msg += ", tags #{tags.inspect}" if tags && !tags.empty? && (filter.nil? || filter.include?(:tags))
|
204
213
|
log_msg += ", payload #{payload.inspect}" if filter.nil? || filter.include?(:payload)
|
@@ -360,6 +369,33 @@ module Nanite
|
|
360
369
|
end
|
361
370
|
|
362
371
|
end
|
363
|
-
|
372
|
+
|
373
|
+
# packet that is sent by agents to the mapper
|
374
|
+
# to update their tags
|
375
|
+
class TagUpdate < Packet
|
376
|
+
|
377
|
+
attr_accessor :identity, :new_tags, :obsolete_tags
|
378
|
+
|
379
|
+
def initialize(identity, new_tags, obsolete_tags, size=nil)
|
380
|
+
@identity = identity
|
381
|
+
@new_tags = new_tags
|
382
|
+
@obsolete_tags = obsolete_tags
|
383
|
+
@size = size
|
384
|
+
end
|
385
|
+
|
386
|
+
def self.json_create(o)
|
387
|
+
i = o['data']
|
388
|
+
new(i['identity'], i['new_tags'], i['obsolete_tags'], o['size'])
|
389
|
+
end
|
390
|
+
|
391
|
+
def to_s
|
392
|
+
log_msg = "#{super} #{id_to_s(identity)}"
|
393
|
+
log_msg += ", new tags: #{new_tags.join(', ')}" if new_tags && !new_tags.empty?
|
394
|
+
log_msg += ", obsolete tags: #{obsolete_tags.join(', ')}" if obsolete_tags && !obsolete_tags.empty?
|
395
|
+
log_msg
|
396
|
+
end
|
397
|
+
|
398
|
+
end
|
399
|
+
|
364
400
|
end
|
365
401
|
|