rightscale-nanite 0.4.1.4 → 0.4.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Rakefile +3 -1
- data/lib/nanite/agent.rb +24 -8
- data/lib/nanite/amqp.rb +5 -1
- data/lib/nanite/cluster.rb +74 -54
- data/lib/nanite/daemonize.rb +4 -4
- data/lib/nanite/local_state.rb +5 -1
- data/lib/nanite/log.rb +1 -1
- data/lib/nanite/mapper.rb +17 -11
- data/lib/nanite/mapper_proxy.rb +9 -1
- data/lib/nanite/{dispatcher.rb → nanite_dispatcher.rb} +0 -0
- data/lib/nanite/packets.rb +55 -19
- data/lib/nanite/reaper.rb +15 -14
- data/lib/nanite/redis_tag_store.rb +141 -0
- data/lib/nanite/state.rb +87 -116
- data/lib/nanite/util.rb +21 -1
- data/lib/nanite.rb +2 -2
- metadata +34 -58
- data/spec/actor_registry_spec.rb +0 -60
- data/spec/actor_spec.rb +0 -77
- data/spec/agent_spec.rb +0 -240
- data/spec/cached_certificate_store_proxy_spec.rb +0 -34
- data/spec/certificate_cache_spec.rb +0 -49
- data/spec/certificate_spec.rb +0 -27
- data/spec/cluster_spec.rb +0 -485
- data/spec/dispatcher_spec.rb +0 -136
- data/spec/distinguished_name_spec.rb +0 -24
- data/spec/encrypted_document_spec.rb +0 -21
- data/spec/job_spec.rb +0 -251
- data/spec/local_state_spec.rb +0 -112
- data/spec/log_spec.rb +0 -49
- data/spec/mapper_proxy_spec.rb +0 -72
- data/spec/mapper_spec.rb +0 -70
- data/spec/nanite_spec.rb +0 -36
- data/spec/packet_spec.rb +0 -220
- data/spec/rsa_key_pair_spec.rb +0 -33
- data/spec/secure_serializer_spec.rb +0 -41
- data/spec/serializer_spec.rb +0 -107
- data/spec/signature_spec.rb +0 -30
- data/spec/spec_helper.rb +0 -30
- data/spec/static_certificate_store_spec.rb +0 -30
- data/spec/util_spec.rb +0 -63
data/Rakefile
CHANGED
@@ -10,7 +10,7 @@ end
|
|
10
10
|
require 'rake/clean'
|
11
11
|
require 'lib/nanite'
|
12
12
|
|
13
|
-
GEM = "nanite"
|
13
|
+
GEM = "rightscale-nanite"
|
14
14
|
AUTHOR = "Ezra Zygmuntowicz"
|
15
15
|
EMAIL = "ezra@engineyard.com"
|
16
16
|
HOMEPAGE = "http://github.com/ezmobius/nanite"
|
@@ -74,3 +74,5 @@ task :docs => :rdoc do
|
|
74
74
|
sh 'firefox rdoc/index.html'
|
75
75
|
end
|
76
76
|
end
|
77
|
+
|
78
|
+
require 'rightscale-nanite.rb'
|
data/lib/nanite/agent.rb
CHANGED
@@ -86,24 +86,20 @@ module Nanite
|
|
86
86
|
def initialize(opts)
|
87
87
|
set_configuration(opts)
|
88
88
|
@tags = []
|
89
|
-
@tags << opts[:tag]
|
89
|
+
@tags << opts[:tag] if opts[:tag]
|
90
90
|
@tags.flatten!
|
91
91
|
@options.freeze
|
92
92
|
end
|
93
93
|
|
94
94
|
def run
|
95
|
-
log_path
|
96
|
-
if @options[:daemonize]
|
97
|
-
log_path = (@options[:log_dir] || @options[:root] || Dir.pwd)
|
98
|
-
end
|
99
|
-
Log.init(@identity, log_path)
|
95
|
+
Log.init(@identity, @options[:log_path])
|
100
96
|
Log.level = @options[:log_level] if @options[:log_level]
|
101
97
|
@serializer = Serializer.new(@options[:format])
|
102
98
|
@status_proc = lambda { parse_uptime(`uptime 2> /dev/null`) rescue 'no status' }
|
103
99
|
pid_file = PidFile.new(@identity, @options)
|
104
100
|
pid_file.check
|
105
101
|
if @options[:daemonize]
|
106
|
-
daemonize
|
102
|
+
daemonize(@identity, @options)
|
107
103
|
pid_file.write
|
108
104
|
at_exit { pid_file.remove }
|
109
105
|
end
|
@@ -135,6 +131,16 @@ module Nanite
|
|
135
131
|
@deny_token = deny_token
|
136
132
|
end
|
137
133
|
|
134
|
+
# Update set of tags published by agent and notify mapper
|
135
|
+
# Add tags in 'new_tags' and remove tags in 'old_tags'
|
136
|
+
def update_tags(new_tags, old_tags)
|
137
|
+
@tags += (new_tags || [])
|
138
|
+
@tags -= (old_tags || [])
|
139
|
+
@tags.uniq!
|
140
|
+
tag_update = TagUpdate.new(identity, new_tags, old_tags)
|
141
|
+
amq.fanout('registration', :no_declare => options[:secure]).publish(serializer.dump(tag_update))
|
142
|
+
end
|
143
|
+
|
138
144
|
protected
|
139
145
|
|
140
146
|
def set_configuration(opts)
|
@@ -149,6 +155,11 @@ module Nanite
|
|
149
155
|
opts.delete(:identity) unless opts[:identity]
|
150
156
|
@options.update(custom_config.merge(opts))
|
151
157
|
@options[:file_root] ||= File.join(@options[:root], 'files')
|
158
|
+
@options[:log_path] = false
|
159
|
+
if @options[:daemonize]
|
160
|
+
@options[:log_path] = (@options[:log_dir] || @options[:root] || Dir.pwd)
|
161
|
+
end
|
162
|
+
|
152
163
|
return @identity = "nanite-#{@options[:identity]}" if @options[:identity]
|
153
164
|
token = Identity.generate
|
154
165
|
@identity = "nanite-#{token}"
|
@@ -160,6 +171,7 @@ module Nanite
|
|
160
171
|
def load_actors
|
161
172
|
return unless options[:root]
|
162
173
|
actors_dir = @options[:actors_dir] || "#{@options[:root]}/actors"
|
174
|
+
Nanite::Log.warn("Actors dir #{actors_dir} does not exist or is not reachable") unless File.directory?(actors_dir)
|
163
175
|
actors = @options[:actors]
|
164
176
|
Dir["#{actors_dir}/*.rb"].each do |actor|
|
165
177
|
next if actors && !actors.include?(File.basename(actor, ".rb"))
|
@@ -167,7 +179,11 @@ module Nanite
|
|
167
179
|
require actor
|
168
180
|
end
|
169
181
|
init_path = @options[:initrb] || File.join(options[:root], 'init.rb')
|
170
|
-
|
182
|
+
if File.exist?(init_path)
|
183
|
+
instance_eval(File.read(init_path), init_path)
|
184
|
+
else
|
185
|
+
Nanite::Log.warn("init.rb #{init_path} does not exist or is not reachable") unless File.exists?(init_path)
|
186
|
+
end
|
171
187
|
end
|
172
188
|
|
173
189
|
def receive(packet)
|
data/lib/nanite/amqp.rb
CHANGED
@@ -46,7 +46,11 @@ module Nanite
|
|
46
46
|
:host => options[:host],
|
47
47
|
:port => (options[:port] || ::AMQP::PORT).to_i,
|
48
48
|
:insist => options[:insist] || false,
|
49
|
-
:retry => options[:retry] || 5
|
49
|
+
:retry => options[:retry] || 5,
|
50
|
+
:connection_status => options[:connection_callback] || proc {|event|
|
51
|
+
Nanite::Log.debug("CONNECTED to MQ") if event == :connected
|
52
|
+
Nanite::Log.debug("DISCONNECTED from MQ") if event == :disconnected
|
53
|
+
}
|
50
54
|
})
|
51
55
|
MQ.new(connection)
|
52
56
|
end
|
data/lib/nanite/cluster.rb
CHANGED
@@ -2,13 +2,14 @@ module Nanite
|
|
2
2
|
class Cluster
|
3
3
|
attr_reader :agent_timeout, :nanites, :reaper, :serializer, :identity, :amq, :redis, :mapper, :callbacks
|
4
4
|
|
5
|
-
def initialize(amq, agent_timeout, identity, serializer, mapper, state_configuration=nil, callbacks = {})
|
5
|
+
def initialize(amq, agent_timeout, identity, serializer, mapper, state_configuration=nil, tag_store=nil, callbacks = {})
|
6
6
|
@amq = amq
|
7
7
|
@agent_timeout = agent_timeout
|
8
8
|
@identity = identity
|
9
9
|
@serializer = serializer
|
10
10
|
@mapper = mapper
|
11
11
|
@state = state_configuration
|
12
|
+
@tag_store = tag_store
|
12
13
|
@security = SecurityProvider.get
|
13
14
|
@callbacks = callbacks
|
14
15
|
setup_state
|
@@ -19,7 +20,7 @@ module Nanite
|
|
19
20
|
# determine which nanites should receive the given request
|
20
21
|
def targets_for(request)
|
21
22
|
return [request.target] if request.target
|
22
|
-
__send__(request.selector, request.type, request.tags).collect {|name, state| name }
|
23
|
+
__send__(request.selector, request.from, request.type, request.tags).collect {|name, state| name }
|
23
24
|
end
|
24
25
|
|
25
26
|
# adds nanite to nanites map: key is nanite's identity
|
@@ -30,38 +31,51 @@ module Nanite
|
|
30
31
|
when Register
|
31
32
|
if @security.authorize_registration(reg)
|
32
33
|
Nanite::Log.info("RECV #{reg.to_s}")
|
33
|
-
nanites[reg.identity] = { :services => reg.services, :status => reg.status, :tags => reg.tags }
|
34
|
-
reaper.
|
34
|
+
nanites[reg.identity] = { :services => reg.services, :status => reg.status, :tags => reg.tags, :timestamp => Time.now.utc.to_i }
|
35
|
+
reaper.register(reg.identity, agent_timeout + 1) { nanite_timed_out(reg.identity) }
|
35
36
|
callbacks[:register].call(reg.identity, mapper) if callbacks[:register]
|
36
37
|
else
|
37
38
|
Nanite::Log.warn("RECV NOT AUTHORIZED #{reg.to_s}")
|
38
39
|
end
|
39
40
|
when UnRegister
|
40
41
|
Nanite::Log.info("RECV #{reg.to_s}")
|
42
|
+
reaper.unregister(reg.identity)
|
41
43
|
nanites.delete(reg.identity)
|
42
44
|
callbacks[:unregister].call(reg.identity, mapper) if callbacks[:unregister]
|
45
|
+
when TagUpdate
|
46
|
+
Nanite::Log.info("RECV #{reg.to_s}")
|
47
|
+
nanites.update_tags(reg.identity, reg.new_tags, reg.obsolete_tags)
|
43
48
|
else
|
44
49
|
Nanite::Log.warn("RECV [register] Invalid packet type: #{reg.class}")
|
45
50
|
end
|
46
51
|
end
|
47
52
|
|
48
53
|
def nanite_timed_out(token)
|
49
|
-
nanite = nanites
|
50
|
-
|
54
|
+
nanite = nanites[token]
|
55
|
+
if nanite && timed_out?(nanite)
|
56
|
+
Nanite::Log.info("Nanite #{token} timed out")
|
57
|
+
nanite = nanites.delete(token)
|
58
|
+
callbacks[:timeout].call(token, mapper) if callbacks[:timeout]
|
59
|
+
true
|
60
|
+
end
|
51
61
|
end
|
52
|
-
|
62
|
+
|
53
63
|
def route(request, targets)
|
54
64
|
EM.next_tick { targets.map { |target| publish(request, target) } }
|
55
65
|
end
|
56
66
|
|
57
67
|
def publish(request, target)
|
58
|
-
# We need to initialize the 'target' field of the request object so that the serializer
|
59
|
-
# access to it.
|
68
|
+
# We need to initialize the 'target' field of the request object so that the serializer and
|
69
|
+
# the security provider have access to it.
|
60
70
|
begin
|
61
71
|
old_target = request.target
|
62
72
|
request.target = target unless target == 'mapper-offline'
|
63
|
-
|
64
|
-
|
73
|
+
if @security.authorize_request(request)
|
74
|
+
Nanite::Log.info("SEND #{request.to_s([:from, :on_behalf, :tags, :target])}")
|
75
|
+
amq.queue(target).publish(serializer.dump(request), :persistent => request.persistent)
|
76
|
+
else
|
77
|
+
Nanite::Log.warn("RECV NOT AUTHORIZED #{request.to_s}")
|
78
|
+
end
|
65
79
|
ensure
|
66
80
|
request.target = old_target
|
67
81
|
end
|
@@ -74,8 +88,8 @@ module Nanite
|
|
74
88
|
def handle_ping(ping)
|
75
89
|
begin
|
76
90
|
if nanite = nanites[ping.identity]
|
77
|
-
|
78
|
-
reaper.
|
91
|
+
nanites.update_status(ping.identity, ping.status)
|
92
|
+
reaper.update(ping.identity, agent_timeout + 1) { nanite_timed_out(ping.identity) }
|
79
93
|
else
|
80
94
|
packet = Advertise.new
|
81
95
|
Nanite::Log.info("SEND #{packet.to_s} to #{ping.identity}")
|
@@ -83,58 +97,54 @@ module Nanite
|
|
83
97
|
end
|
84
98
|
end
|
85
99
|
end
|
86
|
-
|
100
|
+
|
87
101
|
# forward request coming from agent
|
88
102
|
def handle_request(request)
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
mapper.send_push(request)
|
95
|
-
else
|
96
|
-
intm_handler = lambda do |result, job|
|
97
|
-
result = IntermediateMessage.new(request.token, job.request.from, mapper.identity, nil, result)
|
98
|
-
forward_response(result, request.persistent)
|
99
|
-
end
|
100
|
-
|
101
|
-
result = Result.new(request.token, request.from, nil, mapper.identity)
|
102
|
-
ok = mapper.send_request(request, :intermediate_handler => intm_handler) do |res|
|
103
|
-
result.results = res
|
104
|
-
forward_response(result, request.persistent)
|
105
|
-
end
|
106
|
-
|
107
|
-
if ok == false
|
108
|
-
forward_response(result, request.persistent)
|
109
|
-
end
|
110
|
-
end
|
103
|
+
Nanite::Log.info("RECV #{request.to_s([:from, :on_behalf, :target, :tags])}") unless Nanite::Log.level == :debug
|
104
|
+
Nanite::Log.debug("RECV #{request.to_s}")
|
105
|
+
case request
|
106
|
+
when Push
|
107
|
+
mapper.send_push(request)
|
111
108
|
else
|
112
|
-
|
109
|
+
intm_handler = lambda do |result, job|
|
110
|
+
result = IntermediateMessage.new(request.token, job.request.from, mapper.identity, nil, result)
|
111
|
+
forward_response(result, request.persistent)
|
112
|
+
end
|
113
|
+
|
114
|
+
result = Result.new(request.token, request.from, nil, mapper.identity)
|
115
|
+
ok = mapper.send_request(request, :intermediate_handler => intm_handler) do |res|
|
116
|
+
result.results = res
|
117
|
+
forward_response(result, request.persistent)
|
118
|
+
end
|
119
|
+
|
120
|
+
if ok == false
|
121
|
+
forward_response(result, request.persistent)
|
122
|
+
end
|
113
123
|
end
|
114
124
|
end
|
115
|
-
|
125
|
+
|
116
126
|
# forward response back to agent that originally made the request
|
117
127
|
def forward_response(res, persistent)
|
118
128
|
Nanite::Log.info("SEND #{res.to_s([:to])}")
|
119
129
|
amq.queue(res.to).publish(serializer.dump(res), :persistent => persistent)
|
120
130
|
end
|
121
|
-
|
131
|
+
|
122
132
|
# returns least loaded nanite that provides given service
|
123
|
-
def least_loaded(service, tags=[])
|
124
|
-
candidates = nanites_providing(service,tags)
|
133
|
+
def least_loaded(from, service, tags=[])
|
134
|
+
candidates = nanites_providing(from, service, tags)
|
125
135
|
return [] if candidates.empty?
|
126
136
|
|
127
137
|
[candidates.min { |a,b| a[1][:status] <=> b[1][:status] }]
|
128
138
|
end
|
129
139
|
|
130
140
|
# returns all nanites that provide given service
|
131
|
-
def all(service, tags=[])
|
132
|
-
nanites_providing(service,tags)
|
141
|
+
def all(from, service, tags=[])
|
142
|
+
nanites_providing(from, service,tags)
|
133
143
|
end
|
134
144
|
|
135
145
|
# returns a random nanite
|
136
|
-
def random(service, tags=[])
|
137
|
-
candidates = nanites_providing(service,tags)
|
146
|
+
def random(from, service, tags=[])
|
147
|
+
candidates = nanites_providing(from, service,tags)
|
138
148
|
return [] if candidates.empty?
|
139
149
|
|
140
150
|
[candidates[rand(candidates.size)]]
|
@@ -142,10 +152,10 @@ module Nanite
|
|
142
152
|
|
143
153
|
# selects next nanite that provides given service
|
144
154
|
# using round robin rotation
|
145
|
-
def rr(service, tags=[])
|
155
|
+
def rr(from, service, tags=[])
|
146
156
|
@last ||= {}
|
147
157
|
@last[service] ||= 0
|
148
|
-
candidates = nanites_providing(service,tags)
|
158
|
+
candidates = nanites_providing(from, service,tags)
|
149
159
|
return [] if candidates.empty?
|
150
160
|
@last[service] = 0 if @last[service] >= candidates.size
|
151
161
|
candidate = candidates[@last[service]]
|
@@ -153,9 +163,18 @@ module Nanite
|
|
153
163
|
[candidate]
|
154
164
|
end
|
155
165
|
|
166
|
+
def timed_out?(nanite)
|
167
|
+
nanite[:timestamp].to_i < (Time.now.utc - agent_timeout).to_i
|
168
|
+
end
|
169
|
+
|
156
170
|
# returns all nanites that provide the given service
|
157
|
-
def nanites_providing(service,
|
158
|
-
nanites.nanites_for(service,
|
171
|
+
def nanites_providing(from, service, tags)
|
172
|
+
nanites.nanites_for(from, service, tags).delete_if do |nanite|
|
173
|
+
if res = timed_out?(nanite[1])
|
174
|
+
Nanite::Log.debug("Ignoring timed out nanite #{nanite[0]} in target selection - last seen at #{nanite[1][:timestamp]}")
|
175
|
+
end
|
176
|
+
res
|
177
|
+
end
|
159
178
|
end
|
160
179
|
|
161
180
|
def setup_queues
|
@@ -172,6 +191,7 @@ module Nanite
|
|
172
191
|
handle_ping(ping)
|
173
192
|
rescue Exception => e
|
174
193
|
Nanite::Log.error("RECV [ping] #{e.message}")
|
194
|
+
callbacks[:exception].call(e, msg, mapper) rescue nil if callbacks[:exception]
|
175
195
|
end
|
176
196
|
end
|
177
197
|
hb_fanout = amq.fanout('heartbeat', :durable => true)
|
@@ -188,6 +208,7 @@ module Nanite
|
|
188
208
|
register(serializer.load(msg))
|
189
209
|
rescue Exception => e
|
190
210
|
Nanite::Log.error("RECV [register] #{e.message}")
|
211
|
+
callbacks[:exception].call(e, msg, mapper) rescue nil if callbacks[:exception]
|
191
212
|
end
|
192
213
|
end
|
193
214
|
reg_fanout = amq.fanout('registration', :durable => true)
|
@@ -197,13 +218,14 @@ module Nanite
|
|
197
218
|
amq.queue("registration-#{identity}", :exclusive => true).bind(reg_fanout).subscribe &handler
|
198
219
|
end
|
199
220
|
end
|
200
|
-
|
221
|
+
|
201
222
|
def setup_request_queue
|
202
223
|
handler = lambda do |msg|
|
203
224
|
begin
|
204
225
|
handle_request(serializer.load(msg))
|
205
226
|
rescue Exception => e
|
206
227
|
Nanite::Log.error("RECV [request] #{e.message}")
|
228
|
+
callbacks[:exception].call(e, msg, mapper) rescue nil if callbacks[:exception]
|
207
229
|
end
|
208
230
|
end
|
209
231
|
req_fanout = amq.fanout('request', :durable => true)
|
@@ -219,16 +241,14 @@ module Nanite
|
|
219
241
|
when String
|
220
242
|
# backwards compatibility, we assume redis if the configuration option
|
221
243
|
# was a string
|
222
|
-
Nanite::Log.info("[setup] using redis for state storage")
|
223
244
|
require 'nanite/state'
|
224
|
-
@nanites = Nanite::State.new(@state)
|
225
|
-
when Hash
|
245
|
+
@nanites = Nanite::State.new(@state, @tag_store)
|
226
246
|
else
|
227
247
|
require 'nanite/local_state'
|
228
248
|
@nanites = Nanite::LocalState.new
|
229
249
|
end
|
230
250
|
end
|
231
|
-
|
251
|
+
|
232
252
|
def shared_state?
|
233
253
|
!@state.nil?
|
234
254
|
end
|
data/lib/nanite/daemonize.rb
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
module Nanite
|
2
2
|
module DaemonizeHelper
|
3
|
-
def daemonize
|
3
|
+
def daemonize(identity, options = {})
|
4
4
|
exit if fork
|
5
5
|
Process.setsid
|
6
6
|
exit if fork
|
7
|
-
File.umask 0000
|
8
7
|
STDIN.reopen "/dev/null"
|
9
|
-
STDOUT.reopen "/
|
10
|
-
STDERR.reopen
|
8
|
+
STDOUT.reopen "#{options[:log_path]}/nanite.#{identity}.out", "a"
|
9
|
+
STDERR.reopen "#{options[:log_path]}/nanite.#{identity}.err", "a"
|
10
|
+
File.umask 0000
|
11
11
|
end
|
12
12
|
end
|
13
13
|
end
|
data/lib/nanite/local_state.rb
CHANGED
@@ -14,7 +14,7 @@ module Nanite
|
|
14
14
|
all(:tags)
|
15
15
|
end
|
16
16
|
|
17
|
-
def nanites_for(service,
|
17
|
+
def nanites_for(from, service, tags)
|
18
18
|
tags = tags.dup.flatten
|
19
19
|
nanites = select { |name, state| state[:services].include?(service) }
|
20
20
|
unless tags.empty?
|
@@ -24,6 +24,10 @@ module Nanite
|
|
24
24
|
end.to_a
|
25
25
|
end
|
26
26
|
|
27
|
+
def update_status(name, status)
|
28
|
+
self[name].update(:status => status, :timestamp => Time.now.utc.to_i)
|
29
|
+
end
|
30
|
+
|
27
31
|
private
|
28
32
|
|
29
33
|
def all(key)
|
data/lib/nanite/log.rb
CHANGED
@@ -35,7 +35,7 @@ module Nanite
|
|
35
35
|
# Throws an ArgumentError if you feed it a bogus log level (that is not
|
36
36
|
# one of :debug, :info, :warn, :error, :fatal or the corresponding strings or a valid Logger level)
|
37
37
|
def level=(loglevel)
|
38
|
-
init
|
38
|
+
init unless @logger
|
39
39
|
lvl = case loglevel
|
40
40
|
when String then loglevel.intern
|
41
41
|
when Integer then LEVELS.invert[loglevel]
|
data/lib/nanite/mapper.rb
CHANGED
@@ -76,10 +76,15 @@ module Nanite
|
|
76
76
|
# secure : use Security features of rabbitmq to restrict nanites to themselves
|
77
77
|
#
|
78
78
|
# prefetch : Sets prefetch (only supported in RabbitMQ >= 1.6)
|
79
|
-
#
|
80
|
-
#
|
81
|
-
#
|
82
|
-
#
|
79
|
+
#
|
80
|
+
# callbacks : A set of callbacks to be executed on specific events. Supported events are :register,
|
81
|
+
# :unregister, :timeout and :exception. This option must be a hash with event names as
|
82
|
+
# as keys and Procs as values. The Proc's arity (number of parameters) depends on the
|
83
|
+
# type of callback:
|
84
|
+
# exception -- the exception, the message being processed, a reference to the mapper
|
85
|
+
# all others -- the corresponding nanite's identity, a reference to the mapper
|
86
|
+
#
|
87
|
+
# tag_store : Name of class which implements tag store backend interface, RedisTagStore by default
|
83
88
|
#
|
84
89
|
# Connection options:
|
85
90
|
#
|
@@ -116,6 +121,10 @@ module Nanite
|
|
116
121
|
@options.update(custom_config.merge(options))
|
117
122
|
@identity = "mapper-#{@options[:identity]}"
|
118
123
|
@options[:file_root] ||= File.join(@options[:root], 'files')
|
124
|
+
@options[:log_path] = false
|
125
|
+
if @options[:daemonize]
|
126
|
+
@options[:log_path] = (@options[:log_dir] || @options[:root] || Dir.pwd)
|
127
|
+
end
|
119
128
|
@options.freeze
|
120
129
|
end
|
121
130
|
|
@@ -125,7 +134,7 @@ module Nanite
|
|
125
134
|
pid_file = PidFile.new(@identity, @options)
|
126
135
|
pid_file.check
|
127
136
|
if @options[:daemonize]
|
128
|
-
daemonize
|
137
|
+
daemonize(@identity, @options)
|
129
138
|
pid_file.write
|
130
139
|
at_exit { pid_file.remove }
|
131
140
|
else
|
@@ -288,21 +297,18 @@ module Nanite
|
|
288
297
|
job_warden.process(msg)
|
289
298
|
rescue Exception => e
|
290
299
|
Nanite::Log.error("RECV [result] #{e.message}")
|
300
|
+
callbacks[:exception].call(e, msg, mapper) rescue nil if callbacks[:exception]
|
291
301
|
end
|
292
302
|
end
|
293
303
|
end
|
294
304
|
|
295
305
|
def setup_logging
|
296
|
-
log_path
|
297
|
-
if @options[:daemonize]
|
298
|
-
log_path = (@options[:log_dir] || @options[:root] || Dir.pwd)
|
299
|
-
end
|
300
|
-
Nanite::Log.init(@identity, log_path)
|
306
|
+
Nanite::Log.init(@identity, @options[:log_path])
|
301
307
|
Nanite::Log.level = @options[:log_level] if @options[:log_level]
|
302
308
|
end
|
303
309
|
|
304
310
|
def setup_cluster
|
305
|
-
@cluster = Cluster.new(@amq, @options[:agent_timeout], @options[:identity], @serializer, self, @options[:redis], @options[:callbacks])
|
311
|
+
@cluster = Cluster.new(@amq, @options[:agent_timeout], @options[:identity], @serializer, self, @options[:redis], @options[:tag_store], @options[:callbacks])
|
306
312
|
end
|
307
313
|
end
|
308
314
|
end
|
data/lib/nanite/mapper_proxy.rb
CHANGED
@@ -49,7 +49,15 @@ module Nanite
|
|
49
49
|
push.persistent = opts.key?(:persistent) ? opts[:persistent] : options[:persistent]
|
50
50
|
Nanite::Log.info("SEND #{push.to_s([:tags, :target])}")
|
51
51
|
amqp.fanout('request', :no_declare => options[:secure]).publish(serializer.dump(push))
|
52
|
-
end
|
52
|
+
end
|
53
|
+
|
54
|
+
# Update tags registered by mapper for agent
|
55
|
+
def update_tags(new_tags, obsolete_tags)
|
56
|
+
raise "Mapper proxy not initialized" unless identity && options
|
57
|
+
update = TagUpdate.new(identity, new_tags, obsolete_tags)
|
58
|
+
Nanite::Log.info("SEND #{update.to_s}")
|
59
|
+
amqp.fanout('registration', :no_declare => options[:secure]).publish(serializer.dump(update))
|
60
|
+
end
|
53
61
|
|
54
62
|
# Handle intermediary result
|
55
63
|
def handle_intermediate_result(res)
|
File without changes
|
data/lib/nanite/packets.rb
CHANGED
@@ -112,15 +112,16 @@ module Nanite
|
|
112
112
|
# payload is arbitrary data that is transferred from mapper to actor
|
113
113
|
#
|
114
114
|
# Options:
|
115
|
-
# from
|
116
|
-
#
|
117
|
-
#
|
118
|
-
#
|
119
|
-
#
|
115
|
+
# from is sender identity
|
116
|
+
# on_behalf is agent identity that should be used to authorize request
|
117
|
+
# token is a generated request id that mapper uses to identify replies
|
118
|
+
# reply_to is identity of the node actor replies to, usually a mapper itself
|
119
|
+
# selector is the selector used to route the request
|
120
|
+
# target is the target nanite for the request
|
120
121
|
# persistent signifies if this request should be saved to persistent storage by the AMQP broker
|
121
122
|
class Request < Packet
|
122
123
|
|
123
|
-
attr_accessor :from, :payload, :type, :token, :reply_to, :selector, :target, :persistent, :tags
|
124
|
+
attr_accessor :from, :on_behalf, :payload, :type, :token, :reply_to, :selector, :target, :persistent, :tags
|
124
125
|
|
125
126
|
DEFAULT_OPTIONS = {:selector => :least_loaded}
|
126
127
|
|
@@ -130,6 +131,7 @@ module Nanite
|
|
130
131
|
@payload = payload
|
131
132
|
@size = size
|
132
133
|
@from = opts[:from]
|
134
|
+
@on_behalf = opts[:on_behalf]
|
133
135
|
@token = opts[:token]
|
134
136
|
@reply_to = opts[:reply_to]
|
135
137
|
@selector = opts[:selector]
|
@@ -140,15 +142,17 @@ module Nanite
|
|
140
142
|
|
141
143
|
def self.json_create(o)
|
142
144
|
i = o['data']
|
143
|
-
new(i['type'], i['payload'], { :from => i['from'],
|
144
|
-
:
|
145
|
-
:
|
146
|
-
:tags => i['tags'] },
|
145
|
+
new(i['type'], i['payload'], { :from => i['from'], :on_behalf => i['on_behalf'],
|
146
|
+
:token => i['token'], :reply_to => i['reply_to'],
|
147
|
+
:selector => i['selector'], :target => i['target'],
|
148
|
+
:persistent => i['persistent'], :tags => i['tags'] },
|
149
|
+
o['size'])
|
147
150
|
end
|
148
151
|
|
149
152
|
def to_s(filter=nil)
|
150
153
|
log_msg = "#{super} <#{token}> #{type}"
|
151
154
|
log_msg += " from #{id_to_s(from)}" if filter.nil? || filter.include?(:from)
|
155
|
+
log_msg += " on behalf of #{id_to_s(on_behalf)}" if on_behalf && (filter.nil? || filter.include?(:on_behalf))
|
152
156
|
log_msg += " to #{id_to_s(target)}" if target && (filter.nil? || filter.include?(:target))
|
153
157
|
log_msg += ", reply_to #{id_to_s(reply_to)}" if reply_to && (filter.nil? || filter.include?(:reply_to))
|
154
158
|
log_msg += ", tags #{tags.inspect}" if tags && !tags.empty? && (filter.nil? || filter.include?(:tags))
|
@@ -165,14 +169,15 @@ module Nanite
|
|
165
169
|
# payload is arbitrary data that is transferred from mapper to actor
|
166
170
|
#
|
167
171
|
# Options:
|
168
|
-
# from
|
169
|
-
#
|
170
|
-
#
|
171
|
-
#
|
172
|
+
# from is sender identity
|
173
|
+
# on_behalf is agent identity that should be used to authorize request
|
174
|
+
# token is a generated request id that mapper uses to identify replies
|
175
|
+
# selector is the selector used to route the request
|
176
|
+
# target is the target nanite for the request
|
172
177
|
# persistent signifies if this request should be saved to persistent storage by the AMQP broker
|
173
178
|
class Push < Packet
|
174
179
|
|
175
|
-
attr_accessor :from, :payload, :type, :token, :selector, :target, :persistent, :tags
|
180
|
+
attr_accessor :from, :on_behalf, :payload, :type, :token, :selector, :target, :persistent, :tags
|
176
181
|
|
177
182
|
DEFAULT_OPTIONS = {:selector => :least_loaded}
|
178
183
|
|
@@ -182,6 +187,7 @@ module Nanite
|
|
182
187
|
@payload = payload
|
183
188
|
@size = size
|
184
189
|
@from = opts[:from]
|
190
|
+
@on_behalf = opts[:on_behalf]
|
185
191
|
@token = opts[:token]
|
186
192
|
@selector = opts[:selector]
|
187
193
|
@target = opts[:target]
|
@@ -191,14 +197,17 @@ module Nanite
|
|
191
197
|
|
192
198
|
def self.json_create(o)
|
193
199
|
i = o['data']
|
194
|
-
new(i['type'], i['payload'], { :from
|
195
|
-
:
|
196
|
-
:
|
200
|
+
new(i['type'], i['payload'], { :from => i['from'], :on_behalf => i['on_behalf'],
|
201
|
+
:token => i['token'], :selector => i['selector'],
|
202
|
+
:target => i['target'], :persistent => i['persistent'],
|
203
|
+
:tags => i['tags'] },
|
204
|
+
o['size'])
|
197
205
|
end
|
198
206
|
|
199
207
|
def to_s(filter=nil)
|
200
208
|
log_msg = "#{super} <#{token}> #{type}"
|
201
209
|
log_msg += " from #{id_to_s(from)}" if filter.nil? || filter.include?(:from)
|
210
|
+
log_msg += " on behalf of #{id_to_s(on_behalf)}" if on_behalf && (filter.nil? || filter.include?(:on_behalf))
|
202
211
|
log_msg += ", target #{id_to_s(target)}" if target && (filter.nil? || filter.include?(:target))
|
203
212
|
log_msg += ", tags #{tags.inspect}" if tags && !tags.empty? && (filter.nil? || filter.include?(:tags))
|
204
213
|
log_msg += ", payload #{payload.inspect}" if filter.nil? || filter.include?(:payload)
|
@@ -360,6 +369,33 @@ module Nanite
|
|
360
369
|
end
|
361
370
|
|
362
371
|
end
|
363
|
-
|
372
|
+
|
373
|
+
# packet that is sent by agents to the mapper
|
374
|
+
# to update their tags
|
375
|
+
class TagUpdate < Packet
|
376
|
+
|
377
|
+
attr_accessor :identity, :new_tags, :obsolete_tags
|
378
|
+
|
379
|
+
def initialize(identity, new_tags, obsolete_tags, size=nil)
|
380
|
+
@identity = identity
|
381
|
+
@new_tags = new_tags
|
382
|
+
@obsolete_tags = obsolete_tags
|
383
|
+
@size = size
|
384
|
+
end
|
385
|
+
|
386
|
+
def self.json_create(o)
|
387
|
+
i = o['data']
|
388
|
+
new(i['identity'], i['new_tags'], i['obsolete_tags'], o['size'])
|
389
|
+
end
|
390
|
+
|
391
|
+
def to_s
|
392
|
+
log_msg = "#{super} #{id_to_s(identity)}"
|
393
|
+
log_msg += ", new tags: #{new_tags.join(', ')}" if new_tags && !new_tags.empty?
|
394
|
+
log_msg += ", obsolete tags: #{obsolete_tags.join(', ')}" if obsolete_tags && !obsolete_tags.empty?
|
395
|
+
log_msg
|
396
|
+
end
|
397
|
+
|
398
|
+
end
|
399
|
+
|
364
400
|
end
|
365
401
|
|