nanite 0.4.1.2 → 0.4.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -39,7 +39,7 @@ require 'nanite/security/static_certificate_store'
39
39
  require 'nanite/serializer'
40
40
 
41
41
  module Nanite
42
- VERSION = '0.4.1.2' unless defined?(Nanite::VERSION)
42
+ VERSION = '0.4.1.10' unless defined?(Nanite::VERSION)
43
43
 
44
44
  class MapperNotRunning < StandardError; end
45
45
 
@@ -86,7 +86,7 @@ module Nanite
86
86
  def initialize(opts)
87
87
  set_configuration(opts)
88
88
  @tags = []
89
- @tags << opts[:tag]
89
+ @tags << opts[:tag] if opts[:tag]
90
90
  @tags.flatten!
91
91
  @options.freeze
92
92
  end
@@ -131,6 +131,16 @@ module Nanite
131
131
  @deny_token = deny_token
132
132
  end
133
133
 
134
+ # Update set of tags published by agent and notify mapper
135
+ # Add tags in 'new_tags' and remove tags in 'old_tags'
136
+ def update_tags(new_tags, old_tags)
137
+ @tags += (new_tags || [])
138
+ @tags -= (old_tags || [])
139
+ @tags.uniq!
140
+ tag_update = TagUpdate.new(identity, new_tags, old_tags)
141
+ amq.fanout('registration', :no_declare => options[:secure]).publish(serializer.dump(tag_update))
142
+ end
143
+
134
144
  protected
135
145
 
136
146
  def set_configuration(opts)
@@ -2,13 +2,14 @@ module Nanite
2
2
  class Cluster
3
3
  attr_reader :agent_timeout, :nanites, :reaper, :serializer, :identity, :amq, :redis, :mapper, :callbacks
4
4
 
5
- def initialize(amq, agent_timeout, identity, serializer, mapper, state_configuration=nil, callbacks = {})
5
+ def initialize(amq, agent_timeout, identity, serializer, mapper, state_configuration=nil, tag_store=nil, callbacks = {})
6
6
  @amq = amq
7
7
  @agent_timeout = agent_timeout
8
8
  @identity = identity
9
9
  @serializer = serializer
10
10
  @mapper = mapper
11
11
  @state = state_configuration
12
+ @tag_store = tag_store
12
13
  @security = SecurityProvider.get
13
14
  @callbacks = callbacks
14
15
  setup_state
@@ -19,7 +20,7 @@ module Nanite
19
20
  # determine which nanites should receive the given request
20
21
  def targets_for(request)
21
22
  return [request.target] if request.target
22
- __send__(request.selector, request.type, request.tags).collect {|name, state| name }
23
+ __send__(request.selector, request.from, request.type, request.tags).collect {|name, state| name }
23
24
  end
24
25
 
25
26
  # adds nanite to nanites map: key is nanite's identity
@@ -41,6 +42,9 @@ module Nanite
41
42
  reaper.unregister(reg.identity)
42
43
  nanites.delete(reg.identity)
43
44
  callbacks[:unregister].call(reg.identity, mapper) if callbacks[:unregister]
45
+ when TagUpdate
46
+ Nanite::Log.info("RECV #{reg.to_s}")
47
+ nanites.update_tags(reg.identity, reg.new_tags, reg.obsolete_tags)
44
48
  else
45
49
  Nanite::Log.warn("RECV [register] Invalid packet type: #{reg.class}")
46
50
  end
@@ -55,19 +59,23 @@ module Nanite
55
59
  true
56
60
  end
57
61
  end
58
-
62
+
59
63
  def route(request, targets)
60
64
  EM.next_tick { targets.map { |target| publish(request, target) } }
61
65
  end
62
66
 
63
67
  def publish(request, target)
64
- # We need to initialize the 'target' field of the request object so that the serializer has
65
- # access to it.
68
+ # We need to initialize the 'target' field of the request object so that the serializer and
69
+ # the security provider have access to it.
66
70
  begin
67
71
  old_target = request.target
68
72
  request.target = target unless target == 'mapper-offline'
69
- Nanite::Log.info("SEND #{request.to_s([:from, :tags, :target])}")
70
- amq.queue(target).publish(serializer.dump(request), :persistent => request.persistent)
73
+ if @security.authorize_request(request)
74
+ Nanite::Log.info("SEND #{request.to_s([:from, :on_behalf, :tags, :target])}")
75
+ amq.queue(target).publish(serializer.dump(request), :persistent => request.persistent)
76
+ else
77
+ Nanite::Log.warn("RECV NOT AUTHORIZED #{request.to_s}")
78
+ end
71
79
  ensure
72
80
  request.target = old_target
73
81
  end
@@ -89,58 +97,54 @@ module Nanite
89
97
  end
90
98
  end
91
99
  end
92
-
100
+
93
101
  # forward request coming from agent
94
102
  def handle_request(request)
95
- if @security.authorize_request(request)
96
- Nanite::Log.info("RECV #{request.to_s([:from, :target, :tags])}") unless Nanite::Log.level == :debug
97
- Nanite::Log.debug("RECV #{request.to_s}")
98
- case request
99
- when Push
100
- mapper.send_push(request)
101
- else
102
- intm_handler = lambda do |result, job|
103
- result = IntermediateMessage.new(request.token, job.request.from, mapper.identity, nil, result)
104
- forward_response(result, request.persistent)
105
- end
106
-
107
- result = Result.new(request.token, request.from, nil, mapper.identity)
108
- ok = mapper.send_request(request, :intermediate_handler => intm_handler) do |res|
109
- result.results = res
110
- forward_response(result, request.persistent)
111
- end
112
-
113
- if ok == false
114
- forward_response(result, request.persistent)
115
- end
116
- end
103
+ Nanite::Log.info("RECV #{request.to_s([:from, :on_behalf, :target, :tags])}") unless Nanite::Log.level == :debug
104
+ Nanite::Log.debug("RECV #{request.to_s}")
105
+ case request
106
+ when Push
107
+ mapper.send_push(request)
117
108
  else
118
- Nanite::Log.warn("RECV NOT AUTHORIZED #{request.to_s}")
109
+ intm_handler = lambda do |result, job|
110
+ result = IntermediateMessage.new(request.token, job.request.from, mapper.identity, nil, result)
111
+ forward_response(result, request.persistent)
112
+ end
113
+
114
+ result = Result.new(request.token, request.from, nil, mapper.identity)
115
+ ok = mapper.send_request(request, :intermediate_handler => intm_handler) do |res|
116
+ result.results = res
117
+ forward_response(result, request.persistent)
118
+ end
119
+
120
+ if ok == false
121
+ forward_response(result, request.persistent)
122
+ end
119
123
  end
120
124
  end
121
-
125
+
122
126
  # forward response back to agent that originally made the request
123
127
  def forward_response(res, persistent)
124
128
  Nanite::Log.info("SEND #{res.to_s([:to])}")
125
129
  amq.queue(res.to).publish(serializer.dump(res), :persistent => persistent)
126
130
  end
127
-
131
+
128
132
  # returns least loaded nanite that provides given service
129
- def least_loaded(service, tags=[])
130
- candidates = nanites_providing(service,tags)
133
+ def least_loaded(from, service, tags=[])
134
+ candidates = nanites_providing(from, service, tags)
131
135
  return [] if candidates.empty?
132
136
 
133
137
  [candidates.min { |a,b| a[1][:status] <=> b[1][:status] }]
134
138
  end
135
139
 
136
140
  # returns all nanites that provide given service
137
- def all(service, tags=[])
138
- nanites_providing(service,tags)
141
+ def all(from, service, tags=[])
142
+ nanites_providing(from, service,tags)
139
143
  end
140
144
 
141
145
  # returns a random nanite
142
- def random(service, tags=[])
143
- candidates = nanites_providing(service,tags)
146
+ def random(from, service, tags=[])
147
+ candidates = nanites_providing(from, service,tags)
144
148
  return [] if candidates.empty?
145
149
 
146
150
  [candidates[rand(candidates.size)]]
@@ -148,27 +152,28 @@ module Nanite
148
152
 
149
153
  # selects next nanite that provides given service
150
154
  # using round robin rotation
151
- def rr(service, tags=[])
155
+ def rr(from, service, tags=[])
152
156
  @last ||= {}
153
157
  @last[service] ||= 0
154
- candidates = nanites_providing(service,tags)
158
+ candidates = nanites_providing(from, service,tags)
155
159
  return [] if candidates.empty?
156
160
  @last[service] = 0 if @last[service] >= candidates.size
157
161
  candidate = candidates[@last[service]]
158
162
  @last[service] += 1
159
163
  [candidate]
160
164
  end
161
-
165
+
162
166
  def timed_out?(nanite)
163
167
  nanite[:timestamp].to_i < (Time.now.utc - agent_timeout).to_i
164
168
  end
165
169
 
166
170
  # returns all nanites that provide the given service
167
- def nanites_providing(service, *tags)
168
- nanites.nanites_for(service, *tags).delete_if do |nanite|
169
- if timed_out?(nanite[1])
171
+ def nanites_providing(from, service, tags)
172
+ nanites.nanites_for(from, service, tags).delete_if do |nanite|
173
+ if res = timed_out?(nanite[1])
170
174
  Nanite::Log.debug("Ignoring timed out nanite #{nanite[0]} in target selection - last seen at #{nanite[1][:timestamp]}")
171
175
  end
176
+ res
172
177
  end
173
178
  end
174
179
 
@@ -186,6 +191,7 @@ module Nanite
186
191
  handle_ping(ping)
187
192
  rescue Exception => e
188
193
  Nanite::Log.error("RECV [ping] #{e.message}")
194
+ callbacks[:exception].call(e, msg, mapper) rescue nil if callbacks[:exception]
189
195
  end
190
196
  end
191
197
  hb_fanout = amq.fanout('heartbeat', :durable => true)
@@ -202,6 +208,7 @@ module Nanite
202
208
  register(serializer.load(msg))
203
209
  rescue Exception => e
204
210
  Nanite::Log.error("RECV [register] #{e.message}")
211
+ callbacks[:exception].call(e, msg, mapper) rescue nil if callbacks[:exception]
205
212
  end
206
213
  end
207
214
  reg_fanout = amq.fanout('registration', :durable => true)
@@ -211,13 +218,14 @@ module Nanite
211
218
  amq.queue("registration-#{identity}", :exclusive => true).bind(reg_fanout).subscribe &handler
212
219
  end
213
220
  end
214
-
221
+
215
222
  def setup_request_queue
216
223
  handler = lambda do |msg|
217
224
  begin
218
225
  handle_request(serializer.load(msg))
219
226
  rescue Exception => e
220
227
  Nanite::Log.error("RECV [request] #{e.message}")
228
+ callbacks[:exception].call(e, msg, mapper) rescue nil if callbacks[:exception]
221
229
  end
222
230
  end
223
231
  req_fanout = amq.fanout('request', :durable => true)
@@ -233,16 +241,14 @@ module Nanite
233
241
  when String
234
242
  # backwards compatibility, we assume redis if the configuration option
235
243
  # was a string
236
- Nanite::Log.info("[setup] using redis for state storage")
237
244
  require 'nanite/state'
238
- @nanites = Nanite::State.new(@state)
239
- when Hash
245
+ @nanites = Nanite::State.new(@state, @tag_store)
240
246
  else
241
247
  require 'nanite/local_state'
242
248
  @nanites = Nanite::LocalState.new
243
249
  end
244
250
  end
245
-
251
+
246
252
  def shared_state?
247
253
  !@state.nil?
248
254
  end
@@ -14,7 +14,7 @@ module Nanite
14
14
  all(:tags)
15
15
  end
16
16
 
17
- def nanites_for(service, *tags)
17
+ def nanites_for(from, service, tags)
18
18
  tags = tags.dup.flatten
19
19
  nanites = select { |name, state| state[:services].include?(service) }
20
20
  unless tags.empty?
@@ -76,10 +76,15 @@ module Nanite
76
76
  # secure : use Security features of rabbitmq to restrict nanites to themselves
77
77
  #
78
78
  # prefetch : Sets prefetch (only supported in RabbitMQ >= 1.6)
79
- # callbacks : A set of callbacks to have code executed on specific events, supported events are :register,
80
- # :unregister and :timeout. Parameter must be a hash with the corresponding events as keys and
81
- # a block as value. The block will get the corresponding nanite's identity and a copy of the
82
- # mapper
79
+ #
80
+ # callbacks : A set of callbacks to be executed on specific events. Supported events are :register,
81
+ # :unregister, :timeout and :exception. This option must be a hash with event names as
82
+ # as keys and Procs as values. The Proc's arity (number of parameters) depends on the
83
+ # type of callback:
84
+ # exception -- the exception, the message being processed, a reference to the mapper
85
+ # all others -- the corresponding nanite's identity, a reference to the mapper
86
+ #
87
+ # tag_store : Name of class which implements tag store backend interface, RedisTagStore by default
83
88
  #
84
89
  # Connection options:
85
90
  #
@@ -245,7 +250,7 @@ module Nanite
245
250
  private
246
251
 
247
252
  def build_deliverable(deliverable_type, type, payload, opts)
248
- deliverable = deliverable_type.new(type, payload, nil, opts)
253
+ deliverable = deliverable_type.new(type, payload, opts)
249
254
  deliverable.from = identity
250
255
  deliverable.token = Identity.generate
251
256
  deliverable.persistent = opts.key?(:persistent) ? opts[:persistent] : options[:persistent]
@@ -292,6 +297,7 @@ module Nanite
292
297
  job_warden.process(msg)
293
298
  rescue Exception => e
294
299
  Nanite::Log.error("RECV [result] #{e.message}")
300
+ callbacks[:exception].call(e, msg, mapper) rescue nil if callbacks[:exception]
295
301
  end
296
302
  end
297
303
  end
@@ -302,7 +308,7 @@ module Nanite
302
308
  end
303
309
 
304
310
  def setup_cluster
305
- @cluster = Cluster.new(@amq, @options[:agent_timeout], @options[:identity], @serializer, self, @options[:redis], @options[:callbacks])
311
+ @cluster = Cluster.new(@amq, @options[:agent_timeout], @options[:identity], @serializer, self, @options[:redis], @options[:tag_store], @options[:callbacks])
306
312
  end
307
313
  end
308
314
  end
@@ -31,7 +31,7 @@ module Nanite
31
31
  # Send request to given agent through the mapper
32
32
  def request(type, payload = '', opts = {}, &blk)
33
33
  raise "Mapper proxy not initialized" unless identity && options
34
- request = Request.new(type, payload, nil, opts)
34
+ request = Request.new(type, payload, opts)
35
35
  request.from = identity
36
36
  request.token = Identity.generate
37
37
  request.persistent = opts.key?(:persistent) ? opts[:persistent] : options[:persistent]
@@ -43,13 +43,21 @@ module Nanite
43
43
 
44
44
  def push(type, payload = '', opts = {})
45
45
  raise "Mapper proxy not initialized" unless identity && options
46
- push = Push.new(type, payload, nil, opts)
46
+ push = Push.new(type, payload, opts)
47
47
  push.from = identity
48
48
  push.token = Identity.generate
49
49
  push.persistent = opts.key?(:persistent) ? opts[:persistent] : options[:persistent]
50
50
  Nanite::Log.info("SEND #{push.to_s([:tags, :target])}")
51
51
  amqp.fanout('request', :no_declare => options[:secure]).publish(serializer.dump(push))
52
- end
52
+ end
53
+
54
+ # Update tags registered by mapper for agent
55
+ def update_tags(new_tags, obsolete_tags)
56
+ raise "Mapper proxy not initialized" unless identity && options
57
+ update = TagUpdate.new(identity, new_tags, obsolete_tags)
58
+ Nanite::Log.info("SEND #{update.to_s}")
59
+ amqp.fanout('registration', :no_declare => options[:secure]).publish(serializer.dump(update))
60
+ end
53
61
 
54
62
  # Handle intermediary result
55
63
  def handle_intermediate_result(res)
@@ -89,7 +89,7 @@ module Nanite
89
89
 
90
90
  attr_accessor :chunk, :token
91
91
 
92
- def initialize(token, size=nil, chunk=nil)
92
+ def initialize(token, chunk=nil, size=nil)
93
93
  @chunk = chunk
94
94
  @token = token
95
95
  @size = size
@@ -97,7 +97,7 @@ module Nanite
97
97
 
98
98
  def self.json_create(o)
99
99
  i = o['data']
100
- new(i['token'], o['size'], i['chunk'])
100
+ new(i['token'], i['chunk'], o['size'])
101
101
  end
102
102
 
103
103
  def to_s
@@ -112,24 +112,26 @@ module Nanite
112
112
  # payload is arbitrary data that is transferred from mapper to actor
113
113
  #
114
114
  # Options:
115
- # from is sender identity
116
- # token is a generated request id that mapper uses to identify replies
117
- # reply_to is identity of the node actor replies to, usually a mapper itself
118
- # selector is the selector used to route the request
119
- # target is the target nanite for the request
115
+ # from is sender identity
116
+ # on_behalf is agent identity that should be used to authorize request
117
+ # token is a generated request id that mapper uses to identify replies
118
+ # reply_to is identity of the node actor replies to, usually a mapper itself
119
+ # selector is the selector used to route the request
120
+ # target is the target nanite for the request
120
121
  # persistent signifies if this request should be saved to persistent storage by the AMQP broker
121
122
  class Request < Packet
122
123
 
123
- attr_accessor :from, :payload, :type, :token, :reply_to, :selector, :target, :persistent, :tags
124
+ attr_accessor :from, :on_behalf, :payload, :type, :token, :reply_to, :selector, :target, :persistent, :tags
124
125
 
125
126
  DEFAULT_OPTIONS = {:selector => :least_loaded}
126
127
 
127
- def initialize(type, payload, size=nil, opts={})
128
+ def initialize(type, payload, opts={}, size=nil)
128
129
  opts = DEFAULT_OPTIONS.merge(opts)
129
130
  @type = type
130
131
  @payload = payload
131
132
  @size = size
132
133
  @from = opts[:from]
134
+ @on_behalf = opts[:on_behalf]
133
135
  @token = opts[:token]
134
136
  @reply_to = opts[:reply_to]
135
137
  @selector = opts[:selector]
@@ -140,15 +142,17 @@ module Nanite
140
142
 
141
143
  def self.json_create(o)
142
144
  i = o['data']
143
- new(i['type'], i['payload'], o['size'], { :from => i['from'], :token => i['token'],
144
- :reply_to => i['reply_to'], :selector => i['selector'],
145
- :target => i['target'], :persistent => i['persistent'],
146
- :tags => i['tags'] })
145
+ new(i['type'], i['payload'], { :from => i['from'], :on_behalf => i['on_behalf'],
146
+ :token => i['token'], :reply_to => i['reply_to'],
147
+ :selector => i['selector'], :target => i['target'],
148
+ :persistent => i['persistent'], :tags => i['tags'] },
149
+ o['size'])
147
150
  end
148
151
 
149
152
  def to_s(filter=nil)
150
153
  log_msg = "#{super} <#{token}> #{type}"
151
154
  log_msg += " from #{id_to_s(from)}" if filter.nil? || filter.include?(:from)
155
+ log_msg += " on behalf of #{id_to_s(on_behalf)}" if on_behalf && (filter.nil? || filter.include?(:on_behalf))
152
156
  log_msg += " to #{id_to_s(target)}" if target && (filter.nil? || filter.include?(:target))
153
157
  log_msg += ", reply_to #{id_to_s(reply_to)}" if reply_to && (filter.nil? || filter.include?(:reply_to))
154
158
  log_msg += ", tags #{tags.inspect}" if tags && !tags.empty? && (filter.nil? || filter.include?(:tags))
@@ -165,23 +169,25 @@ module Nanite
165
169
  # payload is arbitrary data that is transferred from mapper to actor
166
170
  #
167
171
  # Options:
168
- # from is sender identity
169
- # token is a generated request id that mapper uses to identify replies
170
- # selector is the selector used to route the request
171
- # target is the target nanite for the request
172
+ # from is sender identity
173
+ # on_behalf is agent identity that should be used to authorize request
174
+ # token is a generated request id that mapper uses to identify replies
175
+ # selector is the selector used to route the request
176
+ # target is the target nanite for the request
172
177
  # persistent signifies if this request should be saved to persistent storage by the AMQP broker
173
178
  class Push < Packet
174
179
 
175
- attr_accessor :from, :payload, :type, :token, :selector, :target, :persistent, :tags
180
+ attr_accessor :from, :on_behalf, :payload, :type, :token, :selector, :target, :persistent, :tags
176
181
 
177
182
  DEFAULT_OPTIONS = {:selector => :least_loaded}
178
183
 
179
- def initialize(type, payload, size=nil, opts={})
184
+ def initialize(type, payload, opts={}, size=nil)
180
185
  opts = DEFAULT_OPTIONS.merge(opts)
181
186
  @type = type
182
187
  @payload = payload
183
188
  @size = size
184
189
  @from = opts[:from]
190
+ @on_behalf = opts[:on_behalf]
185
191
  @token = opts[:token]
186
192
  @selector = opts[:selector]
187
193
  @target = opts[:target]
@@ -191,14 +197,17 @@ module Nanite
191
197
 
192
198
  def self.json_create(o)
193
199
  i = o['data']
194
- new(i['type'], i['payload'], o['size'], { :from => i['from'], :token => i['token'],
195
- :selector => i['selector'], :target => i['target'],
196
- :persistent => i['persistent'], :tags => i['tags'] })
200
+ new(i['type'], i['payload'], { :from => i['from'], :on_behalf => i['on_behalf'],
201
+ :token => i['token'], :selector => i['selector'],
202
+ :target => i['target'], :persistent => i['persistent'],
203
+ :tags => i['tags'] },
204
+ o['size'])
197
205
  end
198
206
 
199
207
  def to_s(filter=nil)
200
208
  log_msg = "#{super} <#{token}> #{type}"
201
209
  log_msg += " from #{id_to_s(from)}" if filter.nil? || filter.include?(:from)
210
+ log_msg += " on behalf of #{id_to_s(on_behalf)}" if on_behalf && (filter.nil? || filter.include?(:on_behalf))
202
211
  log_msg += ", target #{id_to_s(target)}" if target && (filter.nil? || filter.include?(:target))
203
212
  log_msg += ", tags #{tags.inspect}" if tags && !tags.empty? && (filter.nil? || filter.include?(:tags))
204
213
  log_msg += ", payload #{payload.inspect}" if filter.nil? || filter.include?(:payload)
@@ -360,6 +369,33 @@ module Nanite
360
369
  end
361
370
 
362
371
  end
363
-
372
+
373
+ # packet that is sent by agents to the mapper
374
+ # to update their tags
375
+ class TagUpdate < Packet
376
+
377
+ attr_accessor :identity, :new_tags, :obsolete_tags
378
+
379
+ def initialize(identity, new_tags, obsolete_tags, size=nil)
380
+ @identity = identity
381
+ @new_tags = new_tags
382
+ @obsolete_tags = obsolete_tags
383
+ @size = size
384
+ end
385
+
386
+ def self.json_create(o)
387
+ i = o['data']
388
+ new(i['identity'], i['new_tags'], i['obsolete_tags'], o['size'])
389
+ end
390
+
391
+ def to_s
392
+ log_msg = "#{super} #{id_to_s(identity)}"
393
+ log_msg += ", new tags: #{new_tags.join(', ')}" if new_tags && !new_tags.empty?
394
+ log_msg += ", obsolete tags: #{obsolete_tags.join(', ')}" if obsolete_tags && !obsolete_tags.empty?
395
+ log_msg
396
+ end
397
+
398
+ end
399
+
364
400
  end
365
401
 
@@ -0,0 +1,141 @@
1
+ require 'redis'
2
+
3
+ module Nanite
4
+
5
+ # Implementation of a tag store on top of Redis
6
+ # For a nanite with the identity 'nanite-foobar', we store the following:
7
+ #
8
+ # s-nanite-foobar: { /foo/bar, /foo/nik } # a SET of the provided services
9
+ # tg-nanite-foobar: { foo-42, customer-12 } # a SET of the tags for this agent
10
+ #
11
+ # Also we do an inverted index for quick lookup of agents providing a certain
12
+ # service, so for each service the agent provides, we add the nanite to a SET
13
+ # of all the nanites that provide said service:
14
+ #
15
+ # foo/bar: { nanite-foobar, nanite-nickelbag, nanite-another } # redis SET
16
+ #
17
+ # We do that same thing for tags:
18
+ #
19
+ # some-tag: { nanite-foobar, nanite-nickelbag, nanite-another } # redis SET
20
+ #
21
+ # This way we can do a lookup of what nanites provide a set of services and tags based
22
+ # on redis SET intersection:
23
+ #
24
+ # nanites_for('/gems/list', 'some-tag')
25
+ # => returns an array of nanites that provide the intersection of these two service tags
26
+
27
+ class RedisTagStore
28
+
29
+ # Initialize tag store with given redis handle
30
+ def initialize(redis)
31
+ @redis = redis
32
+ end
33
+
34
+ # Store services and tags for given agent
35
+ def store(nanite, services, tags)
36
+ services = nil if services.compact.empty?
37
+ tags = nil if tags.compact.empty?
38
+ log_redis_error do
39
+ if services
40
+ obsolete_services = @redis.set_members("s-#{nanite}") - services
41
+ update_elems(nanite, services, obsolete_services, "s-#{nanite}", 'naniteservices')
42
+ end
43
+ if tags
44
+ obsolete_tags = @redis.set_members("tg-#{nanite}") - tags
45
+ update_elems(nanite, tags, obsolete_tags, "tg-#{nanite}", 'nanitestags')
46
+ end
47
+ end
48
+ end
49
+
50
+ # Update tags for given agent
51
+ def update(nanite, new_tags, obsolete_tags)
52
+ update_elems(nanite, new_tags, obsolete_tags, "tg-#{nanite}", 'nanitestags')
53
+ end
54
+
55
+ # Delete services and tags for given agent
56
+ def delete(nanite)
57
+ delete_elems(nanite, "s-#{nanite}", 'naniteservices')
58
+ delete_elems(nanite, "tg-#{nanite}", 'nanitestags')
59
+ end
60
+
61
+ # Services implemented by given agent
62
+ def services(nanite)
63
+ @redis.set_members("s-#{nanite}")
64
+ end
65
+
66
+ # Tags exposed by given agent
67
+ def tags(nanite)
68
+ @redis.set_members("tg-#{nanite}")
69
+ end
70
+
71
+ # Retrieve all agents services
72
+ def all_services
73
+ log_redis_error do
74
+ @redis.set_members('naniteservices')
75
+ end
76
+ end
77
+
78
+ # Retrieve all agents tags
79
+ def all_tags
80
+ log_redis_error do
81
+ @redis.set_members('nanitetags')
82
+ end
83
+ end
84
+
85
+ # Retrieve nanites implementing given service and exposing given tags
86
+ def nanites_for(from, service, tags)
87
+ keys = tags && tags.dup || []
88
+ keys << service
89
+ log_redis_error do
90
+ @redis.set_intersect(keys.compact)
91
+ end
92
+ end
93
+
94
+ private
95
+
96
+ # Update values stored for given agent
97
+ # Also store reverse lookup information using both a unique and
98
+ # a global key (so it's possible to retrieve that agent value or
99
+ # all related values)
100
+ def update_elems(nanite, new_tags, obsolete_tags, elem_key, global_key)
101
+ new_tags = nil if new_tags.compact.empty?
102
+ obsolete_tags = nil if obsolete_tags.compact.empty?
103
+ log_redis_error do
104
+ obsolete_tags.each do |val|
105
+ @redis.set_delete(val, nanite)
106
+ @redis.set_delete(elem_key, val)
107
+ @redis.set_delete(global_key, val)
108
+ end if obsolete_tags
109
+ new_tags.each do |val|
110
+ @redis.set_add(val, nanite)
111
+ @redis.set_add(elem_key, val)
112
+ @redis.set_add(global_key, val)
113
+ end if new_tags
114
+ end
115
+ end
116
+
117
+ # Delete all values for given nanite agent
118
+ # Also delete reverse lookup information
119
+ def delete_elems(nanite, elem_key, global_key)
120
+ log_redis_error do
121
+ (@redis.set_members(elem_key)||[]).each do |val|
122
+ @redis.set_delete(val, nanite)
123
+ if @redis.set_count(val) == 0
124
+ @redis.delete(val)
125
+ @redis.set_delete(global_key, val)
126
+ end
127
+ end
128
+ @redis.delete(elem_key)
129
+ end
130
+ end
131
+
132
+ # Helper method, catch and log errors
133
+ def log_redis_error(&blk)
134
+ blk.call
135
+ rescue Exception => e
136
+ Nanite::Log.warn("redis error in method: #{caller[0]}")
137
+ raise e
138
+ end
139
+
140
+ end
141
+ end
@@ -1,168 +1,135 @@
1
1
  require 'redis'
2
+ require 'redis_tag_store'
2
3
 
3
4
  module Nanite
4
5
  class State
5
6
  include Enumerable
6
7
 
7
- # this class encapsulates the state of a nanite system using redis as the
8
- # data store. here is the schema, for each agent we store a number of items,
9
- # for a nanite with the identity: nanite-foobar we store the following things:
8
+ # This class encapsulates the state of a nanite system using redis as the
9
+ # data store and a provided tag store. For a nanite with the identity
10
+ # 'nanite-foobar' we store the following:
10
11
  #
11
- # nanite-foobar: 0.72 # load average or 'status'
12
- # s-nanite-foobar: { /foo/bar, /foo/nik } # a SET of the provided services
13
- # tg-nanite-foobar: { foo-42, customer-12 } # a SET of the tags for this agent
12
+ # nanite-foobar: 0.72 # load average or 'status'
14
13
  # t-nanite-foobar: 123456789 # unix timestamp of the last state update
15
14
  #
16
- # also we do an inverted index for quick lookup of agents providing a certain
17
- # service, so for each service the agent provides, we add the nanite to a SET
18
- # of all the nanites that provide said service:
15
+ # The tag store is used to store the associated services and tags.
19
16
  #
20
- # foo/bar: { nanite-foobar, nanite-nickelbag, nanite-another } # redis SET
17
+ # A tag store should provide the following methods:
18
+ # - initialize(redis): Initialize tag store, may use provided redis handle
19
+ # - services(nanite): Retrieve services implemented by given agent
20
+ # - tags(nanite): Retrieve tags implemented by given agent
21
+ # - all_services: Retrieve all services implemented by all agents
22
+ # - all_tags: Retrieve all tags exposed by all agents
23
+ # - store(nanite, services, tags): Store agent's services and tags
24
+ # - update(name, new_tags,obsolete_tags): Update agent's tags
25
+ # - delete(nanite): Delete all entries associated with given agent
26
+ # - nanites_for(service, tags): Retrieve agents implementing given service
27
+ # and exposing given tags
21
28
  #
22
- # we do that same thing for tags:
23
- # some-tag: { nanite-foobar, nanite-nickelbag, nanite-another } # redis SET
24
- #
25
- # This way we can do a lookup of what nanites provide a set of services and tags based
26
- # on redis SET intersection:
27
- #
28
- # nanites_for('/gems/list', 'some-tag')
29
- # => returns a nested array of nanites and their state that provide the intersection
30
- # of these two service tags
31
-
32
- def initialize(redis)
33
- Nanite::Log.info("[setup] initializing redis state: #{redis}")
34
- host, port = redis.split(':')
29
+ # The default implementation for the tag store reuses Redis.
30
+
31
+ def initialize(redis, tag_store=nil)
32
+ host, port, tag_store_type = redis.split(':')
35
33
  host ||= '127.0.0.1'
36
34
  port ||= '6379'
37
- @redis = Redis.new :host => host, :port => port
38
- end
39
-
40
- def log_redis_error(meth,&blk)
41
- blk.call
42
- rescue Exception => e
43
- Nanite::Log.info("redis error in method: #{meth}")
44
- raise e
35
+ tag_store||= 'Nanite::RedisTagStore'
36
+ @redis = Redis.new(:host => host, :port => port)
37
+ @tag_store = tag_store.to_const.new(@redis)
38
+ Nanite::Log.info("[setup] Initializing redis state using host '#{host}', port '#{port}' and tag store #{tag_store}")
45
39
  end
46
-
40
+
41
+ # Retrieve given agent services, tags, status and timestamp
47
42
  def [](nanite)
48
- log_redis_error("[]") do
49
- status = @redis[nanite]
43
+ log_redis_error do
44
+ status = @redis[nanite]
50
45
  timestamp = @redis["t-#{nanite}"]
51
- services = @redis.set_members("s-#{nanite}")
52
- tags = @redis.set_members("tg-#{nanite}")
46
+ services = @tag_store.services(nanite)
47
+ tags = @tag_store.tags(nanite)
53
48
  return nil unless status && timestamp && services
54
49
  {:services => services, :status => status, :timestamp => timestamp.to_i, :tags => tags}
55
50
  end
56
51
  end
57
-
52
+
53
+ # Set given attributes for given agent
54
+ # Attributes may include services, tags and status
58
55
  def []=(nanite, attributes)
59
- log_redis_error("[]=") do
60
- update_state(nanite, attributes[:status], attributes[:services], attributes[:tags])
61
- end
56
+ @tag_store.store(nanite, attributes[:services], attributes[:tags])
57
+ update_status(nanite, attributes[:status])
62
58
  end
63
-
59
+
60
+ # Delete all information related to given agent
64
61
  def delete(nanite)
65
- log_redis_error("delete") do
66
- (@redis.set_members("s-#{nanite}")||[]).each do |srv|
67
- @redis.set_delete(srv, nanite)
68
- if @redis.set_count(srv) == 0
69
- @redis.delete(srv)
70
- @redis.set_delete("naniteservices", srv)
71
- end
72
- end
73
- (@redis.set_members("tg-#{nanite}")||[]).each do |tag|
74
- @redis.set_delete(tag, nanite)
75
- if @redis.set_count(tag) == 0
76
- @redis.delete(tag)
77
- @redis.set_delete("nanitetags", tag)
78
- end
79
- end
80
- @redis.delete nanite
81
- @redis.delete "s-#{nanite}"
82
- @redis.delete "t-#{nanite}"
83
- @redis.delete "tg-#{nanite}"
62
+ @tag_store.delete(nanite)
63
+ log_redis_error do
64
+ @redis.delete(nanite)
65
+ @redis.delete("t-#{nanite}")
84
66
  end
85
67
  end
86
-
68
+
69
+ # Return all services exposed by all agents
87
70
  def all_services
88
- log_redis_error("all_services") do
89
- @redis.set_members("naniteservices")
90
- end
71
+ @tag_store.all_services
91
72
  end
92
73
 
74
+ # Return all tags exposed by all agents
93
75
  def all_tags
94
- log_redis_error("all_tags") do
95
- @redis.set_members("nanitetags")
96
- end
76
+ @tag_store.all_tags
97
77
  end
98
-
99
- def update_state(name, status, services, tags)
100
- old_services = @redis.set_members("s-#{name}")
101
- if old_services
102
- (old_services - services).each do |s|
103
- @redis.set_delete(s, name)
104
- @redis.set_delete("naniteservices", s)
105
- end
106
- end
107
- old_tags = @redis.set_members("tg-#{name}")
108
- if old_tags
109
- (old_tags - tags).each do |t|
110
- @redis.set_delete(t, name)
111
- @redis.set_delete("nanitetags", t)
112
- end
113
- end
114
- @redis.delete("s-#{name}")
115
- services.each do |srv|
116
- @redis.set_add(srv, name)
117
- @redis.set_add("s-#{name}", srv)
118
- @redis.set_add("naniteservices", srv)
119
- end
120
- @redis.delete("tg-#{name}")
121
- tags.each do |tag|
122
- next if tag.nil?
123
- @redis.set_add(tag, name)
124
- @redis.set_add("tg-#{name}", tag)
125
- @redis.set_add("nanitetags", tag)
78
+
79
+ # Update status and timestamp for given agent
80
+ def update_status(name, status)
81
+ log_redis_error do
82
+ @redis[name] = status
83
+ @redis["t-#{name}"] = Time.now.utc.to_i
126
84
  end
127
- update_status(name, status)
128
85
  end
129
86
 
130
- def update_status(name, status)
131
- @redis[name] = status
132
- @redis["t-#{name}"] = Time.now.utc.to_i
87
+ # Update tags for given agent
88
+ def update_tags(name, new_tags, obsolete_tags)
89
+ @tag_store.update(name, new_tags, obsolete_tags)
133
90
  end
134
-
91
+
92
+ # Return all registered agents
135
93
  def list_nanites
136
- log_redis_error("list_nanites") do
94
+ log_redis_error do
137
95
  @redis.keys("nanite-*")
138
96
  end
139
97
  end
140
-
98
+
99
+ # Number of registered agents
141
100
  def size
142
101
  list_nanites.size
143
102
  end
144
-
145
- def clear_state
146
- log_redis_error("clear_state") do
147
- @redis.keys("*").each {|k| @redis.delete k}
148
- end
149
- end
150
-
103
+
104
+ # Iterate through all agents, yielding services, tags
105
+ # status and timestamp keyed by agent name
151
106
  def each
152
107
  list_nanites.each do |nan|
153
108
  yield nan, self[nan]
154
109
  end
155
110
  end
156
-
157
- def nanites_for(service, *tags)
158
- keys = tags.dup << service
159
- log_redis_error("nanites_for") do
160
- res = []
161
- (@redis.set_intersect(keys)||[]).each do |nan|
162
- res << [nan, self[nan]]
111
+
112
+ # Return agents that implement given service and expose
113
+ # all given tags
114
+ def nanites_for(from, service, tags)
115
+ res = []
116
+ @tag_store.nanites_for(from, service, tags).each do |nanite_id|
117
+ if nanite = self[nanite_id]
118
+ res << [nanite_id, nanite]
163
119
  end
164
- res
165
120
  end
121
+ res
122
+ end
123
+
124
+ private
125
+
126
+ # Helper method, catch and log errors
127
+ def log_redis_error(&blk)
128
+ blk.call
129
+ rescue Exception => e
130
+ Nanite::Log.warn("redis error in method: #{caller[0]}")
131
+ raise e
166
132
  end
133
+
167
134
  end
168
135
  end
@@ -27,6 +27,26 @@ class String
27
27
  def to_const_path
28
28
  snake_case.gsub(/::/, "/")
29
29
  end
30
+
31
+ # Convert constant name to constant
32
+ #
33
+ # "FooBar::Baz".to_const => FooBar::Baz
34
+ #
35
+ # @return [Constant] Constant corresponding to given name or nil if no
36
+ # constant with that name exists
37
+ #
38
+ # @api public
39
+ def to_const
40
+ names = split('::')
41
+ names.shift if names.empty? || names.first.empty?
42
+
43
+ constant = Object
44
+ names.each do |name|
45
+ # modified to return nil instead of raising an const_missing error
46
+ constant = constant && constant.const_defined?(name) ? constant.const_get(name) : nil
47
+ end
48
+ constant
49
+ end
30
50
  end
31
51
 
32
52
  class Object
@@ -55,4 +75,4 @@ class Object
55
75
  end
56
76
  end
57
77
  end
58
- end
78
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: nanite
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.1.2
4
+ version: 0.4.1.10
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ezra Zygmuntowicz
@@ -9,7 +9,7 @@ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
11
 
12
- date: 2009-10-22 00:00:00 +13:00
12
+ date: 2009-11-06 00:00:00 -08:00
13
13
  default_executable:
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
@@ -39,6 +39,7 @@ files:
39
39
  - README.rdoc
40
40
  - Rakefile
41
41
  - TODO
42
+ - lib/nanite
42
43
  - lib/nanite/actor.rb
43
44
  - lib/nanite/actor_registry.rb
44
45
  - lib/nanite/admin.rb
@@ -51,6 +52,7 @@ files:
51
52
  - lib/nanite/identity.rb
52
53
  - lib/nanite/job.rb
53
54
  - lib/nanite/local_state.rb
55
+ - lib/nanite/log
54
56
  - lib/nanite/log/formatter.rb
55
57
  - lib/nanite/log.rb
56
58
  - lib/nanite/mapper.rb
@@ -59,6 +61,8 @@ files:
59
61
  - lib/nanite/packets.rb
60
62
  - lib/nanite/pid_file.rb
61
63
  - lib/nanite/reaper.rb
64
+ - lib/nanite/redis_tag_store.rb
65
+ - lib/nanite/security
62
66
  - lib/nanite/security/cached_certificate_store_proxy.rb
63
67
  - lib/nanite/security/certificate.rb
64
68
  - lib/nanite/security/certificate_cache.rb
@@ -79,8 +83,6 @@ files:
79
83
  - bin/nanite-mapper
80
84
  has_rdoc: true
81
85
  homepage: http://github.com/ezmobius/nanite
82
- licenses: []
83
-
84
86
  post_install_message:
85
87
  rdoc_options: []
86
88
 
@@ -101,9 +103,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
101
103
  requirements: []
102
104
 
103
105
  rubyforge_project:
104
- rubygems_version: 1.3.5
106
+ rubygems_version: 1.3.1
105
107
  signing_key:
106
- specification_version: 3
108
+ specification_version: 2
107
109
  summary: self assembling fabric of ruby daemons
108
110
  test_files: []
109
111