rightscale-nanite-dev 0.4.1.11 → 0.4.1.15

Sign up to get free protection for your applications and to get access to all the features.
data/Rakefile CHANGED
@@ -73,6 +73,4 @@ task :docs => :rdoc do
73
73
  else
74
74
  sh 'firefox rdoc/index.html'
75
75
  end
76
- end
77
-
78
- require 'rightscale-nanite.rb'
76
+ end
data/lib/nanite.rb CHANGED
@@ -39,7 +39,7 @@ require 'nanite/security/static_certificate_store'
39
39
  require 'nanite/serializer'
40
40
 
41
41
  module Nanite
42
- VERSION = '0.4.1.11' unless defined?(Nanite::VERSION)
42
+ VERSION = '0.4.1.15' unless defined?(Nanite::VERSION)
43
43
 
44
44
  class MapperNotRunning < StandardError; end
45
45
 
data/lib/nanite/agent.rb CHANGED
@@ -5,7 +5,7 @@ module Nanite
5
5
  include ConsoleHelper
6
6
  include DaemonizeHelper
7
7
 
8
- attr_reader :identity, :options, :serializer, :dispatcher, :registry, :amq, :tags
8
+ attr_reader :identity, :options, :serializer, :dispatcher, :registry, :amq, :tags, :callbacks
9
9
  attr_accessor :status_proc
10
10
 
11
11
  DEFAULT_OPTIONS = COMMON_DEFAULT_OPTIONS.merge({
@@ -74,6 +74,10 @@ module Nanite
74
74
  # this defaults to 5672, port used by some widely
75
75
  # used AMQP brokers (RabbitMQ and ZeroMQ)
76
76
  #
77
+ # callback : Hash of proc objects defining well known callbacks
78
+ # Currently only the :exception callback is supported
79
+ # This block gets called whenever a packet generates an exception
80
+ #
77
81
  # On start Nanite reads config.yml, so it is common to specify
78
82
  # options in the YAML file. However, when both Ruby code options
79
83
  # and YAML file specify option, Ruby code options take precedence.
@@ -166,6 +170,8 @@ module Nanite
166
170
  File.open(File.expand_path(File.join(@options[:root], 'config.yml')), 'w') do |fd|
167
171
  fd.write(YAML.dump(custom_config.merge(:identity => token)))
168
172
  end
173
+
174
+ @callbacks = options[:callbacks]
169
175
  end
170
176
 
171
177
  def load_actors
@@ -224,6 +230,7 @@ module Nanite
224
230
  receive(serializer.load(msg))
225
231
  rescue Exception => e
226
232
  Nanite::Log.error("RECV #{e.message}")
233
+ callbacks[:exception].call(e, msg, self) rescue nil if callbacks[:exception]
227
234
  end
228
235
  end
229
236
  end
@@ -18,9 +18,9 @@ module Nanite
18
18
  end
19
19
 
20
20
  # determine which nanites should receive the given request
21
- def targets_for(request)
21
+ def targets_for(request, include_timed_out)
22
22
  return [request.target] if request.target
23
- __send__(request.selector, request.from, request.type, request.tags).collect {|name, state| name }
23
+ __send__(request.selector, request, include_timed_out)
24
24
  end
25
25
 
26
26
  # adds nanite to nanites map: key is nanite's identity
@@ -71,7 +71,7 @@ module Nanite
71
71
  old_target = request.target
72
72
  request.target = target unless target == 'mapper-offline'
73
73
  if @security.authorize_request(request)
74
- Nanite::Log.info("SEND #{request.to_s([:from, :on_behalf, :tags, :target])}")
74
+ Nanite::Log.info("SEND #{request.to_s([:from, :scope, :tags, :target])}")
75
75
  amq.queue(target).publish(serializer.dump(request), :persistent => request.persistent)
76
76
  else
77
77
  Nanite::Log.warn("RECV NOT AUTHORIZED #{request.to_s}")
@@ -100,12 +100,12 @@ module Nanite
100
100
 
101
101
  # forward request coming from agent
102
102
  def handle_request(request)
103
- Nanite::Log.info("RECV #{request.to_s([:from, :on_behalf, :target, :tags])}") unless Nanite::Log.level == :debug
103
+ Nanite::Log.info("RECV #{request.to_s([:from, :scope, :target, :tags])}") unless Nanite::Log.level == :debug
104
104
  Nanite::Log.debug("RECV #{request.to_s}")
105
105
  case request
106
106
  when Push
107
107
  mapper.send_push(request)
108
- else
108
+ when Request
109
109
  intm_handler = lambda do |result, job|
110
110
  result = IntermediateMessage.new(request.token, job.request.from, mapper.identity, nil, result)
111
111
  forward_response(result, request.persistent)
@@ -120,6 +120,10 @@ module Nanite
120
120
  if ok == false
121
121
  forward_response(result, request.persistent)
122
122
  end
123
+ when TagQuery
124
+ results = nanites.nanites_for(request)
125
+ result = Result.new(request.token, request.from, results, mapper.identity)
126
+ forward_response(result, request.persistent)
123
127
  end
124
128
  end
125
129
 
@@ -130,37 +134,38 @@ module Nanite
130
134
  end
131
135
 
132
136
  # returns least loaded nanite that provides given service
133
- def least_loaded(from, service, tags=[])
134
- candidates = nanites_providing(from, service, tags)
137
+ def least_loaded(request, include_timed_out)
138
+ candidates = nanites_providing(request, include_timed_out)
135
139
  return [] if candidates.empty?
136
-
137
- [candidates.min { |a,b| a[1][:status] <=> b[1][:status] }]
140
+ res = candidates.to_a.min { |a, b| a[1][:status] <=> b[1][:status] }
141
+ [res[0]]
138
142
  end
139
143
 
140
144
  # returns all nanites that provide given service
141
- def all(from, service, tags=[])
142
- nanites_providing(from, service,tags)
145
+ # potentially including timed out agents
146
+ def all(request, include_timed_out)
147
+ nanites_providing(request, include_timed_out).keys
143
148
  end
144
149
 
145
150
  # returns a random nanite
146
- def random(from, service, tags=[])
147
- candidates = nanites_providing(from, service,tags)
151
+ def random(request, include_timed_out)
152
+ candidates = nanites_providing(request, include_timed_out)
148
153
  return [] if candidates.empty?
149
-
150
- [candidates[rand(candidates.size)]]
154
+ [candidates.keys[rand(candidates.size)]]
151
155
  end
152
156
 
153
157
  # selects next nanite that provides given service
154
158
  # using round robin rotation
155
- def rr(from, service, tags=[])
159
+ def rr(request, include_timed_out)
156
160
  @last ||= {}
161
+ service = request.type
157
162
  @last[service] ||= 0
158
- candidates = nanites_providing(from, service,tags)
163
+ candidates = nanites_providing(request, include_timed_out)
159
164
  return [] if candidates.empty?
160
165
  @last[service] = 0 if @last[service] >= candidates.size
161
- candidate = candidates[@last[service]]
166
+ key = candidates.keys[@last[service]]
162
167
  @last[service] += 1
163
- [candidate]
168
+ [key]
164
169
  end
165
170
 
166
171
  def timed_out?(nanite)
@@ -168,10 +173,10 @@ module Nanite
168
173
  end
169
174
 
170
175
  # returns all nanites that provide the given service
171
- def nanites_providing(from, service, tags)
172
- nanites.nanites_for(from, service, tags).delete_if do |nanite|
173
- if res = timed_out?(nanite[1])
174
- Nanite::Log.debug("Ignoring timed out nanite #{nanite[0]} in target selection - last seen at #{nanite[1][:timestamp]}")
176
+ def nanites_providing(request, include_timed_out)
177
+ nanites.nanites_for(request).delete_if do |nanite, info|
178
+ if res = !include_timed_out && timed_out?(info)
179
+ Nanite::Log.debug("Ignoring timed out nanite #{nanite} in target selection - last seen at #{info[:timestamp]}")
175
180
  end
176
181
  res
177
182
  end
@@ -191,7 +196,7 @@ module Nanite
191
196
  handle_ping(ping)
192
197
  rescue Exception => e
193
198
  Nanite::Log.error("RECV [ping] #{e.message}")
194
- callbacks[:exception].call(e, msg, mapper) rescue nil if callbacks[:exception]
199
+ callbacks[:exception].call(e, ping, mapper) rescue nil if callbacks[:exception]
195
200
  end
196
201
  end
197
202
  hb_fanout = amq.fanout('heartbeat', :durable => true)
@@ -6,28 +6,32 @@ module Nanite
6
6
  end
7
7
  end
8
8
 
9
- def all_services
10
- all(:services)
11
- end
12
-
13
- def all_tags
14
- all(:tags)
15
- end
9
+ def nanites_for(request)
10
+ tags = request.tags
11
+ service = request.service
12
+ if service
13
+ nanites = reject { |_, state| !state[:services].include?(service) }
14
+ else
15
+ nanites = self
16
+ end
16
17
 
17
- def nanites_for(from, service, tags)
18
- tags = tags.dup.flatten
19
- nanites = select { |name, state| state[:services].include?(service) }
20
- unless tags.empty?
21
- nanites.select { |a, b| !(b[:tags] & tags).empty? }
18
+ if tags.nil? || tags.empty?
19
+ service ? nanites : {}
22
20
  else
23
- nanites
24
- end.to_a
21
+ nanites.reject { |_, state| (state[:tags] & tags).empty? }
22
+ end
25
23
  end
26
24
 
27
25
  def update_status(name, status)
28
26
  self[name].update(:status => status, :timestamp => Time.now.utc.to_i)
29
27
  end
30
28
 
29
+ def update_tags(name, new_tags, obsolete_tags)
30
+ prev_tags = self[name] && self[name][:tags]
31
+ updated_tags = (new_tags || []) + (prev_tags || []) - (obsolete_tags || [])
32
+ self[name].update(:tags => updated_tags.uniq)
33
+ end
34
+
31
35
  private
32
36
 
33
37
  def all(key)
data/lib/nanite/mapper.rb CHANGED
@@ -195,7 +195,7 @@ module Nanite
195
195
  def send_request(request, opts = {}, &blk)
196
196
  request.reply_to = identity
197
197
  intm_handler = opts.delete(:intermediate_handler)
198
- targets = cluster.targets_for(request)
198
+ targets = cluster.targets_for(request, include_timed_out=false)
199
199
  if !targets.empty?
200
200
  job = job_warden.new_job(request, targets, intm_handler, blk)
201
201
  cluster.route(request, job.targets)
@@ -235,7 +235,8 @@ module Nanite
235
235
  end
236
236
 
237
237
  def send_push(push, opts = {})
238
- targets = cluster.targets_for(push)
238
+ include_timed_out = push.selector == :all
239
+ targets = cluster.targets_for(push, include_timed_out)
239
240
  if !targets.empty?
240
241
  cluster.route(push, targets)
241
242
  true
@@ -270,7 +271,8 @@ module Nanite
270
271
  offline_queue = amq.queue('mapper-offline', :durable => true)
271
272
  offline_queue.subscribe(:ack => true) do |info, deliverable|
272
273
  deliverable = serializer.load(deliverable)
273
- targets = cluster.targets_for(deliverable)
274
+ include_timed_out = deliverable.is_a?(Push) && deliverable.selector == :all
275
+ targets = cluster.targets_for(deliverable, include_timed_out)
274
276
  unless targets.empty?
275
277
  info.ack
276
278
  if deliverable.kind_of?(Request)
@@ -113,7 +113,7 @@ module Nanite
113
113
  #
114
114
  # Options:
115
115
  # from is sender identity
116
- # on_behalf is agent identity that should be used to authorize request
116
+ # scope define behavior that should be used to resolve tag based routing
117
117
  # token is a generated request id that mapper uses to identify replies
118
118
  # reply_to is identity of the node actor replies to, usually a mapper itself
119
119
  # selector is the selector used to route the request
@@ -121,7 +121,7 @@ module Nanite
121
121
  # persistent signifies if this request should be saved to persistent storage by the AMQP broker
122
122
  class Request < Packet
123
123
 
124
- attr_accessor :from, :on_behalf, :payload, :type, :token, :reply_to, :selector, :target, :persistent, :tags
124
+ attr_accessor :from, :scope, :payload, :type, :token, :reply_to, :selector, :target, :persistent, :tags
125
125
 
126
126
  DEFAULT_OPTIONS = {:selector => :least_loaded}
127
127
 
@@ -131,7 +131,7 @@ module Nanite
131
131
  @payload = payload
132
132
  @size = size
133
133
  @from = opts[:from]
134
- @on_behalf = opts[:on_behalf]
134
+ @scope = opts[:scope]
135
135
  @token = opts[:token]
136
136
  @reply_to = opts[:reply_to]
137
137
  @selector = opts[:selector]
@@ -142,9 +142,9 @@ module Nanite
142
142
 
143
143
  def self.json_create(o)
144
144
  i = o['data']
145
- new(i['type'], i['payload'], { :from => i['from'], :on_behalf => i['on_behalf'],
146
- :token => i['token'], :reply_to => i['reply_to'],
147
- :selector => i['selector'], :target => i['target'],
145
+ new(i['type'], i['payload'], { :from => i['from'], :scope => i['scope'],
146
+ :token => i['token'], :reply_to => i['reply_to'],
147
+ :selector => i['selector'], :target => i['target'],
148
148
  :persistent => i['persistent'], :tags => i['tags'] },
149
149
  o['size'])
150
150
  end
@@ -152,7 +152,7 @@ module Nanite
152
152
  def to_s(filter=nil)
153
153
  log_msg = "#{super} <#{token}> #{type}"
154
154
  log_msg += " from #{id_to_s(from)}" if filter.nil? || filter.include?(:from)
155
- log_msg += " on behalf of #{id_to_s(on_behalf)}" if on_behalf && (filter.nil? || filter.include?(:on_behalf))
155
+ log_msg += " with scope #{scope}" if scope && (filter.nil? || filter.include?(:scope))
156
156
  log_msg += " to #{id_to_s(target)}" if target && (filter.nil? || filter.include?(:target))
157
157
  log_msg += ", reply_to #{id_to_s(reply_to)}" if reply_to && (filter.nil? || filter.include?(:reply_to))
158
158
  log_msg += ", tags #{tags.inspect}" if tags && !tags.empty? && (filter.nil? || filter.include?(:tags))
@@ -170,14 +170,14 @@ module Nanite
170
170
  #
171
171
  # Options:
172
172
  # from is sender identity
173
- # on_behalf is agent identity that should be used to authorize request
173
+ # scope define behavior that should be used to resolve tag based routing
174
174
  # token is a generated request id that mapper uses to identify replies
175
175
  # selector is the selector used to route the request
176
176
  # target is the target nanite for the request
177
177
  # persistent signifies if this request should be saved to persistent storage by the AMQP broker
178
178
  class Push < Packet
179
179
 
180
- attr_accessor :from, :on_behalf, :payload, :type, :token, :selector, :target, :persistent, :tags
180
+ attr_accessor :from, :scope, :payload, :type, :token, :selector, :target, :persistent, :tags
181
181
 
182
182
  DEFAULT_OPTIONS = {:selector => :least_loaded}
183
183
 
@@ -187,7 +187,7 @@ module Nanite
187
187
  @payload = payload
188
188
  @size = size
189
189
  @from = opts[:from]
190
- @on_behalf = opts[:on_behalf]
190
+ @scope = opts[:scope]
191
191
  @token = opts[:token]
192
192
  @selector = opts[:selector]
193
193
  @target = opts[:target]
@@ -197,9 +197,9 @@ module Nanite
197
197
 
198
198
  def self.json_create(o)
199
199
  i = o['data']
200
- new(i['type'], i['payload'], { :from => i['from'], :on_behalf => i['on_behalf'],
201
- :token => i['token'], :selector => i['selector'],
202
- :target => i['target'], :persistent => i['persistent'],
200
+ new(i['type'], i['payload'], { :from => i['from'], :scope => i['scope'],
201
+ :token => i['token'], :selector => i['selector'],
202
+ :target => i['target'], :persistent => i['persistent'],
203
203
  :tags => i['tags'] },
204
204
  o['size'])
205
205
  end
@@ -207,7 +207,7 @@ module Nanite
207
207
  def to_s(filter=nil)
208
208
  log_msg = "#{super} <#{token}> #{type}"
209
209
  log_msg += " from #{id_to_s(from)}" if filter.nil? || filter.include?(:from)
210
- log_msg += " on behalf of #{id_to_s(on_behalf)}" if on_behalf && (filter.nil? || filter.include?(:on_behalf))
210
+ log_msg += " with scope #{scope}" if scope && (filter.nil? || filter.include?(:scope))
211
211
  log_msg += ", target #{id_to_s(target)}" if target && (filter.nil? || filter.include?(:target))
212
212
  log_msg += ", tags #{tags.inspect}" if tags && !tags.empty? && (filter.nil? || filter.include?(:tags))
213
213
  log_msg += ", payload #{payload.inspect}" if filter.nil? || filter.include?(:payload)
@@ -215,6 +215,31 @@ module Nanite
215
215
  end
216
216
  end
217
217
 
218
+ # Tag query: retrieve agent ids with associated tags that match given tags
219
+ #
220
+ # Options:
221
+ # from is sender identity
222
+ # tags list of tags that each agent in the result must match
223
+ class TagQuery < Packet
224
+
225
+ attr_accessor :from, :tags
226
+
227
+ def initialize(from, tags, size=nil)
228
+ @from = from
229
+ @tags = tags
230
+ @size = size
231
+ end
232
+
233
+ def self.json_create(o)
234
+ i = o['data']
235
+ new(i['from'], i['tags'], o['size'])
236
+ end
237
+
238
+ def to_s(filter=nil)
239
+ "#{super} <#{token}> #{type} from #{id_to_s(from)}, tags #{tags.inspect}"
240
+ end
241
+ end
242
+
218
243
  # packet that means a work result notification sent from actor to mapper
219
244
  #
220
245
  # from is sender identity
@@ -68,27 +68,13 @@ module Nanite
68
68
  @redis.set_members("tg-#{nanite}")
69
69
  end
70
70
 
71
- # Retrieve all agents services
72
- def all_services
73
- log_redis_error do
74
- @redis.set_members('naniteservices')
75
- end
76
- end
77
-
78
- # Retrieve all agents tags
79
- def all_tags
80
- log_redis_error do
81
- @redis.set_members('nanitetags')
82
- end
83
- end
84
-
85
71
  # Retrieve nanites implementing given service and exposing given tags
86
- def nanites_for(from, service, tags)
87
- keys = tags && tags.dup || []
88
- keys << service
89
- log_redis_error do
90
- @redis.set_intersect(keys.compact)
91
- end
72
+ def nanites_ids_for(request)
73
+ keys = request.tags ? request.tags.dup : []
74
+ keys << request.service if request.service
75
+ keys.compact!
76
+ return {} if keys.empty?
77
+ log_redis_error { @redis.set_intersect(keys) }
92
78
  end
93
79
 
94
80
  private
@@ -27,7 +27,9 @@ module Nanite
27
27
 
28
28
  # Load certificate from file
29
29
  def self.load(file)
30
- from_data(File.new(file))
30
+ res = nil
31
+ File.open(file, 'r') { |f| res = from_data(f) }
32
+ res
31
33
  end
32
34
 
33
35
  # Initialize with raw certificate
@@ -28,13 +28,14 @@ module Nanite
28
28
  end
29
29
 
30
30
  # Serialize message and sign it using X.509 certificate
31
- def self.dump(obj)
31
+ def self.dump(obj, encrypt=nil)
32
32
  raise "Missing certificate identity" unless @identity
33
33
  raise "Missing certificate" unless @cert
34
34
  raise "Missing certificate key" unless @key
35
35
  raise "Missing certificate store" unless @store || !@encrypt
36
+ must_encrypt = encrypt.nil? ? @encrypt : encrypt
36
37
  json = obj.to_json
37
- if @encrypt
38
+ if must_encrypt
38
39
  certs = @store.get_recipients(obj)
39
40
  json = EncryptedDocument.new(json, certs).encrypted_data if certs
40
41
  end
@@ -51,9 +52,10 @@ module Nanite
51
52
  data = JSON.load(json)
52
53
  sig = Signature.from_data(data['signature'])
53
54
  certs = @store.get_signer(data['id'])
55
+ raise "Failed to check signature for signer #{data['id']}" unless certs.any? { |c| sig.match?(c) }
54
56
  raise "Could not find a cert for signer #{data['id']}" unless certs
55
57
  certs = [ certs ] unless certs.respond_to?(:each)
56
- jsn = data['data'] if certs.any? { |c| sig.match?(c) }
58
+ jsn = data['data']
57
59
  if jsn && @encrypt && data['encrypted']
58
60
  jsn = EncryptedDocument.from_data(jsn).decrypted_data(@key, @cert)
59
61
  end
data/lib/nanite/state.rb CHANGED
@@ -66,16 +66,6 @@ module Nanite
66
66
  end
67
67
  end
68
68
 
69
- # Return all services exposed by all agents
70
- def all_services
71
- @tag_store.all_services
72
- end
73
-
74
- # Return all tags exposed by all agents
75
- def all_tags
76
- @tag_store.all_tags
77
- end
78
-
79
69
  # Update status and timestamp for given agent
80
70
  def update_status(name, status)
81
71
  log_redis_error do
@@ -111,11 +101,11 @@ module Nanite
111
101
 
112
102
  # Return agents that implement given service and expose
113
103
  # all given tags
114
- def nanites_for(from, service, tags)
115
- res = []
116
- @tag_store.nanites_for(from, service, tags).each do |nanite_id|
104
+ def nanites_for(request)
105
+ res = {}
106
+ @tag_store.nanites_ids_for(request).each do |nanite_id|
117
107
  if nanite = self[nanite_id]
118
- res << [nanite_id, nanite]
108
+ res[nanite_id] = nanite
119
109
  end
120
110
  end
121
111
  res
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rightscale-nanite-dev
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.1.11
4
+ version: 0.4.1.15
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ezra Zygmuntowicz
@@ -9,7 +9,7 @@ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
11
 
12
- date: 2009-11-10 00:00:00 -08:00
12
+ date: 2009-11-30 00:00:00 -08:00
13
13
  default_executable:
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
@@ -39,7 +39,6 @@ files:
39
39
  - README.rdoc
40
40
  - Rakefile
41
41
  - TODO
42
- - lib/nanite
43
42
  - lib/nanite/actor.rb
44
43
  - lib/nanite/actor_registry.rb
45
44
  - lib/nanite/admin.rb
@@ -52,7 +51,6 @@ files:
52
51
  - lib/nanite/identity.rb
53
52
  - lib/nanite/job.rb
54
53
  - lib/nanite/local_state.rb
55
- - lib/nanite/log
56
54
  - lib/nanite/log/formatter.rb
57
55
  - lib/nanite/log.rb
58
56
  - lib/nanite/mapper.rb
@@ -62,7 +60,6 @@ files:
62
60
  - lib/nanite/pid_file.rb
63
61
  - lib/nanite/reaper.rb
64
62
  - lib/nanite/redis_tag_store.rb
65
- - lib/nanite/security
66
63
  - lib/nanite/security/cached_certificate_store_proxy.rb
67
64
  - lib/nanite/security/certificate.rb
68
65
  - lib/nanite/security/certificate_cache.rb
@@ -83,6 +80,8 @@ files:
83
80
  - bin/nanite-mapper
84
81
  has_rdoc: true
85
82
  homepage: http://github.com/ezmobius/nanite
83
+ licenses: []
84
+
86
85
  post_install_message:
87
86
  rdoc_options: []
88
87
 
@@ -103,9 +102,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
103
102
  requirements: []
104
103
 
105
104
  rubyforge_project:
106
- rubygems_version: 1.3.1
105
+ rubygems_version: 1.3.5
107
106
  signing_key:
108
- specification_version: 2
107
+ specification_version: 3
109
108
  summary: self assembling fabric of ruby daemons
110
109
  test_files: []
111
110