rightscale-nanite-dev 0.4.1.11 → 0.4.1.15

data/Rakefile

@@ -73,6 +73,4 @@ task :docs => :rdoc do
   else 
     sh 'firefox rdoc/index.html'
   end
-end
-
-require 'rightscale-nanite.rb'
+end

data/lib/nanite.rb

@@ -39,7 +39,7 @@ require 'nanite/security/static_certificate_store'
 require 'nanite/serializer'
 
 module Nanite
-  VERSION = '0.4.1.11' unless defined?(Nanite::VERSION)
+  VERSION = '0.4.1.15' unless defined?(Nanite::VERSION)
 
   class MapperNotRunning < StandardError; end
 

data/lib/nanite/agent.rb

@@ -5,7 +5,7 @@ module Nanite
     include ConsoleHelper
     include DaemonizeHelper
 
-    attr_reader :identity, :options, :serializer, :dispatcher, :registry, :amq, :tags
+    attr_reader :identity, :options, :serializer, :dispatcher, :registry, :amq, :tags, :callbacks
     attr_accessor :status_proc
 
     DEFAULT_OPTIONS = COMMON_DEFAULT_OPTIONS.merge({
@@ -74,6 +74,10 @@ module Nanite
     #            this defaults to 5672, port used by some widely
     #            used AMQP brokers (RabbitMQ and ZeroMQ)
     #
+    # callback : Hash of proc objects defining well known callbacks
+    #            Currently only the :exception callback is supported
+    #            This block gets called whenever a packet generates an exception
+    #
     # On start Nanite reads config.yml, so it is common to specify
     # options in the YAML file. However, when both Ruby code options
     # and YAML file specify option, Ruby code options take precedence.
@@ -166,6 +170,8 @@ module Nanite
       File.open(File.expand_path(File.join(@options[:root], 'config.yml')), 'w') do |fd|
         fd.write(YAML.dump(custom_config.merge(:identity => token)))
       end
+
+      @callbacks = options[:callbacks]      
     end
 
     def load_actors
@@ -224,6 +230,7 @@ module Nanite
           receive(serializer.load(msg))
         rescue Exception => e
           Nanite::Log.error("RECV #{e.message}")
+          callbacks[:exception].call(e, msg, self) rescue nil if callbacks[:exception]
         end
       end
     end

data/lib/nanite/cluster.rb

@@ -18,9 +18,9 @@ module Nanite
     end
 
     # determine which nanites should receive the given request
-    def targets_for(request)
+    def targets_for(request, include_timed_out)
       return [request.target] if request.target
-      __send__(request.selector, request.from, request.type, request.tags).collect {|name, state| name }
+      __send__(request.selector, request, include_timed_out)
     end
 
     # adds nanite to nanites map: key is nanite's identity
@@ -71,7 +71,7 @@ module Nanite
         old_target = request.target
         request.target = target unless target == 'mapper-offline'
         if @security.authorize_request(request)
-          Nanite::Log.info("SEND #{request.to_s([:from, :on_behalf, :tags, :target])}")
+          Nanite::Log.info("SEND #{request.to_s([:from, :scope, :tags, :target])}")
           amq.queue(target).publish(serializer.dump(request), :persistent => request.persistent)
         else
           Nanite::Log.warn("RECV NOT AUTHORIZED #{request.to_s}")
@@ -100,12 +100,12 @@ module Nanite
 
     # forward request coming from agent
     def handle_request(request)
-      Nanite::Log.info("RECV #{request.to_s([:from, :on_behalf, :target, :tags])}") unless Nanite::Log.level == :debug
+      Nanite::Log.info("RECV #{request.to_s([:from, :scope, :target, :tags])}") unless Nanite::Log.level == :debug
       Nanite::Log.debug("RECV #{request.to_s}")
       case request
       when Push
         mapper.send_push(request)
-      else
+      when Request
         intm_handler = lambda do |result, job|
           result = IntermediateMessage.new(request.token, job.request.from, mapper.identity, nil, result)
           forward_response(result, request.persistent)
@@ -120,6 +120,10 @@ module Nanite
         if ok == false
           forward_response(result, request.persistent)
         end
+      when TagQuery
+        results = nanites.nanites_for(request)
+        result = Result.new(request.token, request.from, results, mapper.identity)
+        forward_response(result, request.persistent)
       end
     end
 
@@ -130,37 +134,38 @@ module Nanite
     end
 
     # returns least loaded nanite that provides given service
-    def least_loaded(from, service, tags=[])
-      candidates = nanites_providing(from, service, tags)
+    def least_loaded(request, include_timed_out)
+      candidates = nanites_providing(request, include_timed_out)
       return [] if candidates.empty?
-
-      [candidates.min { |a,b| a[1][:status] <=> b[1][:status] }]
+      res = candidates.to_a.min { |a, b| a[1][:status] <=> b[1][:status] }
+      [res[0]]
     end
 
     # returns all nanites that provide given service
-    def all(from, service, tags=[])
-      nanites_providing(from, service,tags)
+    # potentially including timed out agents
+    def all(request, include_timed_out)
+      nanites_providing(request, include_timed_out).keys
     end
 
     # returns a random nanite
-    def random(from, service, tags=[])
-      candidates = nanites_providing(from, service,tags)
+    def random(request, include_timed_out)
+      candidates = nanites_providing(request, include_timed_out)
       return [] if candidates.empty?
-
-      [candidates[rand(candidates.size)]]
+      [candidates.keys[rand(candidates.size)]]
     end
 
     # selects next nanite that provides given service
     # using round robin rotation
-    def rr(from, service, tags=[])
+    def rr(request, include_timed_out)
       @last ||= {}
+      service = request.type
       @last[service] ||= 0
-      candidates = nanites_providing(from, service,tags)
+      candidates = nanites_providing(request, include_timed_out)
       return [] if candidates.empty?
       @last[service] = 0 if @last[service] >= candidates.size
-      candidate = candidates[@last[service]]
+      key = candidates.keys[@last[service]]
       @last[service] += 1
-      [candidate]
+      [key]
     end
 
     def timed_out?(nanite)
@@ -168,10 +173,10 @@ module Nanite
     end
 
     # returns all nanites that provide the given service
-    def nanites_providing(from, service, tags)
-      nanites.nanites_for(from, service, tags).delete_if do |nanite|
-        if res = timed_out?(nanite[1])
-          Nanite::Log.debug("Ignoring timed out nanite #{nanite[0]} in target selection - last seen at #{nanite[1][:timestamp]}")
+    def nanites_providing(request, include_timed_out)
+      nanites.nanites_for(request).delete_if do |nanite, info|
+        if res = !include_timed_out && timed_out?(info)
+          Nanite::Log.debug("Ignoring timed out nanite #{nanite} in target selection - last seen at #{info[:timestamp]}")
         end
         res
       end
@@ -191,7 +196,7 @@ module Nanite
           handle_ping(ping)
         rescue Exception => e
           Nanite::Log.error("RECV [ping] #{e.message}")
-          callbacks[:exception].call(e, msg, mapper) rescue nil if callbacks[:exception]
+          callbacks[:exception].call(e, ping, mapper) rescue nil if callbacks[:exception]
         end
       end
       hb_fanout = amq.fanout('heartbeat', :durable => true)

data/lib/nanite/local_state.rb

@@ -6,28 +6,32 @@ module Nanite
       end
     end
 
-    def all_services
-      all(:services)
-    end
-
-    def all_tags
-      all(:tags)
-    end
+    def nanites_for(request)
+      tags = request.tags
+      service = request.service
+      if service
+        nanites = reject { |_, state| !state[:services].include?(service) }
+      else
+        nanites = self
+      end
 
-    def nanites_for(from, service, tags)
-      tags = tags.dup.flatten
-      nanites = select { |name, state| state[:services].include?(service) }
-      unless tags.empty?
-        nanites.select { |a, b| !(b[:tags] & tags).empty? }
+      if tags.nil? || tags.empty?
+        service ? nanites : {}
       else
-        nanites
-      end.to_a
+        nanites.reject { |_, state| (state[:tags] & tags).empty? }
+      end
     end
 
     def update_status(name, status)
       self[name].update(:status => status, :timestamp => Time.now.utc.to_i)
     end
     
+    def update_tags(name, new_tags, obsolete_tags)
+      prev_tags = self[name] && self[name][:tags]
+      updated_tags = (new_tags || []) + (prev_tags || []) - (obsolete_tags || [])
+      self[name].update(:tags => updated_tags.uniq)
+    end
+    
     private
 
     def all(key)

data/lib/nanite/mapper.rb

@@ -195,7 +195,7 @@ module Nanite
     def send_request(request, opts = {}, &blk)
       request.reply_to = identity
       intm_handler = opts.delete(:intermediate_handler)
-      targets = cluster.targets_for(request)
+      targets = cluster.targets_for(request, include_timed_out=false)
       if !targets.empty?
         job = job_warden.new_job(request, targets, intm_handler, blk)
         cluster.route(request, job.targets)
@@ -235,7 +235,8 @@ module Nanite
     end
 
     def send_push(push, opts = {})
-      targets = cluster.targets_for(push)
+      include_timed_out = push.selector == :all
+      targets = cluster.targets_for(push, include_timed_out)
       if !targets.empty?
         cluster.route(push, targets)
         true
@@ -270,7 +271,8 @@ module Nanite
       offline_queue = amq.queue('mapper-offline', :durable => true)
       offline_queue.subscribe(:ack => true) do |info, deliverable|
         deliverable = serializer.load(deliverable)
-        targets = cluster.targets_for(deliverable)
+        include_timed_out = deliverable.is_a?(Push) && deliverable.selector == :all
+        targets = cluster.targets_for(deliverable, include_timed_out)
         unless targets.empty?
           info.ack
           if deliverable.kind_of?(Request)

data/lib/nanite/packets.rb

@@ -113,7 +113,7 @@ module Nanite
   #
   # Options:
   # from      is sender identity
-  # on_behalf is agent identity that should be used to authorize request
+  # scope     define behavior that should be used to resolve tag based routing
   # token     is a generated request id that mapper uses to identify replies
   # reply_to  is identity of the node actor replies to, usually a mapper itself
   # selector  is the selector used to route the request
@@ -121,7 +121,7 @@ module Nanite
   # persistent signifies if this request should be saved to persistent storage by the AMQP broker
   class Request < Packet
 
-    attr_accessor :from, :on_behalf, :payload, :type, :token, :reply_to, :selector, :target, :persistent, :tags
+    attr_accessor :from, :scope, :payload, :type, :token, :reply_to, :selector, :target, :persistent, :tags
 
     DEFAULT_OPTIONS = {:selector => :least_loaded}
 
@@ -131,7 +131,7 @@ module Nanite
       @payload    = payload
       @size       = size
       @from       = opts[:from]
-      @on_behalf  = opts[:on_behalf]
+      @scope      = opts[:scope]
       @token      = opts[:token]
       @reply_to   = opts[:reply_to]
       @selector   = opts[:selector]
@@ -142,9 +142,9 @@ module Nanite
 
     def self.json_create(o)
       i = o['data']
-      new(i['type'], i['payload'], { :from     => i['from'],         :on_behalf => i['on_behalf'],
-                                     :token      => i['token'],      :reply_to => i['reply_to'], 
-                                     :selector   => i['selector'],   :target   => i['target'],
+      new(i['type'], i['payload'], { :from       => i['from'],       :scope    => i['scope'],
+                                     :token      => i['token'],      :reply_to => i['reply_to'],
+                                     :selector   => i['selector'],   :target   => i['target'],   
                                      :persistent => i['persistent'], :tags     => i['tags'] },
                                    o['size'])
     end
@@ -152,7 +152,7 @@ module Nanite
     def to_s(filter=nil)
       log_msg = "#{super} <#{token}> #{type}"
       log_msg += " from #{id_to_s(from)}" if filter.nil? || filter.include?(:from)
-      log_msg += " on behalf of #{id_to_s(on_behalf)}" if on_behalf && (filter.nil? || filter.include?(:on_behalf))
+      log_msg += " with scope #{scope}" if scope && (filter.nil? || filter.include?(:scope))
       log_msg += " to #{id_to_s(target)}" if target && (filter.nil? || filter.include?(:target))
       log_msg += ", reply_to #{id_to_s(reply_to)}" if reply_to && (filter.nil? || filter.include?(:reply_to))
       log_msg += ", tags #{tags.inspect}" if tags && !tags.empty? && (filter.nil? || filter.include?(:tags))
@@ -170,14 +170,14 @@ module Nanite
   #
   # Options:
   # from      is sender identity
-  # on_behalf is agent identity that should be used to authorize request
+  # scope     define behavior that should be used to resolve tag based routing
   # token     is a generated request id that mapper uses to identify replies
   # selector  is the selector used to route the request
   # target    is the target nanite for the request
   # persistent signifies if this request should be saved to persistent storage by the AMQP broker
   class Push < Packet
 
-    attr_accessor :from, :on_behalf, :payload, :type, :token, :selector, :target, :persistent, :tags
+    attr_accessor :from, :scope, :payload, :type, :token, :selector, :target, :persistent, :tags
 
     DEFAULT_OPTIONS = {:selector => :least_loaded}
 
@@ -187,7 +187,7 @@ module Nanite
       @payload    = payload
       @size       = size
       @from       = opts[:from]
-      @on_behalf  = opts[:on_behalf]
+      @scope      = opts[:scope]
       @token      = opts[:token]
       @selector   = opts[:selector]
       @target     = opts[:target]
@@ -197,9 +197,9 @@ module Nanite
 
     def self.json_create(o)
       i = o['data']
-      new(i['type'], i['payload'], { :from   => i['from'],   :on_behalf  => i['on_behalf'],
-                                     :token  => i['token'],  :selector   => i['selector'],   
-                                     :target => i['target'], :persistent => i['persistent'], 
+      new(i['type'], i['payload'], { :from   => i['from'],   :scope      => i['scope'],
+                                     :token  => i['token'],  :selector   => i['selector'],
+                                     :target => i['target'], :persistent => i['persistent'],
                                      :tags   => i['tags'] },
                                    o['size'])
     end
@@ -207,7 +207,7 @@ module Nanite
     def to_s(filter=nil)
       log_msg = "#{super} <#{token}> #{type}"
       log_msg += " from #{id_to_s(from)}" if filter.nil? || filter.include?(:from)
-      log_msg += " on behalf of #{id_to_s(on_behalf)}" if on_behalf && (filter.nil? || filter.include?(:on_behalf))
+      log_msg += " with scope #{scope}" if scope && (filter.nil? || filter.include?(:scope))
       log_msg += ", target #{id_to_s(target)}" if target && (filter.nil? || filter.include?(:target))
       log_msg += ", tags #{tags.inspect}" if tags && !tags.empty? && (filter.nil? || filter.include?(:tags))
       log_msg += ", payload #{payload.inspect}" if filter.nil? || filter.include?(:payload)
@@ -215,6 +215,31 @@ module Nanite
     end
   end
 
+  # Tag query: retrieve agent ids with associated tags that match given tags
+  #
+  # Options:
+  # from  is sender identity
+  # tags  list of tags that each agent in the result must match
+  class TagQuery < Packet
+
+    attr_accessor :from, :tags
+
+    def initialize(from, tags, size=nil)
+      @from = from
+      @tags = tags
+      @size = size
+    end
+
+    def self.json_create(o)
+      i = o['data']
+      new(i['from'], i['tags'], o['size'])
+    end
+
+    def to_s(filter=nil)
+      "#{super} <#{token}> #{type} from #{id_to_s(from)}, tags #{tags.inspect}"
+    end
+  end
+
   # packet that means a work result notification sent from actor to mapper
   #
   # from     is sender identity

data/lib/nanite/redis_tag_store.rb

@@ -68,27 +68,13 @@ module Nanite
       @redis.set_members("tg-#{nanite}")
     end
 
-    # Retrieve all agents services
-    def all_services
-      log_redis_error do
-        @redis.set_members('naniteservices')
-      end
-    end
-
-    # Retrieve all agents tags
-    def all_tags
-      log_redis_error do
-        @redis.set_members('nanitetags')
-      end
-    end
-
     # Retrieve nanites implementing given service and exposing given tags
-    def nanites_for(from, service, tags)
-      keys = tags && tags.dup || []
-      keys << service
-      log_redis_error do
-        @redis.set_intersect(keys.compact)
-      end
+    def nanites_ids_for(request)
+      keys = request.tags ? request.tags.dup : []
+      keys << request.service if request.service
+      keys.compact!
+      return {} if keys.empty?
+      log_redis_error { @redis.set_intersect(keys) }
     end
 
     private

data/lib/nanite/security/certificate.rb

@@ -27,7 +27,9 @@ module Nanite
     
     # Load certificate from file
     def self.load(file)
-      from_data(File.new(file))
+      res = nil
+      File.open(file, 'r') { |f| res = from_data(f) }
+      res
     end
     
     # Initialize with raw certificate

data/lib/nanite/security/secure_serializer.rb

@@ -28,13 +28,14 @@ module Nanite
     end
 
     # Serialize message and sign it using X.509 certificate
-    def self.dump(obj)
+    def self.dump(obj, encrypt=nil)
       raise "Missing certificate identity" unless @identity
       raise "Missing certificate" unless @cert
       raise "Missing certificate key" unless @key
       raise "Missing certificate store" unless @store || !@encrypt
+      must_encrypt = encrypt.nil? ? @encrypt : encrypt
       json = obj.to_json
-      if @encrypt
+      if must_encrypt
         certs = @store.get_recipients(obj)
         json = EncryptedDocument.new(json, certs).encrypted_data if certs
       end
@@ -51,9 +52,10 @@ module Nanite
         data = JSON.load(json)
         sig = Signature.from_data(data['signature'])
         certs = @store.get_signer(data['id'])
+        raise "Failed to check signature for signer #{data['id']}" unless certs.any? { |c| sig.match?(c) }
         raise "Could not find a cert for signer #{data['id']}" unless certs
         certs = [ certs ] unless certs.respond_to?(:each)
-        jsn = data['data'] if certs.any? { |c| sig.match?(c) }
+        jsn = data['data']
         if jsn && @encrypt && data['encrypted']
           jsn = EncryptedDocument.from_data(jsn).decrypted_data(@key, @cert)
         end

data/lib/nanite/state.rb

@@ -66,16 +66,6 @@ module Nanite
       end
     end
 
-    # Return all services exposed by all agents
-    def all_services
-      @tag_store.all_services
-    end
-
-    # Return all tags exposed by all agents
-    def all_tags
-      @tag_store.all_tags
-    end
-
     # Update status and timestamp for given agent
     def update_status(name, status)
       log_redis_error do
@@ -111,11 +101,11 @@ module Nanite
 
     # Return agents that implement given service and expose
     # all given tags
-    def nanites_for(from, service, tags)
-      res = []
-      @tag_store.nanites_for(from, service, tags).each do |nanite_id|
+    def nanites_for(request)
+      res = {}
+      @tag_store.nanites_ids_for(request).each do |nanite_id|
         if nanite = self[nanite_id]
-          res << [nanite_id, nanite]
+          res[nanite_id] = nanite
         end
       end
       res

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: rightscale-nanite-dev
 version: !ruby/object:Gem::Version 
-  version: 0.4.1.11
+  version: 0.4.1.15
 platform: ruby
 authors: 
 - Ezra Zygmuntowicz
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-11-10 00:00:00 -08:00
+date: 2009-11-30 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -39,7 +39,6 @@ files:
 - README.rdoc
 - Rakefile
 - TODO
-- lib/nanite
 - lib/nanite/actor.rb
 - lib/nanite/actor_registry.rb
 - lib/nanite/admin.rb
@@ -52,7 +51,6 @@ files:
 - lib/nanite/identity.rb
 - lib/nanite/job.rb
 - lib/nanite/local_state.rb
-- lib/nanite/log
 - lib/nanite/log/formatter.rb
 - lib/nanite/log.rb
 - lib/nanite/mapper.rb
@@ -62,7 +60,6 @@ files:
 - lib/nanite/pid_file.rb
 - lib/nanite/reaper.rb
 - lib/nanite/redis_tag_store.rb
-- lib/nanite/security
 - lib/nanite/security/cached_certificate_store_proxy.rb
 - lib/nanite/security/certificate.rb
 - lib/nanite/security/certificate_cache.rb
@@ -83,6 +80,8 @@ files:
 - bin/nanite-mapper
 has_rdoc: true
 homepage: http://github.com/ezmobius/nanite
+licenses: []
+
 post_install_message: 
 rdoc_options: []
 
@@ -103,9 +102,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.1
+rubygems_version: 1.3.5
 signing_key: 
-specification_version: 2
+specification_version: 3
 summary: self assembling fabric of ruby daemons
 test_files: []