nanite 0.4.1.10 → 0.4.1.13

data/Rakefile

@@ -35,6 +35,7 @@ spec = Gem::Specification.new do |s|
   s.executables  = %w( nanite-agent nanite-mapper nanite-admin )
 
   s.add_dependency('amqp', '>= 0.6.0')
+  s.add_dependency('json', '>= 1.1.7')
 
   s.require_path = 'lib'
   s.files = %w(LICENSE README.rdoc Rakefile TODO) + Dir.glob("{lib,bin,specs}/**/*")

data/bin/nanite-agent

@@ -42,6 +42,9 @@ opts = OptionParser.new do |opts|
     options[:threadpool_size] = tps
   end
   
+  opts.on("--prefetch COUNT", Integer, "The number of messages stuffed into the queue at anytime.  Set this to a value of 1 or so for longer running jobs (1 or more seconds), so the agent does not get overwhelmed.  Default is unlimited.") do |pref|
+    options[:prefetch] = pref
+  end
 end
 
 opts.parse!

data/lib/nanite.rb

@@ -39,7 +39,7 @@ require 'nanite/security/static_certificate_store'
 require 'nanite/serializer'
 
 module Nanite
-  VERSION = '0.4.1.10' unless defined?(Nanite::VERSION)
+  VERSION = '0.4.1.13' unless defined?(Nanite::VERSION)
 
   class MapperNotRunning < StandardError; end
 
@@ -54,6 +54,11 @@ module Nanite
       @mapper = Nanite::Mapper.start(options)
     end
 
+    def start_mapper_proxy(options = {})
+      identity = options[:identity] || Nanite::Identity.generate
+      @mapper = Nanite::MapperProxy.new(identity, options)
+    end
+      
     def request(*args, &blk)
       ensure_mapper
       @mapper.request(*args, &blk)

data/lib/nanite/agent.rb

@@ -55,6 +55,9 @@ module Nanite
     # services    : list of services provided by this agent, by default
     #               all methods exposed by actors are listed
     #
+    # prefetch    : Sets prefetch (only supported in RabbitMQ >= 1.6).  Use value of 1 for long
+    #               running jobs (greater than a second) to avoid slamming/stalling your agent.
+    #
     # single_threaded: Run all operations in one thread
     #
     # threadpool_size: Number of threads to run operations in
@@ -86,7 +89,7 @@ module Nanite
     def initialize(opts)
       set_configuration(opts)
       @tags = []
-      @tags << opts[:tag] if opts[:tag]
+      @tags << opts[:tag]
       @tags.flatten!
       @options.freeze
     end
@@ -131,16 +134,6 @@ module Nanite
       @deny_token = deny_token
     end
 
-    # Update set of tags published by agent and notify mapper
-    # Add tags in 'new_tags' and remove tags in 'old_tags'
-    def update_tags(new_tags, old_tags)
-      @tags += (new_tags || [])
-      @tags -= (old_tags || [])
-      @tags.uniq!
-      tag_update = TagUpdate.new(identity, new_tags, old_tags)
-      amq.fanout('registration', :no_declare => options[:secure]).publish(serializer.dump(tag_update))
-    end
-
     protected
 
     def set_configuration(opts)
@@ -218,6 +211,9 @@ module Nanite
     end
 
     def setup_queue
+      if amq.respond_to?(:prefetch) && @options.has_key?(:prefetch)
+        amq.prefetch(@options[:prefetch])
+      end
       amq.queue(identity, :durable => true).subscribe(:ack => true) do |info, msg|
         begin
           info.ack

data/lib/nanite/cluster.rb

@@ -2,14 +2,13 @@ module Nanite
   class Cluster
     attr_reader :agent_timeout, :nanites, :reaper, :serializer, :identity, :amq, :redis, :mapper, :callbacks
 
-    def initialize(amq, agent_timeout, identity, serializer, mapper, state_configuration=nil, tag_store=nil, callbacks = {})
+    def initialize(amq, agent_timeout, identity, serializer, mapper, state_configuration=nil, callbacks = {})
       @amq = amq
       @agent_timeout = agent_timeout
       @identity = identity
       @serializer = serializer
       @mapper = mapper
       @state = state_configuration
-      @tag_store = tag_store
       @security = SecurityProvider.get
       @callbacks = callbacks
       setup_state
@@ -20,7 +19,7 @@ module Nanite
     # determine which nanites should receive the given request
     def targets_for(request)
       return [request.target] if request.target
-      __send__(request.selector, request.from, request.type, request.tags).collect {|name, state| name }
+      __send__(request.selector, request.type, request.tags).collect {|name, state| name }
     end
 
     # adds nanite to nanites map: key is nanite's identity
@@ -30,7 +29,7 @@ module Nanite
       case reg
       when Register
         if @security.authorize_registration(reg)
-          Nanite::Log.info("RECV #{reg.to_s}")
+          Nanite::Log.debug("RECV #{reg.to_s}")
           nanites[reg.identity] = { :services => reg.services, :status => reg.status, :tags => reg.tags, :timestamp => Time.now.utc.to_i }
           reaper.register(reg.identity, agent_timeout + 1) { nanite_timed_out(reg.identity) }
           callbacks[:register].call(reg.identity, mapper) if callbacks[:register]
@@ -42,9 +41,6 @@ module Nanite
         reaper.unregister(reg.identity)
         nanites.delete(reg.identity)
         callbacks[:unregister].call(reg.identity, mapper) if callbacks[:unregister]
-      when TagUpdate
-        Nanite::Log.info("RECV #{reg.to_s}")
-        nanites.update_tags(reg.identity, reg.new_tags, reg.obsolete_tags)
       else
         Nanite::Log.warn("RECV [register] Invalid packet type: #{reg.class}")
       end
@@ -59,23 +55,19 @@ module Nanite
         true
       end
     end
-
+    
     def route(request, targets)
       EM.next_tick { targets.map { |target| publish(request, target) } }
     end
 
     def publish(request, target)
-      # We need to initialize the 'target' field of the request object so that the serializer and
-      # the security provider have access to it.
+      # We need to initialize the 'target' field of the request object so that the serializer has
+      # access to it.
       begin
         old_target = request.target
         request.target = target unless target == 'mapper-offline'
-        if @security.authorize_request(request)
-          Nanite::Log.info("SEND #{request.to_s([:from, :on_behalf, :tags, :target])}")
-          amq.queue(target).publish(serializer.dump(request), :persistent => request.persistent)
-        else
-          Nanite::Log.warn("RECV NOT AUTHORIZED #{request.to_s}")
-        end
+        Nanite::Log.debug("SEND #{request.to_s([:from, :tags, :target])}")
+        amq.queue(target).publish(serializer.dump(request), :persistent => request.persistent)
       ensure
         request.target = old_target
       end
@@ -92,59 +84,62 @@ module Nanite
           reaper.update(ping.identity, agent_timeout + 1) { nanite_timed_out(ping.identity) }
         else
           packet = Advertise.new
-          Nanite::Log.info("SEND #{packet.to_s} to #{ping.identity}")
+          Nanite::Log.debug("SEND #{packet.to_s} to #{ping.identity}")
           amq.queue(ping.identity).publish(serializer.dump(packet))
         end
       end
     end
-
+    
     # forward request coming from agent
     def handle_request(request)
-      Nanite::Log.info("RECV #{request.to_s([:from, :on_behalf, :target, :tags])}") unless Nanite::Log.level == :debug
-      Nanite::Log.debug("RECV #{request.to_s}")
-      case request
-      when Push
-        mapper.send_push(request)
-      else
-        intm_handler = lambda do |result, job|
-          result = IntermediateMessage.new(request.token, job.request.from, mapper.identity, nil, result)
-          forward_response(result, request.persistent)
-        end
-
-        result = Result.new(request.token, request.from, nil, mapper.identity)
-        ok = mapper.send_request(request, :intermediate_handler => intm_handler) do |res|
-          result.results = res
-          forward_response(result, request.persistent)
-        end
-
-        if ok == false
-          forward_response(result, request.persistent)
+      if @security.authorize_request(request)
+        Nanite::Log.debug("RECV #{request.to_s}")
+        case request
+        when Push
+          mapper.send_push(request)
+        else
+          intm_handler = lambda do |result, job|
+            result = IntermediateMessage.new(request.token, job.request.from, mapper.identity, nil, result)
+            forward_response(result, request.persistent)
+          end
+        
+          result = Result.new(request.token, request.from, nil, mapper.identity)
+          ok = mapper.send_request(request, :intermediate_handler => intm_handler) do |res|
+            result.results = res
+            forward_response(result, request.persistent)
+          end
+          
+          if ok == false
+            forward_response(result, request.persistent)
+          end
         end
+      else
+        Nanite::Log.warn("RECV NOT AUTHORIZED #{request.to_s}")
       end
     end
-
+    
     # forward response back to agent that originally made the request
     def forward_response(res, persistent)
-      Nanite::Log.info("SEND #{res.to_s([:to])}")
+      Nanite::Log.debug("SEND #{res.to_s([:to])}")
       amq.queue(res.to).publish(serializer.dump(res), :persistent => persistent)
     end
-
+    
     # returns least loaded nanite that provides given service
-    def least_loaded(from, service, tags=[])
-      candidates = nanites_providing(from, service, tags)
+    def least_loaded(service, tags=[])
+      candidates = nanites_providing(service,tags)
       return [] if candidates.empty?
 
       [candidates.min { |a,b| a[1][:status] <=> b[1][:status] }]
     end
 
     # returns all nanites that provide given service
-    def all(from, service, tags=[])
-      nanites_providing(from, service,tags)
+    def all(service, tags=[])
+      nanites_providing(service,tags)
     end
 
     # returns a random nanite
-    def random(from, service, tags=[])
-      candidates = nanites_providing(from, service,tags)
+    def random(service, tags=[])
+      candidates = nanites_providing(service,tags)
       return [] if candidates.empty?
 
       [candidates[rand(candidates.size)]]
@@ -152,28 +147,30 @@ module Nanite
 
     # selects next nanite that provides given service
     # using round robin rotation
-    def rr(from, service, tags=[])
+    def rr(service, tags=[])
       @last ||= {}
       @last[service] ||= 0
-      candidates = nanites_providing(from, service,tags)
+      candidates = nanites_providing(service,tags)
       return [] if candidates.empty?
       @last[service] = 0 if @last[service] >= candidates.size
       candidate = candidates[@last[service]]
       @last[service] += 1
       [candidate]
     end
-
+    
     def timed_out?(nanite)
       nanite[:timestamp].to_i < (Time.now.utc - agent_timeout).to_i
     end
 
     # returns all nanites that provide the given service
-    def nanites_providing(from, service, tags)
-      nanites.nanites_for(from, service, tags).delete_if do |nanite|
-        if res = timed_out?(nanite[1])
-          Nanite::Log.debug("Ignoring timed out nanite #{nanite[0]} in target selection - last seen at #{nanite[1][:timestamp]}")
+    def nanites_providing(service, *tags)
+      nanites.nanites_for(service, *tags).delete_if do |nanite|
+        nanite_id, nanite_attributes = nanite
+        if timed_out?(nanite_attributes)
+          reaper.unregister(nanite_id)
+          nanites.delete(nanite_id)
+          Nanite::Log.debug("Nanite #{nanite_id} timed out - ignoring in target selection and deleting from state - last seen at #{nanite_attributes[:timestamp]}")
         end
-        res
       end
     end
 
@@ -191,7 +188,6 @@ module Nanite
           handle_ping(ping)
         rescue Exception => e
           Nanite::Log.error("RECV [ping] #{e.message}")
-          callbacks[:exception].call(e, msg, mapper) rescue nil if callbacks[:exception]
         end
       end
       hb_fanout = amq.fanout('heartbeat', :durable => true)
@@ -208,7 +204,6 @@ module Nanite
           register(serializer.load(msg))
         rescue Exception => e
           Nanite::Log.error("RECV [register] #{e.message}")
-          callbacks[:exception].call(e, msg, mapper) rescue nil if callbacks[:exception]
         end
       end
       reg_fanout = amq.fanout('registration', :durable => true)
@@ -218,14 +213,13 @@ module Nanite
         amq.queue("registration-#{identity}", :exclusive => true).bind(reg_fanout).subscribe &handler
       end
     end
-
+    
     def setup_request_queue
       handler = lambda do |msg|
         begin
           handle_request(serializer.load(msg))
         rescue Exception => e
           Nanite::Log.error("RECV [request] #{e.message}")
-          callbacks[:exception].call(e, msg, mapper) rescue nil if callbacks[:exception]
         end
       end
       req_fanout = amq.fanout('request', :durable => true)
@@ -241,14 +235,16 @@ module Nanite
       when String
         # backwards compatibility, we assume redis if the configuration option
         # was a string
+        Nanite::Log.info("[setup] using redis for state storage")
         require 'nanite/state'
-        @nanites = Nanite::State.new(@state, @tag_store)
+        @nanites = Nanite::State.new(@state)
+      when Hash
       else
         require 'nanite/local_state'
         @nanites = Nanite::LocalState.new
       end
     end
-
+    
     def shared_state?
       !@state.nil?
     end

data/lib/nanite/config.rb

@@ -31,7 +31,7 @@ module Nanite
       opts.on("--offline-failsafe", "Store messages in an offline queue when all the nanites are offline. Messages will be redelivered when nanites come online. Can be overriden on a per-message basis using the request methods.") do
         options[:offline_failsafe] = true
       end
-      
+
       opts.on("--redis HOST_PORT", "Use redis as the agent state storage in the mapper: --redis 127.0.0.1:6379; missing host and/or port will be filled with defaults if colon is present") do |redis|
         redishost, redisport = redis.split(':')
         redishost = '127.0.0.1' if (redishost.nil? || redishost.empty?)

data/lib/nanite/local_state.rb

@@ -14,7 +14,7 @@ module Nanite
       all(:tags)
     end
 
-    def nanites_for(from, service, tags)
+    def nanites_for(service, *tags)
       tags = tags.dup.flatten
       nanites = select { |name, state| state[:services].include?(service) }
       unless tags.empty?

data/lib/nanite/mapper.rb

@@ -76,15 +76,10 @@ module Nanite
     # secure      : use Security features of rabbitmq to restrict nanites to themselves
     #
     # prefetch    : Sets prefetch (only supported in RabbitMQ >= 1.6)
-    #
-    # callbacks   : A set of callbacks to be executed on specific events. Supported events are :register,
-    #               :unregister, :timeout and :exception. This option must be a hash with event names as
-    #               as keys and Procs as values. The Proc's arity (number of parameters) depends on the
-    #               type of callback:
-    #                    exception  -- the exception, the message being processed, a reference to the mapper
-    #                    all others -- the corresponding nanite's identity, a reference to the mapper
-    #
-    # tag_store   : Name of class which implements tag store backend interface, RedisTagStore by default
+    # callbacks   : A set of callbacks to have code executed on specific events, supported events are :register,
+    #               :unregister and :timeout. Parameter must be a hash with the corresponding events as keys and
+    #               a block as value. The block will get the corresponding nanite's identity and a copy of the   
+    #               mapper
     #
     # Connection options:
     #
@@ -250,7 +245,7 @@ module Nanite
     private
 
     def build_deliverable(deliverable_type, type, payload, opts)
-      deliverable = deliverable_type.new(type, payload, opts)
+      deliverable = deliverable_type.new(type, payload, nil, opts)
       deliverable.from = identity
       deliverable.token = Identity.generate
       deliverable.persistent = opts.key?(:persistent) ? opts[:persistent] : options[:persistent]
@@ -293,11 +288,9 @@ module Nanite
         begin
           msg = serializer.load(msg)     
           Nanite::Log.debug("RECV #{msg.to_s}")
-          Nanite::Log.info("RECV #{msg.to_s([:from])}") unless Nanite::Log.level == :debug
           job_warden.process(msg)
         rescue Exception => e
           Nanite::Log.error("RECV [result] #{e.message}")
-          callbacks[:exception].call(e, msg, mapper) rescue nil if callbacks[:exception]          
         end
       end
     end
@@ -308,7 +301,7 @@ module Nanite
     end
 
     def setup_cluster
-      @cluster = Cluster.new(@amq, @options[:agent_timeout], @options[:identity], @serializer, self, @options[:redis], @options[:tag_store], @options[:callbacks])
+      @cluster = Cluster.new(@amq, @options[:agent_timeout], @options[:identity], @serializer, self, @options[:redis], @options[:callbacks])
     end
   end
 end

data/lib/nanite/mapper_proxy.rb

@@ -31,33 +31,25 @@ module Nanite
     # Send request to given agent through the mapper
     def request(type, payload = '', opts = {}, &blk)
       raise "Mapper proxy not initialized" unless identity && options
-      request = Request.new(type, payload, opts)
+      request = Request.new(type, payload, nil, opts)
       request.from = identity
       request.token = Identity.generate
       request.persistent = opts.key?(:persistent) ? opts[:persistent] : options[:persistent]
       pending_requests[request.token] = 
         { :intermediate_handler => opts[:intermediate_handler], :result_handler => blk }
-      Nanite::Log.info("SEND #{request.to_s([:tags, :target])}")
+      Nanite::Log.debug("SEND #{request.to_s([:tags, :target])}")
       amqp.fanout('request', :no_declare => options[:secure]).publish(serializer.dump(request))
     end    
 
     def push(type, payload = '', opts = {})
       raise "Mapper proxy not initialized" unless identity && options
-      push = Push.new(type, payload, opts)
+      push = Push.new(type, payload, nil, opts)
       push.from = identity
       push.token = Identity.generate
       push.persistent = opts.key?(:persistent) ? opts[:persistent] : options[:persistent]
-      Nanite::Log.info("SEND #{push.to_s([:tags, :target])}")
+      Nanite::Log.debug("SEND #{push.to_s([:tags, :target])}")
       amqp.fanout('request', :no_declare => options[:secure]).publish(serializer.dump(push))
-    end
-
-    # Update tags registered by mapper for agent
-    def update_tags(new_tags, obsolete_tags)
-      raise "Mapper proxy not initialized" unless identity && options
-      update = TagUpdate.new(identity, new_tags, obsolete_tags)
-      Nanite::Log.info("SEND #{update.to_s}")
-      amqp.fanout('registration', :no_declare => options[:secure]).publish(serializer.dump(update))
-    end
+    end    
     
     # Handle intermediary result
     def handle_intermediate_result(res)

data/lib/nanite/packets.rb

@@ -89,7 +89,7 @@ module Nanite
 
     attr_accessor :chunk, :token
 
-    def initialize(token, chunk=nil, size=nil)
+    def initialize(token, size=nil, chunk=nil)
       @chunk = chunk
       @token = token
       @size = size
@@ -97,7 +97,7 @@ module Nanite
 
     def self.json_create(o)
       i = o['data']
-      new(i['token'], i['chunk'], o['size'])
+      new(i['token'], o['size'], i['chunk'])
     end
 
     def to_s
@@ -112,26 +112,24 @@ module Nanite
   # payload  is arbitrary data that is transferred from mapper to actor
   #
   # Options:
-  # from      is sender identity
-  # on_behalf is agent identity that should be used to authorize request
-  # token     is a generated request id that mapper uses to identify replies
-  # reply_to  is identity of the node actor replies to, usually a mapper itself
-  # selector  is the selector used to route the request
-  # target    is the target nanite for the request
+  # from     is sender identity
+  # token    is a generated request id that mapper uses to identify replies
+  # reply_to is identity of the node actor replies to, usually a mapper itself
+  # selector is the selector used to route the request
+  # target   is the target nanite for the request
   # persistent signifies if this request should be saved to persistent storage by the AMQP broker
   class Request < Packet
 
-    attr_accessor :from, :on_behalf, :payload, :type, :token, :reply_to, :selector, :target, :persistent, :tags
+    attr_accessor :from, :payload, :type, :token, :reply_to, :selector, :target, :persistent, :tags
 
     DEFAULT_OPTIONS = {:selector => :least_loaded}
 
-    def initialize(type, payload, opts={}, size=nil)
+    def initialize(type, payload, size=nil, opts={})
       opts = DEFAULT_OPTIONS.merge(opts)
       @type       = type
       @payload    = payload
       @size       = size
       @from       = opts[:from]
-      @on_behalf  = opts[:on_behalf]
       @token      = opts[:token]
       @reply_to   = opts[:reply_to]
       @selector   = opts[:selector]
@@ -142,17 +140,15 @@ module Nanite
 
     def self.json_create(o)
       i = o['data']
-      new(i['type'], i['payload'], { :from     => i['from'],         :on_behalf => i['on_behalf'],
-                                     :token      => i['token'],      :reply_to => i['reply_to'], 
-                                     :selector   => i['selector'],   :target   => i['target'],
-                                     :persistent => i['persistent'], :tags     => i['tags'] },
-                                   o['size'])
+      new(i['type'], i['payload'], o['size'], { :from     => i['from'],     :token      => i['token'],
+                                                :reply_to => i['reply_to'], :selector   => i['selector'],
+                                                :target   => i['target'],   :persistent => i['persistent'],
+                                                :tags     => i['tags'] })
     end
 
     def to_s(filter=nil)
       log_msg = "#{super} <#{token}> #{type}"
       log_msg += " from #{id_to_s(from)}" if filter.nil? || filter.include?(:from)
-      log_msg += " on behalf of #{id_to_s(on_behalf)}" if on_behalf && (filter.nil? || filter.include?(:on_behalf))
       log_msg += " to #{id_to_s(target)}" if target && (filter.nil? || filter.include?(:target))
       log_msg += ", reply_to #{id_to_s(reply_to)}" if reply_to && (filter.nil? || filter.include?(:reply_to))
       log_msg += ", tags #{tags.inspect}" if tags && !tags.empty? && (filter.nil? || filter.include?(:tags))
@@ -169,25 +165,23 @@ module Nanite
   # payload  is arbitrary data that is transferred from mapper to actor
   #
   # Options:
-  # from      is sender identity
-  # on_behalf is agent identity that should be used to authorize request
-  # token     is a generated request id that mapper uses to identify replies
-  # selector  is the selector used to route the request
-  # target    is the target nanite for the request
+  # from     is sender identity
+  # token    is a generated request id that mapper uses to identify replies
+  # selector is the selector used to route the request
+  # target   is the target nanite for the request
   # persistent signifies if this request should be saved to persistent storage by the AMQP broker
   class Push < Packet
 
-    attr_accessor :from, :on_behalf, :payload, :type, :token, :selector, :target, :persistent, :tags
+    attr_accessor :from, :payload, :type, :token, :selector, :target, :persistent, :tags
 
     DEFAULT_OPTIONS = {:selector => :least_loaded}
 
-    def initialize(type, payload, opts={}, size=nil)
+    def initialize(type, payload, size=nil, opts={})
       opts = DEFAULT_OPTIONS.merge(opts)
       @type       = type
       @payload    = payload
       @size       = size
       @from       = opts[:from]
-      @on_behalf  = opts[:on_behalf]
       @token      = opts[:token]
       @selector   = opts[:selector]
       @target     = opts[:target]
@@ -197,17 +191,14 @@ module Nanite
 
     def self.json_create(o)
       i = o['data']
-      new(i['type'], i['payload'], { :from   => i['from'],   :on_behalf  => i['on_behalf'],
-                                     :token  => i['token'],  :selector   => i['selector'],   
-                                     :target => i['target'], :persistent => i['persistent'], 
-                                     :tags   => i['tags'] },
-                                   o['size'])
+      new(i['type'], i['payload'], o['size'], { :from       => i['from'],       :token  => i['token'],
+                                                :selector   => i['selector'],   :target => i['target'],
+                                                :persistent => i['persistent'], :tags   => i['tags'] })
     end
 
     def to_s(filter=nil)
       log_msg = "#{super} <#{token}> #{type}"
       log_msg += " from #{id_to_s(from)}" if filter.nil? || filter.include?(:from)
-      log_msg += " on behalf of #{id_to_s(on_behalf)}" if on_behalf && (filter.nil? || filter.include?(:on_behalf))
       log_msg += ", target #{id_to_s(target)}" if target && (filter.nil? || filter.include?(:target))
       log_msg += ", tags #{tags.inspect}" if tags && !tags.empty? && (filter.nil? || filter.include?(:tags))
       log_msg += ", payload #{payload.inspect}" if filter.nil? || filter.include?(:payload)
@@ -369,33 +360,6 @@ module Nanite
     end
 
   end
-
-  # packet that is sent by agents to the mapper
-  # to update their tags
-  class TagUpdate < Packet
-
-    attr_accessor :identity, :new_tags, :obsolete_tags
-
-    def initialize(identity, new_tags, obsolete_tags, size=nil)
-      @identity      = identity
-      @new_tags      = new_tags
-      @obsolete_tags = obsolete_tags
-      @size          = size
-    end
-
-    def self.json_create(o)
-      i = o['data']
-      new(i['identity'], i['new_tags'], i['obsolete_tags'], o['size'])
-    end
-
-    def to_s
-      log_msg = "#{super} #{id_to_s(identity)}"
-      log_msg += ", new tags: #{new_tags.join(', ')}" if new_tags && !new_tags.empty?
-      log_msg += ", obsolete tags: #{obsolete_tags.join(', ')}" if obsolete_tags && !obsolete_tags.empty?
-      log_msg
-    end
-
-  end
-
+ 
 end
 

data/lib/nanite/state.rb

@@ -1,135 +1,168 @@
 require 'redis'
-require 'redis_tag_store'
 
 module Nanite
   class State
     include Enumerable
     
-    # This class encapsulates the state of a nanite system using redis as the 
-    # data store and a provided tag store. For a nanite with the identity
-    # 'nanite-foobar' we store the following:
+    # this class encapsulates the state of a nanite system using redis as the 
+    # data store. here is the schema, for each agent we store a number of items,
+    # for a nanite with the identity:  nanite-foobar we store the following things:
     #
-    # nanite-foobar: 0.72        # load average or 'status'
+    # nanite-foobar: 0.72 # load average or 'status'
+    # s-nanite-foobar: { /foo/bar, /foo/nik } # a SET of the provided services
+    # tg-nanite-foobar: { foo-42, customer-12 } # a SET of the tags for this agent
     # t-nanite-foobar: 123456789 # unix timestamp of the last state update
     #
-    # The tag store is used to store the associated services and tags.
+    # also we do an inverted index for quick lookup of agents providing a certain
+    # service, so for each service the agent provides, we add the nanite to a SET 
+    # of all the nanites that provide said service:
     #
-    # A tag store should provide the following methods:
-    #  - initialize(redis): Initialize tag store, may use provided redis handle
-    #  - services(nanite): Retrieve services implemented by given agent
-    #  - tags(nanite): Retrieve tags implemented by given agent
-    #  - all_services: Retrieve all services implemented by all agents
-    #  - all_tags: Retrieve all tags exposed by all agents
-    #  - store(nanite, services, tags): Store agent's services and tags
-    #  - update(name, new_tags,obsolete_tags): Update agent's tags
-    #  - delete(nanite): Delete all entries associated with given agent
-    #  - nanites_for(service, tags): Retrieve agents implementing given service
-    #                                and exposing given tags
+    # foo/bar: { nanite-foobar, nanite-nickelbag, nanite-another } # redis SET
     #
-    # The default implementation for the tag store reuses Redis.
-  
-    def initialize(redis, tag_store=nil)
-      host, port, tag_store_type = redis.split(':')
+    # we do that same thing for tags:
+    # some-tag: { nanite-foobar, nanite-nickelbag, nanite-another } # redis SET
+    #
+    # This way we can do a lookup of what nanites provide a set of services and tags based
+    # on redis SET intersection:
+    #
+    # nanites_for('/gems/list', 'some-tag')
+    # => returns a nested array of nanites and their state that provide the intersection
+    # of these two service tags
+    
+    def initialize(redis)
+      Nanite::Log.info("[setup] initializing redis state: #{redis}")
+      host, port = redis.split(':')
       host ||= '127.0.0.1'
       port ||= '6379'
-      tag_store||= 'Nanite::RedisTagStore'
-      @redis = Redis.new(:host => host, :port => port)
-      @tag_store = tag_store.to_const.new(@redis)
-      Nanite::Log.info("[setup] Initializing redis state using host '#{host}', port '#{port}' and tag store #{tag_store}")
+      @redis = Redis.new :host => host, :port => port
     end
-
-    # Retrieve given agent services, tags, status and timestamp
+    
+    def log_redis_error(meth,&blk)
+      blk.call
+    rescue Exception => e
+      Nanite::Log.info("redis error in method: #{meth}")
+      raise e
+    end
+    
     def [](nanite)
-      log_redis_error do
-        status    = @redis[nanite]
+      log_redis_error("[]") do
+        status = @redis[nanite]
         timestamp = @redis["t-#{nanite}"]
-        services  = @tag_store.services(nanite)
-        tags      = @tag_store.tags(nanite)
+        services = @redis.set_members("s-#{nanite}")
+        tags = @redis.set_members("tg-#{nanite}")
         return nil unless status && timestamp && services
         {:services => services, :status => status, :timestamp => timestamp.to_i, :tags => tags}
       end
     end
-
-    # Set given attributes for given agent
-    # Attributes may include services, tags and status
+    
     def []=(nanite, attributes)
-      @tag_store.store(nanite, attributes[:services], attributes[:tags])
-      update_status(nanite, attributes[:status])
+      log_redis_error("[]=") do
+        update_state(nanite, attributes[:status], attributes[:services], attributes[:tags])
+      end
     end
-
-    # Delete all information related to given agent
+    
     def delete(nanite)
-      @tag_store.delete(nanite)
-      log_redis_error do
-        @redis.delete(nanite)
-        @redis.delete("t-#{nanite}")
+      log_redis_error("delete") do
+        (@redis.set_members("s-#{nanite}")||[]).each do |srv|
+          @redis.set_delete(srv, nanite)
+          if @redis.set_count(srv) == 0
+            @redis.delete(srv)
+            @redis.set_delete("naniteservices", srv)
+          end
+        end
+        (@redis.set_members("tg-#{nanite}")||[]).each do |tag|
+          @redis.set_delete(tag, nanite)
+          if @redis.set_count(tag) == 0
+            @redis.delete(tag)
+            @redis.set_delete("nanitetags", tag)
+          end
+        end
+        @redis.delete nanite
+        @redis.delete "s-#{nanite}"
+        @redis.delete "t-#{nanite}"
+        @redis.delete "tg-#{nanite}"
       end
     end
-
-    # Return all services exposed by all agents
+    
     def all_services
-      @tag_store.all_services
+      log_redis_error("all_services") do
+        @redis.set_members("naniteservices")
+      end
     end
 
-    # Return all tags exposed by all agents
     def all_tags
-      @tag_store.all_tags
+      log_redis_error("all_tags") do
+        @redis.set_members("nanitetags")
+      end
     end
-
-    # Update status and timestamp for given agent
-    def update_status(name, status)
-      log_redis_error do
-        @redis[name] = status
-        @redis["t-#{name}"] = Time.now.utc.to_i
+    
+    def update_state(name, status, services, tags)
+      old_services = @redis.set_members("s-#{name}")
+      if old_services
+        (old_services - services).each do |s|
+          @redis.set_delete(s, name)
+          @redis.set_delete("naniteservices", s)
+        end
+      end
+      old_tags = @redis.set_members("tg-#{name}")
+      if old_tags
+        (old_tags - tags).each do |t|
+          @redis.set_delete(t, name)
+          @redis.set_delete("nanitetags", t)
+        end
       end
+      @redis.delete("s-#{name}")
+      services.each do |srv|
+        @redis.set_add(srv, name)
+        @redis.set_add("s-#{name}", srv)
+        @redis.set_add("naniteservices", srv)
+      end
+      @redis.delete("tg-#{name}")
+      tags.each do |tag|
+        next if tag.nil?
+        @redis.set_add(tag, name)
+        @redis.set_add("tg-#{name}", tag)
+        @redis.set_add("nanitetags", tag)
+      end
+      update_status(name, status)
     end
 
-    # Update tags for given agent
-    def update_tags(name, new_tags, obsolete_tags)
-      @tag_store.update(name, new_tags, obsolete_tags)
+    def update_status(name, status)
+      @redis[name] = status
+      @redis["t-#{name}"] = Time.now.utc.to_i
     end
-
-    # Return all registered agents
+    
     def list_nanites
-      log_redis_error do
+      log_redis_error("list_nanites") do
         @redis.keys("nanite-*")
       end
     end
-
-    # Number of registered agents
+    
     def size
       list_nanites.size
     end
-
-    # Iterate through all agents, yielding services, tags
-    # status and timestamp keyed by agent name
+    
+    def clear_state
+      log_redis_error("clear_state") do
+        @redis.keys("*").each {|k| @redis.delete k}
+      end
+    end
+    
     def each
       list_nanites.each do |nan|
         yield nan, self[nan]
       end
     end
-
-    # Return agents that implement given service and expose
-    # all given tags
-    def nanites_for(from, service, tags)
-      res = []
-      @tag_store.nanites_for(from, service, tags).each do |nanite_id|
-        if nanite = self[nanite_id]
-          res << [nanite_id, nanite]
+    
+    def nanites_for(service, *tags)
+      keys = tags.dup << service
+      log_redis_error("nanites_for") do
+        res = []
+        (@redis.set_intersect(keys)||[]).each do |nan|
+          res << [nan, self[nan]]
         end
+        res
       end
-      res
-    end
-
-    private
-
-    # Helper method, catch and log errors
-    def log_redis_error(&blk)
-      blk.call
-    rescue Exception => e
-      Nanite::Log.warn("redis error in method: #{caller[0]}")
-      raise e
     end
-    
   end
 end  

data/lib/nanite/util.rb

@@ -27,26 +27,6 @@ class String
   def to_const_path
     snake_case.gsub(/::/, "/")
   end
-
-  # Convert constant name to constant
-  #
-  # "FooBar::Baz".to_const => FooBar::Baz
-  #
-  # @return [Constant] Constant corresponding to given name or nil if no
-  #   constant with that name exists
-  #
-  # @api public
-  def to_const
-    names = split('::')
-    names.shift if names.empty? || names.first.empty?
-
-    constant = Object
-    names.each do |name|
-      # modified to return nil instead of raising an const_missing error
-      constant = constant && constant.const_defined?(name) ? constant.const_get(name) : nil
-    end
-    constant
-  end
 end
 
 class Object
@@ -75,4 +55,4 @@ class Object
       end
     end
   end
-end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: nanite
 version: !ruby/object:Gem::Version 
-  version: 0.4.1.10
+  version: 0.4.1.13
 platform: ruby
 authors: 
 - Ezra Zygmuntowicz
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-11-06 00:00:00 -08:00
+date: 2009-11-18 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -22,6 +22,16 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: 0.6.0
     version: 
+- !ruby/object:Gem::Dependency 
+  name: json
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.1.7
+    version: 
 description: self assembling fabric of ruby daemons
 email: ezra@engineyard.com
 executables: 
@@ -39,7 +49,6 @@ files:
 - README.rdoc
 - Rakefile
 - TODO
-- lib/nanite
 - lib/nanite/actor.rb
 - lib/nanite/actor_registry.rb
 - lib/nanite/admin.rb
@@ -52,7 +61,6 @@ files:
 - lib/nanite/identity.rb
 - lib/nanite/job.rb
 - lib/nanite/local_state.rb
-- lib/nanite/log
 - lib/nanite/log/formatter.rb
 - lib/nanite/log.rb
 - lib/nanite/mapper.rb
@@ -61,8 +69,6 @@ files:
 - lib/nanite/packets.rb
 - lib/nanite/pid_file.rb
 - lib/nanite/reaper.rb
-- lib/nanite/redis_tag_store.rb
-- lib/nanite/security
 - lib/nanite/security/cached_certificate_store_proxy.rb
 - lib/nanite/security/certificate.rb
 - lib/nanite/security/certificate_cache.rb
@@ -83,6 +89,8 @@ files:
 - bin/nanite-mapper
 has_rdoc: true
 homepage: http://github.com/ezmobius/nanite
+licenses: []
+
 post_install_message: 
 rdoc_options: []
 
@@ -103,9 +111,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.1
+rubygems_version: 1.3.5
 signing_key: 
-specification_version: 2
+specification_version: 3
 summary: self assembling fabric of ruby daemons
 test_files: []
 

data/lib/nanite/redis_tag_store.rb

@@ -1,141 +0,0 @@
-require 'redis'
-
-module Nanite
-
-  # Implementation of a tag store on top of Redis
-  # For a nanite with the identity 'nanite-foobar', we store the following:
-  #
-  # s-nanite-foobar: { /foo/bar, /foo/nik } # a SET of the provided services
-  # tg-nanite-foobar: { foo-42, customer-12 } # a SET of the tags for this agent
-  #
-  # Also we do an inverted index for quick lookup of agents providing a certain
-  # service, so for each service the agent provides, we add the nanite to a SET
-  # of all the nanites that provide said service:
-  #
-  # foo/bar: { nanite-foobar, nanite-nickelbag, nanite-another } # redis SET
-  #
-  # We do that same thing for tags:
-  #
-  # some-tag: { nanite-foobar, nanite-nickelbag, nanite-another } # redis SET
-  #
-  # This way we can do a lookup of what nanites provide a set of services and tags based
-  # on redis SET intersection:
-  #
-  # nanites_for('/gems/list', 'some-tag')
-  # => returns an array of nanites that provide the intersection of these two service tags
-
-  class RedisTagStore
-
-    # Initialize tag store with given redis handle
-    def initialize(redis)
-      @redis = redis
-    end
-
-    # Store services and tags for given agent
-    def store(nanite, services, tags)
-      services = nil if services.compact.empty?
-      tags = nil if tags.compact.empty?
-      log_redis_error do
-        if services
-          obsolete_services = @redis.set_members("s-#{nanite}") - services
-          update_elems(nanite, services, obsolete_services, "s-#{nanite}", 'naniteservices')
-        end
-        if tags
-          obsolete_tags = @redis.set_members("tg-#{nanite}") - tags
-          update_elems(nanite, tags, obsolete_tags, "tg-#{nanite}", 'nanitestags')
-        end
-      end
-    end
-
-    # Update tags for given agent
-    def update(nanite, new_tags, obsolete_tags)
-      update_elems(nanite, new_tags, obsolete_tags, "tg-#{nanite}", 'nanitestags')
-    end
-
-    # Delete services and tags for given agent
-    def delete(nanite)
-      delete_elems(nanite, "s-#{nanite}", 'naniteservices')
-      delete_elems(nanite, "tg-#{nanite}", 'nanitestags')
-    end
-
-    # Services implemented by given agent
-    def services(nanite)
-      @redis.set_members("s-#{nanite}")
-    end
-
-    # Tags exposed by given agent
-    def tags(nanite)
-      @redis.set_members("tg-#{nanite}")
-    end
-
-    # Retrieve all agents services
-    def all_services
-      log_redis_error do
-        @redis.set_members('naniteservices')
-      end
-    end
-
-    # Retrieve all agents tags
-    def all_tags
-      log_redis_error do
-        @redis.set_members('nanitetags')
-      end
-    end
-
-    # Retrieve nanites implementing given service and exposing given tags
-    def nanites_for(from, service, tags)
-      keys = tags && tags.dup || []
-      keys << service
-      log_redis_error do
-        @redis.set_intersect(keys.compact)
-      end
-    end
-
-    private
-
-    # Update values stored for given agent
-    # Also store reverse lookup information using both a unique and
-    # a global key (so it's possible to retrieve that agent value or
-    # all related values)
-    def update_elems(nanite, new_tags, obsolete_tags, elem_key, global_key)
-      new_tags = nil if new_tags.compact.empty?
-      obsolete_tags = nil if obsolete_tags.compact.empty?
-      log_redis_error do
-        obsolete_tags.each do |val|
-          @redis.set_delete(val, nanite)
-          @redis.set_delete(elem_key, val)
-          @redis.set_delete(global_key, val)
-        end if obsolete_tags
-        new_tags.each do |val|
-          @redis.set_add(val, nanite)
-          @redis.set_add(elem_key, val)
-          @redis.set_add(global_key, val)
-        end if new_tags
-      end
-    end
-
-    # Delete all values for given nanite agent
-    # Also delete reverse lookup information
-    def delete_elems(nanite, elem_key, global_key)
-      log_redis_error do
-        (@redis.set_members(elem_key)||[]).each do |val|
-          @redis.set_delete(val, nanite)
-          if @redis.set_count(val) == 0
-            @redis.delete(val)
-            @redis.set_delete(global_key, val)
-          end
-        end
-        @redis.delete(elem_key)
-      end
-    end
-
-    # Helper method, catch and log errors
-    def log_redis_error(&blk)
-      blk.call
-    rescue Exception => e
-      Nanite::Log.warn("redis error in method: #{caller[0]}")
-      raise e
-    end
-
-  end
-end