rightscale-nanite 0.4.1.4 → 0.4.1.10

data/lib/nanite/reaper.rb

@@ -1,23 +1,29 @@
 module Nanite
   class Reaper
-
+    attr_reader :timeouts
     def initialize(frequency=2)
       @timeouts = {}
       EM.add_periodic_timer(frequency) { EM.next_tick { reap } }
     end
 
-    def timeout(token, seconds, &blk)
+    # Add the specified token to the internal timeout hash.
+    # The reaper will then check this instance on every reap.
+    def register(token, seconds, &blk)
       @timeouts[token] = {:timestamp => Time.now + seconds, :seconds => seconds, :callback => blk}
     end
+    
+    def unregister(token)
+      @timeouts.delete(token)
+    end
 
-    def reset_with_autoregister_hack(token,seconds,&blk)
+    # Updates the timeout timestamp for the given token. If the token is
+    # unknown to this reaper instance it will be auto-registered, usually
+    # happening when you have several mappers and not all of them know
+    # this agent yet, but received a ping from it.
+    def update(token, seconds, &blk)
       unless @timeouts[token]
-        timeout(token, seconds, &blk)
+        register(token, seconds, &blk)
       end
-      reset(token)
-    end
-
-    def reset(token)
       @timeouts[token][:timestamp] = Time.now + @timeouts[token][:seconds]
     end
 
@@ -26,12 +32,7 @@ module Nanite
     def reap
       time = Time.now
       @timeouts.reject! do |token, data|
-        if time > data[:timestamp]
-          data[:callback].call
-          true
-        else
-          false
-        end
+        time > data[:timestamp] and data[:callback].call
       end
     end
   end

data/lib/nanite/redis_tag_store.rb

@@ -0,0 +1,141 @@
+require 'redis'
+
+module Nanite
+
+  # Implementation of a tag store on top of Redis
+  # For a nanite with the identity 'nanite-foobar', we store the following:
+  #
+  # s-nanite-foobar: { /foo/bar, /foo/nik } # a SET of the provided services
+  # tg-nanite-foobar: { foo-42, customer-12 } # a SET of the tags for this agent
+  #
+  # Also we do an inverted index for quick lookup of agents providing a certain
+  # service, so for each service the agent provides, we add the nanite to a SET
+  # of all the nanites that provide said service:
+  #
+  # foo/bar: { nanite-foobar, nanite-nickelbag, nanite-another } # redis SET
+  #
+  # We do that same thing for tags:
+  #
+  # some-tag: { nanite-foobar, nanite-nickelbag, nanite-another } # redis SET
+  #
+  # This way we can do a lookup of what nanites provide a set of services and tags based
+  # on redis SET intersection:
+  #
+  # nanites_for('/gems/list', 'some-tag')
+  # => returns an array of nanites that provide the intersection of these two service tags
+
+  class RedisTagStore
+
+    # Initialize tag store with given redis handle
+    def initialize(redis)
+      @redis = redis
+    end
+
+    # Store services and tags for given agent
+    def store(nanite, services, tags)
+      services = nil if services.compact.empty?
+      tags = nil if tags.compact.empty?
+      log_redis_error do
+        if services
+          obsolete_services = @redis.set_members("s-#{nanite}") - services
+          update_elems(nanite, services, obsolete_services, "s-#{nanite}", 'naniteservices')
+        end
+        if tags
+          obsolete_tags = @redis.set_members("tg-#{nanite}") - tags
+          update_elems(nanite, tags, obsolete_tags, "tg-#{nanite}", 'nanitestags')
+        end
+      end
+    end
+
+    # Update tags for given agent
+    def update(nanite, new_tags, obsolete_tags)
+      update_elems(nanite, new_tags, obsolete_tags, "tg-#{nanite}", 'nanitestags')
+    end
+
+    # Delete services and tags for given agent
+    def delete(nanite)
+      delete_elems(nanite, "s-#{nanite}", 'naniteservices')
+      delete_elems(nanite, "tg-#{nanite}", 'nanitestags')
+    end
+
+    # Services implemented by given agent
+    def services(nanite)
+      @redis.set_members("s-#{nanite}")
+    end
+
+    # Tags exposed by given agent
+    def tags(nanite)
+      @redis.set_members("tg-#{nanite}")
+    end
+
+    # Retrieve all agents services
+    def all_services
+      log_redis_error do
+        @redis.set_members('naniteservices')
+      end
+    end
+
+    # Retrieve all agents tags
+    def all_tags
+      log_redis_error do
+        @redis.set_members('nanitetags')
+      end
+    end
+
+    # Retrieve nanites implementing given service and exposing given tags
+    def nanites_for(from, service, tags)
+      keys = tags && tags.dup || []
+      keys << service
+      log_redis_error do
+        @redis.set_intersect(keys.compact)
+      end
+    end
+
+    private
+
+    # Update values stored for given agent
+    # Also store reverse lookup information using both a unique and
+    # a global key (so it's possible to retrieve that agent value or
+    # all related values)
+    def update_elems(nanite, new_tags, obsolete_tags, elem_key, global_key)
+      new_tags = nil if new_tags.compact.empty?
+      obsolete_tags = nil if obsolete_tags.compact.empty?
+      log_redis_error do
+        obsolete_tags.each do |val|
+          @redis.set_delete(val, nanite)
+          @redis.set_delete(elem_key, val)
+          @redis.set_delete(global_key, val)
+        end if obsolete_tags
+        new_tags.each do |val|
+          @redis.set_add(val, nanite)
+          @redis.set_add(elem_key, val)
+          @redis.set_add(global_key, val)
+        end if new_tags
+      end
+    end
+
+    # Delete all values for given nanite agent
+    # Also delete reverse lookup information
+    def delete_elems(nanite, elem_key, global_key)
+      log_redis_error do
+        (@redis.set_members(elem_key)||[]).each do |val|
+          @redis.set_delete(val, nanite)
+          if @redis.set_count(val) == 0
+            @redis.delete(val)
+            @redis.set_delete(global_key, val)
+          end
+        end
+        @redis.delete(elem_key)
+      end
+    end
+
+    # Helper method, catch and log errors
+    def log_redis_error(&blk)
+      blk.call
+    rescue Exception => e
+      Nanite::Log.warn("redis error in method: #{caller[0]}")
+      raise e
+    end
+
+  end
+end

data/lib/nanite/state.rb

@@ -1,164 +1,135 @@
 require 'redis'
+require 'redis_tag_store'
 
 module Nanite
   class State
     include Enumerable
     
-    # this class encapsulates the state of a nanite system using redis as the 
-    # data store. here is the schema, for each agent we store a number of items,
-    # for a nanite with the identity:  nanite-foobar we store the following things:
+    # This class encapsulates the state of a nanite system using redis as the 
+    # data store and a provided tag store. For a nanite with the identity
+    # 'nanite-foobar' we store the following:
     #
-    # nanite-foobar: 0.72 # load average or 'status'
-    # s-nanite-foobar: { /foo/bar, /foo/nik } # a SET of the provided services
-    # tg-nanite-foobar: { foo-42, customer-12 } # a SET of the tags for this agent
+    # nanite-foobar: 0.72        # load average or 'status'
     # t-nanite-foobar: 123456789 # unix timestamp of the last state update
     #
-    # also we do an inverted index for quick lookup of agents providing a certain
-    # service, so for each service the agent provides, we add the nanite to a SET 
-    # of all the nanites that provide said service:
+    # The tag store is used to store the associated services and tags.
     #
-    # foo/bar: { nanite-foobar, nanite-nickelbag, nanite-another } # redis SET
+    # A tag store should provide the following methods:
+    #  - initialize(redis): Initialize tag store, may use provided redis handle
+    #  - services(nanite): Retrieve services implemented by given agent
+    #  - tags(nanite): Retrieve tags implemented by given agent
+    #  - all_services: Retrieve all services implemented by all agents
+    #  - all_tags: Retrieve all tags exposed by all agents
+    #  - store(nanite, services, tags): Store agent's services and tags
+    #  - update(name, new_tags,obsolete_tags): Update agent's tags
+    #  - delete(nanite): Delete all entries associated with given agent
+    #  - nanites_for(service, tags): Retrieve agents implementing given service
+    #                                and exposing given tags
     #
-    # we do that same thing for tags:
-    # some-tag: { nanite-foobar, nanite-nickelbag, nanite-another } # redis SET
-    #
-    # This way we can do a lookup of what nanites provide a set of services and tags based
-    # on redis SET intersection:
-    #
-    # nanites_for('/gems/list', 'some-tag')
-    # => returns a nested array of nanites and their state that provide the intersection
-    # of these two service tags
-    
-    def initialize(redis)
-      Nanite::Log.info("[setup] initializing redis state: #{redis}")
-      host, port = redis.split(':')
+    # The default implementation for the tag store reuses Redis.
+  
+    def initialize(redis, tag_store=nil)
+      host, port, tag_store_type = redis.split(':')
       host ||= '127.0.0.1'
       port ||= '6379'
-      @redis = Redis.new :host => host, :port => port
-    end
-    
-    def log_redis_error(meth,&blk)
-      blk.call
-    rescue Exception => e
-      Nanite::Log.info("redis error in method: #{meth}")
-      raise e
+      tag_store||= 'Nanite::RedisTagStore'
+      @redis = Redis.new(:host => host, :port => port)
+      @tag_store = tag_store.to_const.new(@redis)
+      Nanite::Log.info("[setup] Initializing redis state using host '#{host}', port '#{port}' and tag store #{tag_store}")
     end
-    
+
+    # Retrieve given agent services, tags, status and timestamp
     def [](nanite)
-      log_redis_error("[]") do
-        status = @redis[nanite]
+      log_redis_error do
+        status    = @redis[nanite]
         timestamp = @redis["t-#{nanite}"]
-        services = @redis.set_members("s-#{nanite}")
-        tags = @redis.set_members("tg-#{nanite}")
+        services  = @tag_store.services(nanite)
+        tags      = @tag_store.tags(nanite)
         return nil unless status && timestamp && services
         {:services => services, :status => status, :timestamp => timestamp.to_i, :tags => tags}
       end
     end
-    
-    def []=(nanite, hsh)
-      log_redis_error("[]=") do
-        update_state(nanite, hsh[:status], hsh[:services], hsh[:tags])
-      end
+
+    # Set given attributes for given agent
+    # Attributes may include services, tags and status
+    def []=(nanite, attributes)
+      @tag_store.store(nanite, attributes[:services], attributes[:tags])
+      update_status(nanite, attributes[:status])
     end
-    
+
+    # Delete all information related to given agent
     def delete(nanite)
-      log_redis_error("delete") do
-        (@redis.set_members("s-#{nanite}")||[]).each do |srv|
-          @redis.set_delete(srv, nanite)
-          if @redis.set_count(srv) == 0
-            @redis.delete(srv)
-            @redis.set_delete("naniteservices", srv)
-          end
-        end
-        (@redis.set_members("tg-#{nanite}")||[]).each do |tag|
-          @redis.set_delete(tag, nanite)
-          if @redis.set_count(tag) == 0
-            @redis.delete(tag)
-            @redis.set_delete("nanitetags", tag)
-          end
-        end
-        @redis.delete nanite
-        @redis.delete "s-#{nanite}"
-        @redis.delete "t-#{nanite}"
-        @redis.delete "tg-#{nanite}"
+      @tag_store.delete(nanite)
+      log_redis_error do
+        @redis.delete(nanite)
+        @redis.delete("t-#{nanite}")
       end
     end
-    
+
+    # Return all services exposed by all agents
     def all_services
-      log_redis_error("all_services") do
-        @redis.set_members("naniteservices")
-      end
+      @tag_store.all_services
     end
 
+    # Return all tags exposed by all agents
     def all_tags
-      log_redis_error("all_tags") do
-        @redis.set_members("nanitetags")
-      end
+      @tag_store.all_tags
     end
-    
-    def update_state(name, status, services, tags)
-      old_services = @redis.set_members("s-#{name}")
-      if old_services
-        (old_services - services).each do |s|
-          @redis.set_delete(s, name)
-          @redis.set_delete("naniteservices", s)
-        end
-      end
-      old_tags = @redis.set_members("tg-#{name}")
-      if old_tags
-        (old_tags - tags).each do |t|
-          @redis.set_delete(t, name)
-          @redis.set_delete("nanitetags", t)
-        end
-      end
-      @redis.delete("s-#{name}")
-      services.each do |srv|
-        @redis.set_add(srv, name)
-        @redis.set_add("s-#{name}", srv)
-        @redis.set_add("naniteservices", srv)
-      end
-      @redis.delete("tg-#{name}")
-      tags.each do |tag|
-        next if tag.nil?
-        @redis.set_add(tag, name)
-        @redis.set_add("tg-#{name}", tag)
-        @redis.set_add("nanitetags", tag)
+
+    # Update status and timestamp for given agent
+    def update_status(name, status)
+      log_redis_error do
+        @redis[name] = status
+        @redis["t-#{name}"] = Time.now.utc.to_i
       end
-      @redis[name] = status
-      @redis["t-#{name}"] = Time.now.to_i
     end
-    
+
+    # Update tags for given agent
+    def update_tags(name, new_tags, obsolete_tags)
+      @tag_store.update(name, new_tags, obsolete_tags)
+    end
+
+    # Return all registered agents
     def list_nanites
-      log_redis_error("list_nanites") do
+      log_redis_error do
         @redis.keys("nanite-*")
       end
     end
-    
+
+    # Number of registered agents
     def size
       list_nanites.size
     end
-    
-    def clear_state
-      log_redis_error("clear_state") do
-        @redis.keys("*").each {|k| @redis.delete k}
-      end
-    end
-    
+
+    # Iterate through all agents, yielding services, tags
+    # status and timestamp keyed by agent name
     def each
       list_nanites.each do |nan|
         yield nan, self[nan]
       end
     end
-    
-    def nanites_for(service, *tags)
-      keys = tags.dup << service
-      log_redis_error("nanites_for") do
-        res = []
-        (@redis.set_intersect(keys)||[]).each do |nan|
-          res << [nan, self[nan]]
+
+    # Return agents that implement given service and expose
+    # all given tags
+    def nanites_for(from, service, tags)
+      res = []
+      @tag_store.nanites_for(from, service, tags).each do |nanite_id|
+        if nanite = self[nanite_id]
+          res << [nanite_id, nanite]
         end
-        res
       end
+      res
+    end
+
+    private
+
+    # Helper method, catch and log errors
+    def log_redis_error(&blk)
+      blk.call
+    rescue Exception => e
+      Nanite::Log.warn("redis error in method: #{caller[0]}")
+      raise e
     end
+    
   end
 end  

data/lib/nanite/util.rb

@@ -27,6 +27,26 @@ class String
   def to_const_path
     snake_case.gsub(/::/, "/")
   end
+
+  # Convert constant name to constant
+  #
+  # "FooBar::Baz".to_const => FooBar::Baz
+  #
+  # @return [Constant] Constant corresponding to given name or nil if no
+  #   constant with that name exists
+  #
+  # @api public
+  def to_const
+    names = split('::')
+    names.shift if names.empty? || names.first.empty?
+
+    constant = Object
+    names.each do |name|
+      # modified to return nil instead of raising an const_missing error
+      constant = constant && constant.const_defined?(name) ? constant.const_get(name) : nil
+    end
+    constant
+  end
 end
 
 class Object
@@ -55,4 +75,4 @@ class Object
       end
     end
   end
-end
+end

data/lib/nanite.rb

@@ -20,7 +20,7 @@ require 'nanite/mapper'
 require 'nanite/actor'
 require 'nanite/actor_registry'
 require 'nanite/streaming'
-require 'nanite/dispatcher'
+require 'nanite/nanite_dispatcher'
 require 'nanite/agent'
 require 'nanite/cluster'
 require 'nanite/reaper'
@@ -39,7 +39,7 @@ require 'nanite/security/static_certificate_store'
 require 'nanite/serializer'
 
 module Nanite
-  VERSION = '0.4.1.4' unless defined?(Nanite::VERSION)
+  VERSION = '0.4.1.10' unless defined?(Nanite::VERSION)
 
   class MapperNotRunning < StandardError; end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: rightscale-nanite
 version: !ruby/object:Gem::Version 
-  version: 0.4.1.4
+  version: 0.4.1.10
 platform: ruby
 authors: 
 - Ezra Zygmuntowicz
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-05-16 00:00:00 -07:00
+date: 2009-11-09 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -39,74 +39,50 @@ files:
 - README.rdoc
 - Rakefile
 - TODO
-- lib/nanite.rb
 - lib/nanite
-- lib/nanite/streaming.rb
-- lib/nanite/serializer.rb
-- lib/nanite/pid_file.rb
-- lib/nanite/mapper.rb
+- lib/nanite/actor.rb
+- lib/nanite/actor_registry.rb
+- lib/nanite/admin.rb
+- lib/nanite/agent.rb
+- lib/nanite/amqp.rb
+- lib/nanite/cluster.rb
+- lib/nanite/config.rb
+- lib/nanite/console.rb
 - lib/nanite/daemonize.rb
+- lib/nanite/identity.rb
+- lib/nanite/job.rb
+- lib/nanite/local_state.rb
+- lib/nanite/log
+- lib/nanite/log/formatter.rb
+- lib/nanite/log.rb
+- lib/nanite/mapper.rb
 - lib/nanite/mapper_proxy.rb
+- lib/nanite/nanite_dispatcher.rb
+- lib/nanite/packets.rb
+- lib/nanite/pid_file.rb
+- lib/nanite/reaper.rb
+- lib/nanite/redis_tag_store.rb
 - lib/nanite/security
-- lib/nanite/security/distinguished_name.rb
-- lib/nanite/security/secure_serializer.rb
-- lib/nanite/security/certificate_cache.rb
-- lib/nanite/security/signature.rb
+- lib/nanite/security/cached_certificate_store_proxy.rb
 - lib/nanite/security/certificate.rb
+- lib/nanite/security/certificate_cache.rb
+- lib/nanite/security/distinguished_name.rb
 - lib/nanite/security/encrypted_document.rb
 - lib/nanite/security/rsa_key_pair.rb
+- lib/nanite/security/secure_serializer.rb
+- lib/nanite/security/signature.rb
 - lib/nanite/security/static_certificate_store.rb
-- lib/nanite/security/cached_certificate_store_proxy.rb
-- lib/nanite/config.rb
-- lib/nanite/util.rb
-- lib/nanite/log
-- lib/nanite/log/formatter.rb
-- lib/nanite/state.rb
-- lib/nanite/cluster.rb
-- lib/nanite/dispatcher.rb
 - lib/nanite/security_provider.rb
-- lib/nanite/packets.rb
-- lib/nanite/actor.rb
-- lib/nanite/console.rb
-- lib/nanite/admin.rb
-- lib/nanite/amqp.rb
-- lib/nanite/agent.rb
-- lib/nanite/local_state.rb
-- lib/nanite/identity.rb
-- lib/nanite/actor_registry.rb
-- lib/nanite/log.rb
-- lib/nanite/reaper.rb
-- lib/nanite/job.rb
-- bin/nanite-agent
+- lib/nanite/serializer.rb
+- lib/nanite/state.rb
+- lib/nanite/streaming.rb
+- lib/nanite/util.rb
+- lib/nanite.rb
 - bin/nanite-admin
+- bin/nanite-agent
 - bin/nanite-mapper
-- spec/util_spec.rb
-- spec/encrypted_document_spec.rb
-- spec/agent_spec.rb
-- spec/certificate_cache_spec.rb
-- spec/cached_certificate_store_proxy_spec.rb
-- spec/dispatcher_spec.rb
-- spec/log_spec.rb
-- spec/rsa_key_pair_spec.rb
-- spec/cluster_spec.rb
-- spec/spec_helper.rb
-- spec/mapper_spec.rb
-- spec/actor_registry_spec.rb
-- spec/actor_spec.rb
-- spec/packet_spec.rb
-- spec/local_state_spec.rb
-- spec/static_certificate_store_spec.rb
-- spec/job_spec.rb
-- spec/signature_spec.rb
-- spec/mapper_proxy_spec.rb
-- spec/secure_serializer_spec.rb
-- spec/serializer_spec.rb
-- spec/certificate_spec.rb
-- spec/nanite_spec.rb
-- spec/distinguished_name_spec.rb
 has_rdoc: true
 homepage: http://github.com/ezmobius/nanite
-licenses: 
 post_install_message: 
 rdoc_options: []
 
@@ -127,7 +103,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.5
+rubygems_version: 1.3.1
 signing_key: 
 specification_version: 2
 summary: self assembling fabric of ruby daemons

data/spec/actor_registry_spec.rb

@@ -1,60 +0,0 @@
-require File.join(File.dirname(__FILE__), 'spec_helper')
-
-describe Nanite::ActorRegistry do
-  
-  class ::WebDocumentImporter
-    include Nanite::Actor
-    expose :import, :cancel
-
-    def import
-      1
-    end
-    def cancel
-      0
-    end
-  end
-
-  module ::Actors
-    class ComedyActor
-      include Nanite::Actor
-      expose :fun_tricks
-      def fun_tricks
-        :rabbit_in_the_hat
-      end
-    end
-  end
-
-  before(:each) do
-    Nanite::Log.stub!(:info)
-    @registry = Nanite::ActorRegistry.new
-  end
-
-  it "should know about all services" do
-    @registry.register(WebDocumentImporter.new, nil)
-    @registry.register(Actors::ComedyActor.new, nil)
-    @registry.services.sort.should == ["/actors/comedy_actor/fun_tricks", "/web_document_importer/cancel", "/web_document_importer/import"]
-  end
-
-  it "should not register anything except Nanite::Actor" do
-    lambda { @registry.register(String.new, nil) }.should raise_error(ArgumentError)
-  end
-
-  it "should register an actor" do
-    importer = WebDocumentImporter.new
-    @registry.register(importer, nil)
-    @registry.actors['web_document_importer'].should == importer
-  end
-
-  it "should log info message that actor was registered" do
-    importer = WebDocumentImporter.new
-    Nanite::Log.should_receive(:info).with("[actor] #{importer.class.to_s}")
-    @registry.register(importer, nil)
-  end
-
-  it "should handle actors registered with a custom prefix" do
-    importer = WebDocumentImporter.new
-    @registry.register(importer, 'monkey')
-    @registry.actor_for('monkey').should == importer
-  end
-  
-end # Nanite::ActorRegistry