rightscale-nanite 0.4.1 → 0.4.1.1

data/lib/nanite/state.rb

@@ -0,0 +1,164 @@
+require 'redis'
+
+module Nanite
+  class State
+    include Enumerable
+    
+    # this class encapsulates the state of a nanite system using redis as the 
+    # data store. here is the schema, for each agent we store a number of items,
+    # for a nanite with the identity:  nanite-foobar we store the following things:
+    #
+    # nanite-foobar: 0.72 # load average or 'status'
+    # s-nanite-foobar: { /foo/bar, /foo/nik } # a SET of the provided services
+    # tg-nanite-foobar: { foo-42, customer-12 } # a SET of the tags for this agent
+    # t-nanite-foobar: 123456789 # unix timestamp of the last state update
+    #
+    # also we do an inverted index for quick lookup of agents providing a certain
+    # service, so for each service the agent provides, we add the nanite to a SET 
+    # of all the nanites that provide said service:
+    #
+    # /gems/list: { nanite-foobar, nanite-nickelbag, nanite-another } # redis SET
+    #
+    # we do that same thing for tags:
+    # some-tag: { nanite-foobar, nanite-nickelbag, nanite-another } # redis SET
+    #
+    # This way we can do a lookup of what nanites provide a set of services and tags based
+    # on redis SET intersection:
+    #
+    # nanites_for('/gems/list', 'some-tag')
+    # => returns a nested array of nanites and their state that provide the intersection
+    # of these two service tags
+    
+    def initialize(redis)
+      Nanite::Log.info("initializing redis state: #{redis}")
+      host, port = redis.split(':')
+      host ||= '127.0.0.1'
+      port ||= '6379'
+      @redis = Redis.new :host => host, :port => port
+    end
+    
+    def log_redis_error(meth,&blk)
+      blk.call
+    rescue RedisError => e
+      Nanite::Log.info("redis error in method: #{meth}")
+      raise e
+    end
+    
+    def [](nanite)
+      log_redis_error("[]") do
+        status = @redis[nanite]
+        timestamp = @redis["t-#{nanite}"]
+        services = @redis.set_members("s-#{nanite}")
+        tags = @redis.set_members("tg-#{nanite}")
+        return nil unless status && timestamp && services
+        {:services => services, :status => status, :timestamp => timestamp.to_i, :tags => tags}
+      end
+    end
+    
+    def []=(nanite, hsh)
+      log_redis_error("[]=") do
+        update_state(nanite, hsh[:status], hsh[:services], hsh[:tags])
+      end
+    end
+    
+    def delete(nanite)
+      log_redis_error("delete") do
+        (@redis.set_members("s-#{nanite}")||[]).each do |srv|
+          @redis.set_delete(srv, nanite)
+          if @redis.set_count(srv) == 0
+            @redis.delete(srv)
+            @redis.set_delete("naniteservices", srv)
+          end
+        end
+        (@redis.set_members("tg-#{nanite}")||[]).each do |tag|
+          @redis.set_delete(tag, nanite)
+          if @redis.set_count(tag) == 0
+            @redis.delete(tag)
+            @redis.set_delete("nanitetags", tag)
+          end
+        end
+        @redis.delete nanite
+        @redis.delete "s-#{nanite}"
+        @redis.delete "t-#{nanite}"
+        @redis.delete "tg-#{nanite}"
+      end
+    end
+    
+    def all_services
+      log_redis_error("all_services") do
+        @redis.set_members("naniteservices")
+      end
+    end
+
+    def all_tags
+      log_redis_error("all_tags") do
+        @redis.set_members("nanitetags")
+      end
+    end
+    
+    def update_state(name, status, services, tags)
+      old_services = @redis.set_members("s-#{name}")
+      if old_services
+        (old_services - services).each do |s|
+          @redis.set_delete(s, name)
+          @redis.set_delete("naniteservices", s)
+        end
+      end
+      old_tags = @redis.set_members("tg-#{name}")
+      if old_tags
+        (old_tags - tags).each do |t|
+          @redis.set_delete(t, name)
+          @redis.set_delete("nanitetags", t)
+        end
+      end
+      @redis.delete("s-#{name}")
+      services.each do |srv|
+        @redis.set_add(srv, name)
+        @redis.set_add("s-#{name}", srv)
+        @redis.set_add("naniteservices", srv)
+      end
+      @redis.delete("tg-#{name}")
+      tags.each do |tag|
+        next if tag.nil?
+        @redis.set_add(tag, name)
+        @redis.set_add("tg-#{name}", tag)
+        @redis.set_add("nanitetags", tag)
+      end
+      @redis[name] = status
+      @redis["t-#{name}"] = Time.now.to_i
+    end
+    
+    def list_nanites
+      log_redis_error("list_nanites") do
+        @redis.keys("nanite-*")
+      end
+    end
+    
+    def size
+      list_nanites.size
+    end
+    
+    def clear_state
+      log_redis_error("clear_state") do
+        @redis.keys("*").each {|k| @redis.delete k}
+      end
+    end
+    
+    def each
+      list_nanites.each do |nan|
+        yield nan, self[nan]
+      end
+    end
+    
+    def nanites_for(service, *tags)
+      keys = tags.dup << service
+      log_redis_error("nanites_for") do
+        res = []
+        (@redis.set_intersect(keys)||[]).each do |nan|
+          res << [nan, self[nan]]
+        end
+        res
+      end
+    end
+  end
+end  

data/lib/nanite/streaming.rb

@@ -0,0 +1,125 @@
+module Nanite
+  # Nanite actors can transfer files to each other.
+  #
+  # ==== Options
+  #
+  # filename    : you guessed it, name of the file!
+  # domain      : part of the routing key used to locate receiver(s)
+  # destination : is a name of the file as it gonna be stored at the destination
+  # meta        :
+  #
+  # File streaming is done in chunks. When file streaming starts,
+  # Nanite::FileStart packet is sent, followed by one or more (usually more ;))
+  # Nanite::FileChunk packets each 16384 (16K) in size. Once file streaming is done,
+  # Nanite::FileEnd packet is sent.
+  #
+  # 16K is a packet size because on certain UNIX-like operating systems, you cannot read/write
+  # more than that in one operation via socket.
+  #
+  # ==== Domains
+  #
+  # Streaming happens using a topic exchange called 'file broadcast', with keys
+  # formatted as "nanite.filepeer.DOMAIN". Domain variable in the key lets senders and
+  # receivers find each other in the cluster. Default domain is 'global'.
+  #
+  # Domains also serve as a way to register a callback Nanite agent executes once file
+  # streaming is completed. If a callback with name of domain is registered, it is called.
+  #
+  # Callbacks are registered by passing a block to subscribe_to_files method.
+  module FileStreaming
+    def broadcast_file(filename, options = {})
+      if File.exist?(filename)
+        File.open(filename, 'rb') do |file|
+           broadcast_data(filename, file, options)
+        end
+      else
+        return "file not found"
+      end
+    end
+
+    def broadcast_data(filename, io, options = {})
+      domain   = options[:domain] || 'global'
+      filename = File.basename(filename)
+      dest     = options[:destination] || filename
+      sent = 0
+
+        begin
+          file_push = Nanite::FileStart.new(filename, dest, Identity.generate)
+          amq.topic('file broadcast').publish(serializer.dump(file_push), :key => "nanite.filepeer.#{domain}")
+          res = Nanite::FileChunk.new(file_push.token)
+          while chunk = io.read(16384)
+            res.chunk = chunk
+            amq.topic('file broadcast').publish(serializer.dump(res), :key => "nanite.filepeer.#{domain}")
+            sent += chunk.length
+          end
+          fend = Nanite::FileEnd.new(file_push.token, options[:meta])
+          amq.topic('file broadcast').publish(serializer.dump(fend), :key => "nanite.filepeer.#{domain}")
+          ""
+        ensure
+          io.close
+        end
+
+        sent
+    end
+
+    # FileState represents a file download in progress.
+    # It incapsulates the following information:
+    #
+    # * unique operation token
+    # * domain (namespace for file streaming operations)
+    # * file IO chunks are written to on receiver's side
+    class FileState
+
+      def initialize(token, dest, domain, write, blk)
+        @token = token
+        @cb = blk
+        @domain = domain
+        @write = write
+        
+        if write
+          @filename = File.join(Nanite.agent.file_root, dest)
+          @dest = File.open(@filename, 'wb')
+        else
+          @dest = dest
+        end
+
+        @data = ""
+      end
+
+      def handle_packet(packet)
+        case packet
+        when Nanite::FileChunk
+          Nanite::Log.debug "written chunk to #{@dest.inspect}"
+          @data << packet.chunk
+        
+          if @write
+            @dest.write(packet.chunk)
+          end
+        when Nanite::FileEnd
+          Nanite::Log.debug "#{@dest.inspect} receiving is completed"
+          if @write
+            @dest.close
+          end
+
+          @cb.call(@data, @dest, packet.meta)
+        end
+      end
+
+    end
+
+    def subscribe_to_files(domain='global', write=false, &blk)
+      Nanite::Log.info "subscribing to file broadcasts for #{domain}"
+      @files ||= {}
+      amq.queue("files#{domain}").bind(amq.topic('file broadcast'), :key => "nanite.filepeer.#{domain}").subscribe do |packet|
+        case msg = serializer.load(packet)
+        when FileStart
+          @files[msg.token] = FileState.new(msg.token, msg.dest, domain, write, blk)
+        when FileChunk, FileEnd
+          if file = @files[msg.token]
+            file.handle_packet(msg)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/nanite/util.rb

@@ -0,0 +1,51 @@
+class String
+  ##
+  # Convert to snake case.
+  #
+  #   "FooBar".snake_case           #=> "foo_bar"
+  #   "HeadlineCNNNews".snake_case  #=> "headline_cnn_news"
+  #   "CNN".snake_case              #=> "cnn"
+  #
+  # @return [String] Receiver converted to snake case.
+  #
+  # @api public
+  def snake_case
+    return self.downcase if self =~ /^[A-Z]+$/
+    self.gsub(/([A-Z]+)(?=[A-Z][a-z]?)|\B[A-Z]/, '_\&') =~ /_*(.*)/
+    return $+.downcase
+  end
+
+  ##
+  # Convert a constant name to a path, assuming a conventional structure.
+  #
+  #   "FooBar::Baz".to_const_path # => "foo_bar/baz"
+  #
+  # @return [String] Path to the file containing the constant named by receiver
+  #   (constantized string), assuming a conventional structure.
+  #
+  # @api public
+  def to_const_path
+    snake_case.gsub(/::/, "/")
+  end
+end
+
+class Object
+  module InstanceExecHelper; end
+  include InstanceExecHelper
+  def instance_exec(*args, &block)
+    begin
+      old_critical, Thread.critical = Thread.critical, true
+      n = 0
+      n += 1 while respond_to?(mname="__instance_exec#{n}")
+      InstanceExecHelper.module_eval{ define_method(mname, &block) }
+    ensure
+      Thread.critical = old_critical
+    end
+    begin
+      ret = send(mname, *args)
+    ensure
+      InstanceExecHelper.module_eval{ remove_method(mname) } rescue nil
+    end
+    ret
+  end
+end

data/spec/actor_registry_spec.rb

@@ -0,0 +1,62 @@
+require File.join(File.dirname(__FILE__), 'spec_helper')
+
+describe Nanite::ActorRegistry do
+  
+  before(:all) do
+    class WebDocumentImporter
+      include Nanite::Actor
+      expose :import, :cancel
+
+      def import
+        1
+      end
+      def cancel
+        0
+      end
+    end
+
+    module Actors
+      class ComedyActor
+        include Nanite::Actor
+        expose :fun_tricks
+        def fun_tricks
+          :rabbit_in_the_hat
+        end
+      end
+    end
+  end
+  
+  before(:each) do
+    Nanite::Log.stub!(:info)
+    @registry = Nanite::ActorRegistry.new
+  end
+
+  it "should know about all services" do
+    @registry.register(WebDocumentImporter.new, nil)
+    @registry.register(Actors::ComedyActor.new, nil)
+    @registry.services.sort.should == ["/actors/comedy_actor/fun_tricks", "/web_document_importer/cancel", "/web_document_importer/import"]
+  end
+
+  it "should not register anything except Nanite::Actor" do
+    lambda { @registry.register(String.new, nil) }.should raise_error(ArgumentError)
+  end
+
+  it "should register an actor" do
+    importer = WebDocumentImporter.new
+    @registry.register(importer, nil)
+    @registry.actors['web_document_importer'].should == importer
+  end
+
+  it "should log info message that actor was registered" do
+    importer = WebDocumentImporter.new
+    Nanite::Log.should_receive(:info).with("Registering #{importer.inspect} with prefix nil")
+    @registry.register(importer, nil)
+  end
+
+  it "should handle actors registered with a custom prefix" do
+    importer = WebDocumentImporter.new
+    @registry.register(importer, 'monkey')
+    @registry.actors['monkey'].should == importer
+  end
+  
+end # Nanite::ActorRegistry

data/spec/actor_spec.rb

@@ -0,0 +1,59 @@
+require File.join(File.dirname(__FILE__), 'spec_helper')
+
+class WebDocumentImporter
+  include Nanite::Actor
+  expose :import, :cancel
+
+  def import
+    1
+  end
+  def cancel
+    0
+  end
+  def continue
+    1
+  end
+end
+
+module Actors
+  class ComedyActor
+    include Nanite::Actor
+    expose :fun_tricks
+    def fun_tricks
+      :rabbit_in_the_hat
+    end
+  end
+end
+
+describe Nanite::Actor do
+  
+  describe ".expose" do
+    it "should single expose method only once" do
+      3.times { WebDocumentImporter.expose(:continue) }
+      WebDocumentImporter.provides_for("webfiles").should == ["/webfiles/import", "/webfiles/cancel", "/webfiles/continue"]
+    end
+  end
+  
+  describe ".default_prefix" do
+    it "is calculated as default prefix as const path of class name" do
+      Actors::ComedyActor.default_prefix.should == "actors/comedy_actor"
+      WebDocumentImporter.default_prefix.should == "web_document_importer"
+    end
+  end
+
+  describe ".provides_for(prefix)" do
+    before :each do
+      @provides = Actors::ComedyActor.provides_for("money")
+    end
+    it "returns an array" do
+      @provides.should be_kind_of(Array)
+    end
+
+    it "maps exposed service methods to prefix" do
+      @provides.should == ["/money/fun_tricks"]
+      wdi_provides = WebDocumentImporter.provides_for("webfiles")
+      wdi_provides.should include("/webfiles/import")
+      wdi_provides.should include("/webfiles/cancel")
+    end
+  end
+end