br-nanite 0.3.0

data/lib/nanite/reaper.rb

@@ -0,0 +1,30 @@
+module Nanite
+  class Reaper
+
+    def initialize(frequency=2)
+      @timeouts = {}
+      EM.add_periodic_timer(frequency) { EM.next_tick { reap } }
+    end
+
+    def timeout(token, seconds, &blk)
+      @timeouts[token] = {:timestamp => Time.now + seconds, :seconds => seconds, :callback => blk}
+    end
+
+    def reset(token)
+      @timeouts[token][:timestamp] = Time.now + @timeouts[token][:seconds]
+    end
+
+    private
+
+    def reap
+      @timeouts.reject! do |token, data|
+        if Time.now > data[:timestamp]
+          data[:callback].call
+          true
+        else
+          false
+        end
+      end
+    end
+  end
+end

data/lib/nanite/serializer.rb

@@ -0,0 +1,40 @@
+module Nanite
+  class Serializer
+
+    class SerializationError < StandardError
+      attr_accessor :action, :packet
+      def initialize(action, packet)
+        @action, @packet = action, packet
+        super("Could not #{action} #{packet.inspect} using #{SERIALIZERS.keys.join(', ')}")
+      end
+    end # SerializationError
+
+    def initialize(preferred_format="marshal")
+      preferred_format ||= "marshal"
+      preferred_serializer = SERIALIZERS[preferred_format.to_sym]
+      @serializers = SERIALIZERS.values.clone
+      @serializers.unshift(@serializers.delete(preferred_serializer)) if preferred_serializer
+    end
+
+    def dump(packet)
+      cascade_serializers(:dump, packet)
+    end
+
+    def load(packet)
+      cascade_serializers(:load, packet)
+    end
+
+    private
+
+    SERIALIZERS = {:json => JSON, :marshal => Marshal, :yaml => YAML}.freeze
+
+    def cascade_serializers(action, packet)
+      @serializers.map do |serializer|
+        o = serializer.send(action, packet) rescue nil
+        return o if o
+      end
+      raise SerializationError.new(action, packet)
+    end
+
+  end # Serializer
+end # Nanite

data/lib/nanite/streaming.rb

@@ -0,0 +1,125 @@
+module Nanite
+  # Nanite actors can transfer files to each other.
+  #
+  # ==== Options
+  #
+  # filename    : you guessed it, name of the file!
+  # domain      : part of the routing key used to locate receiver(s)
+  # destination : is a name of the file as it gonna be stored at the destination
+  # meta        :
+  #
+  # File streaming is done in chunks. When file streaming starts,
+  # Nanite::FileStart packet is sent, followed by one or more (usually more ;))
+  # Nanite::FileChunk packets each 16384 (16K) in size. Once file streaming is done,
+  # Nanite::FileEnd packet is sent.
+  #
+  # 16K is a packet size because on certain UNIX-like operating systems, you cannot read/write
+  # more than that in one operation via socket.
+  #
+  # ==== Domains
+  #
+  # Streaming happens using a topic exchange called 'file broadcast', with keys
+  # formatted as "nanite.filepeer.DOMAIN". Domain variable in the key lets senders and
+  # receivers find each other in the cluster. Default domain is 'global'.
+  #
+  # Domains also serve as a way to register a callback Nanite agent executes once file
+  # streaming is completed. If a callback with name of domain is registered, it is called.
+  #
+  # Callbacks are registered by passing a block to subscribe_to_files method.
+  module FileStreaming
+    def broadcast_file(filename, options = {})
+      if File.exist?(filename)
+        File.open(filename, 'rb') do |file|
+           broadcast_data(filename, file, options)
+        end
+      else
+        return "file not found"
+      end
+    end
+
+    def broadcast_data(filename, io, options = {})
+      domain   = options[:domain] || 'global'
+      filename = File.basename(filename)
+      dest     = options[:destination] || filename
+      sent = 0
+
+        begin
+          file_push = Nanite::FileStart.new(filename, dest)
+          amq.topic('file broadcast').publish(dump_packet(file_push), :key => "nanite.filepeer.#{domain}")
+          res = Nanite::FileChunk.new(file_push.token)
+          while chunk = io.read(16384)
+            res.chunk = chunk
+            amq.topic('file broadcast').publish(dump_packet(res), :key => "nanite.filepeer.#{domain}")
+            sent += chunk.length
+          end
+          fend = Nanite::FileEnd.new(file_push.token, options[:meta])
+          amq.topic('file broadcast').publish(dump_packet(fend), :key => "nanite.filepeer.#{domain}")
+          ""
+        ensure
+          io.close
+        end
+
+        sent
+    end
+
+    # FileState represents a file download in progress.
+    # It incapsulates the following information:
+    #
+    # * unique operation token
+    # * domain (namespace for file streaming operations)
+    # * file IO chunks are written to on receiver's side
+    class FileState
+
+      def initialize(token, dest, domain, write, blk)
+        @token = token
+        @cb = blk
+        @domain = domain
+        @write = write
+        
+        if write
+          @filename = File.join(Nanite.agent.file_root, dest)
+          @dest = File.open(@filename, 'wb')
+        else
+          @dest = dest
+        end
+
+        @data = ""
+      end
+
+      def handle_packet(packet)
+        case packet
+        when Nanite::FileChunk
+          log.debug "written chunk to #{@dest.inspect}"
+          @data << packet.chunk
+        
+          if @write
+            @dest.write(packet.chunk)
+          end
+        when Nanite::FileEnd
+          log.debug "#{@dest.inspect} receiving is completed"
+          if @write
+            @dest.close
+          end
+
+          @cb.call(@data, @dest, packet.meta)
+        end
+      end
+
+    end
+
+    def subscribe_to_files(domain='global', write=false, &blk)
+      log.info "subscribing to file broadcasts for #{domain}"
+      @files ||= {}
+      amq.queue("files#{domain}").bind(amq.topic('file broadcast'), :key => "nanite.filepeer.#{domain}").subscribe do |packet|
+        case msg = load_packet(packet)
+        when FileStart
+          @files[msg.token] = FileState.new(msg.token, msg.dest, domain, write, blk)
+        when FileChunk, FileEnd
+          if file = @files[msg.token]
+            file.handle_packet(msg)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/nanite/util.rb

@@ -0,0 +1,51 @@
+class String
+  ##
+  # Convert to snake case.
+  #
+  #   "FooBar".snake_case           #=> "foo_bar"
+  #   "HeadlineCNNNews".snake_case  #=> "headline_cnn_news"
+  #   "CNN".snake_case              #=> "cnn"
+  #
+  # @return [String] Receiver converted to snake case.
+  #
+  # @api public
+  def snake_case
+    return self.downcase if self =~ /^[A-Z]+$/
+    self.gsub(/([A-Z]+)(?=[A-Z][a-z]?)|\B[A-Z]/, '_\&') =~ /_*(.*)/
+    return $+.downcase
+  end
+
+  ##
+  # Convert a constant name to a path, assuming a conventional structure.
+  #
+  #   "FooBar::Baz".to_const_path # => "foo_bar/baz"
+  #
+  # @return [String] Path to the file containing the constant named by receiver
+  #   (constantized string), assuming a conventional structure.
+  #
+  # @api public
+  def to_const_path
+    snake_case.gsub(/::/, "/")
+  end
+end
+
+class Object
+  module InstanceExecHelper; end
+  include InstanceExecHelper
+  def instance_exec(*args, &block)
+    begin
+      old_critical, Thread.critical = Thread.critical, true
+      n = 0
+      n += 1 while respond_to?(mname="__instance_exec#{n}")
+      InstanceExecHelper.module_eval{ define_method(mname, &block) }
+    ensure
+      Thread.critical = old_critical
+    end
+    begin
+      ret = send(mname, *args)
+    ensure
+      InstanceExecHelper.module_eval{ remove_method(mname) } rescue nil
+    end
+    ret
+  end
+end

metadata

@@ -0,0 +1,104 @@
+--- !ruby/object:Gem::Specification 
+name: br-nanite
+version: !ruby/object:Gem::Version 
+  version: 0.3.0
+platform: ruby
+authors: 
+- Ezra Zygmuntowicz
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-12-17 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: extlib
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: amqp
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.6.0
+    version: 
+description: self assembling fabric of ruby daemons
+email: ezra@engineyard.com
+executables: 
+- nanite-agent
+- nanite-mapper
+- nanite-admin
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+- LICENSE
+- TODO
+files: 
+- LICENSE
+- README.rdoc
+- Rakefile
+- TODO
+- lib/nanite/actor.rb
+- lib/nanite/actor_registry.rb
+- lib/nanite/admin.rb
+- lib/nanite/agent.rb
+- lib/nanite/amqp.rb
+- lib/nanite/cluster.rb
+- lib/nanite/config.rb
+- lib/nanite/console.rb
+- lib/nanite/daemonize.rb
+- lib/nanite/dispatcher.rb
+- lib/nanite/identity.rb
+- lib/nanite/job.rb
+- lib/nanite/log.rb
+- lib/nanite/mapper.rb
+- lib/nanite/packets.rb
+- lib/nanite/reaper.rb
+- lib/nanite/serializer.rb
+- lib/nanite/streaming.rb
+- lib/nanite/util.rb
+- lib/nanite.rb
+- bin/nanite-admin
+- bin/nanite-agent
+- bin/nanite-mapper
+has_rdoc: true
+homepage: http://github.com/ezmobius/nanite
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: self assembling fabric of ruby daemons
+test_files: []
+