ruby_skynet 0.1.0

data/lib/ruby_skynet/client.rb

@@ -0,0 +1,293 @@
+require 'bson'
+require 'sync_attr'
+require 'multi_json'
+
+#
+# RubySkynet Client
+#
+# Supports
+#   RPC calls to Skynet
+#   Skynet Service autodiscovery
+#
+module RubySkynet
+  class Client
+    include SyncAttr
+
+    # Default doozer configuration
+    # To replace this default, set the config as follows:
+    #    RubySkynet::Client.doozer_config = { .... }
+    sync_attr_accessor :doozer_config do
+      {
+        :server                 => '127.0.0.1:8046',
+        :read_timeout           => 5,
+        :connect_timeout        => 3,
+        :connect_retry_interval => 0.1,
+        :connect_retry_count    => 3
+      }
+    end
+
+    # Lazy initialize Doozer Client
+    sync_cattr_reader :doozer do
+      Doozer::Client.new
+    end
+
+    # Create a client connection, call the supplied block and close the connection on
+    # completion of the block
+    #
+    # Example
+    #
+    #  require 'ruby_skynet'
+    #  SemanticLogger.default_level = :trace
+    #  SemanticLogger.appenders << SemanticLogger::Appender::File(STDOUT)
+    #  RubySkynet::Client.connect('TutorialService') do |tutorial_service|
+    #    p tutorial_service.call(:value => 5)
+    #  end
+    def self.connect(service_name, params={})
+      begin
+        client = self.new(service_name, params)
+        yield(client)
+      ensure
+        client.close if client
+      end
+    end
+
+    # Returns a new RubySkynet Client for the named service
+    #
+    # Parameters:
+    #   :service_name
+    #     Name of the service to look for and connect to on Skynet
+    #
+    #   :doozer_servers [Array of String]
+    #     Array of URL's of doozer servers to connect to with port numbers
+    #     ['server1:2000', 'server2:2000']
+    #
+    #     The second server will only be attempted once the first server
+    #     cannot be connected to or has timed out on connect
+    #     A read failure or timeout will not result in switching to the second
+    #     server, only a connection failure or during an automatic reconnect
+    #
+    #   :read_timeout [Float]
+    #     Time in seconds to timeout on read
+    #     Can be overridden by supplying a timeout in the read call
+    #     Default: 60
+    #
+    #   :connect_timeout [Float]
+    #     Time in seconds to timeout when trying to connect to the server
+    #     Default: Half of the :read_timeout ( 30 seconds )
+    #
+    #   :connect_retry_count [Fixnum]
+    #     Number of times to retry connecting when a connection fails
+    #     Default: 10
+    #
+    #   :connect_retry_interval [Float]
+    #     Number of seconds between connection retry attempts after the first failed attempt
+    #     Default: 0.5
+    def initialize(service_name, params = {})
+      @service_name = service_name
+      @logger = SemanticLogger::Logger.new("#{self.class.name}: #{service_name}")
+
+      # User configurable options
+      params[:read_timeout]           ||= 60
+      params[:connect_timeout]        ||= 30
+      params[:connect_retry_interval] ||= 0.1
+      params[:connect_retry_count]    ||= 5
+
+      # If Server name and port of where Skynet Service is running
+      # is not supplied look for it in Doozer
+      unless params[:server] || params[:servers]
+        params[:server] = self.class.server_for(service_name)
+      end
+
+      # Disable buffering the send since it is a RPC call
+      params[:buffered] = false
+
+      @logger.trace "Socket Connection parameters", params
+
+      # For each new connection perform the Skynet handshake
+      params[:on_connect] = Proc.new do |socket|
+        # Reset user_data on each connection
+        socket.user_data = 0
+
+        # Receive Service Handshake
+        # Registered bool
+        #   Registered indicates the state of this service. If it is false, the connection will
+        #   close immediately and the client should look elsewhere for this service.
+        #
+        # ClientID string
+        #   ClientID is a UUID that is used by the client to identify itself in RPC requests.
+        @logger.debug "Waiting for Service Handshake"
+        service_handshake = self.class.read_bson_document(socket)
+        @logger.trace 'Service Handshake', service_handshake
+
+        # #TODO When a reconnect returns registered == false we need to go back to doozer
+        @registered = service_handshake['registered']
+        @client_id = service_handshake['clientid']
+
+        # Send blank ClientHandshake
+        client_handshake = { 'clientid' => @client_id }
+        @logger.debug "Sending Client Handshake"
+        @logger.trace 'Client Handshake', client_handshake
+        socket.send(BSON.serialize(client_handshake))
+      end
+
+      @socket = ResilientSocket::TCPClient.new(params)
+    end
+
+    # Performs a synchronous call to the Skynet Service
+    #
+    # Parameters:
+    #   method_name [String|Symbol]:
+    #     Name of the method to call at the service
+    #   parameters [Hash]:
+    #     Parameters to pass into the service
+    #
+    # Returns the Hash result returned from the Skynet Service
+    #
+    # Raises RubySkynet::ProtocolError
+    # Raises RubySkynet::SkynetException
+    def call(method_name, parameters)
+      # Skynet requires BSON RPC Calls to have the following format:
+      # https://github.com/bketelsen/skynet/blob/protocol/protocol.md
+      request_id = BSON::ObjectId.new.to_s
+      @logger.tagged request_id do
+        @logger.benchmark_info "Called Skynet Service: #{@service_name}.#{method_name}" do
+
+          # Resilient Send
+          retry_count = 0
+          @socket.retry_on_connection_failure do |socket|
+            # user_data is maintained per session and a different session could
+            # be supplied with each retry
+            socket.user_data ||= 0
+            header = {
+              'servicemethod' => "#{@service_name}.Forward",
+              'seq'           => socket.user_data,
+            }
+            @logger.debug "Sending Header"
+            @logger.trace 'Header', header
+            socket.send(BSON.serialize(header))
+
+            @logger.trace 'Parameters:', parameters
+
+            # The parameters are placed in the request object in BSON serialized
+            # form
+            request = {
+              'clientid'    => @client_id,
+              'in'          => BSON.serialize(parameters).to_s,
+              'method'      => method_name.to_s,
+              'requestinfo' => {
+                'requestid'     => request_id,
+                # Increment retry count to indicate that the request may have been tried previously
+                # TODO: this should be incremented if request is retried,
+                'retrycount'    => retry_count,
+                # TODO: this should be forwarded along in case of services also
+                # being a client and calling additional services. If empty it will
+                # be stuffed with connecting address
+                'originaddress' => ''
+              }
+            }
+
+            @logger.debug "Sending Request"
+            @logger.trace 'Request', request
+            socket.send(BSON.serialize(request))
+          end
+
+          # Once send is successful it could have been processed, so we can no
+          # longer retry now otherwise we could create a duplicate
+          # retry_count += 1
+
+          # Read header first as a separate BSON document
+          @logger.debug "Reading header from server"
+          header = self.class.read_bson_document(@socket)
+          @logger.debug 'Header', header
+
+          # Read the BSON response document
+          @logger.debug "Reading response from server"
+          response = self.class.read_bson_document(@socket)
+          @logger.trace 'Response', response
+
+          # Ensure the sequence number in the response header matches the
+          # sequence number sent in the request
+          if seq_no = header['seq']
+            raise ProtocolError.new("Incorrect Response received, expected seq=#{@socket.user_data}, received: #{header.inspect}") if seq_no != @socket.user_data
+          else
+            raise ProtocolError.new("Invalid Response header, missing 'seq': #{header.inspect}")
+          end
+
+          # Increment Sequence number only on successful response
+          @socket.user_data += 1
+
+          # If an error is returned from Skynet raise a Skynet exception
+          if error = header['error']
+            raise SkynetException.new(error) if error.to_s.length > 0
+          end
+
+          # If an error is returned from the service raise a Service exception
+          if error = response['error']
+            raise ServiceException.new(error) if error.to_s.length > 0
+          end
+
+          # Return Value
+          # The return value is inside the response object, it's a byte array of it's own and needs to be deserialized
+          result = BSON.deserialize(response['out'])
+          @logger.trace 'Return Value', result
+          result
+        end
+      end
+    end
+
+    # Returns a BSON document read from the socket.
+    # Returns nil if the operation times out or if a network
+    #         connection failure occurs
+    def self.read_bson_document(socket)
+      bytebuf = BSON::ByteBuffer.new
+      # Read 4 byte size of following BSON document
+      bytes = ''
+      socket.read(4, bytes)
+
+      # Read BSON document
+      sz = bytes.unpack("V")[0]
+      raise "Invalid Data received from server:#{bytes.inspect}" unless sz
+
+      bytebuf.append!(bytes)
+      bytes = ''
+      sz -= 4
+      until bytes.size >= sz
+        buf = ''
+        socket.read(sz, buf)
+        bytes << buf
+      end
+      bytebuf.append!(bytes)
+      return BSON.deserialize(bytebuf)
+    end
+
+    def close()
+      @socket.close
+    end
+
+    ##############################
+    #protected
+
+    # Returns [Array] of the hostname and port pair [String] that implements a particular service
+    # Performs a doozer lookup to find the servers
+    #
+    #   service_name:
+    #   version: Version of service to locate
+    #            Default: Find latest version
+    def self.registered_implementers(service_name, version = '*', region = 'Development')
+      hosts = []
+      doozer.walk("/services/#{service_name}/#{version}/#{region}/*/*").each do |node|
+        entry = MultiJson.load(node.value)
+        hosts << entry if entry['Registered']
+      end
+      hosts
+    end
+
+    # Randomly returns a server that implements the requested service
+    def self.server_for(service_name, version = '*', region = 'Development')
+      hosts = registered_implementers(service_name, version, region)
+      service = hosts[rand(hosts.size)]['Config']['ServiceAddr']
+      "#{service['IPAddress']}:#{service['Port']}"
+    end
+
+  end
+end

data/lib/ruby_skynet/doozer/client.rb

@@ -0,0 +1,202 @@
+require 'ruby_skynet/doozer/msg.pb'
+require 'semantic_logger'
+require 'resilient_socket'
+require 'ruby_skynet/doozer/exceptions'
+require 'ruby_skynet/doozer/msg.pb'
+
+module RubySkynet
+  module Doozer
+    class Client
+
+      # Create a resilient client connection to a Doozer server
+      def initialize(params={})
+        @logger = SemanticLogger::Logger.new(self.class)
+
+        # User configurable options
+        params[:read_timeout]           ||= 5
+        params[:connect_timeout]        ||= 3
+        params[:connect_retry_interval] ||= 0.1
+        params[:connect_retry_count]    ||= 3
+
+        # Server name and port where Doozer is running
+        # Defaults to 127.0.0.1:8046
+        params[:server] ||= '127.0.0.1:8046' unless params[:servers]
+
+        # Disable buffering the send since it is a RPC call
+        params[:buffered] = false
+
+        @logger.trace "Socket Connection parameters", params
+
+        # For each new connection
+        params[:on_connect] = Proc.new do |socket|
+          # Reset user_data on each connection
+          socket.user_data = 0
+        end
+
+        @socket = ResilientSocket::TCPClient.new(params)
+      end
+
+      # Close this client connection to doozer
+      def close
+        @socket.close if @socket
+      end
+
+      # Returns the current Doozer revision
+      def current_revision
+        invoke(Request.new(:verb => Request::Verb::REV)).rev
+      end
+
+      # Set a value in Doozer
+      #   path:  Path to the value to be set
+      #   value: Value to set
+      #   rev:   Revision at which to set the value
+      #         If not supplied it will replace the latest version on the server
+      #
+      # Returns the new revision of the updated value
+      #
+      # It is recommended to set the revision so that multiple clients do not
+      # attempt to update the value at the same time.
+      # Setting the revision also allows the call to be retried automatically
+      # in the event of a network failure
+      def set(path, value, rev=-1)
+        invoke(Request.new(:path => path, :value => value, :rev => rev, :verb => Request::Verb::SET), false).rev
+      end
+
+      # Sets the current value at the supplied path
+      def []=(path,value)
+        set(path, value)
+      end
+
+      # Return the value at the supplied path and revision
+      def get(path, rev = nil)
+        invoke(Request.new(:path => path, :rev => rev, :verb => Request::Verb::GET))
+      end
+
+      # Returns just the value at the supplied path, not the revision
+      def [](path)
+        get(path).value
+      end
+
+      # Deletes the file at path if rev is greater than or equal to the file's revision.
+      # Returns nil when the file was removed
+      # Raises an exception if an attempt to remove the file and its revision
+      #   is greater than that supplied
+      def delete(path, rev=-1)
+        invoke(Request.new(:path => path, :rev => rev, :verb => Request::Verb::DEL))
+        nil
+      end
+
+      # Returns the directory in the supplied path
+      # Use offset to get the next
+      # returns nil if no further paths are available
+      def directory(path, offset = 0, rev = nil)
+        begin
+          invoke(Request.new(:path => path, :rev => rev, :offset => offset, :verb => Request::Verb::GETDIR))
+        rescue RubySkynet::Doozer::ResponseError => exc
+          raise exc unless exc.message.include?('RANGE')
+          nil
+        end
+      end
+
+      def stat(path, rev = nil)
+        invoke(Request.new(:path => path, :rev => rev, :verb => Request::Verb::STAT))
+      end
+
+      def access(secret)
+        invoke(Request.new(:path => secret, :verb => Request::Verb::ACCESS))
+      end
+
+      # Returns every entry in the supplied path
+      # path can also contain wildcard characters such as '*'
+      # Example:
+      #   hosts = []
+      #   walk('/ctl/node/*/addr', current_revision).each do |node|
+      #     hosts << node.value unless hosts.include? node.value
+      #   end
+      def walk(path, rev = nil, offset = 0)
+        paths = []
+        revision = rev || current_revision
+        # Resume walk on network connection failure
+        @socket.retry_on_connection_failure do
+          while true
+            send(Request.new(:path => path, :rev => revision , :offset => offset, :verb => Request::Verb::WALK))
+            response = read
+            if response.err_code
+              break if response.err_code == Response::Err::RANGE
+            else
+              raise ResponseError.new("#{Response::Err.name_by_value(response.err_code)}: #{response.err_detail}") if response.err_code != 0
+            end
+            paths << response
+            offset += 1
+          end
+        end
+        paths
+      end
+
+      # Returns [Array] of hostname [String] with each string
+      # representing another Doozer server that can be connected to
+      def doozer_hosts
+        hosts = []
+        walk('/ctl/node/*/addr', current_revision).each do |node|
+          hosts << node.value unless hosts.include? node.value
+        end
+      end
+
+      # TODO Implement watching for changes in a separate thread with it's own
+      #      client connection
+      #def watch(path, rev = nil)
+      #  invoke(Request.new(:path => secret, :verb => Request::Verb::WAIT))
+      #end
+
+      #####################
+      # protected
+
+      # Call the Doozer server
+      #
+      # When readonly ==> true the request is always retried on network failure
+      # When readonly ==> false the request is retried on network failure
+      #   _only_ if a rev has been supplied
+      #
+      # When modifier is true
+      def invoke(request, readonly=true)
+        retry_read = readonly || !request.rev.nil?
+        response = nil
+        @socket.retry_on_connection_failure do
+          send(request)
+          response = read if retry_read
+        end
+        # Network error on read must be sent back to caller since we do not
+        # know if the modification was made
+        response = read unless retry_read
+        raise ResponseError.new("#{Response::Err.name_by_value(response.err_code)}: #{response.err_detail}") if response.err_code != 0
+        response
+      end
+
+      # Send the protobuf Request to Doozer
+      def send(request)
+        request.tag = 0
+        data = request.serialize_to_string
+        # An additional header is added to the request indicating the size of the request
+        head = [data.length].pack("N")
+        @socket.send(head+data)
+      end
+
+      # Read the protobuf Response from Doozer
+      def read
+        # First strip the additional header indicating the size of the subsequent response
+        head = @socket.read(4)
+        length = head.unpack("N")[0]
+
+        # Since can returns upto 'length' bytes we need to make sure it returns
+        # at least 'length' bytes
+        # TODO: Make this a binary buffer
+        data = ''
+        until data.size >= length
+          data << @socket.read(length)
+        end
+        Response.new.parse_from_string(data)
+      end
+
+    end
+  end
+end

data/lib/ruby_skynet/doozer/exceptions.rb

@@ -0,0 +1,5 @@
+module RubySkynet
+  module Doozer
+    class ResponseError < ::RuntimeError; end
+  end
+end

data/lib/ruby_skynet/doozer/msg.pb.rb

@@ -0,0 +1,118 @@
+### Generated by rprotoc. DO NOT EDIT!
+### <proto file: doozerd/server/msg.proto>
+# package server;
+#
+# // see doc/proto.md
+# message Request {
+#   optional int32 tag = 1;
+#
+#   enum Verb {
+#       GET      = 1;
+#       SET      = 2;
+#       DEL      = 3;
+#       REV      = 5;
+#       WAIT     = 6;
+#       NOP      = 7;
+#       WALK     = 9;
+#       GETDIR   = 14;
+#       STAT     = 16;
+#       ACCESS   = 99;
+#   }
+#   optional Verb verb = 2;
+#
+#   optional string path = 4;
+#   optional bytes value = 5;
+#   optional int32 other_tag = 6;
+#
+#   optional int32 offset = 7;
+#
+#   optional int64 rev = 9;
+# }
+#
+# // see doc/proto.md
+# message Response {
+#   optional int32 tag = 1;
+#   optional int32 flags = 2;
+#
+#   optional int64 rev = 3;
+#   optional string path = 5;
+#   optional bytes value = 6;
+#   optional int32 len = 8;
+#
+#   enum Err {
+#     // don't use value 0
+#     OTHER        = 127;
+#     TAG_IN_USE   = 1;
+#     UNKNOWN_VERB = 2;
+#     READONLY     = 3;
+#     TOO_LATE     = 4;
+#     REV_MISMATCH = 5;
+#     BAD_PATH     = 6;
+#     MISSING_ARG  = 7;
+#     RANGE        = 8;
+#     NOTDIR       = 20;
+#     ISDIR        = 21;
+#     NOENT        = 22;
+#   }
+#   optional Err err_code = 100;
+#   optional string err_detail = 101;
+# }
+
+require 'protobuf/message/message'
+require 'protobuf/message/enum'
+require 'protobuf/message/service'
+require 'protobuf/message/extend'
+
+module RubySkynet
+  module Doozer
+    class Request < ::Protobuf::Message
+      defined_in __FILE__
+      optional :int32, :tag, 1
+      class Verb < ::Protobuf::Enum
+        defined_in __FILE__
+        GET = value(:GET, 1)
+        SET = value(:SET, 2)
+        DEL = value(:DEL, 3)
+        REV = value(:REV, 5)
+        WAIT = value(:WAIT, 6)
+        NOP = value(:NOP, 7)
+        WALK = value(:WALK, 9)
+        GETDIR = value(:GETDIR, 14)
+        STAT = value(:STAT, 16)
+        ACCESS = value(:ACCESS, 99)
+      end
+      optional :Verb, :verb, 2
+      optional :string, :path, 4
+      optional :bytes, :value, 5
+      optional :int32, :other_tag, 6
+      optional :int32, :offset, 7
+      optional :int64, :rev, 9
+    end
+    class Response < ::Protobuf::Message
+      defined_in __FILE__
+      optional :int32, :tag, 1
+      optional :int32, :flags, 2
+      optional :int64, :rev, 3
+      optional :string, :path, 5
+      optional :bytes, :value, 6
+      optional :int32, :len, 8
+      class Err < ::Protobuf::Enum
+        defined_in __FILE__
+        OTHER = value(:OTHER, 127)
+        TAG_IN_USE = value(:TAG_IN_USE, 1)
+        UNKNOWN_VERB = value(:UNKNOWN_VERB, 2)
+        READONLY = value(:READONLY, 3)
+        TOO_LATE = value(:TOO_LATE, 4)
+        REV_MISMATCH = value(:REV_MISMATCH, 5)
+        BAD_PATH = value(:BAD_PATH, 6)
+        MISSING_ARG = value(:MISSING_ARG, 7)
+        RANGE = value(:RANGE, 8)
+        NOTDIR = value(:NOTDIR, 20)
+        ISDIR = value(:ISDIR, 21)
+        NOENT = value(:NOENT, 22)
+      end
+      optional :Err, :err_code, 100
+      optional :string, :err_detail, 101
+    end
+  end
+end

data/lib/ruby_skynet/exceptions.rb

@@ -0,0 +1,6 @@
+module RubySkynet
+  class Exception < ::RuntimeError; end
+  class ProtocolError < Exception; end
+  class SkynetException < Exception; end
+  class ServiceException < Exception; end
+end

data/lib/ruby_skynet/version.rb

@@ -0,0 +1,3 @@
+module RubySkynet #:nodoc
+  VERSION = "0.1.0"
+end

data/lib/ruby_skynet.rb

@@ -0,0 +1,11 @@
+require 'semantic_logger'
+require 'resilient_socket'
+
+require 'ruby_skynet/exceptions'
+require 'ruby_skynet/version'
+module RubySkynet
+  module Doozer
+    autoload :Client, 'ruby_skynet/doozer/client'
+  end
+  autoload :Client, 'ruby_skynet/client'
+end

data/nbproject/private/private.properties

@@ -0,0 +1 @@
+platform.active=Ruby

data/nbproject/private/rake-d.txt

@@ -0,0 +1,4 @@
+clean=Remove any temporary products.
+clobber=Remove any generated file.
+gem=Build gem
+test=Run Test Suite

data/nbproject/project.properties

@@ -0,0 +1,6 @@
+javac.classpath=
+main.file=
+platform.active=Ruby
+source.encoding=UTF-8
+src.lib.dir=lib
+test.test.dir=test

data/nbproject/project.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://www.netbeans.org/ns/project/1">
+    <type>org.netbeans.modules.ruby.rubyproject</type>
+    <configuration>
+        <data xmlns="http://www.netbeans.org/ns/ruby-project/1">
+            <name>ruby_skynet</name>
+            <source-roots>
+                <root id="src.lib.dir" name="Source Files"/>
+            </source-roots>
+            <test-roots>
+                <root id="test.test.dir"/>
+            </test-roots>
+        </data>
+    </configuration>
+</project>