em-zmq-tp10 0.1.7

data/lib/em/protocols/zmq2/inproc.rb

@@ -0,0 +1,147 @@
+require 'em/protocols/zmq2/connection'
+module EventMachine
+  module Protocols
+    module Zmq2
+      module InProc
+        class AlreadyBinded < RuntimeError; end
+        class NotBinded < RuntimeError; end
+        BindPoints = {}
+        def self.bind(name, socket)
+          unless BindPoints[name].nil? || BindPoints[name] == socket
+            raise AlreadyBinded, "ZMQ inproc://#{name} already binded"
+          end
+          BindPoints[name] = socket
+          unless BindPoints[:shutdown_hook_set]
+            BindPoints[:shutdown_hook_set] = true
+            EM.add_shutdown_hook { BindPoints.clear }
+          end
+          name
+        end
+
+        def self.unbind(name, socket)
+          unless BindPoints[name] == socket
+            unless BindPoints[name]
+              raise NotBinded, "ZMQ bind point inproc://#{name} binded to other socket"
+            else
+              raise NotBinded, "ZMQ bind point inproc://#{name} is not binded"
+            end
+          end
+          BindPoints.delete name
+        end
+
+        def self.connect(name, socket)
+          connect = Connection.new(socket)
+          EM.next_tick {
+            if server = BindPoints[name]
+              sconnect = Connection.new(server)
+              sconnect.peer = connect
+              connect.peer = sconnect
+            else
+              connect.unbind
+            end
+          }
+          connect
+        end
+
+        class Connection
+          include ConnectionMixin
+          attr :peer
+          attr_accessor :peer_waiting, :notify_when_free
+          def initialize(socket)
+            @socket = socket
+            @peer = nil
+            @peer_waiting = false
+            @outgoing_queue = []
+            @recursion = 0
+            @state = :connecting
+          end
+
+          def peer=(peer)
+            raise "Already has peer"  if @peer
+            @state = :connected
+            @peer = peer
+            peer.peer_waiting!
+            EM.next_tick { post_init }
+          end
+
+          def _not_too_busy?
+            @state == :connected && @outgoing_queue.size <= 32
+          end
+
+          def shift_outgoing_queue
+            message = @outgoing_queue.shift
+            EM.next_tick { sent_data }  if @notify_when_free && _not_too_busy?
+            message
+          end
+
+          def call
+            if message = @peer.shift_outgoing_queue
+              receive_strings(message)
+            else
+              @peer.peer_waiting!
+            end
+          end
+
+          def receive_strings(message)
+            super
+            @peer.peer_waiting!
+          end
+
+          def peer_waiting!
+            unless @outgoing_queue.empty?
+              @peer_waiting = false
+              if (@recursion += 1) == 10
+                EM.next_tick @peer
+              else
+                @peer.call
+              end
+            else
+              unbind  if @state == :closing
+              @peer_waiting = true
+            end
+          end
+
+          def send_strings(strings)
+            if @state == :connected
+              unless Array === strings
+                strings = Array(strings)
+              end
+              @outgoing_queue << strings
+            end
+            if @peer_waiting
+              @peer_waiting = false
+              EM.next_tick @peer
+            end
+          end
+
+          def send_strings_or_prepared(strings, prepared)
+            send_strings(strings)
+          end
+
+          def close_connection(after_writting = false)
+            @peer.close_connection(:peer)  unless after_writting == :peer
+            unless after_writting == true
+              @outgoing_queue.clear
+            end
+            if @peer_waiting
+              EM.next_tick @peer
+            end
+            @state = :closing
+          end
+
+          def unbind
+            old, @state = @state, :closed
+            if old == :closing
+              @peer.peer_waiting!
+            end
+            super
+          end
+
+          def error?
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/em/protocols/zmq2/pub_sub.rb

@@ -0,0 +1,102 @@
+require 'em/protocols/zmq2/socket'
+require 'em/protocols/zmq2/queue_per_peer'
+module EventMachine
+  module Protocols
+    module Zmq2
+      # ZMQ socket which acts as SUB
+      # subscriptions are done by subscribe method
+      class Sub < Socket
+        class DefaultAction
+          def initialize(sub)
+            @sub = sub
+          end
+
+          def call(message)
+            @sub.receive_message(message)
+          end
+        end
+
+        def initialize(opts = {})
+          super
+          @subscriptions = {}
+          @default_action = DefaultAction.new(self)
+          subscribe_many [*opts[:subscribe]]
+        end
+
+        def subscribe_many(subscriptions)
+          subscriptions.each{|sub| subscribe *sub}  if subscriptions
+        end
+
+        def subscribe(s, cb = nil, &block)
+          @subscriptions[s] = cb || block || @default_action
+        end
+
+        def receive_message_and_peer(message, peer)
+          for sub, callback in @subscriptions
+            matched = if String === sub
+                        message.first.start_with?(sub)
+                      elsif sub.respond_to?(:call)
+                        sub.call(message.first)
+                      else
+                        sub === message.first  # Regexp and anything you want
+                      end
+            callback.call(message) if matched
+          end
+        end
+
+        def receive_message(message)
+          raise NoMethodError
+        end
+
+        private :send_message
+      end
+
+      class PrePub < Socket
+        include PackString
+        def send_message(message, even_if_busy = false)
+          sent = false
+          prepared = prepare_message(message)
+          peers = even_if_busy ? @peers : @free_peers
+          for identity, peer in peers
+            if !peer.error?
+              peer.send_strings_or_prepared(message, prepared)
+              sent = true
+            end
+          end
+        end
+      end
+
+      class Pub < Socket
+        include QueuePerPeer
+        include PackString
+
+        def send_message(message)
+          sent = false
+          prepared = prepare_message(message)
+          idents = @peers.keys | @queues.keys
+          for identity in idents
+            peer = @free_peers[identity]
+            queue = @queues[identity]
+            if peer && (queue.empty? || flush_queue(queue, peer)) && 
+               !peer.error? && peer.not_too_busy?
+              peer.send_strings_or_prepared(message, prepared)
+              sent = true
+            else
+              pushed = push_to_queue(queue, [message, prepared])
+              sent ||= pushed
+            end
+          end
+          sent
+        end
+      private
+        def cancel_message(message)
+          false
+        end
+
+        def send_formed_message(peer, from_queue)
+          peer.send_strings_or_prepared(*from_queue)
+        end
+      end
+    end
+  end
+end

data/lib/em/protocols/zmq2/queue_per_peer.rb

@@ -0,0 +1,57 @@
+module EventMachine
+  module Protocols
+    module Zmq2
+      module QueuePerPeer
+        def initialize(opts = {})
+          super
+          @queues = {}
+        end
+
+        def register_peer(peer_identity, connection)
+          peer_identity = super
+          @queues[peer_identity] ||= []
+          peer_identity
+        end
+
+        def unregister_peer(peer_identity)
+          super
+          if generated_identity?(peer_identity)
+            @queues.delete peer_identity
+          end
+        end
+
+        def peer_free(peer, connection)
+          super
+          peer_conn = @peers[peer]
+          queue = @queues[peer]
+          flush_queue(queue, peer_conn)
+        end
+
+      private
+        def react_on_hwm_decrease
+          @queues.each{|_, queue| push_to_queue(queue)}
+        end
+
+        def flush_queue(queue, peer, even_if_busy = false)
+          until queue.empty?
+            return false if peer.error? || !(even_if_busy || peer.not_too_busy?)
+            send_formed_message(peer, queue.shift)
+          end
+          true
+        end
+
+        def send_formed_message(peer, from_queue)
+          peer.send_strings(from_queue)
+        end
+
+        def flush_all_queue
+          @peers.each{|peer_identity, peer|
+            if queue = @queues[peer_identity]
+              flush_queue(queue, peer, true)
+            end
+          }
+        end
+      end
+    end
+  end
+end

data/lib/em/protocols/zmq2/rep.rb

@@ -0,0 +1,64 @@
+require 'em/protocols/zmq2/router'
+
+module EventMachine
+  module Protocols
+    module Zmq2
+      # mixin which transforms +PreRouter+ to +PreRep+ and +Router+ to +Rep+
+      module RepMixin
+        # you should not override +#receive_message+ in +PreRep+ and +Rep+
+        # see +#receive_request+ instead
+        def receive_message(message)
+          envelope, message = split_message(message)
+          receive_request(message, envelope)
+        end
+
+        # callback on incoming message, splits message and envelope
+        # use envelope later in +#send_reply+
+        def receive_request(message, envelope)
+          raise NoMethodError
+        end
+
+        # joins message and envelope into single message and sends it
+        def send_reply(message, envelope)
+          send_message([*envelope, EMPTY, *message])
+        end
+      end
+
+      # ZMQ socket which acts like REP (without outgoing queue)
+      #
+      #     class EchoBangRep < EM::Protocols::Zmq2::PreRep
+      #       def receive_request(message, envelope)
+      #         message << "!"
+      #         if send_reply(message, envelope)
+      #           puts "reply sent successfuly"
+      #         end
+      #       end
+      #     end
+      #     rep = EchoBangRep.new
+      #     rep.bind('ipc://rep')
+      class PreRep < PreRouter
+        include RepMixin
+        private :send_message
+      end
+
+      # ZMQ socket which acts like REP
+      #
+      #
+      #     class EchoBangRep < EM::Protocols::Zmq2::Rep
+      #       def receive_request(message, envelope)
+      #         message << "!"
+      #         if send_reply(message, envelope)
+      #           puts "reply sent successfuly (or placed into queue)"
+      #         end
+      #       end
+      #     end
+      #     rep = EchoBangRep.new
+      #     rep.bind('ipc://rep')
+      class Rep < Router
+        include RepMixin
+        private :send_message
+      end
+
+    end
+  end
+end

data/lib/em/protocols/zmq2/req.rb

@@ -0,0 +1,325 @@
+require 'em/protocols/zmq2/dealer'
+module EventMachine
+  module Protocols
+    module Zmq2
+      # ZMQ socket which acts like REQ, but without outgoing message queueing,
+      # It generates unique request ids and uses ZMQ routing scheme for mapping
+      # replies.
+      #
+      # Note, that on subclassing, you should override +#receive_reply+ and not
+      # +#receive_message+ , and you should use +#send_request+ instead of
+      # +#send_message+
+      #
+      # @example
+      #   class MyReq < EM::Protocols::Zmq2::PreReq
+      #     def receive_reply(message, data, request_id)
+      #       puts "received message #{message} and stored data #{data}
+      #     end
+      #   end
+      #
+      #   req = MyReq.new
+      #   if request_id = req.send_request(['hi'], 'ho')
+      #     puts "Message sent"
+      #   end
+      #
+      # @example
+      #   class TimeoutedPreReq < EM::Protocols::Zmq2::PreReq
+      #     def initialize(opts={})
+      #       super
+      #       @timeout = opts[:timeout] || 1
+      #     end
+      #     def cancel_request(request_id)
+      #       data = super
+      #       EM.cancel_timer(data[:timer])  if data[:timer]
+      #       data[:data]
+      #     end
+      #     def receive_reply(message, data, request_id)
+      #       if timer = data[:timer]
+      #         EM.cancel_timer(data[:timer])
+      #       end
+      #       puts "receive message #{message.inspect}, associated data #{data[:data].inspect}"
+      #     end
+      #     def send_request(message, data)
+      #       data = {data: data}
+      #       if request_id = super(message, data)
+      #         data[:timer] = EM.add_timer(@timeout){ cancel_request(request_id) }
+      #       end
+      #       request_id
+      #     end
+      #   end
+      #   req = TimeoutedPreReq.new
+      #   req.bind('ipc://req')
+      #   callback_data = { some_data: "" }
+      #   if request_id = req.send_request(['hello', 'world'], callback_data)
+      #     puts "Request sent with request_id #{request_id}"
+      #   else
+      #     puts "No free peers"
+      #   end
+      class PreReq < PreDealer
+        def initialize(opts = {}) # :nodoc:
+          super
+          @data = {}
+        end
+
+        # cancel pending request, so that callback will not be called on incoming
+        # message.
+        # @return associated data with request_id
+        def cancel_request(request_id)
+          @data.delete request_id
+        end
+
+        # do not override +#receive_message+ or for PreReq subclasses
+        def receive_message(message)
+          request_id, message = split_message(message)
+          request_id = request_id.first
+          if data = @data.delete(request_id)
+            receive_reply(message, data, request_id)
+          end
+        end
+
+        # override it to react on incoming message
+        # Also accept saved data by +#send_request+
+        def receive_reply(message, data, request_id)
+          raise NoMethodError
+        end
+
+        def form_request(message) # :nodoc:
+          request_id = next_uniq_identity # I believe, it large enough
+          [request_id, EMPTY, *message]
+        end
+
+        # send request and store associated callback data
+        # Returns assigned request id or nil, if sending is unsuccessfull
+        def send_request(message, data, even_if_busy = false)
+          request = form_request(message)
+          request_id = request.first
+          @data[request_id] = data
+          if send_message(request, even_if_busy)
+            request_id
+          else
+            @data.delete request_id
+            false
+          end
+        end
+
+      private :send_message
+      private
+        def cancel_message(message) # :nodoc:
+          cancel_request(message.first)
+        end
+      end
+
+      # ZMQ socket which acts like REQ. It also reacts on connection busyness, so
+      # that it is a bit smarter, than ZMQ REQ.
+      # It generates unique request ids and uses ZMQ routing scheme for mapping
+      # replies.
+      #
+      # The only visible change from PreReq is less frequent +send_request+ false return
+      #
+      # Note, that on subclassing, you should override +#receive_reply+ and not
+      # +#receive_message+ , and you should use +#send_request+ instead of
+      # +#send_message+
+      #
+      #   class MyReq < EM::Protocols::Zmq2::Req
+      #     def receive_reply(message, data, request_id)
+      #       puts "received message #{message} and stored data #{data}
+      #     end
+      #   end
+      #
+      #   req = MyReq.new
+      #   if request_id = req.send_request('hi', 'ho')
+      #     puts "Message sent"
+      #   end
+      #
+      #   class TimeoutedReq < EM::Protocols::Zmq2::PreReq
+      #     def initialize(opts={})
+      #       super
+      #       @timeout = opts[:timeout] || 1
+      #     end
+      #     def cancel_request(request_id)
+      #       data = super
+      #       EM.cancel_timer(data[:timer])  if data[:timer]
+      #       data[:data]
+      #     end
+      #     def receive_reply(message, data)
+      #       if timer = data[:timer]
+      #         EM.cancel_timer(data[:timer])
+      #       end
+      #       puts "receive message #{message.inspect}, associated data #{data[:data].inspect}"
+      #     end
+      #     def send_request(message, data)
+      #       data = {data: data}
+      #       if request_id = super(message, data)
+      #         data[:timer] = EM.add_timer(@timeout){ cancel_request(request_id) }
+      #       end
+      #       request_id
+      #     end
+      #   end
+      #   req = TimeoutedReq.new
+      #   req.bind('ipc://req')
+      #   callback_data = { some_data: "" }
+      #   if request_id = req.send_request(['hello', 'world'], callback_data)
+      #     puts "Request sent with request_id #{request_id}"
+      #   else
+      #     puts "No free peers and highwatermark reached"
+      #   end
+      class Req < PreReq
+        def initialize(opts = {})
+          super
+          @requests = {}
+        end
+
+        # cancel pending request, so that callback will not be called on incoming
+        # message
+        # @return associated data with request_id
+        def cancel_request(request_id)
+          @requests.delete request_id
+          @data.delete request_id
+        end
+
+        def send_request(message, data)
+          request = form_request(message)
+          request_id = request.first
+          @data[request_id] = data
+          if flush_queue && send_message(request) || push_to_queue(request)
+            request_id
+          end
+        end
+
+        def flush_queue(even_if_busy = false)
+          until @requests.empty?
+            request_id, request = @requests.first
+            return false  unless send_message(request, even_if_busy)
+            @requests.delete(request_id)
+          end
+          true
+        end
+
+        def flush_all_queue # :nodoc:
+          flush_queue(true)
+        end
+
+        def peer_free(peer, connection)
+          super
+          flush_queue
+        end
+
+        private
+
+        def react_on_hwm_decrease
+          push_to_queue
+        end
+
+        def cancel_message(message)
+          cancel_request(message.first)
+        end
+
+        def push_to_queue(request = nil)
+          if @requests.size >= @hwm
+            case @hwm_strategy
+            when :drop_last
+              if @requests.size > @hwm
+                new_requests = {}
+                @requests.each do |k,message|
+                  if new_requests.size >= @hwm
+                    cancel_message(message)
+                  else
+                    new_requests[k] = message
+                  end
+                end
+                @requests = new_requests
+              end
+              false
+            when :drop_first
+              hwm = @hwm - (request ? 1 : 0)
+              while @requests.size > hwm
+                k, message = @requests.shift
+                cancel_message(message)
+              end
+              @requests[request.first] = request.dup  if request
+              true
+            end
+          else
+            @requests[request.first] = request.dup  if request
+            true
+          end
+        end
+      end
+
+      # Convinient Req class which accepts callback as data
+      #
+      # @example
+      #   req = EM::Protocols::Zmq2::ReqCb.new
+      #   req.bind('ipc://req')
+      #   timer = nil
+      #   request_id = req.send_request(['hello', 'world']) do |message|
+      #     EM.cancel_timer(timer)
+      #     puts "Message #{message}"
+      #   end
+      #   if request_id
+      #     timer = EM.add_timer(1) {
+      #       req.cancel_request(request_id)
+      #     }
+      #   end
+      class ReqCb < Req
+        def cancel_request(request_id)
+          callback = super
+          callback.call(nil, request_id)
+        end
+
+        def receive_reply(message, callback, request_id)
+          callback.call(message, request_id)
+        end
+
+        def send_request(message, callback = nil, &block)
+          super message, callback || block
+        end
+      end
+
+      # Convinient Req class, which returns EM::DefaultDeferable on +#send_request+
+      #
+      # @example
+      #   req = EM::Protocolse::Zmq2::ReqDefer.new
+      #   req.bind('ipc://req')
+      #   data = {hi: 'ho'}
+      #   deferable = req.send_request(['hello', 'world'], data) do |reply, data|
+      #     puts "Reply received #{reply} #{data}"
+      #   end
+      #   deferable.timeout 1
+      #   deferable.errback do
+      #     puts "Message canceled"
+      #   end
+      #   deferable.callback do |reply, data|
+      #     puts "Another callback #{reply} #{data}"
+      #   end
+      class ReqDefer < Req
+        class Wrapped < ::Struct.new(:data, :deferrable); end
+
+        def cancel_request(request_id)
+          wrapped = super
+          wrapped.deferrable.fail(nil, wrapped.data)
+        end
+        def receive_reply(reply, wrapped, request_id)
+          wrapped.deferrable.succeed(reply, wrapped.data)
+        end
+        def send_request(message, data, callback = nil, &block)
+          deferrable = EM::DefaultDeferrable.new
+          wrapped = Wrapped.new(data, deferrable)
+          callback ||= block
+          if Proc === callback
+            deferrable.callback &callback
+          else
+            deferrable.callback{|reply, data| callback.call(reply, data)}
+          end
+          if request_id = super(message, wrapped)
+            deferrable.errback{ cancel_request(request_id) }
+          else
+            deferrable.fail(nil, data)
+          end
+          deferrable
+        end
+      end
+
+    end
+  end
+end