librmdp 0.0.2 → 0.1.0

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    MjdkMTlkOGQ3YjY5ZDliNWYyYWIzNTA2NzE4ZDhlYzVlYTI5ZTFmYw==
-  data.tar.gz: !binary |-
-    MzA0NGMwODViMDU3YTM2MmQwYzEzNjk5Yzk4YTlhYWE2Mzg5MzBkMQ==
-!binary "U0hBNTEy":
-  metadata.gz: !binary |-
-    ZjFjN2ZlMjY5NjljYjE0ZDdkYWYxYjIzNWZiNmVmNDIyNDEwOGUxZWUxMzUw
-    YzFkMTk2NmE5MjA2YTU1NjIyZjU5OTMwZGQ5NDgyMzYwYjllOWVlNjE5Yjhj
-    YmJjNjVmZWI5Nzk2MmQxNDNhNGRjNDIzYjQ0ZDM2MjA0MmIwYmE=
-  data.tar.gz: !binary |-
-    OTkyMTNmMDgyNjg1MDAyNzFiZDA4NjU2NjhkMWM5ODA4ZTY1ZTAxMDdhOTI3
-    OTM0MTQxM2M3OGQ4ZGEyNjVhMjBlOGExMzg1ZjViYWQ5ZWMyMzAzNGFmOWI1
-    OWUxMTI1OThkMTljNWE3YWIxYWYwNDc5Y2Y0Mjk5OTE3ZWU4NmY=
+SHA1:
+  metadata.gz: 39d2a5d3842164319cbf4dff19d01933352d3629
+  data.tar.gz: f70b5f0e6451edef4dba01083896071b1e09bdc1
+SHA512:
+  metadata.gz: 4babd79540a49f9a7299d91342e1bcdbaa00aefe56427e7f6bd5d69e4baaf2f6386890c0f31b2c735e4f7e8e046951fac5d1f5eb5b3a7868ddd3e5bcbcfabcfe
+  data.tar.gz: 4c7283d196714c0d8f831bed90778c580439e400b9df77e7def2d31baf471eac3be208023bb382de4bb0b5a83cf3b35c9febfffd4dc0515f9c4a3abfe021c48b

data/.gitignore

@@ -16,3 +16,4 @@ test/tmp
 test/version_tmp
 tmp
 .idea
+log/

data/Gemfile

@@ -4,3 +4,4 @@ gem 'minitest'
 gem 'simplecov', :require => false, :group => :test
 # Specify your gem's dependencies in librmdp.gemspec
 gemspec
+gem 'rack'

data/README.md

@@ -30,7 +30,7 @@ Start a broker
 Build a worker with 'echo' as service name
 ==========================================
 
-    worker = Majordomo::Worker.new('tcp://0.0.0.0:5555', 'echo')
+    worker = Majordomo::Worker.new(Majordomo::Config.new, 'echo')
 
     loop do
       request = worker.receive_message(reply_to = '')
@@ -40,7 +40,7 @@ Build a worker with 'echo' as service name
 
 Build a client
 ==============
-    client = Majordomo::Client.new('tcp://0.0.0.0:5555')
+    client = Majordomo::AsyncClient.new(Majordomo::Config.new)
     client.send_message('echo', 'a')
 
     response = client.receive_message

data/broker.rb

@@ -0,0 +1,4 @@
+require 'librmdp'
+
+config = Majordomo::Config.new
+Majordomo::Broker.new(config).mediate

data/client.rb

@@ -0,0 +1,27 @@
+require 'librmdp'
+
+context = ZMQ::Context.new
+
+requests = 1
+threads  = 1
+
+start = Time.now
+p 'Sending message'
+threads.times.collect do
+  Thread.new do
+    client = Majordomo::AsyncClient.new(Majordomo::Config.new, context)
+    requests.times do |i|
+      client.send_message('blaat', 'echo')
+      client.send_message('mmi.service', 'list-services')
+
+      begin
+        puts JSON.parse(client.receive_message(10).first)
+      rescue Majordomo::Timeout
+        p "Timeout #{i}"
+      end
+    end
+  end
+end.map(&:join)
+
+
+puts (Time.now - start) / (threads*requests)

data/lib/librmdp.rb

@@ -1,10 +1,15 @@
 require 'ffi-rzmq'
 require 'majordomo/version'
+require 'active_support/all'
+require 'yaml'
+require 'json'
 
 module Majordomo
   CLIENT = 'MDPC01'
   WORKER = 'MDPW01'
 
+  Timeout = Class.new(StandardError)
+
   READY              = "\x01"
   REQUEST            = "\x02"
   REPLY              = "\x03"
@@ -15,6 +20,7 @@ module Majordomo
   HEARTBEAT_EXPIRY   = HEARTBEAT_INTERVAL * HEARTBEAT_LIVENESS
 end
 
+require 'majordomo/config'
 require 'majordomo/broker'
 require 'majordomo/worker'
-require 'majordomo/client'
+require 'majordomo/async_client'

data/lib/majordomo/async_client.rb

@@ -0,0 +1,55 @@
+module Majordomo
+  class AsyncClient
+    attr_accessor :timeout, :client, :logger
+
+    # @param [Majordomo::Config] config
+    def initialize(config, context = ZMQ::Context.new)
+      @config  = config
+      @context = context
+      @poller  = ZMQ::Poller.new
+      @broker  = config.broker_endpoint
+      @logger  = ActiveSupport::Logger.new(STDOUT)
+
+      @timeout = 2500
+
+      connect_to_broker
+    end
+
+    def connect_to_broker
+      if @client
+        @poller.deregister(@client, ZMQ::POLLIN)
+        @client.close
+      end
+      @client = @context.socket(ZMQ::DEALER)
+      @client.connect(@broker)
+      @poller.register(@client, ZMQ::POLLIN)
+    end
+
+    def send_message(service, request)
+      request = [request] unless request.is_a?(Array)
+      request.unshift(service)
+      request.unshift(CLIENT)
+      request.unshift('')
+      @client.send_strings(request)
+    end
+
+    def receive_message(timeout = 60)
+      items = @poller.poll(timeout * 1000)
+      if items > 0
+        logger.debug "Broker has #{items} items"
+
+        @client.recv_strings(message = [])
+        message.shift
+        header = message.shift
+        raise unless header == CLIENT
+
+        message.shift #service
+
+        message
+      else
+        connect_to_broker
+        raise Timeout.new('Client timed out, reconnected to broker')
+      end
+    end
+  end
+end

data/lib/majordomo/broker.rb

@@ -6,24 +6,33 @@ module Majordomo
 
     attr_accessor :workers, :waiting, :services, :socket
 
-    def initialize(bind = 'tcp://*:5555', context = ZMQ::Context.new)
+    # @param [Majordomo::Config] config
+    def initialize(config, context = ZMQ::Context.new)
+      @config       = config
       @context      = context
       @socket       = @context.socket(ZMQ::ROUTER)
       @poller       = ZMQ::Poller.new
       @services     = {}
       @workers      = {}
       @waiting      = []
-      @heartbeat_at = Time.now + 0.001 * HEARTBEAT_INTERVAL
+      @heartbeat_at = Time.now + 0.001 * @config.heartbeat_interval
 
-      @socket.bind(bind)
+      @socket.bind(config.broker_endpoint)
       @poller.register(@socket, ZMQ::POLLIN)
       trap(:INT) { exit }
       at_exit { destroy }
+
+      logger.debug "Broker is bound to: #{config.broker_endpoint}"
+    end
+
+    def logger
+      @config.logger
     end
 
     def mediate
+      logger.debug 'Broker is waiting for incoming messages'
       loop do
-        items = @poller.poll(HEARTBEAT_INTERVAL)
+        items = @poller.poll(@config.heartbeat_interval)
 
         if items > 0
           @socket.recv_strings(message = [])
@@ -43,8 +52,7 @@ module Majordomo
           waiting.each do |worker|
             worker.send_message(HEARTBEAT)
           end
-          @heartbeat_at = Time.now + 0.001 * HEARTBEAT_INTERVAL
-          puts "Heartbeat #{waiting.count}"
+          @heartbeat_at = Time.now + 0.001 * @config.heartbeat_interval
         end
       end
     end
@@ -68,6 +76,7 @@ module Majordomo
       case command
         when READY
           if worker_ready
+            logger.debug "Worker was already marked as ready. Sending DISCONNECT to worker #{identity}"
             worker.delete(true)
           else
             service        = message.shift
@@ -75,12 +84,14 @@ module Majordomo
             @waiting << worker
             worker.service.waiting << worker
             worker.service.workers += 1
-            worker.expires_at      = Time.now + 0.001 * HEARTBEAT_EXPIRY
+            worker.expires_at      = Time.now + 0.001 * @config.heartbeat_expiry
+            logger.debug "Worker #{identity} for service #{worker.service.name} is ready"
             worker.service.dispatch
           end
         when REPLY
           if worker_ready
             client = message.shift
+            logger.debug "Worker #{identity} has a reply for #{client.unpack('H*').first}"
             message.shift
             message.unshift(worker.service.name)
             message.unshift(CLIENT)
@@ -96,8 +107,9 @@ module Majordomo
               @waiting.delete(worker)
               @waiting << worker
             end
-            worker.expires_at = Time.now + 0.001 * HEARTBEAT_EXPIRY
+            worker.expires_at = Time.now + 0.001 * @config.heartbeat_expiry
           else
+            logger.debug "Worker #{identity} send HEARTBEAT before READY"
             worker.delete(true)
           end
         when DISCONNECT
@@ -107,18 +119,48 @@ module Majordomo
 
     def message_client(sender, message)
       service_name = message.shift
-      service      = Service.require(self, service_name)
+      logger.debug "Client #{sender.unpack('H*').first} send MESSAGE to service #{service_name}"
+
+      case service_name
+        when 'mmi.service'
+          message_service(sender, message)
+        else
+          service = Service.require(self, service_name)
 
-      message.unshift nil
-      message.unshift sender
+          message.unshift nil
+          message.unshift sender
 
-      service.requests << message
-      service.dispatch
+          service.requests << message
+          service.dispatch
+          logger.debug "Queued messages for service [#{service.name}]: #{service.requests.count}"
+      end
+    end
+
+    def message_service(sender, message)
+      command = message.shift
+
+      result = case command
+        when 'list-services'
+          services.values
+        else
+          { errors: 'unknown command' }
+      end
+
+      message.unshift(result.to_json)
+      message.unshift(nil)
+      message.unshift(CLIENT)
+      message.unshift(nil)
+      message.unshift(sender)
+
+      @socket.send_strings(message)
     end
 
     def purge
       waiting.each do |worker|
-        worker.delete(false) if Time.now > worker.expires_at
+        if Time.now > worker.expires_at
+          logger.debug "Connection with worker #{worker.identity} lost"
+          worker.delete(false)
+        end
       end
     end
   end

data/lib/majordomo/broker/service.rb

@@ -6,12 +6,11 @@ module Majordomo
       attr_accessor :broker, :name, :requests, :waiting, :workers, :blacklist
 
       def initialize(broker, name)
-        @broker    = broker
-        @name      = name
-        @requests  = []
-        @waiting   = []
-        @workers   = 0
-        @blacklist = ::Set.new
+        @broker   = broker
+        @name     = name
+        @requests = []
+        @waiting  = []
+        @workers  = 0
       end
 
       def self.require(broker, name)
@@ -24,6 +23,15 @@ module Majordomo
         service
       end
 
+      def as_json(options = {})
+        {
+            name:            name,
+            workers:         workers,
+            requests:        requests.count,
+            workers_waiting: waiting.count
+        }
+      end
+
       def dispatch
         broker.purge
         return if waiting.empty?
@@ -35,18 +43,6 @@ module Majordomo
           waiting << worker
         end
       end
-
-      def enable_command(command)
-        @blacklist.delete(command)
-      end
-
-      def disable_command(command)
-        @blacklist.add(command)
-      end
-
-      def command_enabled?(command)
-        @blacklist.include?(command)
-      end
     end
   end
 end

data/lib/majordomo/config.rb

@@ -0,0 +1,34 @@
+module Majordomo
+  class Config
+    attr_accessor :log_level, :heartbeat_interval, :heartbeat_liveness, :broker_endpoint, :logger
+
+    def initialize
+      defaults = {
+          log_level:          Logger::DEBUG,
+          logger:             ActiveSupport::Logger.new(STDOUT),
+          heartbeat_interval: HEARTBEAT_INTERVAL,
+          heartbeat_liveness: HEARTBEAT_LIVENESS,
+          broker_endpoint:    'tcp://127.0.0.1:5555'
+      }
+      configure(defaults)
+
+      logger.level = log_level
+    end
+
+    def configure(attributes = {}, &block)
+      if attributes.present?
+        attributes.each do |key, value|
+          public_send("#{key}=", value)
+        end
+      elsif block_given?
+        block.call(self)
+      else
+        raise ArgumentError.new('Configure requires a Hash or a Block')
+      end
+    end
+
+    def heartbeat_expiry
+      heartbeat_interval * heartbeat_liveness
+    end
+  end
+end

data/lib/majordomo/version.rb

@@ -1,3 +1,3 @@
 module Majordomo
-  VERSION = "0.0.2"
+  VERSION = "0.1.0"
 end

data/lib/majordomo/worker.rb

@@ -2,20 +2,25 @@ module Majordomo
   class Worker
     attr_accessor :broker, :service, :worker, :heartbeat_at, :heartbeat, :liveness, :reconnect
 
-    def initialize(broker, service, context = ZMQ::Context.new)
+    # @param [Majordomo::Config] config
+    def initialize(config, service, context = ZMQ::Context.new)
+      @config  = config
       @context = context
       @poller  = ZMQ::Poller.new
-      @broker  = broker
       @service = service
 
-      @heartbeat = HEARTBEAT_INTERVAL
-      @reconnect = HEARTBEAT_INTERVAL
+      @heartbeat = @config.heartbeat_interval
+      @reconnect = @config.heartbeat_liveness
       connect_to_broker
 
       trap(:INT) { exit }
       at_exit { destroy }
     end
 
+    def logger
+      @config.logger
+    end
+
     def destroy
       @worker.close
       @context.terminate
@@ -23,18 +28,19 @@ module Majordomo
 
     def connect_to_broker
       if @worker
-        puts 'Closing connection to broker'
+        logger.debug 'Closing connection to broker'
         @poller.deregister(@worker, ZMQ::POLLIN)
         @worker.close
       end
       @worker = @context.socket(ZMQ::DEALER)
-      @worker.connect(@broker)
+      @worker.connect(@config.broker_endpoint)
       @worker.setsockopt(ZMQ::LINGER, 0)
       @poller.register(@worker, ZMQ::POLLIN)
+      @liveness     = @config.heartbeat_liveness
+      @heartbeat_at = Time.now + 0.001 * @heartbeat
 
+      logger.debug 'Sending READY to broker'
       send_to_broker(READY, [@service])
-      @liveness     = HEARTBEAT_LIVENESS
-      @heartbeat_at = Time.now + 0.001 * @heartbeat
     end
 
     def send_to_broker(command, message = [])
@@ -51,7 +57,7 @@ module Majordomo
         if items > 0
           @worker.recv_strings(message = [])
 
-          @liveness = HEARTBEAT_LIVENESS
+          @liveness = @config.heartbeat_liveness
           message.shift
           header = message.shift
           raise unless header == WORKER
@@ -59,12 +65,14 @@ module Majordomo
           command = message.shift
           case command
             when REQUEST
+              logger.debug 'REQUEST from broker'
               reply_to << message.shift
               message.shift
               return message
             when HEARTBEAT
-            # Do nothing
+              #
             when DISCONNECT
+              logger.debug 'DISCONNECT from broker'
               connect_to_broker
             else
               raise
@@ -81,6 +89,7 @@ module Majordomo
     end
 
     def send_message(report, reply_to)
+      logger.debug 'Sending REPLY to broker'
       report.unshift('')
       report.unshift(reply_to)
       send_to_broker(REPLY, report)

data/librmdp.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_development_dependency 'bundler', '~> 1.3'
+  spec.add_development_dependency 'activesupport', '~> 4.1'
   spec.add_development_dependency 'rake'
   spec.add_dependency 'ffi-rzmq'
 end

data/test/{client_test.rb → async_client_test.rb}

@@ -3,7 +3,7 @@ require 'test_helper'
 class TestClient < MiniTest::Unit::TestCase
 
   def setup
-    @client = Majordomo::Client.new('tcp://0.0.0.0:5555')
+    @client = Majordomo::AsyncClient.new(Majordomo::Config.new)
   end
 
   def test_connect_to_broker

data/test/integration_test.rb

@@ -8,13 +8,13 @@ class TestIntegration < MiniTest::Unit::TestCase
 
   def start_broker
     @threads << Thread.new do
-      Majordomo::Broker.new.mediate
+      Majordomo::Broker.new(Majordomo::Config.new).mediate
     end
   end
 
   def start_worker
     @threads << Thread.new do
-      worker = Majordomo::Worker.new('tcp://0.0.0.0:5555', 'echo')
+      worker = Majordomo::Worker.new(Majordomo::Config.new, 'echo')
 
       request = worker.receive_message(reply_to = '')
       worker.send_message(request, reply_to)
@@ -31,7 +31,7 @@ class TestIntegration < MiniTest::Unit::TestCase
     start_broker
     start_worker
 
-    client = Majordomo::Client.new('tcp://0.0.0.0:5555')
+    client = Majordomo::AsyncClient.new(Majordomo::Config.new)
     client.send_message('echo', 'a')
 
     assert_equal %w(a), client.receive_message

data/worker.rb

@@ -0,0 +1,13 @@
+require 'librmdp'
+
+worker   = Majordomo::Worker.new(Majordomo::Config.new, 'blaat')
+@running = true
+
+while @running do
+  request = worker.receive_message(reply_to = '')
+  # do something with a request
+
+  # sleep rand(5)
+
+  worker.send_message(request, reply_to)
+end

metadata

@@ -1,55 +1,69 @@
 --- !ruby/object:Gem::Specification
 name: librmdp
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
 platform: ruby
 authors:
 - Benoist
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-20 00:00:00.000000000 Z
+date: 2014-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: ffi-rzmq
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: A ruby implementation of the ZeroMQ majordomo pattern v0.1
@@ -59,22 +73,26 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
+- broker.rb
+- client.rb
 - lib/librmdp.rb
+- lib/majordomo/async_client.rb
 - lib/majordomo/broker.rb
 - lib/majordomo/broker/service.rb
 - lib/majordomo/broker/worker.rb
-- lib/majordomo/client.rb
+- lib/majordomo/config.rb
 - lib/majordomo/version.rb
 - lib/majordomo/worker.rb
 - librmdp.gemspec
-- test/client_test.rb
+- test/async_client_test.rb
 - test/integration_test.rb
 - test/test_helper.rb
+- worker.rb
 homepage: https://github.com/benoist/librmdp
 licenses:
 - MIT
@@ -85,21 +103,21 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.7
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: A ruby implementation of the ZeroMQ majordomo pattern v0.1
 test_files:
-- test/client_test.rb
+- test/async_client_test.rb
 - test/integration_test.rb
 - test/test_helper.rb

data/lib/majordomo/client.rb

@@ -1,44 +0,0 @@
-module Majordomo
-  class Client
-    attr_accessor :timeout, :client
-
-    def initialize(broker, context = ZMQ::Context.new)
-      @context = context
-      @poller  = ZMQ::Poller.new
-      @broker  = broker
-
-      @timeout = 2500
-
-      connect_to_broker
-    end
-
-    def connect_to_broker
-      if @client
-        @poller.deregister(@client, ZMQ::POLLIN)
-        @client.close
-      end
-      @client = @context.socket(ZMQ::DEALER)
-      @client.connect(@broker)
-      @poller.register(@client, ZMQ::POLLIN)
-    end
-
-    def send_message(service, request)
-      request = [request] unless request.is_a?(Array)
-      request.unshift(service)
-      request.unshift(CLIENT)
-      request.unshift('')
-      @client.send_strings(request)
-    end
-
-    def receive_message
-      @client.recv_strings(message = [])
-      message.shift
-      header = message.shift
-      raise unless header == CLIENT
-
-      message.shift #service
-
-      message
-    end
-  end
-end