newque 0.0.1

data/lib/newque/zmq/newque_zmq.rb

@@ -0,0 +1,121 @@
+require 'ffi-rzmq'
+require 'securerandom'
+
+module Newque
+
+  class Newque_zmq
+    attr_reader :sock
+
+    def initialize host, port, options, timeout
+      @ctx = ZMQ::Context.new
+      @options = Util.compute_options Zmq_tools::BASE_OPTIONS, options
+      @timeout = timeout / 1000.0
+
+      @sock = @ctx.socket ZMQ::DEALER
+      Zmq_tools.set_zmq_sock_options @sock, @options
+
+      @poller = ZMQ::Poller.new
+      @poller.register_readable @sock
+
+      @router = {}
+      @sock.connect "tcp://#{host}:#{port}"
+      start_loop
+      @thread
+    end
+
+    def write channel, atomic, msgs, ids=nil
+      input = Input.new(channel: channel, write_input: Write_Input.new(atomic: atomic, ids: ids))
+      send_request input, msgs do |buffers|
+        output = parse_response buffers[0], :write_output
+        Write_response.new output.saved
+      end
+    end
+
+    def read channel, mode, limit=nil
+      input = Input.new(channel: channel, read_input: Read_Input.new(mode: mode, limit: limit))
+      send_request input do |buf, *messages|
+        output = parse_response buf, :read_output
+        Read_response.new output.length, output.last_id, output.last_timens, messages
+      end
+    end
+
+    def read_stream channel, mode, limit=nil
+      raise NewqueError.new "Read_stream is only available in :http mode"
+    end
+
+    def count channel
+      input = Input.new(channel: channel, count_input: Count_Input.new)
+      send_request input do |buffers|
+        output = parse_response buffers[0], :count_output
+        Count_response.new output.count
+      end
+    end
+
+    def delete channel
+      input = Input.new(channel: channel, delete_input: Delete_Input.new)
+      send_request input do |buffers|
+        output = parse_response buffers[0], :delete_output
+        Delete_response.new
+      end
+    end
+
+    def health channel, global=false
+      input = Input.new(channel: channel, health_input: Health_Input.new(global: global))
+      send_request input do |buffers|
+        output = parse_response buffers[0], :health_output
+        Health_response.new
+      end
+    end
+
+    private
+
+    def send_request input, msgs=[], async:false, &block
+      id = SecureRandom.uuid
+      meta = input.encode.to_s
+      thread = register id, &block
+      @sock.send_strings (msgs.size > 0 ? [id, meta] + msgs : [id, meta]), ZMQ::DONTWAIT
+      Future.new thread, @timeout
+    end
+
+    def start_loop
+      @thread = Thread.new do
+        while @poller.poll(:blocking) > 0
+          buffers = []
+          @sock.recv_strings buffers, ZMQ::DONTWAIT
+          id, *frames = buffers
+
+          thread = @router[id]
+          while thread.status == 'run'
+            # If the scheduler didn't run the other thread and execute its Thread.stop
+            # then we have to wait before we can continue. Sleep 0 yields to the scheduler.
+            sleep 0
+          end
+
+          thread.thread_variable_set :result, frames
+          thread.run if thread.status == 'sleep' # 'if' not necessary, it's a sanity check
+        end
+      end
+      @thread.abort_on_exception = true
+    end
+
+    def register id
+      thread = Thread.new do
+        Thread.stop
+        @router.delete id
+        yield Thread.current.thread_variable_get(:result)
+      end
+      Thread.pass # Give a hint to schedule the new thread now
+      @router[id] = thread
+    end
+
+    def parse_response buffer, type
+      output = Output.decode buffer
+      if output.errors
+        raise Util.newque_error output.errors
+      end
+      output.send type
+    end
+
+  end
+
+end

data/lib/newque/zmq/protobuf.rb

@@ -0,0 +1,96 @@
+require 'beefcake'
+
+module Newque
+
+  # ------------------------------------
+  # REQUEST OBJECTS
+
+  class Write_Input
+    include Beefcake::Message
+    optional :atomic, :bool, 1
+    repeated :ids, :bytes, 2
+  end
+
+  class Read_Input
+    include Beefcake::Message
+    required :mode, :bytes, 1
+    optional :limit, :int64, 2
+  end
+
+  class Count_Input
+    include Beefcake::Message
+  end
+
+  class Delete_Input
+    include Beefcake::Message
+  end
+
+  class Health_Input
+    include Beefcake::Message
+    required :global, :bool, 1
+  end
+
+  class Input
+    include Beefcake::Message
+    required :channel, :bytes, 1
+    optional :write_input, Write_Input, 11
+    optional :read_input, Read_Input, 12
+    optional :count_input, Count_Input, 13
+    optional :delete_input, Delete_Input, 14
+    optional :health_input, Health_Input, 15
+  end
+
+  # ------------------------------------
+  # RESPONSE OBJECTS
+
+  class Error_Output
+    include Beefcake::Message
+  end
+
+  class Write_Output
+    include Beefcake::Message
+    optional :saved, :int32, 1
+  end
+
+  class Read_Output
+    include Beefcake::Message
+    required :length, :int32, 1
+    optional :last_id, :bytes, 2
+    optional :last_timens, :int64, 3
+  end
+
+  class Count_Output
+    include Beefcake::Message
+    optional :count, :int64, 1
+  end
+
+  class Delete_Output
+    include Beefcake::Message
+  end
+
+  class Health_Output
+    include Beefcake::Message
+  end
+
+  class Output
+    include Beefcake::Message
+    repeated :errors, :bytes, 1
+    optional :error_output, Error_Output, 11
+    optional :write_output, Write_Output, 12
+    optional :read_output, Read_Output, 13
+    optional :count_output, Count_Output, 14
+    optional :delete_output, Delete_Output, 15
+    optional :health_output, Health_Output, 16
+  end
+
+  # ------------------------------------
+  # WRAPPERS
+
+  class Many
+    include Beefcake::Message
+    repeated :buffers, :bytes, 1
+  end
+
+  # ------------------------------------
+
+end

data/lib/newque/zmq/zmq_tools.rb

@@ -0,0 +1,52 @@
+require 'ffi-rzmq'
+
+module Newque
+
+  class Zmq_tools
+
+    BASE_OPTIONS = {
+      ZMQ_MAXMSGSIZE: -1,
+      ZMQ_LINGER: 60000,
+      ZMQ_RECONNECT_IVL: 100,
+      ZMQ_RECONNECT_IVL_MAX: 60000,
+      ZMQ_BACKLOG: 100,
+      ZMQ_SNDHWM: 5000,
+      ZMQ_RCVHWM: 5000
+    }
+    # BASE_OPTIONS, with the values being the ZMQ constants for those options
+    ZMQ_OPT_MAPPING = Hash[
+        BASE_OPTIONS.map do |name, value|
+        [name, ZMQ.const_get(name.to_s.slice(4..-1))]
+      end
+    ]
+
+    def self.set_zmq_sock_options sock, options
+      options.each do |name, value|
+        sock.setsockopt ZMQ_OPT_MAPPING[name], value
+      end
+    end
+
+    def self.parse_input buffers
+      buf, *messages = buffers
+      input = Input.decode buf
+
+      action = if !input.write_input.nil?
+        Write_request.new input.write_input.atomic, input.write_input.ids
+      elsif !input.read_input.nil?
+        Read_request.new input.read_input.mode, input.read_input.limit
+      elsif !input.count_input.nil?
+        Count_request.new
+      elsif !input.delete_input.nil?
+        Delete_request.new
+      elsif !input.health_input.nil?
+        Health_request.new input.health_input.global
+      else
+        raise NewqueError.new "Cannot find a valid message type"
+      end
+
+      Input_request.new input.channel, action, messages
+    end
+
+  end
+
+end

data/newque.gemspec

@@ -0,0 +1,35 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'newque/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = 'newque'
+  gem.version       = Newque::VERSION
+  gem.authors       = ['Simon Grondin']
+  gem.email         = ['simon.grondin@outlook.com']
+  gem.description   = 'Ruby Client library for Newque'
+  gem.summary       = 'Ruby Client library for Newque'
+  gem.homepage      = 'http://github.com/newque/newque-ruby'
+  gem.license       = 'MPL-2.0'
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = []
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ['lib']
+
+  gem.required_ruby_version = '>= 2.0.0'
+
+  gem.add_dependency 'beefcake', '~> 1.2.0'
+  gem.add_dependency 'ffi-rzmq', '~> 2.0.5'
+  gem.add_dependency 'faraday', '~> 0.12.2'
+
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'pry'
+
+  gem.extra_rdoc_files = [
+    'LICENSE',
+    'README.md'
+  ]
+end

data/spec/client_spec.rb

@@ -0,0 +1,124 @@
+require 'newque'
+require 'pry'
+require './spec/helpers'
+
+module Newque
+  describe 'Client' do
+
+    run_shared_tests = -> (name, client, channel) {
+      def validate_read read, num
+        expect(read.class).to eq Read_response
+        expect(read.length).to eq num
+        expect(read.last_id).to_not be_empty
+        expect(read.last_timens).to be_a_kind_of Numeric
+        expect(read.messages.size).to eq num
+      end
+
+      before(:each) do
+        client.delete(channel).get
+      end
+
+      it "#{name} should write" do
+        write = client.write(channel, false, ['msg1', 'msg2', Helpers.bin_str]).get
+        expect(write.class).to eq Write_response
+        expect(write.saved).to eq 3
+      end
+
+      it "#{name} should read nothing" do
+        read = client.read(channel, "one").get
+        expect(read.class).to eq Read_response
+        expect(read.length).to eq 0
+      end
+
+      it "#{name} should read messages" do
+        client.write(channel, false, Helpers.make_msgs(5)).get
+        read = client.read(channel, "one").get
+        validate_read read, 1
+
+        read = client.read(channel, "many 3").get
+        validate_read read, 3
+
+        read = client.read(channel, "after_id #{read.last_id}").get
+        validate_read read, 2
+      end
+
+      it "#{name} should read binary data correctly" do
+        client.write(channel, false, [Helpers.bin_str]).get
+        read = client.read(channel, "one").get
+        expect(read.messages.first.length).to eq Helpers.bin_str.length
+        expect(read.messages.first).to eq Helpers.bin_str
+      end
+
+      it "#{name} should count" do
+        count = client.count(channel).get
+        expect(count.class).to eq Count_response
+        expect(count.count).to eq 0
+
+        client.write(channel, false, Helpers.make_msgs(5)).get
+        count = client.count(channel).get
+        expect(count.class).to eq Count_response
+        expect(count.count).to eq 5
+      end
+
+      it "#{name} should check health" do
+        health = client.health(channel).get
+        expect(health.class).to eq Health_response
+      end
+
+      it "#{name} should check health globally" do
+        health = client.health(channel, true).get
+        expect(health.class).to eq Health_response
+      end
+
+      it "#{name} should support concurrent calls" do
+        write1 = client.write(channel, false, Helpers.make_msgs(5))
+        write2 = client.write(channel, false, Helpers.make_msgs(3))
+
+        expect(write2.get.saved).to eq 3
+        expect(write1.get.saved).to eq 5
+      end
+
+      it "#{name} should pass errors" do
+        write = client.write('invalid_channel', false, Helpers.make_msgs(5))
+        expect { write.get }.to raise_error NewqueError
+      end
+    }
+
+    host = '127.0.0.1'
+    timeout = 3000
+    zmq_client = Client.new(:zmq, host, 8005, timeout:timeout)
+    http_json_client = Client.new(:http, host, 8000, timeout:timeout)
+    http_plaintext_client = Client.new(:http, host, 8000, protocol_options:{http_format: :plaintext}, timeout:timeout)
+
+    run_shared_tests.('ZMQ', zmq_client, 'example')
+    run_shared_tests.('HTTP JSON', http_json_client, 'example')
+    run_shared_tests.('HTTP Plaintext', http_plaintext_client, 'example_plaintext')
+
+    describe 'HTTP-specific' do
+      run_http_tests = -> (name, client, channel) {
+        it "#{name} should support Read Stream" do
+          client.delete(channel).get
+          num_batches = 25
+          batch_size = 100
+          num_batches.times do |i|
+            write = client.write(channel, false, Helpers.make_msgs(batch_size, from:(i*batch_size))).get
+            expect(write.saved).to eq batch_size
+          end
+
+          counter = 0
+          # TODO: Fix Limit
+          enum = client.read_stream channel, "many #{num_batches * batch_size}" #, limit:100
+          enum.each.with_index do |x, i|
+            expect("msg#{i}").to eq x
+            counter = counter + 1
+          end
+          expect(counter).to eq(num_batches * batch_size)
+        end
+      }
+
+      run_http_tests.('HTTP JSON', http_json_client, 'example')
+      run_http_tests.('HTTP Plaintext', http_plaintext_client, 'example_plaintext')
+    end
+  end
+
+end

data/spec/fifo_client_spec.rb

@@ -0,0 +1,130 @@
+require 'newque'
+require 'pry'
+require './spec/helpers'
+
+module Newque
+  describe 'Fifo_client' do
+    let(:channel) { 'example_fifo' }
+
+    before(:each) do
+      host = '127.0.0.1'
+      timeout = 3000
+      @producer1 = Client.new(:zmq, host, 8005, timeout:timeout)
+      @consumer1 = Fifo_client.new host, 8007
+      @consumer2 = Fifo_client.new host, 8007
+    end
+
+    after(:each) do
+      @consumer1.disconnect
+      @consumer2.disconnect
+    end
+
+    it 'should write' do
+      messages = ['msg1', 'msg2']
+      ready = @consumer1.connect do |input|
+        expect(input.channel).to eq channel
+        expect(input.action.class).to eq Write_request
+        expect(input.action.ids.size).to eq messages.size
+        expect(input.messages).to eq messages
+        Write_response.new 9
+      end
+
+      ready.get
+      write = @producer1.write(channel, false, messages).get
+      expect(write.saved).to eq 9
+    end
+
+    it 'should read' do
+      should_receive = ['msg1', 'msg2']
+      ready = @consumer1.connect do |input|
+        expect(input.channel).to eq channel
+        expect(input.action.class).to eq Read_request
+        expect(input.action.mode).to eq 'Many 2'
+        expect(input.action.limit).to eq 2
+        Read_response.new 2, 'some_id', 12345, ['msg123', 'msg456']
+      end
+
+      ready.get
+      read = @producer1.read(channel, 'many 2').get
+      expect(read.length).to eq 2
+      expect(read.last_id).to_not be_empty
+      expect(read.last_timens).to be_a_kind_of Numeric
+      expect(read.messages.size).to eq 2
+    end
+
+    it 'should count' do
+      ready = @consumer1.connect do |input|
+        expect(input.channel).to eq channel
+        expect(input.action.class).to eq Count_request
+        Count_response.new 8
+      end
+
+      ready.get
+      count = @producer1.count(channel).get
+      expect(count.count).to eq 8
+    end
+
+    it 'should delete' do
+      ready = @consumer1.connect do |input|
+        expect(input.channel).to eq channel
+        expect(input.action.class).to eq Delete_request
+        Delete_response.new
+      end
+
+      ready.get
+      @producer1.delete(channel).get
+    end
+
+    it 'should check health' do
+      ready = @consumer1.connect do |input|
+        expect(input.channel).to eq channel
+        expect(input.action.class).to eq Health_request
+        expect(input.action.global).to eq false
+        Health_response.new
+      end
+
+      ready.get
+      health = @producer1.health(channel, true).get
+    end
+
+    it 'should not be connected twice' do
+      expect {
+        ready = @consumer1.connect { puts; }
+        ready.get
+      }.to_not raise_error
+
+      expect {
+        # We're already connected
+        @consumer1.connect { puts; }
+      }.to raise_error NewqueError
+    end
+
+    it 'should not accept invalid response types' do
+      ready = @consumer1.connect do |input|
+        expect(input.channel).to eq channel
+        expect(input.action.class).to eq Count_request
+        'SOME STRING'
+      end
+
+      ready.get
+      expect {
+        @producer1.count(channel).get
+      }.to raise_error NewqueError
+    end
+
+    it 'should not accept incorrect responses' do
+      ready = @consumer1.connect do |input|
+        expect(input.channel).to eq channel
+        expect(input.action.class).to eq Count_request
+        Delete_response.new
+      end
+
+      ready.get
+      expect {
+        @producer1.count(channel).get
+      }.to raise_error NewqueError
+    end
+
+
+  end
+end