newque 0.0.1

data/Rakefile

@@ -0,0 +1,5 @@
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new('test')
+
+task :default => :test

data/conf/channels/example.json

@@ -0,0 +1,12 @@
+{
+  "listeners": ["http8000", "zmq8005"],
+  "backend": "disk",
+  "acknowledgement": "saved",
+  "readSettings": {},
+  "writeSettings": {
+    "forward": []
+  },
+  "raw": true,
+  "emptiable": true,
+  "maxRead": 5000
+}

data/conf/channels/example_fifo.json

@@ -0,0 +1,18 @@
+{
+  "listeners": ["http8000", "zmq8005"],
+  "backend": "fifo",
+  "backendSettings": {
+    "host": "0.0.0.0",
+    "port": 8007,
+    "timeout": 1000,
+    "healthTimeLimit": 500
+  },
+  "acknowledgement": "saved",
+  "readSettings": {
+  },
+  "writeSettings": {
+    "forward": []
+  },
+  "raw": true,
+  "emptiable": true
+}

data/conf/channels/example_plaintext.json

@@ -0,0 +1,15 @@
+{
+  "listeners": ["http8000", "zmq8005"],
+  "backend": "disk",
+  "acknowledgement": "saved",
+  "readSettings": {
+    "httpFormat": "plaintext"
+  },
+  "writeSettings": {
+    "forward": [],
+    "httpFormat": "plaintext"
+  },
+  "raw": true,
+  "emptiable": true,
+  "maxRead": 5000
+}

data/conf/channels/example_pubsub.json

@@ -0,0 +1,15 @@
+{
+  "listeners": ["http8000", "zmq8005"],
+  "backend": "pubsub",
+  "backendSettings": {
+    "host": "0.0.0.0",
+    "port": 8006
+  },
+  "acknowledgement": "saved",
+  "readSettings": null,
+  "writeSettings": {
+    "forward": []
+  },
+  "raw": true,
+  "emptiable": false
+}

data/conf/newque.json

@@ -0,0 +1,26 @@
+{
+  "logLevel": "debug",
+  "environment": "development",
+  "admin": {
+    "host": "0.0.0.0",
+    "port": 8001
+  },
+  "listeners": [
+    {
+      "protocol": "http",
+      "name": "http8000",
+      "host": "0.0.0.0",
+      "port": 8000,
+      "protocolSettings": {}
+    },
+    {
+      "protocol": "zmq",
+      "name": "zmq8005",
+      "host": "0.0.0.0",
+      "port": 8005,
+      "protocolSettings": {
+        "concurrency": 20
+      }
+    }
+  ]
+}

data/lib/newque.rb

@@ -0,0 +1,16 @@
+require 'newque/errors'
+require 'newque/util'
+require 'newque/responses'
+require 'newque/future'
+
+require 'newque/zmq/protobuf'
+require 'newque/zmq/zmq_tools'
+require 'newque/zmq/newque_zmq'
+
+require 'newque/http/http_json'
+require 'newque/http/http_plaintext'
+require 'newque/http/newque_http'
+
+require 'newque/clients/client'
+require 'newque/clients/pubsub_client'
+require 'newque/clients/fifo_client'

data/lib/newque/clients/client.rb

@@ -0,0 +1,23 @@
+module Newque
+
+  class Client
+    extend Forwardable
+
+    def_delegators :@instance, :write, :read, :read_stream, :count, :delete, :health
+
+    attr_reader :protocol
+
+    def initialize protocol, host, port, protocol_options:nil, timeout:10000
+      @protocol = protocol
+      @instance = if protocol == :zmq
+        Newque_zmq.new host, port, (protocol_options || {}), timeout
+      elsif protocol == :http
+        Newque_http.new host, port, (protocol_options || {}), timeout
+      else
+        raise NewqueError.new "Invalid protocol [#{protocol}]. Must be either :zmq or :http"
+      end
+    end
+
+  end
+
+end

data/lib/newque/clients/fifo_client.rb

@@ -0,0 +1,109 @@
+require 'ffi-rzmq'
+
+module Newque
+
+  class Fifo_client
+
+    STATES = [:NEW, :CONNECTING, :RUNNING, :DISCONNECTING, :CLOSING, :CLOSED]
+
+    def initialize host, port, protocol_options:{}, socket_wait:100
+      @ctx = ZMQ::Context.new
+      @addr = "tcp://#{host}:#{port}"
+      @options = Util.compute_options Zmq_tools::BASE_OPTIONS, protocol_options
+      @socket_wait = socket_wait
+      @state = 0
+    end
+
+    def connect &block
+      raise NewqueError.new "Cannot connect because this client is #{current_state}" unless current_state == :NEW
+      next_state :CONNECTING
+      @handler = block
+      @sock = @ctx.socket ZMQ::DEALER
+      Zmq_tools.set_zmq_sock_options @sock, @options
+
+      @poller = ZMQ::Poller.new
+      @poller.register_readable @sock
+
+      @sock.connect @addr
+
+      ready = Util.wait_t
+      @thread = Thread.new do
+        next_state :RUNNING
+        @poller.poll(@socket_wait)
+        Util.resolve_t ready, ''
+        loop do
+          if current_state == :DISCONNECTING
+            @sock.disconnect @addr
+            @socket_wait = 0
+            next_state :CLOSING
+          end
+          if @poller.poll(@socket_wait) == 0
+            if current_state == :CLOSING
+              @sock.close
+              break
+            else
+              next
+            end
+          end
+          # RECEIVING INCOMING MESSAGE
+          buffers = []
+          @sock.recv_strings buffers, ZMQ::DONTWAIT
+          id, *frames = buffers
+          parsed = Zmq_tools.parse_input frames
+
+          response = begin
+            returned = @handler.(parsed)
+            unless returned.respond_to?(:serialize)
+              raise NewqueError.new "Block given to Fifo_client.connect returned #{returned.class} which is not a valid response object"
+            end
+            serialized = returned.serialize
+            messages = serialized[:messages]
+            serialized.delete :messages
+            {
+              output: Output.new(serialized.merge!(errors: [])),
+              messages: messages
+            }
+          rescue => handler_error
+            {
+              output: Output.new(errors: [handler_error.to_s], error_output: Error_Output.new),
+              messages: []
+            }
+          end
+          @sock.send_strings ([id, response[:output].encode.to_s] + response[:messages]), ZMQ::DONTWAIT
+
+        end
+        next_state :CLOSED
+      end
+      @thread.abort_on_exception = true
+      Future.new ready, 10
+    end
+
+    def disconnect
+      state = current_state
+      if state == :NEW
+        nil
+      elsif state == :RUNNING
+        next_state :DISCONNECTING
+      else
+        raise NewqueError.new "Can't disconnect since the Fifo_client is currently #{state}"
+      end
+      nil
+    end
+
+    private
+
+    def current_state
+      STATES[@state]
+    end
+
+    def next_state should_be
+      goes_to = STATES[@state + 1]
+      unless goes_to == should_be
+        raise NewqueError.new "Inconsistent state in Fifo_client. #{goes_to} should be #{should_be}"
+      end
+      @state = @state + 1
+    end
+
+  end
+
+end

data/lib/newque/clients/pubsub_client.rb

@@ -0,0 +1,101 @@
+require 'ffi-rzmq'
+require 'securerandom'
+
+module Newque
+
+  class Pubsub_client
+
+    def initialize host, port, protocol_options:{}, socket_wait:100
+      @ctx = ZMQ::Context.new
+      @addr = "tcp://#{host}:#{port}"
+      @options = Util.compute_options Zmq_tools::BASE_OPTIONS, protocol_options
+      @socket_wait = socket_wait
+      @disconnect = false
+      @ready
+
+      @listeners = {}
+      @error_handlers = []
+    end
+
+    def add_error_handler &block
+      @error_handlers << block
+    end
+
+    def subscribe &block
+      @disconnect = false
+      id = SecureRandom.uuid
+      @listeners[id] = block
+      start_loop unless is_looping?
+      thread = Thread.new do
+        @ready.join(1)
+        id
+      end
+      Future.new thread, 10
+    end
+
+    def unsubscribe id
+      @listeners.delete id
+      nil
+    end
+
+    def disconnect
+      @disconnect = true
+      nil
+    end
+
+    private
+
+    # The socket connection happens here so that no network traffic occurs while not subscribed
+    def start_loop
+      @disconnect = false
+      @sock = @ctx.socket ZMQ::SUB
+      Zmq_tools.set_zmq_sock_options @sock, @options
+
+      @poller = ZMQ::Poller.new
+      @poller.register_readable @sock
+
+      @sock.connect @addr
+      @sock.setsockopt(ZMQ::SUBSCRIBE, '')
+
+      @ready = Util.wait_t
+      @thread = Thread.new do
+        @poller.poll(@socket_wait)
+        Util.resolve_t @ready, ''
+        loop do
+          break if @disconnect
+          next if @poller.poll(@socket_wait) == 0
+          buffers = []
+          @sock.recv_strings buffers, ZMQ::DONTWAIT
+          input = Zmq_tools.parse_input buffers
+          @listeners.values.each do |listener|
+            begin
+              listener.(input)
+            rescue => listener_error
+              print_uncaught_exception(listener_error, 'subscribe') if @error_handlers.size == 0
+              @error_handlers.each do |err_handler|
+                begin
+                  err_handler.(listener_error)
+                rescue => uncaught_err
+                  print_uncaught_exception uncaught_err, 'add_error_handler'
+                end
+              end
+            end
+          end
+        end
+        @sock.disconnect @addr
+        @sock.close
+      end
+      @thread.abort_on_exception = true
+    end
+
+    def is_looping?
+      !@thread.nil? && @thread.alive?
+    end
+
+    def print_uncaught_exception err, block_name
+      puts "Uncaught exception in Pubsub_client.#{block_name} block: #{err.to_s} #{JSON.pretty_generate(err.backtrace)}"
+    end
+
+  end
+
+end

data/lib/newque/errors.rb

@@ -0,0 +1,6 @@
+module Newque
+
+  class NewqueError < StandardError
+  end
+
+end

data/lib/newque/future.rb

@@ -0,0 +1,33 @@
+module Newque
+
+  class Future
+
+    attr_reader :thread
+
+    def initialize thread, timeout
+      @thread = thread
+      @timeout = timeout
+    end
+
+    def get limit=@timeout
+      result = @thread.join(limit)
+      if result.nil?
+        # Timeout exceeded
+        @thread.kill
+        raise Timeout::Error
+      end
+      result.value
+    end
+
+
+    def to_s
+      "<NewqueFuture timeout: #{@timeout} status: #{@thread.status}>"
+    end
+
+    def inspect
+      to_s
+    end
+
+  end
+
+end

data/lib/newque/http/http_json.rb

@@ -0,0 +1,49 @@
+require 'faraday'
+
+module Newque
+
+  class Http_json
+    def initialize http
+      @http = http
+    end
+
+    def write channel, atomic, msgs, ids=nil
+      thread = Thread.new do
+        body = {
+          'atomic' => false,
+          'messages' => msgs
+        }
+        body["ids"] = ids unless ids.nil?
+        res = @http.conn.post do |req|
+          @http.send :set_req_options, req
+          req.url "/v1/#{channel}"
+          req.body = body.to_json
+        end
+        parsed = @http.send :parse_json_response, res.body
+        Write_response.new parsed['saved']
+      end
+      Future.new thread, @http.timeout
+    end
+
+    def read channel, mode, limit=nil
+      thread = Thread.new do
+        res = @http.conn.get do |req|
+          @http.send :set_req_options, req
+          req.url "/v1/#{channel}"
+          req.headers['newque-mode'] = mode
+          req.headers['newque-read-max'] = limit unless limit.nil?
+        end
+        parsed = @http.send :parse_json_response, res.body
+        Read_response.new(
+          res.headers['newque-response-length'].to_i,
+          res.headers['newque-response-last-id'],
+          res.headers['newque-response-last-ts'].to_i,
+          parsed['messages']
+        )
+      end
+      Future.new thread, @http.timeout
+    end
+
+  end
+
+end