georgi-nsq-ruby 2.1.0

data/lib/nsq/consumer.rb

@@ -0,0 +1,98 @@
+require_relative 'client_base'
+
+module Nsq
+  class Consumer < ClientBase
+
+    attr_reader :max_in_flight
+
+    def initialize(opts = {})
+      if opts[:nsqlookupd]
+        @nsqlookupds = [opts[:nsqlookupd]].flatten
+      else
+        @nsqlookupds = []
+      end
+
+      @topic = opts[:topic] || raise(ArgumentError, 'topic is required')
+      @channel = opts[:channel] || raise(ArgumentError, 'channel is required')
+      @max_in_flight = opts[:max_in_flight] || 1
+      @discovery_interval = opts[:discovery_interval] || 60
+      @msg_timeout = opts[:msg_timeout]
+      @ssl_context = opts[:ssl_context]
+      @tls_options = opts[:tls_options]
+      @tls_v1 = opts[:tls_v1]
+
+      # This is where we queue up the messages we receive from each connection
+      @messages = opts[:queue] || Queue.new
+
+      # This is where we keep a record of our active nsqd connections
+      # The key is a string with the host and port of the instance (e.g.
+      # '127.0.0.1:4150') and the key is the Connection instance.
+      @connections = {}
+
+      if !@nsqlookupds.empty?
+        discover_repeatedly(
+          nsqlookupds: @nsqlookupds,
+          topic: @topic,
+          interval: @discovery_interval
+        )
+      else
+        # normally, we find nsqd instances to connect to via nsqlookupd(s)
+        # in this case let's connect to an nsqd instance directly
+        add_connection(opts[:nsqd] || '127.0.0.1:4150', max_in_flight: @max_in_flight)
+      end
+    end
+
+
+    # pop the next message off the queue
+    def pop
+      @messages.pop
+    end
+
+
+    # By default, if the internal queue is empty, pop will block until
+    # a new message comes in.
+    #
+    # Calling this method won't block. If there are no messages, it just
+    # returns nil.
+    def pop_without_blocking
+      @messages.pop(true)
+    rescue ThreadError
+      # When the Queue is empty calling `Queue#pop(true)` will raise a ThreadError
+      nil
+    end
+
+
+    # returns the number of messages we have locally in the queue
+    def size
+      @messages.size
+    end
+
+
+    private
+    def add_connection(nsqd, options = {})
+      super(nsqd, {
+        topic: @topic,
+        channel: @channel,
+        queue: @messages,
+        msg_timeout: @msg_timeout,
+        max_in_flight: 1
+      }.merge(options))
+    end
+
+    # Be conservative, but don't set a connection's max_in_flight below 1
+    def max_in_flight_per_connection(number_of_connections = @connections.length)
+      [@max_in_flight / number_of_connections, 1].max
+    end
+
+    def connections_changed
+      redistribute_ready
+    end
+
+    def redistribute_ready
+      @connections.values.each do |connection|
+        connection.max_in_flight = max_in_flight_per_connection
+        connection.re_up_ready
+      end
+    end
+  end
+end

data/lib/nsq/discovery.rb

@@ -0,0 +1,98 @@
+require 'json'
+require 'net/http'
+require 'uri'
+
+require_relative 'logger'
+
+# Connects to nsqlookup's to find the nsqd instances for a given topic
+module Nsq
+  class Discovery
+    include Nsq::AttributeLogger
+
+    # lookupd addresses must be formatted like so: '<host>:<http-port>'
+    def initialize(lookupds)
+      @lookupds = lookupds
+    end
+
+    # Returns an array of nsqds instances
+    #
+    # nsqd instances returned are strings in this format: '<host>:<tcp-port>'
+    #
+    #     discovery.nsqds
+    #     #=> ['127.0.0.1:4150', '127.0.0.1:4152']
+    #
+    # If all nsqlookupd's are unreachable, raises Nsq::DiscoveryException
+    #
+    def nsqds
+      gather_nsqds_from_all_lookupds do |lookupd|
+        get_nsqds(lookupd)
+      end
+    end
+
+    # Returns an array of nsqds instances that have messages for
+    # that topic.
+    #
+    # nsqd instances returned are strings in this format: '<host>:<tcp-port>'
+    #
+    #     discovery.nsqds_for_topic('a-topic')
+    #     #=> ['127.0.0.1:4150', '127.0.0.1:4152']
+    #
+    # If all nsqlookupd's are unreachable, raises Nsq::DiscoveryException
+    #
+    def nsqds_for_topic(topic)
+      gather_nsqds_from_all_lookupds do |lookupd|
+        get_nsqds(lookupd, topic)
+      end
+    end
+
+
+    private
+
+    def gather_nsqds_from_all_lookupds
+      nsqd_list = @lookupds.map do |lookupd|
+        yield(lookupd)
+      end.flatten
+
+      # All nsqlookupds were unreachable, raise an error!
+      if nsqd_list.length > 0 && nsqd_list.all? { |nsqd| nsqd.nil? }
+        raise DiscoveryException
+      end
+
+      nsqd_list.compact.uniq
+    end
+
+    # Returns an array of nsqd addresses
+    # If there's an error, return nil
+    def get_nsqds(lookupd, topic = nil)
+      uri_scheme = 'http://' unless lookupd.match(%r(https?://))
+      uri = URI.parse("#{uri_scheme}#{lookupd}")
+
+      uri.query = "ts=#{Time.now.to_i}"
+      if topic
+        uri.path = '/lookup'
+        uri.query += "&topic=#{URI.escape(topic)}"
+      else
+        uri.path = '/nodes'
+      end
+
+      begin
+        body = Net::HTTP.get(uri)
+        data = JSON.parse(body)
+        producers = data['producers'] || # v1.0.0-compat
+                      (data['data'] && data['data']['producers'])
+
+        if producers
+          producers.map do |producer|
+            "#{producer['broadcast_address']}:#{producer['tcp_port']}"
+          end
+        else
+          []
+        end
+      rescue Exception => e
+        error "Error during discovery for #{lookupd}: #{e}"
+        nil
+      end
+    end
+
+  end
+end

data/lib/nsq/exceptions.rb

@@ -0,0 +1,5 @@
+module Nsq
+  # Raised when nsqlookupd discovery fails
+  class DiscoveryException < Exception; end
+end
+

data/lib/nsq/frames/error.rb

@@ -0,0 +1,6 @@
+require_relative 'frame'
+
+module Nsq
+  class Error < Frame
+  end
+end

data/lib/nsq/frames/frame.rb

@@ -0,0 +1,16 @@
+require_relative '../logger'
+
+module Nsq
+  class Frame
+    include Nsq::AttributeLogger
+    @@log_attributes = [:connection]
+
+    attr_reader :data
+    attr_reader :connection
+
+    def initialize(data, connection)
+      @data = data
+      @connection = connection
+    end
+  end
+end

data/lib/nsq/frames/message.rb

@@ -0,0 +1,33 @@
+require_relative 'frame'
+
+module Nsq
+  class Message < Frame
+
+    attr_reader :attempts
+    attr_reader :id
+    attr_reader :body
+
+    def initialize(data, connection)
+      super
+      @timestamp_in_nanoseconds, @attempts, @id, @body = @data.unpack('Q>s>a16a*')
+      @body.force_encoding('UTF-8')
+    end
+
+    def finish
+      connection.fin(id)
+    end
+
+    def requeue(timeout = 0)
+      connection.req(id, timeout)
+    end
+
+    def touch
+      connection.touch(id)
+    end
+
+    def timestamp
+      Time.at(@timestamp_in_nanoseconds / 1_000_000_000.0)
+    end
+
+  end
+end

data/lib/nsq/frames/response.rb

@@ -0,0 +1,6 @@
+require_relative 'frame'
+
+module Nsq
+  class Response < Frame
+  end
+end

data/lib/nsq/logger.rb

@@ -0,0 +1,38 @@
+require 'logger'
+module Nsq
+  @@logger = Logger.new(nil)
+
+
+  def self.logger
+    @@logger
+  end
+
+
+  def self.logger=(new_logger)
+    @@logger = new_logger
+  end
+
+
+  module AttributeLogger
+    def self.included(klass)
+      klass.send :class_variable_set, :@@log_attributes, []
+    end
+
+    %w(fatal error warn info debug).map{|m| m.to_sym}.each do |level|
+      define_method level do |msg|
+        Nsq.logger.send(level, "#{prefix} #{msg}")
+      end
+    end
+
+
+    private
+    def prefix
+      attrs = self.class.send(:class_variable_get, :@@log_attributes)
+      if attrs.count > 0
+        "[#{attrs.map{|a| "#{a.to_s}: #{self.send(a)}"}.join(' ')}] "
+      else
+        ''
+      end
+    end
+  end
+end

data/lib/nsq/producer.rb

@@ -0,0 +1,94 @@
+require_relative 'client_base'
+
+module Nsq
+  class Producer < ClientBase
+    attr_reader :topic
+
+    def initialize(opts = {})
+      @connections = {}
+      @topic = opts[:topic]
+      @discovery_interval = opts[:discovery_interval] || 60
+      @ssl_context = opts[:ssl_context]
+      @tls_options = opts[:tls_options]
+      @tls_v1 = opts[:tls_v1]
+
+      nsqlookupds = []
+      if opts[:nsqlookupd]
+        nsqlookupds = [opts[:nsqlookupd]].flatten
+        discover_repeatedly(
+          nsqlookupds: nsqlookupds,
+          interval: @discovery_interval
+        )
+
+      elsif opts[:nsqd]
+        nsqds = [opts[:nsqd]].flatten
+        nsqds.each{|d| add_connection(d)}
+
+      else
+        add_connection('127.0.0.1:4150')
+      end
+    end
+
+    def write(*raw_messages)
+      if !@topic
+        raise 'No topic specified. Either specify a topic when instantiating the Producer or use write_to_topic.'
+      end
+
+      write_to_topic(@topic, *raw_messages)
+    end
+
+    # Arg 'delay' in seconds
+    def deferred_write(delay, *raw_messages)
+      if !@topic
+        raise 'No topic specified. Either specify a topic when instantiating the Producer or use write_to_topic.'
+      end
+      if delay < 0.0
+        raise "Delay can't be negative, use a positive float."
+      end
+
+      deferred_write_to_topic(@topic, delay, *raw_messages)
+    end
+
+    def write_to_topic(topic, *raw_messages)
+      # return error if message(s) not provided
+      raise ArgumentError, 'message not provided' if raw_messages.empty?
+
+      # stringify the messages
+      messages = raw_messages.map(&:to_s)
+
+      # get a suitable connection to write to
+      connection = connection_for_write
+
+      if messages.length > 1
+        connection.mpub(topic, messages)
+      else
+        connection.pub(topic, messages.first)
+      end
+    end
+
+    def deferred_write_to_topic(topic, delay, *raw_messages)
+      raise ArgumentError, 'message not provided' if raw_messages.empty?
+      messages = raw_messages.map(&:to_s)
+      connection = connection_for_write
+      messages.each do |msg|
+        connection.dpub(topic, (delay * 1000).to_i, msg)
+      end
+    end
+
+    private
+    def connection_for_write
+      # Choose a random Connection that's currently connected
+      # Or, if there's nothing connected, just take any random one
+      connections_currently_connected = connections.select{|_,c| c.connected?}
+      connection = connections_currently_connected.values.sample || connections.values.sample
+
+      # Raise an exception if there's no connection available
+      unless connection
+        raise 'No connections available'
+      end
+
+      connection
+    end
+
+  end
+end

data/lib/version.rb

@@ -0,0 +1,9 @@
+module Nsq
+  module Version
+    MAJOR = 2
+    MINOR = 1
+    PATCH = 0
+    BUILD = nil
+    STRING = [MAJOR, MINOR, PATCH, BUILD].compact.join('.')
+  end
+end

metadata

@@ -0,0 +1,116 @@
+--- !ruby/object:Gem::Specification
+name: georgi-nsq-ruby
+version: !ruby/object:Gem::Version
+  version: 2.1.0
+platform: ruby
+authors:
+- Wistia
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-06-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: nsq-cluster
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: ''
+email: dev@wistia.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+- README.md
+files:
+- LICENSE.txt
+- README.md
+- lib/nsq.rb
+- lib/nsq/client_base.rb
+- lib/nsq/connection.rb
+- lib/nsq/consumer.rb
+- lib/nsq/discovery.rb
+- lib/nsq/exceptions.rb
+- lib/nsq/frames/error.rb
+- lib/nsq/frames/frame.rb
+- lib/nsq/frames/message.rb
+- lib/nsq/frames/response.rb
+- lib/nsq/logger.rb
+- lib/nsq/producer.rb
+- lib/version.rb
+homepage: http://github.com/wistia/nsq-ruby
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Ruby client library for NSQ
+test_files: []