nsq-ruby 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 03b1479ff830b7fa68dcbccaab0e2b9a9866b433
-  data.tar.gz: 3df0a88a0c40d24c252e891e3d054a3278213492
+  metadata.gz: 16ea2d920538289564a71a48e051589bd8977a53
+  data.tar.gz: 4ac36aa2fcf6178267d75f46db965555756337bc
 SHA512:
-  metadata.gz: 7548b29979d49e8b22d74b93a91b1f45eda87ef57d6211d6f4b8dac0fbe44c54d498c3a918a2d914e0cb3073abd63e537b104f0ee87b4540b4f04b57c2a6db4e
-  data.tar.gz: 5d15b617fd62fd51deb3b4902e51a1d60537c30b87fd0e36f76fc49e78329d69c3e8ab83e282d66853844447898d02f827134bf480bc57c4dd039e8bb03e0d08
+  metadata.gz: 6d528a2b2b3752cc9ded663ea4bcc1f8343bb6300481cbeb9587b4e543acccda9a469e8463a1060f951937accb751dfb0e214b6aab86de8d25f5f8b3bd5301b3
+  data.tar.gz: 73f9c978d51d10b3e2a7a61092cf96c1e7d1d5b1c4899f943c0a2a331e35f94c741a1f765831d8bd718b534c7983a71a556851a8bf1bcebb9406fb65dee34563

data/README.md

@@ -50,7 +50,7 @@ consumer.terminate
 
 ## Producer
 
-### Instantialization
+### Initialization
 
 The Nsq::Producer constructor takes the following options:
 
@@ -58,8 +58,9 @@ The Nsq::Producer constructor takes the following options:
 |---------------|----------------------------------------|--------------------|
 | `topic`       | Topic to which to publish messages     |                    |
 | `nsqd`        | Host and port of the nsqd instance     | '127.0.0.1:4150'   |
+| `nsqlookupd`  | Use lookupd to automatically discover nsqds |               |
 
-For example:
+For example, if you'd like to publish messages to a single nsqd.
 
 ```Ruby
 producer = Nsq::Producer.new(
@@ -68,6 +69,19 @@ producer = Nsq::Producer.new(
 )
 ```
 
+Alternatively, you can use nsqlookupd to find all nsqd nodes in the cluster.
+When you instantiate Nsq::Producer in this way, it will automatically maintain
+connections to all nsqd instances. When you publish a message, it will be sent
+to a random nsqd instance.
+
+```Ruby
+producer = Nsq::Producer.new(
+  nsqlookupd: ['1.2.3.4:4161', '6.7.8.9:4161'],
+  topic: 'topic-of-great-esteem'
+)
+```
+
+
 ### `#write`
 
 Publishes one or more message to nsqd. If you give it a single argument, it will
@@ -107,7 +121,7 @@ producers when you're done with them.
 
 ## Consumer
 
-### Instantialization
+### Initialization
 
 | Option               | Description                                   | Default            |
 |----------------------|-----------------------------------------------|--------------------|
@@ -215,7 +229,7 @@ Nsq.logger = Logger.new(STDOUT)
 
 ## Requirements
 
-NSQ v0.2.29 or later (due to IDENTITY metadata specification (0.2.28 and per-
+NSQ v0.2.29 or later due for IDENTITY metadata specification (0.2.28) and per-
 connection timeout support (0.2.29).
 
 
@@ -252,6 +266,13 @@ VERBOSE=true rake spec
 ```
 
 
+## Authors
+
+- Robby Grossman (@freerobby)
+- Brendan Schwartz (@bschwartz)
+- Marshall Moutenot (@mmoutenot)
+
+
 ## MIT License
 
 Copyright (C) 2014 Wistia, Inc.

data/lib/nsq/client_base.rb

@@ -0,0 +1,111 @@
+require_relative 'discovery'
+require_relative 'connection'
+require_relative 'logger'
+
+module Nsq
+  class ClientBase
+    include Nsq::AttributeLogger
+    @@log_attributes = [:topic]
+
+    attr_reader :topic
+    attr_reader :connections
+
+    def connected?
+      @connections.values.any?(&:connected?)
+    end
+
+
+    def terminate
+      @discovery_thread.kill if @discovery_thread
+      drop_all_connections
+    end
+
+
+    private
+
+    # discovers nsqds from an nsqlookupd repeatedly
+    #
+    #   opts:
+    #     nsqlookups: ['127.0.0.1:4161'],
+    #     topic: 'topic-to-find-nsqds-for',
+    #     interval: 60
+    #
+    def discover_repeatedly(opts = {})
+      @discovery_thread = Thread.new do
+
+        @discovery = Discovery.new(opts[:nsqlookupds])
+
+        loop do
+          nsqds = nsqds_from_lookupd(opts[:topic])
+          drop_and_add_connections(nsqds)
+          sleep opts[:interval]
+        end
+
+      end
+
+      @discovery_thread.abort_on_exception = true
+    end
+
+
+    def nsqds_from_lookupd(topic = nil)
+      if topic
+        @discovery.nsqds_for_topic(topic)
+      else
+        @discovery.nsqds
+      end
+    end
+
+
+    def drop_and_add_connections(nsqds)
+      # drop nsqd connections that are no longer in lookupd
+      missing_nsqds = @connections.keys - nsqds
+      missing_nsqds.each do |nsqd|
+        drop_connection(nsqd)
+      end
+
+      # add new ones
+      new_nsqds = nsqds - @connections.keys
+      new_nsqds.each do |nsqd|
+        begin
+          add_connection(nsqd)
+        rescue Exception => ex
+          error "Failed to connect to nsqd @ #{nsqd}: #{ex}"
+        end
+      end
+
+      # balance RDY state amongst the connections
+      connections_changed
+    end
+
+
+    def add_connection(nsqd, options = {})
+      info "+ Adding connection #{nsqd}"
+      host, port = nsqd.split(':')
+      connection = Connection.new({
+        host: host,
+        port: port
+      }.merge(options))
+      @connections[nsqd] = connection
+    end
+
+
+    def drop_connection(nsqd)
+      info "- Dropping connection #{nsqd}"
+      connection = @connections.delete(nsqd)
+      connection.close if connection
+      connections_changed
+    end
+
+
+    def drop_all_connections
+      @connections.keys.each do |nsqd|
+        drop_connection(nsqd)
+      end
+    end
+
+
+    # optional subclass hook
+    def connections_changed
+    end
+  end
+end

data/lib/nsq/consumer.rb

@@ -1,16 +1,9 @@
-require_relative 'connection'
-require_relative 'discovery'
-require_relative 'logger'
+require_relative 'client_base'
 
 module Nsq
-  class Consumer
-    include Nsq::AttributeLogger
-    @@log_attributes = [:topic]
+  class Consumer < ClientBase
 
-    attr_reader :topic
     attr_reader :max_in_flight
-    attr_reader :discovery_interval
-    attr_reader :connections
 
     def initialize(opts = {})
       if opts[:nsqlookupd]
@@ -34,24 +27,21 @@ module Nsq
       @connections = {}
 
       if !@nsqlookupds.empty?
-        @discovery = Discovery.new(@nsqlookupds)
-        discover_repeatedly
+        discover_repeatedly(
+          nsqlookupds: @nsqlookupds,
+          topic: @topic,
+          interval: @discovery_interval
+        )
       else
         # normally, we find nsqd instances to connect to via nsqlookupd(s)
         # in this case let's connect to an nsqd instance directly
-        add_connection(opts[:nsqd] || '127.0.0.1:4150', @max_in_flight)
+        add_connection(opts[:nsqd] || '127.0.0.1:4150', max_in_flight: @max_in_flight)
       end
 
       at_exit{terminate}
     end
 
 
-    def terminate
-      @discovery_thread.kill if @discovery_thread
-      drop_all_connections
-    end
-
-
     # pop the next message off the queue
     def pop
       @messages.pop
@@ -65,82 +55,30 @@ module Nsq
 
 
     private
-    def discover_repeatedly
-      @discovery_thread = Thread.new do
-        loop do
-          discover
-          sleep @discovery_interval
-        end
-      end
-      @discovery_thread.abort_on_exception = true
-    end
-
-
-    def discover
-      nsqds = @discovery.nsqds_for_topic(@topic)
-
-      # drop nsqd connections that are no longer in lookupd
-      missing_nsqds = @connections.keys - nsqds
-      missing_nsqds.each do |nsqd|
-        drop_connection(nsqd)
-      end
-
-      # add new ones
-      new_nsqds = nsqds - @connections.keys
-      new_nsqds.each do |nsqd|
-        # Be conservative and start new connections with RDY 1
-        # This helps ensure we don't exceed @max_in_flight across all our
-        # connections momentarily.
-        add_connection(nsqd, 1)
-      end
-
-      # balance RDY state amongst the connections
-      redistribute_ready
-    end
-
-
-    def add_connection(nsqd, max_in_flight)
-      info "+ Adding connection #{nsqd}"
-      host, port = nsqd.split(':')
-      connection = Connection.new(
-        host: host,
-        port: port,
+    def add_connection(nsqd, options = {})
+      super(nsqd, {
         topic: @topic,
         channel: @channel,
         queue: @messages,
         msg_timeout: @msg_timeout,
-        max_in_flight: max_in_flight
-      )
-      @connections[nsqd] = connection
+        max_in_flight: 1
+      }.merge(options))
     end
 
+    # Be conservative, but don't set a connection's max_in_flight below 1
+    def max_in_flight_per_connection(number_of_connections = @connections.length)
+      [@max_in_flight / number_of_connections, 1].max
+    end
 
-    def drop_connection(nsqd)
-      info "- Dropping connection #{nsqd}"
-      connection = @connections.delete(nsqd)
-      connection.close
+    def connections_changed
       redistribute_ready
     end
 
-
     def redistribute_ready
       @connections.values.each do |connection|
         connection.max_in_flight = max_in_flight_per_connection
         connection.re_up_ready
       end
     end
-
-
-    def drop_all_connections
-      @connections.keys.each do |nsqd|
-        drop_connection(nsqd)
-      end
-    end
-
-
-    # Be conservative, but don't set a connection's max_in_flight below 1
-    def max_in_flight_per_connection(number_of_connections = @connections.length)
-      [@max_in_flight / number_of_connections, 1].max
-    end
   end
 end

data/lib/nsq/discovery.rb

@@ -14,27 +14,47 @@ module Nsq
       @lookupds = lookupds
     end
 
-    # Given a topic, returns an array of nsqds instances that have messages for
+    # Returns an array of nsqds instances
+    #
+    # nsqd instances returned are strings in this format: '<host>:<tcp-port>'
+    #
+    #     discovery.nsqds
+    #     #=> ['127.0.0.1:4150', '127.0.0.1:4152']
+    #
+    def nsqds
+      @lookupds.map do |lookupd|
+        get_nsqds(lookupd)
+      end.flatten.uniq
+    end
+
+    # Returns an array of nsqds instances that have messages for
     # that topic.
     #
     # nsqd instances returned are strings in this format: '<host>:<tcp-port>'
     #
-    #     discovery.nsqds_for_topic('some-topic')
+    #     discovery.nsqds_for_topic('a-topic')
     #     #=> ['127.0.0.1:4150', '127.0.0.1:4152']
     #
     def nsqds_for_topic(topic)
       @lookupds.map do |lookupd|
-        get_nsqds_for_topic(lookupd, topic)
+        get_nsqds(lookupd, topic)
       end.flatten.uniq
     end
 
-
     private
 
-    def get_nsqds_for_topic(lookupd, topic)
-      uri = URI.parse("http://#{lookupd}")
-      uri.path = '/lookup'
-      uri.query = "topic=#{topic}&ts=#{Time.now.to_i}"
+    def get_nsqds(lookupd, topic = nil)
+      uri_scheme = 'http://' unless lookupd.match(%r(https?://))
+      uri = URI.parse("#{uri_scheme}#{lookupd}")
+
+      uri.query = "ts=#{Time.now.to_i}"
+      if topic
+        uri.path = '/lookup'
+        uri.query += "&topic=#{topic}"
+      else
+        uri.path = '/nodes'
+      end
+
       begin
         body = Net::HTTP.get(uri)
         data = JSON.parse(body)

data/lib/nsq/producer.rb

@@ -1,47 +1,62 @@
-require_relative 'connection'
-require_relative 'logger'
+require_relative 'client_base'
 
 module Nsq
-  class Producer
-    include Nsq::AttributeLogger
-    @@log_attributes = [:host, :port, :topic]
-
-    attr_reader :host
-    attr_reader :port
+  class Producer < ClientBase
     attr_reader :topic
 
-
     def initialize(opts = {})
-      @nsqd = opts[:nsqd] || '127.0.0.1:4150'
-      @host, @port = @nsqd.split(':')
-
+      @connections = {}
       @topic = opts[:topic] || raise(ArgumentError, 'topic is required')
+      @discovery_interval = opts[:discovery_interval] || 60
+
+      nsqlookupds = []
+      if opts[:nsqlookupd]
+        nsqlookupds = [opts[:nsqlookupd]].flatten
+        discover_repeatedly(
+          nsqlookupds: nsqlookupds,
+          interval: @discovery_interval
+        )
+
+      elsif opts[:nsqd]
+        nsqds = [opts[:nsqd]].flatten
+        nsqds.each{|d| add_connection(d)}
 
-      @connection = Connection.new(host: @host, port: @port)
+      else
+        add_connection('127.0.0.1:4150')
+      end
 
       at_exit{terminate}
     end
 
 
     def write(*raw_messages)
-      # stringify them
+      # stringify the messages
       messages = raw_messages.map(&:to_s)
 
+      # get a suitable connection to write to
+      connection = connection_for_write
+
       if messages.length > 1
-        @connection.mpub(@topic, messages)
+        connection.mpub(@topic, messages)
       else
-        @connection.pub(@topic, messages.first)
+        connection.pub(@topic, messages.first)
       end
     end
 
 
-    def connected?
-      @connection.connected?
-    end
+    private
+    def connection_for_write
+      # Choose a random Connection that's currently connected
+      # Or, if there's nothing connected, just take any random one
+      connections_currently_connected = connections.select{|_,c| c.connected?}
+      connection = connections_currently_connected.values.sample || connections.values.sample
 
+      # Raise an exception if there's no connection available
+      unless connection
+        raise 'No connections available'
+      end
 
-    def terminate
-      @connection.close
+      connection
     end
 
   end

data/lib/version.rb

@@ -1,7 +1,7 @@
 module Nsq
   module Version
     MAJOR = 0
-    MINOR = 1
+    MINOR = 2
     PATCH = 0
     BUILD = nil
     STRING = [MAJOR, MINOR, PATCH, BUILD].compact.join('.')

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nsq-ruby
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Wistia
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-28 00:00:00.000000000 Z
+date: 2014-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.7
+        version: 1.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.7
+        version: 1.1.0
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -77,6 +77,7 @@ files:
 - LICENSE.txt
 - README.md
 - lib/nsq.rb
+- lib/nsq/client_base.rb
 - lib/nsq/connection.rb
 - lib/nsq/consumer.rb
 - lib/nsq/discovery.rb