fastly_nsq 1.16.0 → 1.18.0

data/lib/fastly_nsq/priority_thread_pool.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class FastlyNsq::PriorityThreadPool < Concurrent::ThreadPoolExecutor
-  alias max_threads max_length
+  alias_method :max_threads, :max_length
 
   def initialize(*)
     super
@@ -15,7 +15,7 @@ class FastlyNsq::PriorityThreadPool < Concurrent::ThreadPoolExecutor
   # @!visibility private
   def ns_enqueue(*args, &task)
     if !ns_limited_queue? || @queue.size < @max_queue
-      @queue.push([task, args[1..-1]], args[0])
+      @queue.push([task, args[1..]], args[0])
       true
     else
       false
@@ -27,6 +27,6 @@ class FastlyNsq::PriorityThreadPool < Concurrent::ThreadPoolExecutor
   #
   # @!visibility private
   def ns_assign_worker(*args, &task)
-    super(args[1..-1], &task)
+    super(args[1..], &task)
   end
 end

data/lib/fastly_nsq/producer.rb

@@ -24,6 +24,9 @@ class FastlyNsq::Producer
   ##
   # Create a FastlyNsq::Producer
   #
+  # Will connect to NSQDs in this priority: 1. direct from FastlyNsq.producer_nsqds 2. discovered via FastlyNsq.lookupd_http_addresses.
+  # If both `producer_nsqds` and `lookupd_http_addresses` are set only the FastlyNsq.producer_nsqds will be used.
+  #
   # @param topic [String] NSQ topic on which to deliver the message
   # @param tls_options [Hash] Hash of TSL options passed the connection.
   #   In most cases this should be nil unless you need to override the
@@ -31,10 +34,10 @@ class FastlyNsq::Producer
   # @param logger [Logger] defaults to FastlyNsq.logger
   # @param connect_timeout [Integer] NSQ connection timeout in seconds
   def initialize(topic:, tls_options: nil, logger: FastlyNsq.logger, connect_timeout: DEFAULT_CONNECTION_TIMEOUT)
-    @topic           = topic
-    @tls_options     = FastlyNsq::TlsOptions.as_hash(tls_options)
+    @topic = topic
+    @tls_options = FastlyNsq::TlsOptions.as_hash(tls_options)
     @connect_timeout = connect_timeout
-    @logger          = logger
+    @logger = logger
 
     connect
   end
@@ -71,27 +74,33 @@ class FastlyNsq::Producer
   # Create an Nsq::Producer and set as +@connection+ instance variable
   # @return [Boolean]
   def connect
+    producers = FastlyNsq.producer_nsqds
     lookupd = FastlyNsq.lookupd_http_addresses
 
-    @connection ||= Nsq::Producer.new(
-      tls_options.merge(
-        nsqlookupd:  lookupd,
-        topic:       topic,
-      ),
-    )
-
-    timeout_args = [connect_timeout, FastlyNsq::ConnectionFailed]
+    opts = tls_options.merge(topic: topic)
 
-    if RUBY_VERSION > '2.4.0'
-      timeout_args << "Failed connection to #{lookupd} within #{connect_timeout} seconds"
+    if !producers.empty?
+      opts[:nsqd] = producers
+    elsif !lookupd.empty?
+      opts[:nsqlookupd] = lookupd
+    else
+      raise FastlyNsq::ConnectionFailed, "One of FastlyNsq.producer_nsqds or FastlyNsq.lookupd_http_addresses must be present"
     end
 
+    @connection ||= Nsq::Producer.new(opts)
+
+    timeout_args = [
+      connect_timeout,
+      FastlyNsq::ConnectionFailed,
+      "Failed connection to #{opts[:nsqd] || opts[:nsqlookupd]} within #{connect_timeout} seconds"
+    ]
+
     Timeout.timeout(*timeout_args) { Thread.pass until connection.connected? }
 
     true
   rescue FastlyNsq::ConnectionFailed
     logger.error { "Producer for #{topic} failed to connect!" }
-    terminate
+    terminate if @connection
     raise
   end
 

data/lib/fastly_nsq/safe_thread.rb

@@ -3,7 +3,7 @@
 module FastlyNsq::SafeThread
   def safe_thread(name, &block)
     Thread.new do
-      Thread.current['fastly_nsq_label'] = name
+      Thread.current["fastly_nsq_label"] = name
       watchdog(name, &block)
     end
   end

data/lib/fastly_nsq/testing.rb

@@ -98,7 +98,7 @@ module FastlyNsq
       #   processor_klass.call(test_message)
       #   expect(Post.find(post_data['id']).not_to be nil
       def message(data:, meta: nil)
-        test_message = FastlyNsq::TestMessage.new(JSON.dump('data' => data, 'meta' => meta))
+        test_message = FastlyNsq::TestMessage.new(JSON.dump("data" => data, "meta" => meta))
         FastlyNsq::Message.new(test_message)
       end
     end
@@ -121,7 +121,7 @@ module FastlyNsq
 
     def initialize(raw_body)
       @raw_body = raw_body
-      @id       = Digest::SHA1.hexdigest(raw_body.to_s + Time.now.to_s)
+      @id = Digest::SHA1.hexdigest(raw_body.to_s + Time.now.to_s)
       @attempts = 0
     end
 
@@ -250,7 +250,8 @@ module FastlyNsq
   FastlyNsq::Listener.prepend(ListenerTesting)
 
   class FakeConnection
-    def connected?; end
+    def connected?
+    end
   end
 
   module ConsumerTesting

data/lib/fastly_nsq/tls_options.rb

@@ -20,7 +20,7 @@ module FastlyNsq
       else
         {
           tls_v1: true,
-          tls_options: @context,
+          tls_options: @context
         }
       end
     end
@@ -28,19 +28,19 @@ module FastlyNsq
     private
 
     def env_key
-      ENV.fetch('NSQ_SSL_KEY', nil)
+      ENV.fetch("NSQ_SSL_KEY", nil)
     end
 
     def env_certificate
-      ENV.fetch('NSQ_SSL_CERTIFICATE', nil)
+      ENV.fetch("NSQ_SSL_CERTIFICATE", nil)
     end
 
     def env_ca_certificate
-      ENV.fetch('NSQ_SSL_CA_CERTIFICATE', nil)
+      ENV.fetch("NSQ_SSL_CA_CERTIFICATE", nil)
     end
 
     def verify_mode
-      ENV.fetch('NSQ_SSL_VERIFY_MODE', nil)
+      ENV.fetch("NSQ_SSL_VERIFY_MODE", nil)
     end
 
     def env_default_hash
@@ -48,7 +48,7 @@ module FastlyNsq
         key: env_key,
         certificate: env_certificate,
         ca_certificate: env_ca_certificate,
-        verify_mode: verify_mode,
+        verify_mode: verify_mode
       }
     end
 

data/lib/fastly_nsq/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FastlyNsq
-  VERSION = '1.16.0'
+  VERSION = "1.18.0"
 end

data/lib/fastly_nsq.rb

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
-require 'nsq'
-require 'concurrent'
-require 'fc'
-require 'set'
-require 'logger'
-require 'forwardable'
-require 'digest/md5'
+require "nsq"
+require "concurrent"
+require "fc"
+require "set"
+require "logger"
+require "forwardable"
+require "digest/md5"
 
 module FastlyNsq
   NotConnectedError = Class.new(StandardError)
@@ -58,7 +58,7 @@ module FastlyNsq
     def logger
       return @logger if @logger
 
-      self.logger = Logger.new(STDERR)
+      self.logger = Logger.new($stderr)
     end
 
     ##
@@ -112,21 +112,58 @@ module FastlyNsq
     # @return [Integer]
     # @see https://nsq.io/components/nsqd.html#command-line-options
     def max_req_timeout
-      @max_req_timeout ||= ENV.fetch('MAX_REQ_TIMEOUT', 60 * 60 * 1_000).to_i
+      @max_req_timeout ||= ENV.fetch("MAX_REQ_TIMEOUT", 60 * 60 * 1_000).to_i
     end
 
     # Maximum number of threads for FastlyNsq::PriorityThreadPool
     # Default setting is 5 and can be set via ENV['MAX_PROCESSING_POOL_THREADS']
     # @return [Integer]
     def max_processing_pool_threads
-      @max_processing_pool_threads ||= ENV.fetch('MAX_PROCESSING_POOL_THREADS', 5).to_i
+      @max_processing_pool_threads ||= ENV.fetch("MAX_PROCESSING_POOL_THREADS", 5).to_i
     end
 
     ##
     # Return an array of NSQ lookupd http addresses sourced from ENV['NSQLOOKUPD_HTTP_ADDRESS']
     # @return [Array<String>] list of nsqlookupd http addresses
     def lookupd_http_addresses
-      ENV.fetch('NSQLOOKUPD_HTTP_ADDRESS').split(',').map(&:strip)
+      @lookups ||= ENV.fetch("NSQLOOKUPD_HTTP_ADDRESS", "").split(/, ?|\s+/).map(&:strip)
+    end
+
+    ##
+    # Set the lookupd_http_addresses
+    # @param lookups [Array] List of http lookupd addresses to use.
+    def lookupd_http_addresses=(lookups)
+      @lookups = lookups.nil? ? nil : Array(lookups)
+    end
+
+    ##
+    # Return an array of NSQD TCP addresses for NSQ consumers. Defaults to the value of ENV['NSQD_CONSUMERS'].
+    # ENV['NSQD_CONSUMERS'] must be a comma or space seperated string of NSQD addresses
+    # @return [Array<String>] list of nsqd addresses
+    def consumer_nsqds
+      @consumer_nsqds ||= ENV.fetch("NSQD_CONSUMERS", "").split(/, ?|\s+/).map(&:strip)
+    end
+
+    ##
+    # Set the consumer_nsqd addresses
+    # @param nsqd_addresses [Array] List of consumer nsqd addresses to use
+    def consumer_nsqds=(nsqd_addresses)
+      @consumer_nsqds = nsqd_addresses.nil? ? nil : Array(nsqd_addresses)
+    end
+
+    ##
+    # Return an array of NSQD TCP addresses for NSQ producers. Defaults to the value of ENV['NSQD_PRODUCERS'].
+    # ENV['NSQD_PRODUCERS'] must be a comma or space seperated string of NSQD addresses
+    # @return [Array<String>] list of nsqd addresses
+    def producer_nsqds
+      @producer_nsqds ||= ENV.fetch("NSQD_PRODUCERS", "").split(/, ?|\s+/).map(&:strip)
+    end
+
+    ##
+    # Set the producer_nsqd addresses
+    # @param nsqd_addresses [Array] List of producer nsqd addresses to use
+    def producer_nsqds=(nsqd_addresses)
+      @producer_nsqds = nsqd_addresses.nil? ? nil : Array(nsqd_addresses)
     end
 
     # Register a block to run at a point in the lifecycle.
@@ -151,11 +188,9 @@ module FastlyNsq
     def fire_event(event)
       blocks = FastlyNsq.events.fetch(event)
       blocks.each do |block|
-        begin
-          block.call
-        rescue => e
-          logger.error "[#{event}] #{e.inspect}"
-        end
+        block.call
+      rescue => e
+        logger.error "[#{event}] #{e.inspect}"
       end
     end
 
@@ -168,16 +203,16 @@ module FastlyNsq
   end
 end
 
-require 'fastly_nsq/consumer'
-require 'fastly_nsq/feeder'
-require 'fastly_nsq/launcher'
-require 'fastly_nsq/listener'
-require 'fastly_nsq/manager'
-require 'fastly_nsq/message'
-require 'fastly_nsq/messenger'
-require 'fastly_nsq/new_relic'
-require 'fastly_nsq/priority_queue'
-require 'fastly_nsq/priority_thread_pool'
-require 'fastly_nsq/producer'
-require 'fastly_nsq/tls_options'
-require 'fastly_nsq/version'
+require "fastly_nsq/consumer"
+require "fastly_nsq/feeder"
+require "fastly_nsq/launcher"
+require "fastly_nsq/listener"
+require "fastly_nsq/manager"
+require "fastly_nsq/message"
+require "fastly_nsq/messenger"
+require "fastly_nsq/new_relic"
+require "fastly_nsq/priority_queue"
+require "fastly_nsq/priority_thread_pool"
+require "fastly_nsq/producer"
+require "fastly_nsq/tls_options"
+require "fastly_nsq/version"

data/spec/cli_spec.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require 'spec_helper'
-require 'fastly_nsq/cli'
+require "spec_helper"
+require "fastly_nsq/cli"
 
 RSpec.describe FastlyNsq::CLI do
 end

data/spec/consumer_spec.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
-require 'spec_helper'
+require "spec_helper"
 
 RSpec.describe FastlyNsq::Consumer do
-  let!(:topic)   { 'fnsq' }
-  let!(:channel) { 'fnsq' }
-  let!(:queue)   { nil }
+  let!(:topic) { "fnsq" }
+  let!(:channel) { "fnsq" }
+  let!(:queue) { nil }
 
   subject { described_class.new(topic: topic, channel: channel, queue: queue) }
 
@@ -16,15 +16,15 @@ RSpec.describe FastlyNsq::Consumer do
 
   it { should be_connected }
 
-  it 'should terminate' do
+  it "should terminate" do
     expect { subject.terminate }.to change(subject, :connected?).to(false)
   end
 
-  describe 'with a specified queue' do
+  describe "with a specified queue" do
     let!(:queue) { Queue.new }
 
-    it 'passes #queue to Nsq::Consumer' do
-      message = 'foo'
+    it "passes #queue to Nsq::Consumer" do
+      message = "foo"
 
       FastlyNsq::Messenger.deliver(message: message, topic: topic)
 
@@ -41,19 +41,60 @@ RSpec.describe FastlyNsq::Consumer do
   it { should delegate(:pop).to(:connection) }
   it { should delegate(:pop_without_blocking).to(:connection) }
 
-  describe 'with a message' do
-    let(:message) { 'foo' }
+  describe "with a message" do
+    let(:message) { "foo" }
 
     before { FastlyNsq::Messenger.deliver(message: message, topic: topic) }
 
-    it 'should not be empty' do
+    it "should not be empty" do
       expect { subject }.to eventually(be_empty).within(15)
     end
 
-    describe 'that has finished' do
+    describe "that has finished" do
       before { subject.pop.finish }
 
       it { should be_empty }
     end
   end
+
+  describe "connection priority" do
+    after do
+      FastlyNsq.lookupd_http_addresses = nil
+      FastlyNsq.consumer_nsqds = nil
+    end
+
+    it "connects to consumer_nsqds if provided" do
+      allow(Nsq::Consumer).to receive(:new)
+
+      expect(FastlyNsq.lookupd_http_addresses).not_to be_empty
+      expect(FastlyNsq.consumer_nsqds).not_to be_empty
+
+      FastlyNsq::Consumer.new(topic: topic, channel: channel)
+      expect(Nsq::Consumer).to have_received(:new).with a_hash_including(nsqd: FastlyNsq.consumer_nsqds).and(excluding(:nsqlookupd))
+    end
+
+    it "connects to lookupd_http_addresses if consumer_nsqds is empty" do
+      FastlyNsq.consumer_nsqds = []
+      allow(Nsq::Consumer).to receive(:new)
+
+      expect(FastlyNsq.lookupd_http_addresses).not_to be_empty
+      expect(FastlyNsq.consumer_nsqds).to be_empty
+
+      FastlyNsq::Consumer.new(topic: topic, channel: channel)
+      expect(Nsq::Consumer).to have_received(:new).with a_hash_including(nsqlookupd: FastlyNsq.lookupd_http_addresses).and(excluding(:nsqd))
+    end
+
+    it "raises when neither consumer_nsqds or lookupd_http_addresses are available" do
+      FastlyNsq.consumer_nsqds = []
+      FastlyNsq.lookupd_http_addresses = []
+      allow(Nsq::Consumer).to receive(:new)
+
+      expect(FastlyNsq.lookupd_http_addresses).to be_empty
+      expect(FastlyNsq.consumer_nsqds).to be_empty
+
+      expect { FastlyNsq::Consumer.new(topic: topic, channel: channel) }
+        .to raise_error(FastlyNsq::ConnectionFailed, "One of FastlyNsq.consumer_nsqds or FastlyNsq.lookupd_http_addresses must be present")
+      expect(Nsq::Consumer).not_to have_received(:new)
+    end
+  end
 end

data/spec/fastly_nsq_spec.rb

@@ -1,89 +1,93 @@
 # frozen_string_literal: true
 
-require 'spec_helper'
+require "spec_helper"
 
 RSpec.describe FastlyNsq do
-  describe '#configure' do
+  describe "#configure" do
     specify { expect { |b| described_class.configure(&b) }.to yield_with_args(described_class) }
   end
 
-  describe '#listen' do
+  describe "#listen" do
     let!(:default_channel) { subject.channel }
-    let!(:topic) { 'fnsq' }
+    let!(:topic) { "fnsq" }
 
-    before { subject.channel = 'fnsq' }
+    before { subject.channel = "fnsq" }
     after { subject.channel = default_channel }
 
-    it 'creates a listener' do
+    it "creates a listener" do
       expect { subject.listen topic, ->(*) {} }.to change { subject.manager.topics }.to([topic])
     end
 
-    it 'creates a listener with a specific priority' do
+    it "creates a listener with a specific priority" do
       listener = subject.listen topic, ->(*) {}, priority: 10
       expect(listener.priority).to eq(10)
     end
   end
 
-  describe '#channel=' do
+  describe "#channel=" do
     let!(:default_channel) { subject.channel }
     after { subject.channel = default_channel }
 
-    it 'allows the channel to be set and retrieved' do
+    it "allows the channel to be set and retrieved" do
       expect(subject.channel).to be_nil
-      subject.channel = 'foo'
-      expect(subject.channel).to eq('foo')
+      subject.channel = "foo"
+      expect(subject.channel).to eq("foo")
     end
   end
 
-  describe '#logger' do
+  describe "#logger" do
     let!(:default_logger) { subject.logger }
-    after { subject.logger = default_logger }
+    after do
+      subject.logger = default_logger
+      $stderr = STDERR
+    end
 
-    it 'returns the set logger' do
+    it "returns the set logger" do
       logger = Logger.new(nil)
       subject.logger = logger
 
       expect(subject.logger).to eq logger
     end
 
-    it 'sets the default logger if none is set' do
+    it "sets the default logger if none is set" do
+      $stderr = File.new("/dev/null", "w")
       subject.instance_variable_set(:@logger, nil)
       expect(subject.instance_variable_get(:@logger)).to be nil
       logger = subject.logger
 
       expect(logger).to be_instance_of(Logger)
-      expect(logger.instance_variable_get(:@logdev).dev).to eq(STDERR)
+      expect(logger.instance_variable_get(:@logdev).dev).to eq($stderr)
       expect(logger).to eq(Nsq.logger)
     end
   end
 
-  describe '#logger=' do
+  describe "#logger=" do
     let!(:default_logger) { subject.logger }
     after { subject.logger = default_logger }
 
-    it 'allows the logger to be set and retrieved' do
-      logger = Logger.new(STDOUT)
+    it "allows the logger to be set and retrieved" do
+      logger = Logger.new("/dev/null")
       subject.logger = logger
 
       expect(subject.logger).to eq logger
     end
 
-    it 'sets Nsq.logger' do
-      logger = Logger.new(STDOUT)
+    it "sets Nsq.logger" do
+      logger = Logger.new("/dev/null")
       subject.logger = logger
 
       expect(Nsq.logger).to eq logger
     end
   end
 
-  describe '#manager' do
-    it 'represents the active default manager' do
+  describe "#manager" do
+    it "represents the active default manager" do
       expect(subject.manager).not_to be_stopped
     end
   end
 
-  describe '#manager=' do
-    it 'transfers to specified manager' do
+  describe "#manager=" do
+    it "transfers to specified manager" do
       old_manager = subject.manager
       new_manager = FastlyNsq::Manager.new
 
@@ -93,17 +97,89 @@ RSpec.describe FastlyNsq do
     end
   end
 
-  describe '#lookupd_http_addresses' do
-    it 'retreives NSQLOOKUPD_HTTP_ADDRESS' do
-      expect(subject.lookupd_http_addresses).to eq(ENV['NSQLOOKUPD_HTTP_ADDRESS'].split(','))
+  describe "#lookupd_http_addresses" do
+    after { subject.instance_variable_set(:@lookups, nil) }
+
+    it "retreives NSQLOOKUPD_HTTP_ADDRESS by default" do
+      expect(subject.lookupd_http_addresses).to eq(ENV["NSQLOOKUPD_HTTP_ADDRESS"].split(","))
+    end
+
+    it "returns the value of the instance variable" do
+      subject.instance_variable_set(:@lookups, ["lolcathost"])
+
+      expect(subject.lookupd_http_addresses).to eq(["lolcathost"])
+    end
+  end
+
+  describe "#lookupd_http_addresses=" do
+    let!(:default_loookups) { subject.lookupd_http_addresses }
+    after { subject.lookupd_http_addresses = default_loookups }
+
+    it "allows the lookups to be set and retrieved" do
+      lookups = ["lolcathost:1234"]
+      subject.lookupd_http_addresses = lookups
+
+      expect(subject.lookupd_http_addresses).to eq lookups
+    end
+  end
+
+  describe "#producer_nsqds" do
+    after { subject.instance_variable_set(:@producer_nsqds, nil) }
+
+    it "retreives NSQD_PRODUCERS by default" do
+      expect(subject.producer_nsqds).to eq(ENV["NSQD_PRODUCERS"].split(","))
+    end
+
+    it "returns the value of the instance variable" do
+      subject.instance_variable_set(:@producer_nsqds, ["producer:1234"])
+
+      expect(subject.producer_nsqds).to eq(["producer:1234"])
+    end
+  end
+
+  describe "#producer_nsqds=" do
+    let!(:default_producers) { subject.producer_nsqds }
+    after { subject.producer_nsqds = default_producers }
+
+    it "allows the producer_nsqds to be set and retrieved" do
+      producers = ["producer:1234"]
+      subject.producer_nsqds = producers
+
+      expect(subject.producer_nsqds).to eq producers
+    end
+  end
+
+  describe "#consumer_nsqds" do
+    after { subject.instance_variable_set(:@consumer_nsqds, nil) }
+
+    it "retreives NSQD_CONSUMERS by default" do
+      expect(subject.consumer_nsqds).to eq(ENV["NSQD_CONSUMERS"].split(","))
+    end
+
+    it "returns the value of the instance variable" do
+      subject.instance_variable_set(:@consumer_nsqds, ["consumer:1234"])
+
+      expect(subject.consumer_nsqds).to eq(["consumer:1234"])
+    end
+  end
+
+  describe "#consumer_nsqds=" do
+    let!(:default_consumers) { subject.consumer_nsqds }
+    after { subject.consumer_nsqds = default_consumers }
+
+    it "allows the consumer_nsqds to be set and retrieved" do
+      consumers = ["consumer:1234"]
+      subject.consumer_nsqds = consumers
+
+      expect(subject.consumer_nsqds).to eq consumers
     end
   end
 
-  describe '#on' do
+  describe "#on" do
     before { FastlyNsq.events.each { |(_, v)| v.clear } }
-    after  { FastlyNsq.events.each { |(_, v)| v.clear } }
+    after { FastlyNsq.events.each { |(_, v)| v.clear } }
 
-    it 'registers callbacks for events' do
+    it "registers callbacks for events" do
       %i[startup shutdown heartbeat].each do |event|
         block = -> {}
         FastlyNsq.on(event, &block)
@@ -111,7 +187,7 @@ RSpec.describe FastlyNsq do
       end
     end
 
-    it 'limits callback registration to valid events' do
+    it "limits callback registration to valid events" do
       expect { FastlyNsq.on(:foo, &-> {}) }.to raise_error(ArgumentError, /Invalid event name/)
     end
   end

data/spec/feeder_spec.rb

@@ -1,14 +1,14 @@
 # frozen_string_literal: true
 
-require 'spec_helper'
+require "spec_helper"
 
 RSpec.describe FastlyNsq::Feeder do
-  describe '#push' do
-    it 'sends message to processor with the specified priority' do
+  describe "#push" do
+    it "sends message to processor with the specified priority" do
       messages = []
       processor = ->(m) { messages << m }
       priority = 5
-      message = 'foo'
+      message = "foo"
 
       feeder = described_class.new(processor, priority)