fastly_nsq 0.6.0 → 0.7.0

data/lib/fastly_nsq/rake_task.rb

@@ -1,9 +1,10 @@
 require 'rake'
 require 'rake/tasklib'
 
-module MessageQueue
+module FastlyNsq
   class RakeTask < Rake::TaskLib
-    attr_accessor :name, :channel
+    attr_accessor :name, :channel, :topics, :preprocessor
+    attr_writer :listener, :logger
 
     def initialize(*args, &task_block)
       @name = args.shift || :begin_listening
@@ -15,68 +16,63 @@ module MessageQueue
             yield(*[self, task_args].slice(0, task_block.arity))
           end
 
-          if task_args[:channel]
-            @channel = task_args[:channel]
-          end
-
-          guard_missing_channel
-          run_tasks
+          initialize_values task_args
+          listen_to_configured_topics
         end
       end
     end
 
     private
 
-    def add_rake_task_description_if_one_needed
-      unless ::Rake.application.last_description
-        desc 'Listen to NSQ on topic using channel'
+    def listen_to_configured_topics
+      topic_per_thread do |topic, processor|
+        logger.info "Listening to queue, topic:'#{topic}' and channel: '#{channel}'"
+        listener.listen_to topic:        topic,
+                           channel:      channel,
+                           logger:       logger,
+                           processor:    processor,
+                           preprocessor: preprocessor
+        logger.info "... done listening on topic:'#{topic}' and channel: '#{channel}'."
       end
     end
 
-    def run_tasks
-      topics.each do |topic|
-        Thread.new do
-          wrap_helpful_output(topic) do
-            MessageQueue::Listener.new(topic: topic, channel: channel).go
-          end
-        end
-      end
-
-      non_main_threads.map(&:join)
+    def require_arg(arg, arg_list)
+      arg_list.fetch(arg) { raise ArgumentError, "required configuration '#{arg}' is missing." }
     end
 
-    def non_main_threads
-      (Thread.list - [Thread.main])
+    def add_rake_task_description_if_one_needed
+      unless ::Rake.application.last_description
+        desc 'Listen to NSQ on topic using channel'
+      end
     end
 
-    def guard_missing_channel
-      unless channel
-        raise ArgumentError, "channel is required. Received channel: #{channel}"
+    def topic_per_thread
+      listener_threads = []
+      topics.each do |(topic, processor)|
+        thread = Thread.new do
+          yield topic, processor
+        end
+        thread.abort_on_exception = true
+        listener_threads << thread
       end
-    end
 
-    def wrap_helpful_output(topic)
-      output "Listening to queue, topic:'#{topic}' and channel: '#{channel}'"
-      yield
-      output "... done listening on topic:'#{topic}' and channel: '#{channel}'."
+      listener_threads.map(&:join)
     end
 
-    def topics
-      MessageProcessor.topics
-    rescue NoMethodError => exception
-      if exception.message =~ /undefined method `topics'/
-        raise ArgumentError, 'MessageProcessor.topics is not defined.'
-      else
-        raise exception
-      end
+    def listener
+      @listener || FastlyNsq::Listener
     end
 
-    def output(string)
-      logger.info(string)
+    def logger
+      @logger || FastlyNsq.logger || Logger.new(STDOUT)
     end
 
-    def logger
-      MessageQueue.logger = Logger.new(STDOUT)
+    def initialize_values(task_args)
+      @channel      ||= require_arg :channel, task_args
+      @topics       ||= require_arg :topics, task_args
+      @listener     ||= task_args[:listener]
+      @logger       ||= task_args[:logger]
+      @preprocessor ||= task_args[:preprocessor]
     end
   end
 end

data/lib/fastly_nsq/ssl_context.rb

@@ -1,44 +1,46 @@
-class SSLContext
-  def initialize(context = nil)
-    @context = context || {}
-  end
+module FastlyNsq
+  class SSLContext
+    def initialize(context = nil)
+      @context = context || {}
+    end
 
-  def to_h
-    merge_contexts
-    if empty_context?
-      nil
-    else
-      @context
+    def to_h
+      merge_contexts
+      if empty_context?
+        nil
+      else
+        @context
+      end
     end
-  end
 
-  private
+    private
 
-  def env_key
-    ENV.fetch('NSQ_SSL_KEY', nil)
-  end
+    def env_key
+      ENV.fetch('NSQ_SSL_KEY', nil)
+    end
 
-  def env_certificate
-    ENV.fetch('NSQ_SSL_CERTIFICATE', nil)
-  end
+    def env_certificate
+      ENV.fetch('NSQ_SSL_CERTIFICATE', nil)
+    end
 
-  def env_ca_certificate
-    ENV.fetch('NSQ_SSL_CA_CERTIFICATE', nil)
-  end
+    def env_ca_certificate
+      ENV.fetch('NSQ_SSL_CA_CERTIFICATE', nil)
+    end
 
-  def env_default_hash
-    {
-      key: env_key,
-      certificate: env_certificate,
-      ca_certificate: env_ca_certificate,
-    }
-  end
+    def env_default_hash
+      {
+        key: env_key,
+        certificate: env_certificate,
+        ca_certificate: env_ca_certificate,
+      }
+    end
 
-  def merge_contexts
-    @context = env_default_hash.merge(@context)
-  end
+    def merge_contexts
+      @context = env_default_hash.merge(@context)
+    end
 
-  def empty_context?
-    @context.all? { |_key, value| value.nil? }
+    def empty_context?
+      @context.all? { |_key, value| value.nil? }
+    end
   end
 end

data/lib/fastly_nsq/strategy.rb

@@ -0,0 +1,38 @@
+module FastlyNsq
+  module Strategy
+    class InvalidParameterError < StandardError; end
+
+    module_function
+
+    def for_queue
+      real_queue || fake_queue || error
+    end
+
+    private_class_method
+
+    ERR_MESSAGE = "You must set ENV['FAKE_QUEUE'] to either true or false".freeze
+
+    def error
+      raise InvalidParameterError, ERR_MESSAGE
+    end
+
+    FALSY_VALUES  = [false, 0, '0', 'false', 'FALSE', 'off', 'OFF', nil].freeze
+    TRUTHY_VALUES = [true, 1, '1', 'true', 'TRUE', 'on', 'ON'].freeze
+
+    def fake_queue
+      FastlyNsq::FakeBackend if should_use_fake_queue?
+    end
+
+    def should_use_real_queue?
+      FALSY_VALUES.include? ENV['FAKE_QUEUE']
+    end
+
+    def real_queue
+      Nsq if should_use_real_queue?
+    end
+
+    def should_use_fake_queue?
+      TRUTHY_VALUES.include? ENV['FAKE_QUEUE']
+    end
+  end
+end

data/lib/fastly_nsq/version.rb

@@ -1,3 +1,3 @@
 module FastlyNsq
-  VERSION = '0.6.0'.freeze
+  VERSION = '0.7.0'.freeze
 end

data/lib/fastly_nsq.rb

@@ -1,3 +1,23 @@
+require 'nsq'
+require 'fastly_nsq/consumer'
+require 'fastly_nsq/fake_backend'
+require 'fastly_nsq/listener'
+require 'fastly_nsq/message'
+require 'fastly_nsq/producer'
+require 'fastly_nsq/strategy'
+require 'fastly_nsq/ssl_context'
 require 'fastly_nsq/version'
 
-require 'fastly_nsq/message_queue'
+module FastlyNsq
+  def self.logger=(logger)
+    strategy.logger = logger
+  end
+
+  def self.logger
+    strategy.logger
+  end
+
+  def self.strategy
+    Strategy.for_queue
+  end
+end

data/spec/lib/fastly_nsq/consumer_spec.rb

@@ -0,0 +1,68 @@
+require 'spec_helper'
+
+RSpec.describe FastlyNsq::Consumer do
+  let(:channel)  { 'star_killer_base' }
+  let(:topic)    { 'death_star' }
+  let(:consumer) { FastlyNsq::Consumer.new topic: topic, channel: channel }
+
+  describe 'when connected to a backend Consumer' do
+    let(:backend)   { instance_double FastlyNsq::FakeBackend::Consumer, pop: nil, pop_without_blocking: nil, size: nil, terminate: nil }
+    let(:connector) { double 'Connector strategy', new: backend }
+
+    let(:consumer) do
+      FastlyNsq::Consumer.new topic: topic, channel: channel, connector: connector
+    end
+
+    it 'forwards #pop' do
+      expect(backend).to receive(:pop)
+      consumer.pop
+    end
+
+    it 'forwards #pop_without_blocking' do
+      expect(backend).to receive(:pop_without_blocking)
+      consumer.pop_without_blocking
+    end
+
+    it 'forwards #size' do
+      expect(backend).to receive(:size)
+      consumer.size
+    end
+
+    it 'forwards #terminate' do
+      expect(backend).to receive(:terminate)
+      consumer.terminate
+    end
+  end
+
+  describe 'using strategy to determine the consumer' do
+    module TestStrategy
+      module Consumer
+        @@never_terminated = true
+
+        module_function
+
+        def new(*_)
+          self
+        end
+
+        def terminate
+          raise 'Already terminated once' unless @@never_terminated
+          @@never_terminated = false
+        end
+
+        def was_terminated
+          !@@never_terminated
+        end
+      end
+    end
+
+    before do
+      allow(FastlyNsq).to receive(:strategy).and_return(TestStrategy)
+    end
+
+    it 'instantiates a consumer via Strategy' do
+      consumer.terminate
+      expect(TestStrategy::Consumer.was_terminated).to be_truthy
+    end
+  end
+end

data/spec/lib/fastly_nsq/{fake_message_queue_spec.rb → fake_backend_spec.rb}

@@ -1,49 +1,49 @@
 require 'spec_helper'
 
-RSpec.describe FakeMessageQueue do
+RSpec.describe FastlyNsq::FakeBackend do
   describe '@@queue' do
     it 'is initalized as an empty array' do
-      expect(FakeMessageQueue.queue).to eq []
+      expect(FastlyNsq::FakeBackend.queue).to eq []
     end
   end
 
   describe '@@logger' do
     after do
-      FakeMessageQueue.logger = Logger.new(nil)
+      FastlyNsq::FakeBackend.logger = Logger.new(nil)
     end
 
     it 'is initalized as an empty Ruby Logger' do
-      expect(FakeMessageQueue.logger).to be_a Logger
+      expect(FastlyNsq::FakeBackend.logger).to be_a Logger
     end
 
     it 'can be set and retrieved' do
       logger = double('some logger')
-      FakeMessageQueue.logger = logger
+      FastlyNsq::FakeBackend.logger = logger
 
-      expect(FakeMessageQueue.logger).to eq logger
+      expect(FastlyNsq::FakeBackend.logger).to eq logger
     end
   end
 
   describe '.reset!' do
     it 'resets the fake message queue' do
-      FakeMessageQueue.queue = ['hello']
-      expect(FakeMessageQueue.queue.size).to eq 1
+      FastlyNsq::FakeBackend.queue = ['hello']
+      expect(FastlyNsq::FakeBackend.queue.size).to eq 1
 
-      FakeMessageQueue.reset!
+      FastlyNsq::FakeBackend.reset!
 
-      expect(FakeMessageQueue.queue).to be_empty
+      expect(FastlyNsq::FakeBackend.queue).to be_empty
     end
   end
 end
 
-RSpec.describe FakeMessageQueue::Producer do
+RSpec.describe FastlyNsq::FakeBackend::Producer do
   let(:topic)    { 'death_star' }
-  let(:producer) { FakeMessageQueue::Producer.new topic: topic }
+  let(:producer) { FastlyNsq::FakeBackend::Producer.new topic: topic }
 
   it 'adds a new message to the queue' do
     producer.write('hello')
 
-    expect(FakeMessageQueue.queue.size).to eq 1
+    expect(FastlyNsq::FakeBackend.queue.size).to eq 1
   end
 
   it 'has a `terminate` method which is a noop' do
@@ -51,18 +51,18 @@ RSpec.describe FakeMessageQueue::Producer do
   end
 end
 
-RSpec.describe FakeMessageQueue::Message do
+RSpec.describe FastlyNsq::FakeBackend::Message do
   describe '#body' do
     it 'returns the body of the message' do
       topic = 'death_star'
       content = 'hello'
-      producer = FakeMessageQueue::Producer.new(
+      producer = FastlyNsq::FakeBackend::Producer.new(
         nsqd: ENV.fetch('NSQD_TCP_ADDRESS'),
         topic: topic,
       )
       producer.write(content)
 
-      message = FakeMessageQueue.queue.pop
+      message = FastlyNsq::FakeBackend.queue.pop
       body = message.body
 
       expect(content).to eq body
@@ -70,10 +70,10 @@ RSpec.describe FakeMessageQueue::Message do
   end
 end
 
-RSpec.describe FakeMessageQueue::Consumer do
+RSpec.describe FastlyNsq::FakeBackend::Consumer do
   let(:topic)    { 'death_star' }
   let(:channel)  { 'star_killer_base' }
-  let(:consumer) { FakeMessageQueue::Consumer.new topic: topic, channel: channel }
+  let(:consumer) { FastlyNsq::FakeBackend::Consumer.new topic: topic, channel: channel }
 
   describe 'when there are no messages on the queue' do
     it 'tells you there are 0 messages in the queue' do
@@ -81,8 +81,8 @@ RSpec.describe FakeMessageQueue::Consumer do
     end
 
     it 'blocks forever (until timeout) from #pop' do
-      FakeMessageQueue.delay = 0.1
-      delay = FakeMessageQueue.delay + 0.1
+      FastlyNsq::FakeBackend.delay = 0.1
+      delay = FastlyNsq::FakeBackend.delay + 0.1
 
       expect do
         Timeout.timeout(delay) do
@@ -99,8 +99,8 @@ RSpec.describe FakeMessageQueue::Consumer do
   end
 
   describe 'when there is a message on the queue' do
-    let(:message) { FakeMessageQueue::Message.new 'hello' }
-    before { FakeMessageQueue.queue = [message] }
+    let(:message) { FastlyNsq::FakeBackend::Message.new 'hello' }
+    before { FastlyNsq::FakeBackend.queue = [message] }
 
     it 'tells you there are messages in the queue' do
       expect(consumer.size).to eq 1
@@ -121,7 +121,7 @@ RSpec.describe FakeMessageQueue::Consumer do
 
   describe '#terminate' do
     it 'has a terminate method which is a noop' do
-      consumer = instance_double('FakeMessageQueue::Consumer')
+      consumer = instance_double('FastlyNsq::FakeBackend::Consumer')
       allow(consumer).to receive(:terminate)
     end
   end

data/spec/lib/fastly_nsq/listener_spec.rb

@@ -0,0 +1,116 @@
+require 'spec_helper'
+
+RSpec.describe FastlyNsq::Listener do
+  let(:topic)    { 'testing_topic' }
+  let(:channel)  { 'testing_channel' }
+  let(:consumer) { FastlyNsq::FakeBackend::Consumer.new topic: topic, channel: channel }
+  let(:logger)   { double 'Logger', info: nil }
+
+  module TestMessageProcessor
+    @@messages_processed = []
+    Message = Struct.new(:body, :topic) do
+      def finish
+        @did_finish = true
+      end
+    end
+
+    def self.process(incoming_message)
+      @@messages_processed.push Message.new(incoming_message.to_s)
+    end
+
+    def self.messages_processed
+      @@messages_processed
+    end
+
+    def self.clear
+      @@messages_processed = []
+    end
+  end
+
+  let(:listener) do
+    FastlyNsq::Listener.new topic:     topic,
+                            processor: TestMessageProcessor,
+                            consumer:  consumer,
+                            logger:    logger
+  end
+
+  let(:message)            { TestMessageProcessor::Message.new 'this is message body' }
+  let(:messages_processed) { TestMessageProcessor.messages_processed }
+  let(:expected_message)   { TestMessageProcessor::Message.new('this is message body') }
+  let(:expected_messages)  { [expected_message] }
+
+  describe 'instantiating without a consumer' do
+    it 'instantiates a consumer, passing the topic and channel' do
+      allow(FastlyNsq::Consumer).to receive(:new)
+
+      FastlyNsq::Listener.new topic:     topic,
+                              channel:   channel,
+                              processor: TestMessageProcessor,
+                              consumer:  nil
+
+      expect(FastlyNsq::Consumer).to have_received(:new).
+        with(topic: topic, channel: channel)
+    end
+  end
+
+  context 'when using the fake queue and it is empty', fake_queue: true do
+    before do
+      TestMessageProcessor.clear
+      FastlyNsq::FakeBackend.delay = 0.1
+    end
+
+    it 'blocks on the process for longer than the check cycle' do
+      delay = FastlyNsq::FakeBackend.delay + 0.1
+
+      expect do
+        Timeout.timeout(delay) do
+          listener.go run_once: true
+        end
+      end.to raise_error(Timeout::Error)
+    end
+  end
+
+  describe 'when processing next message' do
+    before(:each) do
+      TestMessageProcessor.clear
+      allow(consumer).to receive(:pop).and_return(message)
+    end
+
+    it 'processes the next message' do
+      listener.go run_once: true
+
+      expect(messages_processed).to eql(expected_messages)
+    end
+
+    it 'finishes the message' do
+      allow(message).to receive(:finish)
+
+      listener.go run_once: true
+
+      expect(message).to have_received(:finish).once
+    end
+
+    it 'logs info for the message body' do
+      allow(logger).to receive(:info)
+      listener.go run_once: true
+
+      expect(logger).to have_received(:info).once.with(/\[NSQ\] Message Received: #{message.body}/)
+    end
+
+    context 'when preprocessor is provided' do
+      it 'calls the preprocessor' do
+        preprocessor_was_called = false
+        preprocessor = ->(*_args) { preprocessor_was_called = true }
+
+        listener = FastlyNsq::Listener.new topic:        topic,
+                                           processor:    TestMessageProcessor,
+                                           consumer:     consumer,
+                                           logger:       logger,
+                                           preprocessor: preprocessor
+
+        listener.go run_once: true
+        expect(preprocessor_was_called).to be_truthy
+      end
+    end
+  end
+end

data/spec/lib/fastly_nsq/message_spec.rb

@@ -0,0 +1,24 @@
+require 'spec_helper'
+require 'json'
+
+RSpec.describe FastlyNsq::Message do
+  let(:body)      { { 'data' => 'goes here', 'other_field' => 'is over here' } }
+  let(:json_body) { body.to_json }
+  subject         { FastlyNsq::Message.new json_body }
+
+  it 'preserves original message body as raw_body' do
+    expect(subject.raw_body).to eq(json_body)
+  end
+
+  it 'presents parsed message body as body' do
+    expect(subject.body).to eq(body)
+  end
+
+  it 'plucks data as data' do
+    expect(subject.data).to eq('goes here')
+  end
+
+  it 'aliases raw_body to to_s' do
+    expect(subject.to_s).to eq(json_body)
+  end
+end

data/spec/lib/fastly_nsq/producer_spec.rb

@@ -0,0 +1,56 @@
+require 'spec_helper'
+
+RSpec.describe FastlyNsq::Producer do
+  let(:topic)    { 'death_star' }
+  let(:producer) { FastlyNsq::Producer.new(topic: topic) }
+
+  describe 'when connector connects to a backend Producer' do
+    let(:backend)   { instance_double FastlyNsq::FakeBackend::Producer, write: nil, terminate: nil }
+    let(:connector) { double 'Connector', new: backend }
+    let(:producer) do
+      FastlyNsq::Producer.new topic: topic, connector: connector
+    end
+
+    it 'forwards #write' do
+      expect(backend).to receive(:write).with("it's a message")
+      producer.write "it's a message"
+    end
+
+    it 'forwards #terminate' do
+      expect(backend).to receive(:terminate)
+      producer.terminate
+    end
+  end
+
+  describe 'using the default connector' do
+    module TestStrategy
+      module Producer
+        @@never_terminated = true
+
+        module_function
+
+        def new(*_)
+          self
+        end
+
+        def terminate
+          raise 'Already terminated once' unless @@never_terminated
+          @@never_terminated = false
+        end
+
+        def was_terminated
+          !@@never_terminated
+        end
+      end
+    end
+
+    before do
+      allow(FastlyNsq).to receive(:strategy).and_return(TestStrategy)
+    end
+
+    it 'instantiates a producer via Strategy' do
+      producer.terminate
+      expect(TestStrategy::Producer.was_terminated).to be_truthy
+    end
+  end
+end