fastly_nsq 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 912a3507f9a1f2aa279cae728bbfef969339efa4
-  data.tar.gz: 58b52a9f841f21cd58385b87e81e9a73a7397648
+  metadata.gz: 2118f8fbefbf2cdd4986db89e65f2fab4c215ac8
+  data.tar.gz: dc20875b82f3b58a4e8eda09148f50f5d1d4e409
 SHA512:
-  metadata.gz: c4b5b7230487629c39cfc004f5f0cdac24c2912bbefc8f4938a38ca1d975fccf320f7347c210b769fb8b00bfe4d28461dabc9824fce2dc8fd2ea11063a3a6a7b
-  data.tar.gz: dedc25afac6237e9efc24ce889e57d5fa3b27fc658bd36d92cccb3207064fd6582d21def6580707cb0dfd0682f9b5c1b2d05b79b43130965f28605e1cee1bb85
+  metadata.gz: d7f6bf648089f9686bed42a7c2e0c16d9bb177843c9b396847139b50ed6e36b0016f7365371c1d51b5503e8ee1080ddcb2ce402a9c8481bc8459948bec09acb8
+  data.tar.gz: e65fd5fbf597af38de0163e30901dfcf8493e3213ae3727067a6bb2660974e8bc191a43898026f5b31d2ecb9f04f9a04450dc11270b1f0bf39b138b7c2b267fe

data/.github/PULL_REQUEST_TEMPLATE.md

@@ -1,6 +1,8 @@
+__NOTE: Do NOT add any Jira links to this PR description or commits.  This is a public repository.__
+
 Reason for Change
 ===================
-Describe the big picture of your changes here to communicate why this pull
+* Describe the big picture of your changes here to communicate why this pull
 request should be accepted. If it fixes a bug or resolves a feature request,
 be sure to link to that issue.
 
@@ -11,7 +13,7 @@ List of Changes
 
 Risks
 =====
-Describe any risks you see to existing systems this fix or feature might have
+* Describe any risks you see to existing systems this fix or feature might have
 if those systems where to incorporate this change.
 
 Checklist
@@ -19,6 +21,7 @@ Checklist
 _Put an `x` in the boxes that apply. 
 You can also fill these out after creating the PR._
 
+- [ ] I have NOT linked to any Jira tickets.
 - [ ] I have linked to all relevant reference issues or work requests
 - [ ] I have added tests that prove my fix is effective or that my feature works
 - [ ] I have added or updated necessary documentation (if appropriate)
@@ -27,4 +30,4 @@ You can also fill these out after creating the PR._
 
 Recommended Reviewers
 =====================
-@alieander, ...
+@fastly/billing

data/.travis.yml

@@ -9,7 +9,7 @@ script:
 notifications:
   slack:
     rooms:
-      - 'fastly:hFvhBQKliYl9QAO3VujcrLhe#billy'
+      secure: Uo78FoH8dwSbt7BjTcfb62RXW8aU5h2se+bs0fHSXNrzh6D3Zl+JmwzMkgN8u7M6F0WsOy6RhilYggRYAGds4K1CUi0wGJOG86NaXB4igqBIdHa6f6vzMQZoXxJk8ekc/0NFCj2UV+2Obt6FpeUZvF0Y+u029k5gN0SROfrN+ANtSq+jbI8sdsCCsedXHgDWnGsZS1hHleiRF1Q6VfKnBFcewPP/W49gzbABlKk2MjkVhse6kg+7+LRXfnDxh+aaRXBGYfzmGn4+gS6Jey63U0woAa8TdP/THlwhgvvBS0kwMMaCss+Uq1690lnVV5MX84Fsm4C+A9gbdJmlansLrYRRWm4aaAf8IGlqRTwx3F2yNHTaPA2zmkooFtn1kTuz1J8L9NjsVPPkATMAg3rTIhel+YkVK8q0Q9N9A70+P2u+Rp3htgjxVPMN7Ve9GqejUEaw690Wsp4hCK5tINj8Aq42ET/RpQt4Rzx78Vk4jEK+9UsFuuRH+awAJu/UsnA0JMTq9Smgk2kE69McVNfYPXRIRQXWAkV2dc6FwI8AAmqRusTeyWMK9SOQNXhlmry5ltWfAnHtuKdq9delbc1iOkcj8u9sCBkTz0ZogsI2oHpLlrw2oHFe49kSOkB5HMSQo8IOR7yXHAZ41lmd87/T367bak9HGqsCrweJL7Vz2Mw=
     on_success: change
     on_failure: change
   email:

data/README.md

@@ -37,12 +37,7 @@ and `bundle install`.
 
 ## Usage
 
-*IMPORTANT NOTE:* You must create your own `MessageProcessor` class
-for this gem to work in your application.
-
-See more information below.
-
-### `MessageQueue::Producer`
+### `FastlyNsq::Producer`
 
 This is a class
 which provides an adapter to the
@@ -57,7 +52,7 @@ message_data = {
   }
 }
 
-producer = MessageQueue::Producer.new(
+producer = FastlyNsq::Producer.new(
   nsqd: ENV.fetch('NSQD_TCP_ADDRESS'),
   topic: topic,
 )
@@ -77,7 +72,7 @@ ENV['FAKE_QUEUE'] = true
 ENV['FAKE_QUEUE'] = false
 ```
 
-### `MessageQueue::Consumer`
+### `FastlyNsq::Consumer`
 This is a class
 which provides an adapter to the
 fake and real NSQ consumers.
@@ -85,7 +80,7 @@ These are used to
 read messages off of the queue:
 
 ```ruby
-consumer = MessageQueue::Consumer.new(
+consumer = FastlyNsq::Consumer.new(
   topic: 'topic',
   channel: 'channel'
 )
@@ -103,48 +98,44 @@ the mock/real strategy used
 can be switched by setting the
 `FAKE_QUEUE` environment variable appropriately.
 
-### `MessageQueue::Listener`
+### `FastlyNsq::Listener`
 
 To process the next message on the queue:
 
 ```ruby
-topic = 'user_created'
-channel = 'my_consuming_service'
+topic     = 'user_created'
+channel   = 'my_consuming_service'
+processor = MessageProcessor
 
-MessageQueue::Listener.new(topic: topic, channel: channel).process_next_message
+FastlyNsq::Listener.new(topic: topic, channel: channel, processor: processor).go(run_once: true)
 ```
 
 This will pop the next message
 off of the queue
 and send the JSON text body
-to `MessageProcessor.new(message_body: message_body, topic: topic).go`.
+to `MessageProcessor.process(message_body, topic)`.
 
 To initiate a blocking loop to process messages continuously:
 
 ```ruby
-topic = 'user_created'
-channel = 'my_consuming_service'
+topic     = 'user_created'
+channel   = 'my_consuming_service'
+processor = MessageProcessor
 
-MessageQueue::Listener.new(topic: topic, channel: channel).go
+FastlyNsq::Listener.new(topic: topic, channel: channel, processor: processor).go
 ```
 
 This will block until
 there is a new message on the queue,
       pop the next message
       off of the queue
-      and send it to `MessageProcessor.new(message_body).go`.
+      and send it to `MessageProcessor.process(message_body, topic)`.
 
-### `MessageQueue::RakeTask`
+### `FastlyNsq::RakeTask`
 
-To help facilitate running the `MessageQueue::Listener` in a blocking fashion
+To help facilitate running the `FastlyNsq::Listener` in a blocking fashion
 outside your application, a simple `RakeTask` is provided.
 
-NOTE: The rake task expects a
-`MessageProcessor.topics` method,
-which must return an array of strings
-defining the topics to which
-we would like to listen and process messages.
-
 The task will listen
 to all specified topics,
 each in a separate thread.
@@ -155,26 +146,14 @@ Using a block:
 ```ruby
 require 'fastly_nsq/rake_task'
 
-MessageQueue::RakeTask.new(:listen_task) do |task|
+FastlyNsq::RakeTask.new(:listen_task) do |task|
   task.channel = 'some_channel'
+  task.topics  = {
+    'some_topic' => SomeMessageProcessor
+  }
 end
 
-# usage:
-`rake listen_task`
-```
-
-or using passed in values:
-```ruby
-require 'fastly_nsq/rake_task'
-
-MessageQueue::RakeTask.new(:listen_task, [:channel])
-
-# usage:
-`rake listen_task['my_channel']`
-```
-
-Both methods can be used at the same time with the passed in values taking
-priority over block assigned values
+The task can also define a `call`-able "preprocessor" (called before any `Processor.process`) and a custom `logger`.
 
 See the [`Rakefile`](examples/Rakefile) file
 for more detail.
@@ -195,34 +174,6 @@ as the real adapter.
 
 ## Configuration
 
-### Processing Messages
-
-This gem expects you to create a
-new class called `MessageProcessor`
-which will process messages
-once they are consumed off of the queue topic.
-
-This class needs to adhere to the following API:
-
-```ruby
-class MessageProcessor
-  # This an instance of NSQ::Message or FakeMessageQueue::Message
-  def initialize(message_body)
-    @message_body = message_body
-  end
-
-  def start
-    # Do things
-  end
-
-  private
-
-  def message
-    JSON.parse(@message_body)
-  end
-end
-```
-
 ### Environment Variables
 
 The URLs for the various
@@ -235,7 +186,6 @@ stock NSQ on OS X,
 installed via Homebrew:
 
 ```shell
-BROADCAST_ADDRESS='127.0.0.1'
 NSQD_TCP_ADDRESS='127.0.0.1:4150'
 NSQD_HTTP_ADDRESS='127.0.0.1:4151'
 NSQLOOKUPD_TCP_ADDRESS='127.0.0.1:4160'
@@ -263,7 +213,7 @@ FAKE_QUEUE=true
 ```
 
 Also be sure call
-`FakeMessageQueue.reset!`
+`FakeBackend.reset!`
 before each test in your app to ensure
 there are no leftover messages.
 

data/env_configuration_for_local_gem_tests.yml

@@ -1,4 +1,3 @@
-BROADCAST_ADDRESS: '127.0.0.1'
 NSQD_TCP_ADDRESS: '127.0.0.1:4150'
 NSQD_HTTP_ADDRESS: '127.0.0.1:4151'
 NSQLOOKUPD_TCP_ADDRESS: '127.0.0.1:4160'

data/examples/.sample.env

@@ -1,13 +1,11 @@
 # NSQ
 # In the Fastly Dev VM
-BROADCAST_ADDRESS='billing'
 NSQD_TCP_ADDRESS='billing:1910'
 NSQD_HTTP_ADDRESS='billing:1911'
 NSQLOOKUPD_TCP_ADDRESS='billing:4160'
 NSQLOOKUPD_HTTP_ADDRESS='billing:4161'
 
 # On OSX with Homebrew defaults (brew install nsq)
-BROADCAST_ADDRESS='127.0.0.1'
 NSQD_TCP_ADDRESS='127.0.0.1:4150'
 NSQD_HTTP_ADDRESS='127.0.0.1:4151'
 NSQLOOKUPD_TCP_ADDRESS='127.0.0.1:4160'

data/examples/Rakefile

@@ -1,88 +1,41 @@
 require 'fastly_nsq/rake_task'
 
-#-------------------------------------------------------------------------------
-# You are required to to set the topics in MessageProcessor.topics as an array:
-
-class MessageProcessor
-  def self.topics
-    %w(customer_created customer_deleted)
-  end
-
-  # ...
-end
-
-# A channel will need to be passed in
-# to the Rake task
+# A list of topics and a channel will need to be passed in to the Rake task
 # when it is called.
-#
-# The task looks like:
-
-desc 'Listen to the messaging queue with the given channel.'
-
-MessageQueue::RakeTask.new(:listen_task, [:channel])
-
-# Call `rake listen_task[my_channel]`.
-# The topics will be ['customer_created', 'customer_deleted']
-# and channel will be 'my_channel'.
-
-#-------------------------------------------------------------------------------
-# In Rails, you can include the application environment. The task looks like:
-
-desc 'Listen to the messaging queue with the given channel.'
-
-MessageQueue::RakeTask.new(:listen_task, [:channel] => :environment)
-
-# Call `rake listen_task[your_channel]`.
-# The topics will be ['customer_created', 'customer_deleted']
-# and channel will be 'your_channel'.
-
-#-------------------------------------------------------------------------------
-# The channel can also be preset in the task
-# by passing a block and assigning the value. The task looks like:
 
 desc 'Listen to the messaging queue with the given channel.'
 
-MessageQueue::RakeTask.new(:listen_task) do |task|
+FastlyNsq::RakeTask.new(:listen_task) do |task|
   task.channel = 'some_channel'
+  task.topics  = { 'a_topic' => RespondsToProcess }
 end
 
-# Call `rake listen_task`.
-# The topics will be ['customer_created', 'customer_deleted']
-# and channel will be 'some_channel'.
-
-#-------------------------------------------------------------------------------
-# Do the same thing and include the environment:
+# A custom logger and preprocessor
 
 desc 'Listen to the messaging queue with the given channel.'
 
-MessageQueue::RakeTask.new(:listen_task, [] => :environment) do |task|
-  task.channel = 'her_channel'
+FastlyNsq::RakeTask.new(:listen_task) do |task|
+  task.channel      = 'some_channel'
+  task.topics       = { 'a_topic' => RespondsToProcess }
+  task.preprocessor = ->(message) { StatsClient.log(message) }
+  task.logger       = Logger.new(STDERR)
 end
 
-# Call `rake listen_task`.
-# The topics will be ['customer_created', 'customer_deleted']
-# and channel will be 'her_channel'.
+#-------------------------------------------------------------------------------
+# In Rails, you can include the application environment. The task looks like:
+
+FastlyNsq::RakeTask.new(:listen_task => :environment)
 
 #-------------------------------------------------------------------------------
-# Both forms can be combined
-# to provide defaults
-# with the ability to override at time of Rake call:
+# It's also possible to define the rake task to allow for overriding channel
+# and topics when calling the rake task:
 
-MessageQueue::RakeTask.new(:listen_task, [:channel]) do |task|
-  task.channel = 'default_channel'
-end
+FastlyNsq::RakeTask.new(:listen_task, [:channel, :topics])
 
-# Call `rake listen_task[overridden_channel]`.
-# The topics will be ['customer_created', 'customer_deleted']
-# and channel will be 'overridden_channel'.
+# Then call the task:
+#   rake listen_task[my_channel, {topic: Processor}]
 
 #-------------------------------------------------------------------------------
-# The same, but including the environment:
-
-MessageQueue::RakeTask.new(:listen_task, [:channel] => :environment) do |task|
-  task.channel = 'default_channel'
-end
+# Do the same thing and include the environment:
 
-# Call `rake listen_task[overridden_channel]`.
-# The topics will be ['customer_created', 'customer_deleted']
-# and channel will be 'overridden_channel'.
+FastlyNsq::RakeTask.new(:listen_task, [:channel, :topics] => :environment)

data/lib/fastly_nsq/{message_queue/consumer.rb → consumer.rb}

@@ -1,8 +1,6 @@
 require 'forwardable'
 
-class InvalidParameterError < StandardError; end
-
-module MessageQueue
+module FastlyNsq
   class Consumer
     extend Forwardable
     def_delegator :connection, :pop
@@ -14,17 +12,19 @@ module MessageQueue
       @topic       = topic
       @channel     = channel
       @ssl_context = SSLContext.new(ssl_context)
-      @connector   = connector || DEFAULT_CONNECTOR
+      @connector   = connector
     end
 
     private
 
-    attr_reader :channel, :connector, :topic, :ssl_context
-
-    DEFAULT_CONNECTOR = ->(params) { MessageQueue.strategy::Consumer.new(params) }
+    attr_reader :channel, :topic, :ssl_context
 
     def connection
-      @connection ||= connector.call(params)
+      @connection ||= connector.new(params)
+    end
+
+    def connector
+      @connector || FastlyNsq.strategy::Consumer
     end
 
     def params

data/lib/fastly_nsq/fake_backend.rb

@@ -0,0 +1,100 @@
+module FastlyNsq
+  module FakeBackend
+    @@logger = Logger.new(nil)
+    @@delay  = 0.5
+    @@queue  = []
+
+    def self.queue
+      @@queue
+    end
+
+    def self.queue=(message)
+      @@queue = message
+    end
+
+    def self.reset!
+      self.queue = []
+    end
+
+    def self.logger=(logger)
+      @@logger = logger
+    end
+
+    def self.logger
+      @@logger
+    end
+
+    def self.delay
+      @@delay
+    end
+
+    def self.delay=(delay)
+      @@delay = delay
+    end
+
+    class Producer
+      def initialize(topic:, nsqd: nil, ssl_context: nil)
+      end
+
+      def write(string)
+        message = Message.new(string)
+        queue.push(message)
+      end
+
+      def terminate
+        # noop
+      end
+
+      private
+
+      def queue
+        FakeBackend.queue
+      end
+    end
+
+    class Consumer
+      def initialize(nsqlookupd: nil, topic:, channel:, ssl_context: nil)
+      end
+
+      def pop(delay = FakeBackend.delay)
+        message = nil
+
+        until message
+          message = queue.pop
+          sleep delay
+        end
+
+        message
+      end
+
+      def pop_without_blocking
+        queue.pop
+      end
+
+      def size
+        queue.size
+      end
+
+      def terminate
+        # noop
+      end
+
+      private
+
+      def queue
+        FakeBackend.queue
+      end
+    end
+
+    class Message
+      attr_reader :body
+
+      def initialize(body)
+        @body = body
+      end
+
+      def finish
+      end
+    end
+  end
+end

data/lib/fastly_nsq/listener.rb

@@ -0,0 +1,59 @@
+require 'fastly_nsq/message'
+
+module FastlyNsq
+  class Listener
+    def self.listen_to(**args)
+      new(**args).go
+    end
+
+    def initialize(topic:, processor:, channel: nil, consumer: nil, **options)
+      @topic        = topic
+      @processor    = processor
+      @consumer     = consumer || FastlyNsq::Consumer.new(topic: topic, channel: channel)
+      @logger       = options.fetch :logger, FastlyNsq.logger
+      @preprocessor = options[:preprocessor]
+    end
+
+    def go(run_once: false)
+      exit_on 'INT'
+      exit_on 'TERM'
+
+      loop do
+        next_message do |message|
+          log message
+          preprocess message
+          processor.process message
+        end
+
+        break if run_once
+      end
+
+      consumer.terminate
+    end
+
+    private
+
+    attr_reader :topic, :consumer, :preprocessor, :processor, :logger
+
+    def log(message)
+      logger.info "[NSQ] Message Received: #{message}" if logger
+    end
+
+    def next_message
+      message = consumer.pop # TODO: consumer.pop do |message|
+      result  = yield FastlyNsq::Message.new(message.body)
+      message.finish if result
+    end
+
+    def preprocess(message)
+      preprocessor.call(message) if preprocessor
+    end
+
+    def exit_on(signal)
+      Signal.trap(signal) do
+        consumer.terminate
+        exit
+      end
+    end
+  end
+end

data/lib/fastly_nsq/message.rb

@@ -0,0 +1,18 @@
+require 'json'
+
+class FastlyNsq::Message
+  attr_reader :raw_body
+  alias to_s raw_body
+
+  def initialize(raw_body)
+    @raw_body = raw_body
+  end
+
+  def data
+    @data ||= body['data']
+  end
+
+  def body
+    @body ||= JSON.parse(raw_body)
+  end
+end

data/lib/fastly_nsq/{message_queue/producer.rb → producer.rb}

@@ -1,8 +1,6 @@
 require 'forwardable'
 
-class InvalidParameterError < StandardError; end
-
-module MessageQueue
+module FastlyNsq
   class Producer
     extend Forwardable
     def_delegator :connection, :terminate
@@ -11,17 +9,19 @@ module MessageQueue
     def initialize(topic:, ssl_context: nil, connector: nil)
       @topic       = topic
       @ssl_context = SSLContext.new(ssl_context)
-      @connector   = connector || DEFAULT_CONNECTOR
+      @connector   = connector
     end
 
     private
 
-    DEFAULT_CONNECTOR = ->(params) { MessageQueue.strategy::Producer.new params }
-
-    attr_reader :connector, :topic, :ssl_context
+    attr_reader :topic, :ssl_context
 
     def connection
-      @connection ||= connector.call(params)
+      @connection ||= connector.new(params)
+    end
+
+    def connector
+      @connector || FastlyNsq.strategy::Producer
     end
 
     def params