fastly_nsq 0.13.2 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 9e293af0d6cd86795807e1727fb3547085917dbf
-  data.tar.gz: 08243ead04232661269fbb182be2087eaa0a6663
+SHA256:
+  metadata.gz: d4e06ecec388c8a7d3f403fbb38ec03788522f25464a6b4a46815ccc584ba613
+  data.tar.gz: '04392e44b547924dfcba6df02b26ac86bc5580c32d30eb6b6ca8ecab313c39d0'
 SHA512:
-  metadata.gz: a58cf76f4c36e5061080e65c819948ae78ca70595c01377ce4a2ba5745d45e28c7151c72dc23a0880832488ffae2cb0b681bd95a6fe16b1ea86a056cd0b34f5f
-  data.tar.gz: 09915b030a22911a886b4f56ed345c6acd2d85b36f538626dc4e0133a998a77041b19ab6f820039bbbf94c6f2e5062466007c1647805099c7843c730aa7ff693
+  metadata.gz: de8962c909fd1cdd08c0182b4eac693c1d0dec1608c0c60826ace6232198d449e5780a2e4ca9f3aee9cdb0bbc6d73871fb25dabd8a419b62e9aaee65f4fbfdb9
+  data.tar.gz: b1bd13296fb6e13f285099374621c0af43485822b04eb3593ec5635f07fb6ff087aa304399b3f389de8b21ecba4304e16a38a716831cc084077fa29dcd5970ed

data/.env

@@ -0,0 +1,4 @@
+NSQD_TCP_ADDRESS=127.0.0.1:4150
+NSQD_HTTP_ADDRESS=127.0.0.1:4151
+NSQLOOKUPD_TCP_ADDRESS=127.0.0.1:4160
+NSQLOOKUPD_HTTP_ADDRESS=127.0.0.1:4161

data/.overcommit.yml

@@ -27,8 +27,8 @@ CommitMsg:
     description: 'Prevent empty commit messages'
     enabled: true
   TextWidth:
-    description: 'Ensure commit message subjects are <60 chars'
+    description: 'Ensure commit message subjects are <72 chars'
     enabled: true
     description: 'Checking text width'
-    max_subject_width: 60
-    max_body_width: 1000
+    max_subject_width: 72
+    max_body_width: 72

data/.rubocop.yml

@@ -5,7 +5,7 @@ AllCops:
     - 'db/schema.rb'
     - 'vendor/**/*'
     - 'Gemfile'
-  TargetRubyVersion: 2.2
+  TargetRubyVersion: 2.4
 
 Layout/DotPosition:
   Enabled: true
@@ -14,15 +14,25 @@ Layout/DotPosition:
     - leading
     - trailing
 
+Lint/AmbiguousBlockAssociation:
+  Exclude:
+    - 'spec/**/*'
+
 Lint/RescueWithoutErrorClass:
   Enabled: false
 
 Metrics/MethodLength:
   Enabled: false
 
+Metrics/AbcSize:
+  Enabled: false
+
 Metrics/LineLength:
   Enabled: false
 
+Metrics/ParameterLists:
+  Enabled: false
+
 Metrics/BlockLength:
   ExcludedMethods: [describe, context, configure]
 

data/.travis.yml

@@ -1,11 +1,18 @@
 language: ruby
 cache: bundler
+sudo: false
 rvm:
-  - 2.2.4
   - 2.3.5
   - 2.4.2
+  - 2.5.0
 script:
   - bundle exec rake
+services:
+  - docker
+before_install:
+  - docker-compose up -d
+before_script:
+  - gem update --system
 notifications:
   slack:
     rooms:

data/Gemfile

@@ -1,3 +1,12 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 gemspec
+
+gem 'bundler-audit', '~> 0.5.0'
+gem 'overcommit', '~> 0.32.0'
+gem 'rake', '~> 11.1.2'
+gem 'rdoc', '~> 4.2.2'
+gem 'rubocop', '~> 0.51.0'
+gem 'rubygems-tasks', '~> 0.2'

data/README.md

@@ -61,15 +61,39 @@ producer.write(message_data.to_json)
 ```
 The mock/real strategy used
 can be switched
-by adding an environment variable
-to your application:
+by requiring the test file and configuring the mode.
 
 ```ruby
-# for the fake
-ENV['FAKE_QUEUE'] = true
+require 'fastly_nsq/testing'
+FastlyNsq::Testing.enabled? #=> true
+FastlyNsq::Testing.disabled? #=> false
 
-# for the real thing
-ENV['FAKE_QUEUE'] = false
+producer = FastlyNsq::Producer.new(topic: topic)
+listener = FastlyNsq::Listener.new(topic: topic, channel: channel, processor: ->(m) { puts 'got: '+ m.body })
+
+FastlyNsq::Testing.fake! # default, messages accumulate on the listeners
+
+producer.write '{"foo":"bar"}'
+listener.messages.size #=> 1
+
+FastlyNsq::Testing.reset!  # remove all accumulated messages
+
+listener.messages.size #=> 0
+
+producer.write '{"foo":"bar"}'
+listener.messages.size #=> 1
+
+listener.drain
+#  got: {"foo"=>"bar"}
+listener.messages.size #=> 0
+
+FastlyNsq::Testing.inline! # messages are processed as they are produced
+producer.write '{"foo":"bar"}'
+#  got: {"foo"=>"bar"}
+listener.messages.size #=> 0
+
+FastlyNsq::Testing.disable! # do it live
+FastlyNsq::Testing.enable!  # re-enable testing mode
 ```
 
 ### `FastlyNsq::Consumer`
@@ -93,11 +117,6 @@ consumer.size #=> 0
 consumer.terminate
 ```
 
-As above,
-the mock/real strategy used
-can be switched by setting the
-`FAKE_QUEUE` environment variable appropriately.
-
 ### `FastlyNsq::Listener`
 
 To process the next message on the queue:
@@ -107,70 +126,49 @@ topic     = 'user_created'
 channel   = 'my_consuming_service'
 processor = MessageProcessor
 
-FastlyNsq::Listener.new(topic: topic, channel: channel, processor: processor).go(run_once: true)
+FastlyNsq::Listener.new(topic: topic, channel: channel, processor: processor)
 ```
 
-This will pop the next message
+This will send messages through `FastlyNsq.manager.pool`
 off of the queue
 and send the JSON text body
-to `MessageProcessor.process(message_body, topic)`.
+to `MessageProcessor.call(message)`.
 
-To initiate a blocking loop to process messages continuously:
+Specify a topic priority by providing a number (default is 0)
 
 ```ruby
 topic     = 'user_created'
 channel   = 'my_consuming_service'
 processor = MessageProcessor
+priority  = 1 # a little higher
 
-FastlyNsq::Listener.new(topic: topic, channel: channel, processor: processor).go
+FastlyNsq::Listener.new(topic: topic, channel: channel, processor: processor, priority: priority)
 ```
 
-This will block until
-there is a new message on the queue,
-      pop the next message
-      off of the queue
-      and send it to `MessageProcessor.process(message_body, topic)`.
-
-### `FastlyNsq::RakeTask`
+### `FastlyNsq::CLI`
 
 To help facilitate running the `FastlyNsq::Listener` in a blocking fashion
-outside your application, a simple `RakeTask` is provided.
-
-The task will listen
-to all specified topics,
-each in a separate thread.
+outside your application, a `CLI` and bin script [`fastly_nsq`](bin/fastly_nsq)
+are provided.
 
-This task can be added into your `Rakefile` in one of two ways:
+This can be setup ahead of time by calling `FastlyNsq.configure` and passing block.
 
-Using a block:
 ```ruby
-require 'fastly_nsq/rake_task'
-
-FastlyNsq::RakeTask.new(:listen_task) do |task|
-  task.channel = 'some_channel'
-  task.topics  = {
-    'some_topic' => SomeMessageProcessor
-  }
+# config/fastly_nsq.rb
+FastlyNsq.configure do |config|
+  config.channel = 'fnsq'
+  config.logger = Logger.new
+  config.preprocessor = ->(_) { FastlyNsq.logger.info 'PREPROCESSESES' }
+
+  lc.listen 'posts', ->(m) { puts "posts: #{m.body}" }
+  lc.listen 'blogs', ->(m) { puts "blogs: #{m.body}" }, priority: 3
 end
 ```
 
-The task can also define a `call`-able "preprocessor" (called before any `Processor.process`) and a custom `logger`.
-
-See the [`Rakefile`](examples/Rakefile) file
-for more detail.
-
-### `FastlyNsq::CLI`
-
-To help facilitate running the `FastlyNsq::Listener` in a blocking fashion
-outside your application, a `CLI` and bin script [`fastly_nsq`](bin/fastly_nsq)
-are provided.
-
-This can be setup ahead of time by calling `FastlyNsq.configure` and passing
-block. An exmaple of this can be found here: [`Example Config`](exmaple_config_class.rb)
-
 An example of using the cli:
+
 ```bash
-./bin/fastly_nsq -r ./example_config_class.rb -L ./test.log -P ./fastly_nsq.pid -v -d -t 4
+./bin/fastly_nsq -r config/fastly_nsq.rb -L ./test.log -P ./fastly_nsq.pid -v -d -t 4 -c 10
 ```
 
 ### `FastlyNsq::Messenger`
@@ -178,13 +176,13 @@ An example of using the cli:
 Wrapper around a producer for sending messages and persisting producer objects.
 
 ```ruby
-FastlyNsq::Messenger.deliver(message: msg, on_topic: 'my_topic', originating_service: 'my service')
+FastlyNsq::Messenger.deliver(message: msg, topic: 'my_topic', originating_service: 'my service')
 ```
 
 You can also optionally pass custom metadata.
 
 ```ruby
-FastlyNsq::Messenger.deliver(message: msg, on_topic: 'my_topic', originating_service: 'my service', meta: { test: 'test' })
+FastlyNsq::Messenger.deliver(message: msg, topic: 'my_topic', originating_service: 'my service', meta: { test: 'test' })
 ```
 
 This will use a FastlyNsq::Producer for the given topic or create on if it isn't
@@ -282,34 +280,6 @@ NSQLOOKUPD_HTTP_ADDRESS='127.0.0.1:4161, 10.1.1.101:4161'
 See the [`.sample.env`](examples/.sample.env) file
 for more detail.
 
-### Testing Against the Fake
-
-In the gem's test suite,
-the fake message queue is used.
-
-If you would like to force
-use of the real NSQ adapter,
-ensure `FAKE_QUEUE` is set to `false`.
-
-When you are developing your application,
-it is recommended to
-start by using the fake queue:
-
-```shell
-FAKE_QUEUE=true
-```
-
-Also be sure call
-`FakeBackend.reset!`
-before each test in your app to ensure
-there are no leftover messages.
-
-Also note that during gem tests,
-we are aliasing `MessageProcessor` to `SampleMessageProcessor`.
-You can also refer to the latter
-as an example of how
-you might write your own processor.
-
 ## Contributors
 
 * Adarsh Pandit ([@adarsh](https://github.com/adarsh))

data/Rakefile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'rubygems'
 
 begin

data/bin/fastly_nsq

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require 'pry'
 require 'fastly_nsq/cli'

data/docker-compose.yml

@@ -0,0 +1,23 @@
+version: '3'
+services:
+  nsqlookupd:
+    image: nsqio/nsq
+    command: /nsqlookupd
+    ports:
+      - "4160:4160"
+      - "4161:4161"
+  nsqd:
+    image: nsqio/nsq
+    command: /nsqd --lookupd-tcp-address=nsqlookupd:4160 --broadcast-address localhost
+    depends_on:
+      - nsqlookupd
+    ports:
+      - "4150:4150"
+      - "4151:4151"
+  nsqadmin:
+    image: nsqio/nsq
+    command: /nsqadmin --lookupd-http-address=nsqlookupd:4161
+    depends_on:
+      - nsqlookupd
+    ports:
+      - "4171:4171"

data/examples/.sample.env

@@ -11,6 +11,3 @@ NSQD_HTTP_ADDRESS='127.0.0.1:4151'
 NSQLOOKUPD_TCP_ADDRESS='127.0.0.1:4160'
 ## NSQLOOKUPD_HTTP_ADDRESS can be a single address or a comma seperated list
 NSQLOOKUPD_HTTP_ADDRESS='127.0.0.1:4161, 10.0.0.101:4161'
-
-# Uncomment to test against a live queue
-#LIVE_QUEUE=true

data/fastly_nsq.gemspec

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'fastly_nsq/version'
@@ -20,16 +22,13 @@ Gem::Specification.new do |gem|
 
   gem.add_development_dependency 'awesome_print', '~> 1.6'
   gem.add_development_dependency 'bundler', '~> 1.12'
-  gem.add_development_dependency 'bundler-audit', '~> 0.5.0'
   gem.add_development_dependency 'dotenv'
-  gem.add_development_dependency 'overcommit', '~> 0.32.0'
   gem.add_development_dependency 'pry-byebug', '~> 3.3'
-  gem.add_development_dependency 'rake', '~> 11.1.2'
-  gem.add_development_dependency 'rdoc', '~> 4.2.2'
-  gem.add_development_dependency 'rspec', '~> 3.4.0'
-  gem.add_development_dependency 'rubocop', '~> 0.51.0'
-  gem.add_development_dependency 'rubygems-tasks', '~> 0.2'
-  gem.add_development_dependency 'webmock'
+  gem.add_development_dependency 'rspec', '~> 3.4'
+  gem.add_development_dependency 'rspec-eventually', '0.2'
+  gem.add_development_dependency 'webmock', '~> 3.0'
 
+  gem.add_dependency 'concurrent-ruby', '~> 1.0'
   gem.add_dependency 'nsq-ruby', '~> 2.0', '>= 2.0.5'
+  gem.add_dependency 'priority_queue_cxx', '~> 0.3'
 end

data/lib/fastly_nsq.rb

@@ -1,62 +1,56 @@
+# frozen_string_literal: true
+
 require 'nsq'
-require 'fastly_nsq/consumer'
-require 'fastly_nsq/fake_backend'
-require 'fastly_nsq/listener'
-require 'fastly_nsq/message'
-require 'fastly_nsq/messenger'
-require 'fastly_nsq/producer'
-require 'fastly_nsq/strategy'
-require 'fastly_nsq/tls_options'
-require 'fastly_nsq/version'
+require 'concurrent'
+require 'fc'
+require 'set'
+require 'logger'
+require 'forwardable'
 
 module FastlyNsq
-  module_function
+  NotConnectedError = Class.new(StandardError)
+  ConnectionFailed = Class.new(StandardError)
 
-  @preprocessor = nil
+  class << self
+    attr_accessor :channel, :preprocessor
+    attr_writer :logger
 
-  def channel=(channel)
-    @channel ||= channel
-  end
-
-  def logger=(logger)
-    strategy.logger = logger
-  end
+    def listen(topic, processor, **options)
+      FastlyNsq::Listener.new(topic: topic, processor: processor, **options)
+    end
 
-  def preprocessor=(preprocessor)
-    @preprocessor ||= preprocessor
-  end
+    def logger
+      @logger ||= Logger.new(nil)
+    end
 
-  def channel
-    @channel
-  end
+    def configure
+      yield self
+    end
 
-  def logger
-    strategy.logger
-  end
+    def manager
+      @manager ||= FastlyNsq::Manager.new
+    end
 
-  def preprocessor
-    @preprocessor
-  end
+    def manager=(manager)
+      @manager&.transfer(manager)
+      @manager = manager
+    end
 
-  def strategy
-    Strategy.for_queue
-  end
-
-  def configure
-    yield self
-  end
-
-  def topic_map
-    @listener_config.topic_map
-  end
-
-  def listener_config
-    @listener_config ||= FastlyNsq::Listener::Config.new
-    yield @listener_config if block_given?
-    @listener_config
-  end
-
-  def reset_config
-    @listener_config = nil
+    def lookupd_http_addresses
+      ENV.fetch('NSQLOOKUPD_HTTP_ADDRESS').split(',').map(&:strip)
+    end
   end
 end
+
+require 'fastly_nsq/consumer'
+require 'fastly_nsq/feeder'
+require 'fastly_nsq/launcher'
+require 'fastly_nsq/listener'
+require 'fastly_nsq/manager'
+require 'fastly_nsq/message'
+require 'fastly_nsq/messenger'
+require 'fastly_nsq/priority_queue'
+require 'fastly_nsq/priority_thread_pool'
+require 'fastly_nsq/producer'
+require 'fastly_nsq/tls_options'
+require 'fastly_nsq/version'