fastly_nsq 0.13.2 → 1.0.2

data/lib/fastly_nsq/cli.rb

@@ -25,19 +25,17 @@ class FastlyNsq::CLI
 
   def run
     startup
-    begin
-      # Multithreading begins here ----
-      launcher.run
-
-      read_loop
-    rescue Interrupt
-      FastlyNsq.logger.info 'Shutting down'
-      launcher.stop
-      # Explicitly exit so busy Processor threads can't block
-      # process shutdown.
-      FastlyNsq.logger.info 'Bye!'
-      exit(0)
-    end
+
+    launcher.beat
+
+    read_loop
+  rescue Interrupt
+    FastlyNsq.logger.info 'Shutting down'
+    launcher.stop
+    # Explicitly exit so busy Processor threads can't block
+    # process shutdown.
+    FastlyNsq.logger.info 'Bye!'
+    exit(0)
   end
 
   private
@@ -75,6 +73,10 @@ class FastlyNsq::CLI
     opts = {}
 
     @parser = OptionParser.new do |o|
+      o.on '-c', '--concurrency COUNT', 'Number of threads used to process messages' do |arg|
+        opts[:max_threads] = arg
+      end
+
       o.on '-d', '--daemon', 'Daemonize process' do |arg|
         opts[:daemonize] = arg
       end
@@ -178,7 +180,7 @@ class FastlyNsq::CLI
       raise Interrupt
     when 'USR1'
       FastlyNsq.logger.info 'Received USR1, no longer accepting new work'
-      launcher.quiet
+      launcher.stop_listeners
     when 'TTIN'
       handle_ttin
     end
@@ -247,6 +249,10 @@ class FastlyNsq::CLI
     options[:logfile]
   end
 
+  def max_threads
+    options[:max_threads]
+  end
+
   def pidfile
     options[:pidfile]
   end

data/lib/fastly_nsq/consumer.rb

@@ -1,43 +1,39 @@
-require 'forwardable'
+# frozen_string_literal: true
 
-module FastlyNsq
-  class Consumer
-    extend Forwardable
-    def_delegator :connection, :pop
-    def_delegator :connection, :pop_without_blocking
-    def_delegator :connection, :size
-    def_delegator :connection, :terminate
+class FastlyNsq::Consumer
+  extend Forwardable
 
-    def initialize(topic:, channel:, tls_options: nil, connector: nil)
-      @topic       = topic
-      @channel     = channel
-      @tls_options = TlsOptions.as_hash(tls_options)
-      @connector   = connector
-      connection
-    end
+  DEFAULT_CONNECTION_TIMEOUT = 5 # seconds
 
-    def empty?
-      connection.size.zero?
-    end
+  attr_reader :channel, :topic, :connection, :connect_timeout
 
-    private
+  def_delegators :connection, :size, :terminate, :connected?, :pop, :pop_without_blocking
 
-    attr_reader :channel, :topic, :tls_options
+  def initialize(topic:, channel:, queue: nil, tls_options: nil, connect_timeout: DEFAULT_CONNECTION_TIMEOUT)
+    @topic           = topic
+    @channel         = channel
+    @tls_options     = FastlyNsq::TlsOptions.as_hash(tls_options)
+    @connect_timeout = connect_timeout
 
-    def connection
-      @connection ||= connector.new(params)
-    end
+    @connection = connect(queue)
+  end
+
+  def empty?
+    size.zero?
+  end
+
+  private
 
-    def connector
-      @connector || FastlyNsq.strategy::Consumer
-    end
+  attr_reader :tls_options
 
-    def params
+  def connect(queue)
+    Nsq::Consumer.new(
       {
-        nsqlookupd: ENV.fetch('NSQLOOKUPD_HTTP_ADDRESS').split(',').map(&:strip),
+        nsqlookupd: FastlyNsq.lookupd_http_addresses,
         topic: topic,
         channel: channel,
-      }.merge(tls_options)
-    end
+        queue: queue,
+      }.merge(tls_options),
+    )
   end
 end

data/lib/fastly_nsq/feeder.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class FastlyNsq::Feeder
+  attr_reader :processor, :priority
+
+  def initialize(processor, priority)
+    @processor = processor
+    @priority = priority
+  end
+
+  # @see http://ruby-concurrency.github.io/concurrent-ruby/Concurrent/ThreadPoolExecutor.html#post-instance_method
+  # @see {Nsq::Connection#read_loop}
+  def push(message)
+    FastlyNsq.manager.pool.post(priority) { processor.call(message) }
+  end
+end

data/lib/fastly_nsq/http/nsqd.rb

@@ -8,7 +8,13 @@ class FastlyNsq::Http
     def_delegator :client, :get
     def_delegator :client, :post
 
-    BASE_NSQD_URL = ENV.fetch 'NSQD_URL', "https://#{ENV.fetch('NSQD_HTTPS_ADDRESS', '')}"
+    BASE_NSQD_URL = ENV.fetch('NSQD_URL') do
+      if ENV['NSQD_HTTPS_ADDRESS']
+        "https://#{ENV.fetch('NSQD_HTTPS_ADDRESS')}"
+      else
+        "http://#{ENV.fetch('NSQD_HTTP_ADDRESS')}"
+      end
+    end
     VALID_FORMATS = %w[text json].freeze
 
     ##

data/lib/fastly_nsq/http/nsqlookupd.rb

@@ -7,7 +7,7 @@ class FastlyNsq::Http
     extend Forwardable
     def_delegator :client, :get
 
-    BASE_NSQLOOKUPD_URL = "http://#{ENV.fetch('NSQLOOKUPD_HTTP_ADDRESS', '').split(',')[0]}".freeze
+    BASE_NSQLOOKUPD_URL = "http://#{ENV.fetch('NSQLOOKUPD_HTTP_ADDRESS', '').split(',')[0]}"
 
     ##
     # List of producers for a given topic

data/lib/fastly_nsq/launcher.rb

@@ -5,29 +5,34 @@ require 'fastly_nsq/safe_thread'
 class FastlyNsq::Launcher
   include FastlyNsq::SafeThread
 
-  def initialize(options)
-    @done = false
-    @manager = FastlyNsq::Manager.new options
-    @options = options
+  attr_reader :timeout, :logger
+  attr_accessor :pulse
+
+  def manager
+    FastlyNsq.manager
   end
 
-  def run
-    @thread = safe_thread('heartbeat', &method(:start_heartbeat))
-    @manager.start
+  def initialize(timeout: 5, pulse: 5, logger: FastlyNsq.logger, **options)
+    @done    = false
+    @timeout = timeout
+    @pulse   = pulse
+    @logger  = logger
+
+    FastlyNsq.manager = FastlyNsq::Manager.new(options)
   end
 
-  def quiet
-    @done = true
-    @manager.quiet
+  def beat
+    @heartbeat ||= safe_thread('heartbeat', &method(:start_heartbeat))
   end
 
-  # Shuts down the process.  This method does not
-  # return until all work is complete and cleaned up.
-  # It can take up to the timeout to complete.
   def stop
-    deadline = Time.now + @options.fetch(:timeout, 5)
-    quiet
-    @manager.stop deadline
+    @done = true
+    manager.terminate(timeout)
+  end
+
+  def stop_listeners
+    @done = true
+    manager.stop_listeners
   end
 
   def stopping?
@@ -37,25 +42,28 @@ class FastlyNsq::Launcher
   private
 
   def heartbeat
-    FastlyNsq.logger.debug do
+    logger.debug do
       [
         'HEARTBEAT:',
-        'thread_status:', @manager.listeners.map(&:status).join(', '),
-        'listener_count:', @manager.listeners.count
+        'busy:', manager.pool.length,
+        'processed:', manager.pool.completed_task_count,
+        'max_threads:', manager.pool.max_length,
+        'max_queue_size:', manager.pool.largest_length,
+        'listeners:', manager.listeners.count
       ].join(' ')
     end
 
     # TODO: Check the health of the system overall and kill it if needed
     #       ::Process.kill('dieing because...', $$)
   rescue => e
-    FastlyNsq.logger.error "heartbeat error: #{e.message}"
+    logger.error "Heartbeat error: #{e.message}"
   end
 
   def start_heartbeat
-    loop do
+    until manager.stopped?
       heartbeat
-      sleep 5
+      sleep pulse
     end
-    FastlyNsq.logger.info('Heartbeat stopping...')
+    logger.info('Heartbeat stopping...')
   end
 end

data/lib/fastly_nsq/listener.rb

@@ -1,117 +1,48 @@
 # frozen_string_literal: true
 
-require 'fastly_nsq/message'
-require 'fastly_nsq/manager'
-require 'fastly_nsq/safe_thread'
-require 'fastly_nsq/listener/config'
+class FastlyNsq::Listener
+  extend Forwardable
 
-module FastlyNsq
-  class Listener
-    include FastlyNsq::SafeThread
+  DEFAULT_PRIORITY = 0
+  DEFAULT_CONNECTION_TIMEOUT = 5 # seconds
 
-    def self.listen_to(*args)
-      new(*args).go
-    end
+  def_delegators :consumer, :connected?
 
-    def initialize(topic:, processor:, channel: nil, consumer: nil, **options)
-      @consumer     = consumer || FastlyNsq::Consumer.new(topic: topic, channel: channel)
-      @done         = false
-      @logger       = options.fetch :logger, FastlyNsq.logger
-      @manager      = options[:manager] || FastlyNsq::Manager.new
-      @preprocessor = options[:preprocessor]
-      @processor    = processor
-      @thread       = nil
-      @topic        = topic
-    end
+  attr_reader :preprocessor, :topic, :processor, :priority, :channel, :logger, :consumer
 
-    def identity
-      {
-        consumer:     @consumer,
-        manager:      @manager,
-        preprocessor: @preprocessor,
-        processor:    @processor,
-        topic:        @topic,
-      }
-    end
+  def initialize(topic:, processor:, preprocessor: FastlyNsq.preprocessor, channel: FastlyNsq.channel, consumer: nil,
+                 logger: FastlyNsq.logger, priority: DEFAULT_PRIORITY, connect_timeout: DEFAULT_CONNECTION_TIMEOUT)
 
-    def reset_then_dup
-      reset
-      dup
-    end
+    raise ArgumentError, "processor #{processor.inspect} does not respond to #call" unless processor.respond_to?(:call)
+    raise ArgumentError, "priority #{priority.inspect} must be a Integer" unless priority.is_a?(Integer)
 
-    def start
-      @logger.info { "> Listener Started: topic #{@topic}" }
-      @thread ||= safe_thread('listener', &method(:go))
-    end
+    @channel      = channel
+    @logger       = logger
+    @preprocessor = preprocessor
+    @processor    = processor
+    @topic        = topic
+    @priority     = priority
 
-    def go(run_once: false)
-      until @done
-        next_message do |message|
-          log message
-          preprocess message
-          @processor.process message
-        end
+    @consumer = consumer || FastlyNsq::Consumer.new(topic: topic,
+                                                    connect_timeout: connect_timeout,
+                                                    channel: channel,
+                                                    queue: FastlyNsq::Feeder.new(self, priority))
 
-        terminate if run_once
-      end
-
-      @manager.listener_stopped(self)
-    rescue FastlyNsq::Shutdown
-      @manager.listener_stopped(self)
-    rescue Exception => e # rubocop:disable Lint/RescueException
-      @logger.error e.inspect
-      @manager.listener_killed(self)
-    end
-
-    def status
-      @thread.status if @thread
-    end
-
-    def terminate
-      @done = true
-      cleanup
-      return unless @thread
-      @logger.info "< Listener TERM: topic #{@topic}"
-      # Interrupt a Consumer blocking in pop with no messages otherwise it will never shutdown
-      @thread.raise FastlyNsq::Shutdown if @consumer.empty?
-    end
-
-    def kill
-      @done = true
-      cleanup
-      return unless @thread
-      @logger.info "< Listener KILL: topic #{@topic}"
-      @thread.raise FastlyNsq::Shutdown
-    end
-
-    private
-
-    def log(message)
-      @logger.info "[NSQ] Message received on topic [#{@topic}]: #{message}" if @logger
-    end
-
-    def cleanup
-      @consumer.terminate
-      @logger.info "< Consumer terminated: topic [#{@topic}]"
-    end
-
-    def next_message
-      nsq_message = @consumer.pop # TODO: consumer.pop do |message|
-      message = FastlyNsq::Message.new(nsq_message)
-      result  = yield message
-      message.finish if result
-    end
+    FastlyNsq.manager.add_listener(self)
+  end
 
-    def preprocess(message)
-      @preprocessor.call(message) if @preprocessor
-    end
+  def call(nsq_message)
+    message = FastlyNsq::Message.new(nsq_message)
+    logger.info "[NSQ] Message received on topic [#{topic}]: #{message}"
+    preprocessor&.call(message)
+    result = processor.call(message)
+    message.finish if result
+    message
+  end
 
-    def reset
-      @done = false
-      @thread = nil
-      self
-    end
+  def terminate
+    return unless connected?
+    consumer.terminate
+    logger.info "< Consumer terminated: topic [#{topic}]"
   end
 end
-
-class FastlyNsq::Shutdown < StandardError; end

data/lib/fastly_nsq/manager.rb

@@ -1,104 +1,80 @@
 # frozen_string_literal: true
 
-require 'set'
-
 class FastlyNsq::Manager
-  attr_reader :listeners
+  DEADLINE = 30
+  DEFAULT_POOL_SIZE = 5
+
+  attr_reader :done, :pool, :logger
 
-  def initialize(options = {})
-    @options = options
-    @done = false
-    @listeners = Set.new
-    @plock = Mutex.new
+  def initialize(logger: FastlyNsq.logger, **pool_options)
+    @done      = false
+    @logger    = logger
+    @pool      = FastlyNsq::PriorityThreadPool.new(
+      { fallback_policy: :caller_runs, max_threads: DEFAULT_POOL_SIZE }.merge(pool_options),
+    )
   end
 
-  def start
-    setup_configured_listeners
-    @listeners.each(&:start)
+  def topic_listeners
+    @topic_listeners ||= {}
   end
 
-  def quiet
-    return if @done
-    @done = true
+  def topics
+    topic_listeners.keys
+  end
 
-    FastlyNsq.logger.info { 'Terminating quiet listeners' }
-    @listeners.each(&:terminate)
+  def listeners
+    topic_listeners.values.to_set
   end
 
-  PAUSE_TIME = 0.5
+  def terminate(deadline = DEADLINE)
+    return if done
 
-  def stop(deadline)
-    quiet
+    stop_listeners
 
-    sleep PAUSE_TIME
-    return if @listeners.empty?
+    return if pool.shutdown?
 
-    FastlyNsq.logger.info { 'Pausing to allow workers to finish...' }
-    remaining = deadline - Time.now
-    while remaining > PAUSE_TIME
-      return if @listeners.empty?
-      sleep PAUSE_TIME
-      remaining = deadline - Time.now
-    end
-    return if @listeners.empty?
+    stop_processing(deadline)
 
-    hard_shutdown
+    @done = true
   end
 
   def stopped?
-    @done
+    done
   end
 
-  def listener_stopped(listener)
-    @plock.synchronize do
-      @listeners.delete listener
-    end
-  end
+  def add_listener(listener)
+    logger.info { "Listening to topic:'#{listener.topic}' on channel: '#{listener.channel}'" }
 
-  def listener_killed(listener)
-    @plock.synchronize do
-      @listeners.delete listener
-      unless @done
-        FastlyNsq.logger.info { "recreating listener for: #{listener.identity}" }
-        new_listener = listener.reset_then_dup
-        @listeners << new_listener
-        new_listener.start
-      end
+    if topic_listeners[listener.topic]
+      logger.warn { "topic: #{listener.topic} was added more than once" }
     end
-  end
-
-  private
 
-  def setup_configured_listeners
-    FastlyNsq.logger.debug { "options #{@options.inspect}" }
-    FastlyNsq.logger.debug { "starting listeners: #{FastlyNsq.topic_map.inspect}" }
+    topic_listeners[listener.topic] = listener
+  end
 
-    FastlyNsq.topic_map.each_pair do |topic, processor|
-      @listeners << setup_listener(topic, processor)
-    end
+  def transfer(new_manager, deadline: DEADLINE)
+    new_manager.topic_listeners.merge!(topic_listeners)
+    stop_processing(deadline)
+    topic_listeners.clear
+    @done = true
   end
 
-  def setup_listener(topic, processor)
-    FastlyNsq.logger.info { "Listening to topic:'#{topic}' on channel: '#{FastlyNsq.channel}'" }
-    FastlyNsq::Listener.new(
-      topic:        topic,
-      channel:      FastlyNsq.channel,
-      processor:    processor,
-      preprocessor: FastlyNsq.preprocessor,
-      manager:      self,
-    )
+  def stop_listeners
+    logger.info { 'Stopping listeners' }
+    listeners.each(&:terminate)
+    topic_listeners.clear
   end
 
-  def hard_shutdown
-    cleanup = nil
-    @plock.synchronize do
-      cleanup = @listeners.dup
-    end
+  protected
 
-    unless cleanup.empty?
-      FastlyNsq.logger.warn { "Terminating #{cleanup.size} busy worker threads" }
-    end
+  def stop_processing(deadline)
+    logger.info { 'Stopping processors' }
+    pool.shutdown
+
+    logger.info { 'Waiting for processors to finish...' }
+    return if pool.wait_for_termination(deadline)
 
-    cleanup.each(&:kill)
+    logger.info { 'Killing processors...' }
+    pool.kill
   end
 end