subserver 0.1.0

data/lib/subserver/exception_handler.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+require 'subserver'
+
+module Subserver
+  module ExceptionHandler
+
+    class Logger
+      def call(ex, ctxHash)
+        Subserver.logger.warn(Subserver.dump_json(ctxHash)) if !ctxHash.empty?
+        Subserver.logger.warn("#{ex.class.name}: #{ex.message}")
+        Subserver.logger.warn(ex.backtrace.join("\n")) unless ex.backtrace.nil?
+      end
+
+      Subserver.error_handlers << Subserver::ExceptionHandler::Logger.new
+    end
+
+    def handle_exception(ex, ctxHash={})
+      Subserver.error_handlers.each do |handler|
+        begin
+          handler.call(ex, ctxHash)
+        rescue => ex
+          Subserver.logger.error "!!! ERROR HANDLER THREW AN ERROR !!!"
+          Subserver.logger.error ex
+          Subserver.logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
+        end
+      end
+    end
+  end
+end

data/lib/subserver/health.rb

@@ -0,0 +1,33 @@
+require 'socket'      # Sockets are in standard library
+
+module Subserver
+  class Health
+
+    attr_accessor :server
+
+    def initialize
+      @server = TCPServer.new 4481   
+    end
+
+    def start  
+      begin
+        while session = @server.accept
+          request = session.gets
+
+          session.print "HTTP/1.1 200\r\n" # 1
+          session.print "Content-Type: text/html\r\n" # 2
+          session.print "\r\n" # 3
+          session.print "Subserver Online" #4
+          session.close
+        end
+      rescue Errno::ECONNRESET, Errno::EPIPE => e
+        puts e.message
+        retry
+      end
+    end
+
+    def stop
+      @server.close
+    end 
+  end
+end 

data/lib/subserver/launcher.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+require 'subserver/manager'
+
+module Subserver
+  # The Launcher is a very simple Actor whose job is to
+  # start, monitor and stop the core Actors in Subserver.
+  # If any of these actors die, the Subserver process exits
+  # immediately.
+  class Launcher
+    include Util
+
+    attr_accessor :manager
+
+    def initialize(options)
+      @manager = Subserver::Manager.new(options)
+      @done = false
+      @options = options
+    end
+
+    def run
+      @manager.start
+    end
+
+    # Stops this instance from processing any more jobs,
+    #
+    def quiet
+      @done = true
+      @manager.quiet
+    end
+
+    # Shuts down the process.  This method does not
+    # return until all work is complete and cleaned up.
+    # It can take up to the timeout to complete.
+    def stop
+      deadline = Time.now + @options[:timeout]
+
+      @done = true
+      @manager.quiet
+      @manager.stop(deadline)
+    end
+
+    def stopping?
+      @done
+    end
+
+    private unless $TESTING
+
+    def to_data
+      @data ||= begin
+        {
+          'hostname' => hostname,
+          'started_at' => Time.now.to_f,
+          'pid' => $$,
+          'tag' => @options[:tag] || '',
+          'queues' => @options[:queues].uniq,
+          'labels' => @options[:labels],
+          'identity' => identity,
+        }
+      end
+    end
+
+    def to_json
+      @json ||= begin
+        # this data changes infrequently so dump it to a string
+        # now so we don't need to dump it every heartbeat.
+        Subserver.dump_json(to_data)
+      end
+    end
+
+  end
+end

data/lib/subserver/listener.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+require 'subserver/util'
+require 'subserver/message_logger'
+require 'subserver/pubsub'
+require 'thread'
+
+module Subserver
+  ##
+  # The Listener is a standalone thread which:
+  #
+  # 1. Starts Google Pubsub subscription threads which: 
+  #   a. Instantiate the Subscription class
+  #   b. Run the middleware chain
+  #   c. call subscriber #perform
+  #
+  # A Listener can exit due to shutdown (listner_stopped)
+  # or due to an error during message processing (listener_died)
+  #
+  # If an error occurs during message processing, the
+  # Listener calls the Manager to create a new one
+  # to replace itself and exits.
+  #
+  class Listener
+
+    include Util
+
+    attr_reader :thread
+    attr_reader :subscriber
+
+    def initialize(mgr, subscriber)
+      @mgr = mgr
+      @down = false
+      @done = false
+      @thread = nil
+      @reloader = Subserver.options[:reloader]
+      @subscriber = subscriber
+      @subscription = retrive_subscrption
+      @logging = (mgr.options[:message_logger] || Subserver::MessageLogger).new
+    end
+
+    def name
+      @subscriber.name
+    end
+
+    def stop
+      @done = true
+      return if !@thread
+      
+      # Stop the listener and wait for current messages to finish processing.
+      @pubsub_listener.stop.wait!
+      @mgr.listener_stopped(self)
+    end
+
+    def kill
+      @done = true
+      return if !@thread
+      # Hard stop the listener and shutdown thread after timeout passes.
+      @pubsub_listener.stop
+      @thread.raise ::Subserver::Shutdown
+    end
+
+    def start
+      @thread ||= safe_thread("listener", &method(:run))
+    end
+
+    private unless $TESTING
+
+    def retrive_subscrption
+      subscription_name = @subscriber.get_subserver_options[:subscription]
+      begin
+        subscription = Pubsub.client.subscription subscription_name
+      rescue Google::Cloud::Error => e
+        raise ArgumentError, "Invalid Subscription name: #{subscription_name} Please ensure your Pubsub subscription exists."
+      end
+      subscription
+    end
+
+    def connect_subscriber
+      options = @subscriber.get_subserver_options
+      logger.debug("Connecting to subscription with options: #{options}")
+      @pubsub_listener = @subscription.listen streams: options[:streams], threads: options[:threads] do |received_message|
+        logger.debug("Message Received: #{received_message}")
+        process_message(received_message)
+      end
+    end
+
+    def run
+      begin
+        connect_subscriber
+        @pubsub_listener.start
+      rescue Subserver::Shutdown
+        @mgr.listener_stopped(self)
+      rescue Exception => ex
+        @mgr.listener_died(self, @subscriber, ex)
+      end
+    end
+
+    def process_message(received_message)
+      begin
+        logger.debug("Executing Middleware")
+        Subserver.middleware.invoke(@subscriber, received_message) do
+          execute_processor(@subscriber, received_message)
+        end
+      rescue Subserver::Shutdown
+        # Reject message if shutdown
+        received_message.reject!
+      rescue Exception => ex
+        handle_exception(e, { context: "Exception raised during message processing.", message: received_message })
+        raise e
+      end
+    end
+
+    def execute_processor(subscriber, received_message)
+      subscriber.new.perform(received_message)
+    end
+
+    # Ruby doesn't provide atomic counters out of the box so we'll
+    # implement something simple ourselves.
+    # https://bugs.ruby-lang.org/issues/14706
+    class Counter
+      def initialize
+        @value = 0
+        @lock = Mutex.new
+      end
+
+      def incr(amount=1)
+        @lock.synchronize { @value = @value + amount }
+      end
+
+      def reset
+        @lock.synchronize { val = @value; @value = 0; val }
+      end
+    end
+
+    PROCESSED = Counter.new
+    FAILURE = Counter.new
+    # This is mutable global state but because each thread is storing
+    # its own unique key/value, there's no thread-safety issue AFAIK.
+    WORKER_STATE = {}
+
+    def stats(job_hash, queue)
+      tid = Subserver::Logging.tid
+      WORKER_STATE[tid] = {:queue => queue, :payload => job_hash, :run_at => Time.now.to_i }
+
+      begin
+        yield
+      rescue Exception
+        FAILURE.incr
+        raise
+      ensure
+        WORKER_STATE.delete(tid)
+        PROCESSED.incr
+      end
+    end
+
+  end
+end

data/lib/subserver/logging.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+require 'time'
+require 'logger'
+require 'fcntl'
+
+module Subserver
+  module Logging
+
+    class Pretty < Logger::Formatter
+      SPACE = " "
+
+      # Provide a call() method that returns the formatted message.
+      def call(severity, time, program_name, message)
+        "#{time.utc.iso8601(3)} #{::Process.pid} TID-#{Subserver::Logging.tid}#{context} #{severity}: #{message}\n"
+      end
+
+      def context
+        c = Thread.current[:subserver_context]
+        " #{c.join(SPACE)}" if c && c.any?
+      end
+    end
+
+    class WithoutTimestamp < Pretty
+      def call(severity, time, program_name, message)
+        "#{::Process.pid} TID-#{Subserver::Logging.tid}#{context} #{severity}: #{message}\n"
+      end
+    end
+
+    def self.tid
+      Thread.current['subserver_tid'] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
+    end
+
+    def self.job_hash_context(job_hash)
+      # If we're using a wrapper class, like ActiveJob, use the "wrapped"
+      # attribute to expose the underlying thing.
+      klass = job_hash['wrapped'] || job_hash["class"]
+      bid = job_hash['bid']
+      "#{klass} JID-#{job_hash['jid']}#{" BID-#{bid}" if bid}"
+    end
+
+    def self.with_job_hash_context(job_hash, &block)
+      with_context(job_hash_context(job_hash), &block)
+    end
+
+    def self.with_context(msg)
+      Thread.current[:subserver_context] ||= []
+      Thread.current[:subserver_context] << msg
+      yield
+    ensure
+      Thread.current[:subserver_context].pop
+    end
+
+    def self.initialize_logger(log_target = STDOUT)
+      oldlogger = defined?(@logger) ? @logger : nil
+      @logger = Logger.new(log_target)
+      @logger.level = Logger::INFO
+      @logger.formatter = ENV['DYNO'] ? WithoutTimestamp.new : Pretty.new
+      oldlogger.close if oldlogger && !$TESTING # don't want to close testing's STDOUT logging
+      @logger
+    end
+
+    def self.logger
+      defined?(@logger) ? @logger : initialize_logger
+    end
+
+    def self.logger=(log)
+      @logger = (log ? log : Logger.new(File::NULL))
+    end
+
+    # This reopens ALL logfiles in the process that have been rotated
+    # using logrotate(8) (without copytruncate) or similar tools.
+    # A +File+ object is considered for reopening if it is:
+    #   1) opened with the O_APPEND and O_WRONLY flags
+    #   2) the current open file handle does not match its original open path
+    #   3) unbuffered (as far as userspace buffering goes, not O_SYNC)
+    # Returns the number of files reopened
+    def self.reopen_logs
+      to_reopen = []
+      append_flags = File::WRONLY | File::APPEND
+
+      ObjectSpace.each_object(File) do |fp|
+        begin
+          if !fp.closed? && fp.stat.file? && fp.sync && (fp.fcntl(Fcntl::F_GETFL) & append_flags) == append_flags
+            to_reopen << fp
+          end
+        rescue IOError, Errno::EBADF
+        end
+      end
+
+      nr = 0
+      to_reopen.each do |fp|
+        orig_st = begin
+          fp.stat
+        rescue IOError, Errno::EBADF
+          next
+        end
+
+        begin
+          b = File.stat(fp.path)
+          next if orig_st.ino == b.ino && orig_st.dev == b.dev
+        rescue Errno::ENOENT
+        end
+
+        begin
+          File.open(fp.path, 'a') { |tmpfp| fp.reopen(tmpfp) }
+          fp.sync = true
+          nr += 1
+        rescue IOError, Errno::EBADF
+          # not much we can do...
+        end
+      end
+      nr
+    rescue RuntimeError => ex
+      # RuntimeError: ObjectSpace is disabled; each_object will only work with Class, pass -X+O to enable
+      puts "Unable to reopen logs: #{ex.message}"
+    end
+
+    def logger
+      Subserver::Logging.logger
+    end
+  end
+end

data/lib/subserver/manager.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+require 'subserver/util'
+require 'subserver/listener'
+require 'thread'
+require 'set'
+
+module Subserver
+
+  ##
+  # The Manager is the central coordination point in Subserver, controlling
+  # the lifecycle of the Google Cloud Listeners.
+  #
+  # Tasks:
+  #
+  # 1. start: Load subscibers and start listeners.
+  # 2. listener_died: restart listener
+  # 3. quiet: tell listeners to stop listening and finish processing messages then shutdown.
+  # 4. stop: hard stop the listeners by deadline.
+  #
+  # Note that only the last task requires its own Thread since it has to monitor
+  # the shutdown process.  The other tasks are performed by other threads.
+  #
+  class Manager
+    include Util
+
+    attr_reader :listeners
+    attr_reader :options
+
+    def initialize(options={})
+      logger.debug { options.inspect }
+      @options = options
+      
+      @done = false
+      @listeners = Set.new
+
+      subscribers.each do |subscriber|
+        @listeners << Listener.new(self, subscriber)
+      end
+
+      @plock = Mutex.new
+    end
+
+    def start
+      if @listeners.count > 0
+        logger.info("Starting Listeners For: #{@listeners.map(&:name).join(', ')}")
+        @listeners.each do |x|
+          x.start
+        end
+      else
+        logger.warn("No Listeners starting: Couldn't find any subscribers.")
+      end
+    end
+
+    def quiet
+      return if @done
+      @done = true
+
+      logger.info { "Stopping listeners" }
+      @listeners.each { |x| x.stop }
+      fire_event(:quiet, reverse: true)
+    end
+
+    # hack for quicker development / testing environment
+    PAUSE_TIME = STDOUT.tty? ? 0.1 : 0.5
+
+    def stop(deadline)
+      quiet
+      fire_event(:shutdown, reverse: true)
+
+      # some of the shutdown events can be async,
+      # we don't have any way to know when they're done but
+      # give them a little time to take effect
+      sleep PAUSE_TIME
+      return if @listeners.empty?
+
+      logger.info { "Pausing to allow listeners to finish..." }
+      remaining = deadline - Time.now
+      while remaining > PAUSE_TIME
+        return if @listeners.empty?
+        sleep PAUSE_TIME
+        remaining = deadline - Time.now
+      end
+      return if @listeners.empty?
+
+      hard_shutdown
+    end
+
+    def listener_stopped(listener)
+      @plock.synchronize do
+        @listeners.delete(listener)
+      end
+    end
+
+    def listener_died(listener, subscriber, reason)
+      @plock.synchronize do
+        @listeners.delete(listener)
+        unless @done
+          l = Listener.new(self, subscriber)
+          @listeners << l
+          l.start
+        end
+      end
+    end
+
+    def stopped?
+      @done
+    end
+
+    def subscribers
+      @subscribers ||= load_subscribers
+    end
+
+    private
+
+    def hard_shutdown
+      # We've reached the timeout and we still have busy listeners.
+      # They must die but their jobs shall live on.
+      cleanup = nil
+      @plock.synchronize do
+        cleanup = @listeners.dup
+      end
+
+      if cleanup.size > 0
+        logger.warn { "Killing #{cleanup.size} busy worker threads" }
+        # Any message not aknowleged will be avalible for reprocessing
+      end
+
+      cleanup.each do |listener|
+        listener.kill
+      end
+    end
+
+    def load_subscribers
+      # Expand Subscriber Directory from relative require 
+      path = File.expand_path("#{options[:subscriber_dir]}/*.rb")
+
+      # Load existing set of classes
+      existing_classes = ObjectSpace.each_object(Class).to_a
+
+      # Require all subscriber files
+      Dir[path].each { |f| require f } 
+
+      # Create set with only newly created classes from require loop
+      new_classes = ObjectSpace.each_object(Class).to_a - existing_classes
+
+      # Only included named classes that have included the Subscriber module
+      subscribers = new_classes.select do |klass|
+        klass.name && klass < ::Subserver::Subscriber && options[:queues].include?(klass.subserver_options[:queue])
+      end
+    end
+
+  end
+end