faktory_worker_ruby 0.5.0

data/lib/faktory/connection.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+require 'connection_pool'
+
+module Faktory
+  class Connection
+    class << self
+      def create(options={})
+        size = Faktory.worker? ? (Faktory.options[:concurrency] + 2) : 5
+        ConnectionPool.new(:timeout => options[:pool_timeout] || 1, :size => size) do
+          Faktory::Client.new
+        end
+      end
+    end
+  end
+end

data/lib/faktory/exception_handler.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+require 'faktory'
+
+module Faktory
+  module ExceptionHandler
+
+    class Logger
+      def call(ex, ctxHash)
+        Faktory.logger.warn(Faktory.dump_json(ctxHash)) if !ctxHash.empty?
+        Faktory.logger.warn "#{ex.class.name}: #{ex.message}"
+        Faktory.logger.warn ex.backtrace.join("\n") unless ex.backtrace.nil?
+      end
+
+      # Set up default handler which just logs the error
+      Faktory.error_handlers << Faktory::ExceptionHandler::Logger.new
+    end
+
+    def handle_exception(ex, ctxHash={})
+      Faktory.error_handlers.each do |handler|
+        begin
+          handler.call(ex, ctxHash)
+        rescue => ex
+          Faktory.logger.error "!!! ERROR HANDLER THREW AN ERROR !!!"
+          Faktory.logger.error ex
+          Faktory.logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
+        end
+      end
+    end
+
+  end
+end

data/lib/faktory/fetch.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Faktory
+  UnitOfWork = Struct.new(:job) do
+    def acknowledge
+      Faktory.server {|c| c.ack(jid) }
+    end
+
+    def fail(ex)
+      Faktory.server {|c| c.fail(jid, ex) }
+    end
+
+    def jid
+      job['jid']
+    end
+  end
+
+  class Fetcher
+    def initialize(options)
+      @strictly_ordered_queues = !!options[:strict]
+      @queues = options[:queues]
+      @queues = @queues.uniq if @strictly_ordered_queues
+    end
+
+    def retrieve_work
+      work = Faktory.server { |conn| conn.fetch(*queues_cmd) }
+      UnitOfWork.new(work) if work
+    end
+
+    # Creating the pop command takes into account any
+    # configured queue weights. By default pop returns
+    # data from the first queue that has pending elements. We
+    # recreate the queue command each time we invoke pop
+    # to honor weights and avoid queue starvation.
+    def queues_cmd
+      if @strictly_ordered_queues
+        @queues
+      else
+        @queues.shuffle.uniq
+      end
+    end
+
+  end
+end

data/lib/faktory/job.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+module Faktory
+
+  ##
+  # Include this module in your Job class and you can easily create
+  # asynchronous jobs:
+  #
+  # class HardJob
+  #   include Faktory::Job
+  #
+  #   def perform(*args)
+  #     # do some work
+  #   end
+  # end
+  #
+  # Then in your Rails app, you can do this:
+  #
+  #   HardJob.perform_async(1, 2, 3)
+  #
+  # Note that perform_async is a class method, perform is an instance method.
+  module Job
+    attr_accessor :jid
+
+    def self.included(base)
+      raise ArgumentError, "You cannot include Faktory::Job in an ActiveJob: #{base.name}" if base.ancestors.any? {|c| c.name == 'ActiveJob::Base' }
+
+      base.extend(ClassMethods)
+      base.faktory_class_attribute :faktory_options_hash
+    end
+
+    def logger
+      Faktory.logger
+    end
+
+    # This helper class encapsulates the set options for `set`, e.g.
+    #
+    #     SomeJob.set(queue: 'foo').perform_async(....)
+    #
+    class Setter
+      def initialize(opts)
+        @opts = opts
+      end
+
+      def perform_async(*args)
+        @opts['jobtype'.freeze].client_push(@opts.merge!('args'.freeze => args))
+      end
+
+      # +interval+ must be a timestamp, numeric or something that acts
+      #   numeric (like an activesupport time interval).
+      def perform_in(interval, *args)
+        int = interval.to_f
+        now = Time.now.to_f
+        ts = (int < 1_000_000_000 ? now + int : int)
+        at = Time.at(ts).utc.to_datetime.rfc3339(9)
+
+        @opts.merge! 'args'.freeze => args, 'at'.freeze => at
+        # Optimization to enqueue something now that is scheduled to go out now or in the past
+        @opts.delete('at'.freeze) if ts <= now
+        @opts['jobtype'.freeze].client_push(@opts)
+      end
+      alias_method :perform_at, :perform_in
+    end
+
+    module ClassMethods
+
+      def set(options)
+        Setter.new(options.merge!('jobtype'.freeze => self))
+      end
+
+      def perform_async(*args)
+        client_push('jobtype'.freeze => self, 'args'.freeze => args)
+      end
+
+      # +interval+ must be a timestamp, numeric or something that acts
+      #   numeric (like an activesupport time interval).
+      def perform_in(interval, *args)
+        int = interval.to_f
+        now = Time.now.to_f
+        ts = (int < 1_000_000_000 ? now + int : int)
+        item = { 'jobtype'.freeze => self, 'args'.freeze => args }
+
+        item['at'] = Time.at(ts).utc.to_datetime.rfc3339(9) if ts > now
+        client_push(item)
+      end
+      alias_method :perform_at, :perform_in
+
+      ##
+      # Allows customization of Faktory features for this type of Job.
+      # Legal options:
+      #
+      #   queue - use a named queue for this Job, default 'default'
+      #   retry - enable automatic retry for this Job, *Integer* count, default 25
+      #   backtrace - whether to save the error backtrace in the job payload to display in web UI,
+      #      an integer number of lines to save, default *0*
+      #
+      def faktory_options(opts={})
+        # stringify
+        self.faktory_options_hash = get_faktory_options.merge(Hash[opts.map{|k, v| [k.to_s, v]}])
+      end
+
+      def get_faktory_options # :nodoc:
+        self.faktory_options_hash ||= Faktory.default_job_options
+      end
+
+      def client_push(item) # :nodoc:
+        pool = Thread.current[:faktory_via_pool] || get_faktory_options['pool'.freeze] || Faktory.server_pool
+        item = get_faktory_options.merge(item)
+        # stringify
+        item.keys.each do |key|
+          item[key.to_s] = item.delete(key)
+        end
+        item["jid"] ||= SecureRandom.hex(12)
+        item["queue"] ||= "default"
+
+        Faktory.client_middleware.invoke(item, pool) do
+          pool.with do |c|
+            c.push(item)
+          end
+        end
+      end
+
+      def faktory_class_attribute(*attrs)
+        instance_reader = true
+        instance_writer = true
+
+        attrs.each do |name|
+          singleton_class.instance_eval do
+            undef_method(name) if method_defined?(name) || private_method_defined?(name)
+          end
+          define_singleton_method(name) { nil }
+
+          ivar = "@#{name}"
+
+          singleton_class.instance_eval do
+            m = "#{name}="
+            undef_method(m) if method_defined?(m) || private_method_defined?(m)
+          end
+          define_singleton_method("#{name}=") do |val|
+            singleton_class.class_eval do
+              undef_method(name) if method_defined?(name) || private_method_defined?(name)
+              define_method(name) { val }
+            end
+
+            if singleton_class?
+              class_eval do
+                undef_method(name) if method_defined?(name) || private_method_defined?(name)
+                define_method(name) do
+                  if instance_variable_defined? ivar
+                    instance_variable_get ivar
+                  else
+                    singleton_class.send name
+                  end
+                end
+              end
+            end
+            val
+          end
+
+          if instance_reader
+            undef_method(name) if method_defined?(name) || private_method_defined?(name)
+            define_method(name) do
+              if instance_variable_defined?(ivar)
+                instance_variable_get ivar
+              else
+                self.class.public_send name
+              end
+            end
+          end
+
+          if instance_writer
+            m = "#{name}="
+            undef_method(m) if method_defined?(m) || private_method_defined?(m)
+            attr_writer name
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/faktory/job_logger.rb

@@ -0,0 +1,24 @@
+module Faktory
+  class JobLogger
+
+    def call(item)
+      start = Time.now
+      logger.info("start".freeze)
+      yield
+      logger.info("done: #{elapsed(start)} sec")
+    rescue Exception
+      logger.info("fail: #{elapsed(start)} sec")
+      raise
+    end
+
+    private
+
+    def elapsed(start)
+      (Time.now - start).round(3)
+    end
+
+    def logger
+      Faktory.logger
+    end
+  end
+end

data/lib/faktory/launcher.rb

@@ -0,0 +1,66 @@
+# encoding: utf-8
+# frozen_string_literal: true
+require 'faktory/manager'
+
+module Faktory
+  class Launcher
+    include Util
+
+    attr_accessor :manager
+
+    def initialize(options)
+      @manager = Faktory::Manager.new(options)
+      @done = false
+      @options = options
+    end
+
+    def run
+      @thread = safe_thread("heartbeat", &method(:heartbeat))
+      @manager.start
+    end
+
+    # Stops this instance from processing any more jobs,
+    def quiet
+      @done = true
+      @manager.quiet
+    end
+
+    # Shuts down the process.  This method does not
+    # return until all work is complete and cleaned up.
+    # It can take up to the timeout to complete.
+    def stop
+      deadline = Time.now + @options[:timeout]
+
+      @done = true
+      @manager.quiet
+      @manager.stop(deadline)
+    end
+
+    def stopping?
+      @done
+    end
+
+    PROCTITLES = []
+
+    private unless $TESTING
+
+    def heartbeat
+      title = ['faktory-worker', Faktory::VERSION, @options[:tag]].compact.join(" ")
+      PROCTITLES << proc { title }
+      PROCTITLES << proc { "[#{Processor.busy_count} of #{@options[:concurrency]} busy]" }
+      PROCTITLES << proc { "stopping" if stopping? }
+
+      loop do
+        $0 = PROCTITLES.map {|p| p.call }.join(" ")
+
+        begin
+          Faktory.server {|c| c.beat }
+        rescue => ex
+          # best effort, try again in a few secs
+        end
+        sleep 10
+      end
+    end
+
+  end
+end

data/lib/faktory/logging.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+require 'time'
+require 'logger'
+require 'fcntl'
+
+module Faktory
+  module Logging
+
+    class Pretty < Logger::Formatter
+      SPACE = " "
+
+      # Provide a call() method that returns the formatted message.
+      def call(severity, time, program_name, message)
+        "#{time.utc.iso8601(3)} #{::Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
+      end
+
+      def context
+        c = Thread.current[:faktory_context]
+        " #{c.join(SPACE)}" if c && c.any?
+      end
+    end
+
+    class WithoutTimestamp < Pretty
+      def call(severity, time, program_name, message)
+        "#{::Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
+      end
+    end
+
+    def self.job_hash_context(job_hash)
+      # If we're using a wrapper class, like ActiveJob, use the "wrapped"
+      # attribute to expose the underlying thing.
+      klass = job_hash['wrapped'.freeze] || job_hash["jobtype".freeze]
+      "#{klass} JID-#{job_hash['jid'.freeze]}"
+    end
+
+    def self.with_job_hash_context(job_hash, &block)
+      with_context(job_hash_context(job_hash), &block)
+    end
+
+    def self.with_context(msg)
+      Thread.current[:faktory_context] ||= []
+      Thread.current[:faktory_context] << msg
+      yield
+    ensure
+      Thread.current[:faktory_context].pop
+    end
+
+    def self.initialize_logger(log_target = STDOUT)
+      oldlogger = defined?(@logger) ? @logger : nil
+      @logger = Logger.new(log_target)
+      @logger.level = Logger::INFO
+      # We assume that any TTY is logging directly to a terminal and needs timestamps.
+      # We assume that any non-TTY is logging to Upstart/Systemd/syslog/Heroku/etc with a decent
+      # logging subsystem that provides a timestamp for each entry.
+      @logger.formatter = log_target.tty? ? Pretty.new : WithoutTimestamp.new
+      oldlogger.close if oldlogger && !$TESTING # don't want to close testing's STDOUT logging
+      @logger
+    end
+
+    def self.logger
+      defined?(@logger) ? @logger : initialize_logger
+    end
+
+    def self.logger=(log)
+      @logger = (log ? log : Logger.new(File::NULL))
+    end
+
+    def logger
+      Faktory::Logging.logger
+    end
+  end
+end

data/lib/faktory/manager.rb

@@ -0,0 +1,129 @@
+# encoding: utf-8
+# frozen_string_literal: true
+require 'faktory/util'
+require 'faktory/processor'
+require 'faktory/fetch'
+require 'thread'
+require 'set'
+
+module Faktory
+
+  ##
+  # The Manager is the central coordination point in Faktory, controlling
+  # the lifecycle of the Processors.
+  #
+  # Tasks:
+  #
+  # 1. start: Spin up Processors.
+  # 3. processor_died: Handle job failure, throw away Processor, create new one.
+  # 4. quiet: shutdown idle Processors.
+  # 5. stop: hard stop the Processors by deadline.
+  #
+  # Note that only the last task requires its own Thread since it has to monitor
+  # the shutdown process.  The other tasks are performed by other threads.
+  #
+  class Manager
+    include Util
+
+    attr_reader :threads
+    attr_reader :options
+
+    def initialize(options={})
+      logger.debug { options.inspect }
+      @options = options
+      @count = options[:concurrency] || 25
+      raise ArgumentError, "Concurrency of #{@count} is not supported" if @count < 1
+
+      @done = false
+      @threads = Set.new
+      @count.times do
+        @threads << Processor.new(self)
+      end
+      @plock = Mutex.new
+    end
+
+    def start
+      @threads.each do |x|
+        x.start
+      end
+    end
+
+    def quiet
+      return if @done
+      @done = true
+
+      logger.info { "Terminating quiet threads" }
+      @threads.each { |x| x.terminate }
+      fire_event(:quiet, true)
+    end
+
+    # hack for quicker development / testing environment
+    PAUSE_TIME = STDOUT.tty? ? 0.1 : 0.5
+
+    def stop(deadline)
+      quiet
+      fire_event(:shutdown, true)
+
+      # some of the shutdown events can be async,
+      # we don't have any way to know when they're done but
+      # give them a little time to take effect
+      sleep PAUSE_TIME
+      return if @threads.empty?
+
+      logger.info { "Pausing to allow threads to finish..." }
+      remaining = deadline - Time.now
+      while remaining > PAUSE_TIME
+        return if @threads.empty?
+        sleep PAUSE_TIME
+        remaining = deadline - Time.now
+      end
+      return if @threads.empty?
+
+      hard_shutdown
+    end
+
+    def processor_stopped(processor)
+      @plock.synchronize do
+        @threads.delete(processor)
+      end
+    end
+
+    def processor_died(processor, reason)
+      @plock.synchronize do
+        @threads.delete(processor)
+        unless @done
+          p = Processor.new(self)
+          @threads << p
+          p.start
+        end
+      end
+    end
+
+    def stopped?
+      @done
+    end
+
+    private
+
+    def hard_shutdown
+      # We've reached the timeout and we still have busy threads.
+      # They must die but their jobs shall live on.
+      cleanup = nil
+      @plock.synchronize do
+        cleanup = @threads.dup
+      end
+
+      if cleanup.size > 0
+        jobs = cleanup.map {|p| p.job }.compact
+
+        logger.warn { "Terminating #{cleanup.size} busy worker threads" }
+        logger.warn { "Work still in progress #{jobs.inspect}" }
+      end
+
+      cleanup.each do |processor|
+        processor.kill
+      end
+    end
+
+  end
+end