faktory_worker_ruby 0.7.0 → 1.0.1

data/lib/faktory/launcher.rb

@@ -11,7 +11,7 @@ module Faktory
     def initialize(options)
       merged_options = Faktory.options.merge(options)
       @manager = Faktory::Manager.new(merged_options)
-      @done = false
+      @current_state = nil
       @options = merged_options
     end
 
@@ -22,7 +22,7 @@ module Faktory
 
     # Stops this instance from processing any more jobs,
     def quiet
-      @done = true
+      @current_state = 'quiet'
       @manager.quiet
     end
 
@@ -32,13 +32,17 @@ module Faktory
     def stop
       deadline = Time.now + @options[:timeout]
 
-      @done = true
+      @current_state = 'terminate'
       @manager.quiet
       @manager.stop(deadline)
     end
 
     def stopping?
-      @done
+      @current_state == 'terminate'
+    end
+
+    def quiet?
+      @current_state == 'quiet'
     end
 
     PROCTITLES = []
@@ -50,12 +54,23 @@ module Faktory
       PROCTITLES << proc { title }
       PROCTITLES << proc { "[#{Processor.busy_count} of #{@options[:concurrency]} busy]" }
       PROCTITLES << proc { "stopping" if stopping? }
+      PROCTITLES << proc { "quiet" if quiet? }
 
       loop do
         $0 = PROCTITLES.map {|p| p.call }.join(" ")
 
         begin
-          Faktory.server {|c| c.beat }
+          result = Faktory.server {|c| c.beat(@current_state) }
+          case result
+          when "OK"
+            # all good
+          when "terminate"
+            ::Process.kill('TERM', $$)
+          when "quiet"
+            ::Process.kill('TSTP', $$)
+          else
+            Faktory.logger.warn "Got unexpected BEAT: #{result}"
+          end
         rescue => ex
           # best effort, try again in a few secs
         end

data/lib/faktory/logging.rb

@@ -29,8 +29,8 @@ module Faktory
     def self.job_hash_context(job_hash)
       # If we're using a wrapper class, like ActiveJob, use the "wrapped"
       # attribute to expose the underlying thing.
-      klass = job_hash['wrapped'.freeze] || job_hash["jobtype".freeze]
-      "#{klass} JID-#{job_hash['jid'.freeze]}"
+      klass = job_hash.dig('custom', 'wrapped') || job_hash["jobtype"]
+      "#{klass} JID-#{job_hash['jid']}"
     end
 
     def self.with_job_hash_context(job_hash, &block)

data/lib/faktory/manager.rb

@@ -123,6 +123,9 @@ module Faktory
       cleanup.each do |processor|
         processor.kill
       end
+      cleanup.each do |processor|
+        processor.thread.join(1)
+      end
     end
 
   end

data/lib/faktory/middleware/batch.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+#
+# Simple middleware to save the current batch and restore it when the job executes.
+#
+module Faktory::Middleware::Batch
+  class Client
+    def call(payload, pool)
+      b = Thread.current["faktory_batch"]
+      if b
+        payload["custom"] ||= {}
+        payload["custom"]["bid"] = b.bid
+      end
+      yield
+    end
+  end
+
+  class Worker
+    def call(jobinst, payload)
+      jobinst.bid = payload.dig("custom", "bid")
+      yield
+    end
+  end
+end
+
+Faktory.configure_client do |config|
+  config.client_middleware do |chain|
+    chain.add Faktory::Middleware::Batch::Client
+  end
+end
+
+Faktory.configure_worker do |config|
+  config.client_middleware do |chain|
+    chain.add Faktory::Middleware::Batch::Client
+  end
+  config.worker_middleware do |chain|
+    chain.add Faktory::Middleware::Batch::Worker
+  end
+end

data/lib/faktory/middleware/i18n.rb

@@ -19,10 +19,8 @@ module Faktory::Middleware::I18n
   # Pull the msg locale out and set the current thread to use it.
   class Worker
     def call(jobinst, payload)
-      I18n.locale = payload.dig("custom", "locale") || I18n.default_locale
-      yield
-    ensure
-      I18n.locale = I18n.default_locale
+      locale = payload.dig("custom", "locale") || I18n.default_locale
+      I18n.with_locale(locale) { yield }
     end
   end
 end

data/lib/faktory/mutate.rb

@@ -0,0 +1,85 @@
+require 'faktory/client'
+
+# require 'faktory/mutate'
+# cl = Faktory::Client.new
+# cl.discard(Faktory::RETRIES) do |filter|
+#   filter.with_type("QuickBooksSyncJob")
+#   filter.matching("*uid:12345*"))
+# end
+module Faktory
+
+  # Valid targets
+  RETRIES = "retries"
+  SCHEDULED = "scheduled"
+  DEAD = "dead"
+
+  module Mutator
+    class Filter
+      attr_accessor :hash
+
+      def initialize
+        @hash = {}
+      end
+
+      # This must be the exact type of the job, no pattern matching
+      def with_type(jobtype)
+        @hash[:jobtype] = jobtype
+      end
+
+      # This is a regexp that will be passed as is to Redis's SCAN.
+      # Notably you should surround it with * to ensure it matches
+      # substrings within the job payload.
+      # See https://redis.io/commands/scan for details.
+      def matching(regexp)
+        @hash[:regexp] = regexp
+      end
+
+      # One or more JIDs to target:
+      # filter.jids << 'abcdefgh1234'
+      # filter.jids = ['abcdefgh1234', '1234567890']
+      def jids
+        @hash[:jids] ||= []
+      end
+      def jids=(ary)
+        @hash[:jids] = Array(ary)
+      end
+    end
+
+    def discard(target, &block)
+      filter = Filter.new
+      block.call(filter) if block
+      mutate('discard', target, filter)
+    end
+
+    def kill(target, &block)
+      filter = Filter.new
+      block.call(filter) if block
+      mutate('kill', target, filter)
+    end
+
+    def requeue(target, &block)
+      filter = Filter.new
+      block.call(filter) if block
+      mutate('requeue', target, filter)
+    end
+
+    def clear(target)
+      mutate('discard', target, nil)
+    end
+
+    private
+
+    def mutate(cmd, target, filter)
+      payload = {:cmd => cmd,:target => target}
+      payload[:filter] = filter.hash if filter && !filter.hash.empty?
+
+      transaction do
+        command("MUTATE", JSON.dump(payload))
+        ok
+      end
+    end
+
+  end
+end
+
+Faktory::Client.send(:include, Faktory::Mutator)

data/lib/faktory/processor.rb

@@ -88,6 +88,7 @@ module Faktory
           @@busy_count = @@busy_count + 1
         end
         begin
+          @job = work.job
           process(work)
         ensure
           @@busy_lock.synchronize do
@@ -148,10 +149,11 @@ module Faktory
           end
         end
         work.acknowledge
-      rescue Faktory::Shutdown
-        # Had to force kill this job because it didn't finish
-        # within the timeout.  Don't acknowledge the work since
-        # we didn't properly finish it.
+      rescue Faktory::Shutdown => shut
+        # Had to force kill this job because it didn't finish within
+        # the timeout.  Fail it so we can release any locks server-side
+        # and immediately restart it.
+        work.fail(shut)
       rescue Exception => ex
         handle_exception(ex, { :context => "Job raised exception", :job => work.job })
         work.fail(ex)

data/lib/faktory/rails.rb

@@ -1,6 +1,16 @@
 # frozen_string_literal: true
 module Faktory
   class Rails < ::Rails::Engine
+    # This hook happens after `Rails::Application` is inherited within
+    # config/application.rb and before config is touched, usually within the
+    # class block. Definitely before config/environments/*.rb and
+    # config/initializers/*.rb.
+    config.before_configuration do
+      if defined?(::ActiveJob)
+        require 'active_job/queue_adapters/faktory_adapter'
+      end
+    end
+
     config.after_initialize do
       # This hook happens after all initializers are run, just before returning
       # from config/environment.rb back to faktory/cli.rb.
@@ -10,8 +20,22 @@ module Faktory
       # None of this matters on the client-side, only within the Faktory executor itself.
       #
       Faktory.configure_worker do |_|
+        if ::Rails::VERSION::MAJOR < 5
+          raise "Your current version of Rails, #{::Rails::VERSION::STRING}, is not supported"
+        end
+
         Faktory.options[:reloader] = Faktory::Rails::Reloader.new
       end
+
+      begin
+        # https://github.com/rails/rails/pull/41248
+        if defined?(::Mail::SMTP)
+          ::Mail::SMTP::DEFAULTS[:read_timeout] ||= 5
+          ::Mail::SMTP::DEFAULTS[:open_timeout] ||= 5
+        end
+      rescue => ex
+        # ignore
+      end
     end
 
     class Reloader
@@ -30,4 +54,11 @@ module Faktory
       end
     end
   end if defined?(::Rails)
+
+  if defined?(::Rails) && ::Rails::VERSION::MAJOR < 5
+    $stderr.puts("**************************************************")
+    $stderr.puts("🚫 ERROR: Faktory Worker does not support Rails versions under 5.x - please ensure your workers are updated")
+    $stderr.puts("**************************************************")
+    $stderr.puts("")
+  end
 end

data/lib/faktory/testing.rb

@@ -31,7 +31,7 @@ module Faktory
     end
 
     def self.inline!(&block)
-      # Only allow blockless inline via `blockless_inline_is_a_bad_idea_but_I_wanna_do_it_anyway!`
+      # Only allow inline testing inside of a block
       # https://github.com/mperham/sidekiq/issues/3495
       unless block_given?
         raise 'Must provide a block to Faktory::Testing.inline!'
@@ -40,10 +40,6 @@ module Faktory
       __set_test_mode(:inline, &block)
     end
 
-    def self.blockless_inline_is_a_bad_idea_but_I_wanna_do_it_anyway!
-      __set_test_mode(:inline)
-    end
-
     def self.enabled?
       self.__test_mode != :disable
     end
@@ -98,9 +94,9 @@ module Faktory
       end
     end
 
-    def open
+    def open(*args)
       unless Faktory::Testing.enabled?
-        real_open
+        real_open(*args)
       end
     end
   end

data/lib/faktory/tracking.rb

@@ -0,0 +1,41 @@
+module Faktory
+  module Trackable
+
+    ##
+    # Tracking allows a long-running Faktory job to report its progress:
+    #
+    #   def perform(...)
+    #     track_progress(10, "Calculating values")
+    #     # do some work
+    #
+    #     track_progress(20, "Sending emails")
+    #     # do some more work
+    #
+    #     track_progress(20, "Sending emails", reserve_until: 10.minutes.from_now)
+    #     # do some more work
+    #   end
+    #
+    # Note:
+    # 1. jobs should be small and fine-grained (and so fast) if possible.
+    # 2. tracking is useful for long-running jobs, tracking a fast job will only add overhead
+    # 3. tracking only works with a single job, use Batches to monitor a group of jobs
+    # 4. reserve_until allows a job to dynamically extend its reservation so it is not garbage collected by Faktory while running
+    # 5. you can only reserve up to 24 hours.
+    #
+    def track_progress(percent, desc=nil, reserve_until:nil)
+      hash = { 'jid' => jid, 'percent' => percent.to_i, 'desc' => desc }
+      hash["reserve_until"] = convert(reserve_until) if reserve_until
+      Faktory.server {|c| c.set_track(hash) }
+    end
+
+    private
+
+    def convert(ts)
+      raise ArgumentError, "Timestamp in the past: #{ts}" if Time.now > ts
+      raise ArgumentError, "Timestamp too far in the future: #{ts}" if (Time.now + 86400) < ts
+
+      tsf = ts.to_f
+      Time.at(tsf).utc.iso8601
+    end
+  end
+end

data/lib/faktory/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Faktory
-  VERSION = "0.7.0"
+  VERSION = "1.0.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: faktory_worker_ruby
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Mike Perham
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-03-17 00:00:00.000000000 Z
+date: 2021-02-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: connection_pool
@@ -19,7 +19,7 @@ dependencies:
         version: '2.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.1
+        version: 2.2.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,7 +29,21 @@ dependencies:
         version: '2.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.1
+        version: 2.2.2
+- !ruby/object:Gem::Dependency
+  name: activejob
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.2.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.2.0
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -62,16 +76,16 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '12'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '12'
+        version: '0'
 description: Ruby worker for Faktory.
 email:
 - mike@contribsys.com
@@ -81,6 +95,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".travis.yml"
 - Changes.md
 - Gemfile
 - Gemfile.lock
@@ -89,22 +104,28 @@ files:
 - Rakefile
 - bin/faktory-worker
 - faktory_worker_ruby.gemspec
+- lib/active_job/queue_adapters/faktory_adapter.rb
 - lib/faktory.rb
+- lib/faktory/batch.rb
 - lib/faktory/cli.rb
 - lib/faktory/client.rb
 - lib/faktory/connection.rb
 - lib/faktory/exception_handler.rb
 - lib/faktory/fetch.rb
+- lib/faktory/io.rb
 - lib/faktory/job.rb
 - lib/faktory/job_logger.rb
 - lib/faktory/launcher.rb
 - lib/faktory/logging.rb
 - lib/faktory/manager.rb
+- lib/faktory/middleware/batch.rb
 - lib/faktory/middleware/chain.rb
 - lib/faktory/middleware/i18n.rb
+- lib/faktory/mutate.rb
 - lib/faktory/processor.rb
 - lib/faktory/rails.rb
 - lib/faktory/testing.rb
+- lib/faktory/tracking.rb
 - lib/faktory/util.rb
 - lib/faktory/version.rb
 - lib/faktory_worker_ruby.rb
@@ -112,7 +133,7 @@ homepage: https://github.com/contribsys/faktory_worker_ruby
 licenses:
 - LGPL-3.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -120,16 +141,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.2.2
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.13
-signing_key: 
+rubygems_version: 3.2.6
+signing_key:
 specification_version: 4
 summary: Ruby worker for Faktory
 test_files: []