faktory_worker_ruby 0.7.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 672affacf993528eedff478961a96bbc222e62bc
-  data.tar.gz: 8b58b6e46e4d6965690bc4a17f3067548671edd0
+SHA256:
+  metadata.gz: 7739220174ff556abdf6a357cbcd260b6290c732628b27c8dc01ad00be727dd9
+  data.tar.gz: 3c87db6e71078419a761c6b34dbdcdd3914d5551f4b42f04140a1621c6568b50
 SHA512:
-  metadata.gz: 7210a8933baf0a671a608d41b5bb183fec3ea93b5c233c937c3882e921634d1baedb5985be1ecd6142ee1ca9257d3483012a7511f40c4f58fc4c13d0f34a4f4f
-  data.tar.gz: 1a2e769ad7b96d539842e29c7056e36523d878cb81ccda8610247b99e8f239877dfbe7caee93be6d194213801c2a31fbfb876c69f5a5850bad6b15461661c4e4
+  metadata.gz: b2fc14441abfab58c7cde5401e42adcfe3bb558cd5d5dbf8f6a5db6b176cf2b1d9057370dd0b1daf3d3ca629bb829495bc53167e56e02f0543ec4f89556792ba
+  data.tar.gz: 12b5d2fc13701e3c221b9ef8301bdb5b43806b1058336298393b66fc3f8020eb01d9569a43884a9f789a5df754bd24e987a340a74f12f055ae71a244d0d0763d

data/.travis.yml

@@ -0,0 +1,11 @@
+sudo: required
+
+language: ruby
+
+services:
+  - docker
+
+before_install:
+  - docker pull contribsys/faktory:latest
+  - docker run -d -p 127.0.0.1:7419:7419 -p 127.0.0.1:7420:7420 contribsys/faktory:latest /faktory -b :7419 -w :7420
+  - docker ps -a

data/Changes.md

@@ -1,5 +1,32 @@
 # Changes
 
+## 1.0.1
+
+- Run client middleware before pushing a job to Faktory [#48]
+- Implement read timeouts for Faktory::Client for faktory#297
+
+## 1.0.0
+
+- Ruby 2.5+ is now required
+- Support for Faktory Enterprise, job batches and job tracking
+- Support for the MUTATE command.
+- Notify Faktory when a worker process is going quiet so that the UI shows this
+- Refactor Faktory::Client error handling for faktory#208
+
+## 0.8.1
+
+- Fix breakage with non-ActiveJobs [#29]
+- Ruby 2.3+ is now required
+
+## 0.8.0
+
+- Add `-l LABEL` argument for adding labels to a process [#27, jpwinans]
+- Support the quiet and shutdown heartbeat signals from the server [#28]
+
+## 0.7.1
+
+- Add an ActiveJob adapter for FWR. [#17, jagthedrummer]
+
 ## 0.7.0
 
 - Add testing API, almost identical to Sidekiq's `sidekiq/testing` API.

data/Gemfile

@@ -1,3 +1,5 @@
 source 'https://rubygems.org'
 
+gem 'simplecov', require: false, group: :test
+
 gemspec

data/Gemfile.lock

@@ -1,25 +1,51 @@
 PATH
   remote: .
   specs:
-    faktory_worker_ruby (0.7.0)
-      connection_pool (~> 2.2, >= 2.2.1)
+    faktory_worker_ruby (1.0.1)
+      connection_pool (~> 2.2, >= 2.2.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    connection_pool (2.2.1)
-    minitest (5.10.3)
-    minitest-hooks (1.4.2)
-    rake (12.1.0)
+    activejob (6.0.3.2)
+      activesupport (= 6.0.3.2)
+      globalid (>= 0.3.6)
+    activesupport (6.0.3.2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+      zeitwerk (~> 2.2, >= 2.2.2)
+    concurrent-ruby (1.1.6)
+    connection_pool (2.2.3)
+    docile (1.3.2)
+    globalid (0.4.2)
+      activesupport (>= 4.2.0)
+    i18n (1.8.5)
+      concurrent-ruby (~> 1.0)
+    minitest (5.14.1)
+    minitest-hooks (1.5.0)
+      minitest (> 5.3)
+    rake (13.0.1)
+    simplecov (0.18.5)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+    simplecov-html (0.12.2)
+    thread_safe (0.3.6)
+    tzinfo (1.2.7)
+      thread_safe (~> 0.1)
+    zeitwerk (2.4.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  activejob (>= 5.2.0)
   faktory_worker_ruby!
   minitest (~> 5)
   minitest-hooks
-  rake (~> 12)
+  rake
+  simplecov
 
 BUNDLED WITH
-   1.16.0
+   2.1.4

data/README.md

@@ -1,5 +1,7 @@
 # faktory\_worker\_ruby
 
+![travis](https://travis-ci.org/contribsys/faktory_worker_ruby.svg?branch=master)
+
 Faktory\_worker\_ruby is the official Ruby client and worker process for the
 Faktory background job server.  It is similar to [Sidekiq](http://sidekiq.org).
 
@@ -15,7 +17,7 @@ Faktory background job server.  It is similar to [Sidekiq](http://sidekiq.org).
 +-----------------+                            +-------------------+
 |                 |                            |                   |
 |    Client       |                            |     Worker        |
-|    pushes       |                            |     pulls         |
+|    pushes       |                            |     fetches       |
 |     jobs        |                            |      jobs         |
 |                 |                            |                   |
 |                 |                            |                   |
@@ -30,11 +32,14 @@ Faktory background job server.  It is similar to [Sidekiq](http://sidekiq.org).
 This gem contains only the client and worker parts.  The
 server part is [here](https://github.com/contribsys/faktory/)
 
-## Installation
+## Requirements
 
-First, make sure you have the [Faktory server](https://github.com/contribsys/faktory/#installation) installed.
+* Ruby 2.5 or higher
+* Faktory 1.2 or higher [Installation](https://github.com/contribsys/faktory/wiki/Installation)
 
-Next, install this gem:
+Optionally, Rails 5.2+ for ActiveJob.
+
+## Installation
 
     gem install faktory_worker_ruby
 
@@ -92,4 +97,4 @@ PRs to improve this are very welcome).
 
 ## Author
 
-Mike Perham, @mperham, mike @ contribsys.com
+Mike Perham, @getajobmike, mike @ contribsys.com

data/faktory_worker_ruby.gemspec

@@ -14,10 +14,11 @@ Gem::Specification.new do |gem|
   gem.files         = `git ls-files | grep -Ev '^(test|myapp|examples)'`.split("\n")
   gem.test_files    = []
   gem.version       = Faktory::VERSION
-  gem.required_ruby_version = ">= 2.2.2"
+  gem.required_ruby_version = ">= 2.5.0"
 
-  gem.add_dependency                  'connection_pool', '~> 2.2', ">= 2.2.1"
+  gem.add_dependency                  'connection_pool', '~> 2.2', ">= 2.2.2"
+  gem.add_development_dependency      'activejob', '>= 5.2.0'
   gem.add_development_dependency      'minitest', '~> 5'
   gem.add_development_dependency      'minitest-hooks'
-  gem.add_development_dependency      'rake', '~> 12'
+  gem.add_development_dependency      'rake'
 end

data/lib/active_job/queue_adapters/faktory_adapter.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'active_job'
+
+module ActiveJob
+  module QueueAdapters
+    # == Faktory adapter for Active Job
+    #
+    # To use Faktory set the queue_adapter config to +:faktory+.
+    #
+    #   Rails.application.config.active_job.queue_adapter = :faktory
+    class FaktoryAdapter
+      def enqueue(job) #:nodoc:
+        enqueue_at(job, nil)
+      end
+
+      def enqueue_at(job, timestamp) #:nodoc:
+        jid = SecureRandom.hex(12)
+        job.provider_job_id = jid
+        hash = {
+          "jid"     => jid,
+          "jobtype" => JobWrapper.to_s,
+          "custom"  => {
+            "wrapped" => job.class.to_s,
+          },
+          "queue"   => job.queue_name,
+          "args"    => [ job.serialize ],
+        }
+        opts = job.faktory_options_hash.dup
+        hash["at"] = Time.at(timestamp).utc.to_datetime.rfc3339(9) if timestamp
+        if opts.size > 0
+          hash["retry"] = opts.delete("retry") if opts.has_key?("retry")
+          hash["custom"] = opts.merge(hash["custom"])
+        end
+        pool = Thread.current[:faktory_via_pool] || Faktory.server_pool
+        Faktory.client_middleware.invoke(hash, pool) do
+          pool.with do |c|
+            c.push(hash)
+          end
+        end
+      end
+
+      class JobWrapper #:nodoc:
+        include Faktory::Job
+
+        def perform(job_data)
+          Base.execute job_data
+        end
+      end
+    end
+  end
+
+  class Base
+    class_attribute :faktory_options_hash
+    self.faktory_options_hash = {}
+
+    def self.faktory_options(hsh)
+      self.faktory_options_hash = self.faktory_options_hash.stringify_keys.merge(hsh.stringify_keys)
+    end
+  end
+end

data/lib/faktory.rb

@@ -52,6 +52,7 @@ module Faktory
   #     config.worker_middleware do |chain|
   #       chain.add MyServerHook
   #     end
+  #     config.default_job_options = { retry: 3 }
   #   end
   def self.configure_worker
     yield self if worker?
@@ -61,7 +62,7 @@ module Faktory
   # Configuration for Faktory client, use like:
   #
   #   Faktory.configure_client do |config|
-  #     config.faktory = { :size => 1, :url => 'myhost:7419' }
+  #     config.default_job_options = { retry: 3 }
   #   end
   def self.configure_client
     yield self unless worker?
@@ -164,3 +165,4 @@ module Faktory
 end
 
 require 'faktory/rails' if defined?(::Rails::Engine)
+require 'faktory/batch'

data/lib/faktory/batch.rb

@@ -0,0 +1,178 @@
+require "faktory/middleware/batch"
+
+module Faktory
+  ##
+  # A Batch is a set of jobs which can be tracked as a group, with
+  # callbacks that can fire after all the jobs are attempted or successful.
+  # Every batch must define at least one callback.
+  #
+  # * The "complete" callback is fired when all jobs in the batch have been attempted.
+  #   Some might have failed.
+  # * The "success" callback is fired when all jobs in the batch have succeeded. This
+  #   might never be fired if a job continues to error until it runs out of retries.
+  #
+  # **Please note that batches are only available in Faktory Enterprise.** This is
+  # the client-side code required to implement batches, it won't work without
+  # the server-side component.
+  #
+  # Simple example:
+  #
+  #   b = Faktory::Batch.new
+  #   b.description = "Process all documents for user 12345"
+  #   # a callback can be defined as just a Ruby job class
+  #   b.success = "MySuccessCallbackJob"
+  #   # or the full job hash...
+  #   b.complete = { jobtype: "MyCompleteCallbackJob", args: [12345], queue: "critical" }
+  #   b.jobs do
+  #     SomeJob.perform_async(xyz)
+  #     AnotherJob.perform_async(user_id)
+  #   end
+  #
+  # At the end of the `jobs` call, the batch is persisted to the Faktory server. It must
+  # not be modified further with one exception: jobs within the batch can "reopen" the batch
+  # in order to dynamically add more jobs or child batches.
+  #
+  # Any job within a batch may "reopen" its own batch to dynamically add more jobs.
+  # A job can get access to its batch by using the `bid` or `batch` accessor on
+  # `Faktory::Job`. You can use the `bid` accessor to test if the job is part of a batch.
+  #
+  # Reopen example:
+  #
+  #  class MyJob
+  #    include Faktory::Job
+  #
+  #    def perform
+  #      batch.jobs do
+  #        SomeOtherJob.perform_async
+  #      end if bid
+  #    end
+  #
+  # Batches may be nested without limit by setting `parent_bid` when creating a
+  # batch. Generally you create child batches if you wish that subset of jobs to have
+  # their own callback for your application logic purposes. Otherwise you can reopen the
+  # current batch and add more jobs.
+  #
+  # Batch parent/child relationship is never implicit: you must manually set
+  # `parent_bid` if you wish to define a child batch.
+  #
+  # Nested example:
+  #
+  #  class MyJob
+  #    include Faktory::Job
+  #
+  #    def perform
+  #      child = Faktory::Batch.new
+  #
+  #      # MyJob is executing as part of a previously defined batch.
+  #      # Add a new child batch to this batch.
+  #      child.parent_bid = bid
+  #      child.success = ...
+  #      child.jobs do |cb|
+  #        SomeJob.perform_async
+  #
+  #        gchild = Faktory::Batch.new
+  #        gchild.parent_bid = cb.bid
+  #        gchild.success = ...
+  #        gchild.jobs do |gcb|
+  #          ChildJob.perform_async
+  #        end
+  #      end
+  #    end
+  #  end
+  #
+  # Callbacks are guaranteed to be called hierarchically: child's success callback
+  # will not be called until gchild's success callback has executed successfully.
+  #
+  class Batch
+    attr_reader :bid
+    attr_accessor :description, :parent_bid
+
+    def initialize(bid=nil)
+      @bid = bid
+    end
+
+    def success=(val)
+      raise "Batch cannot be modified once created" if bid
+      @success = to_callback(val)
+    end
+
+    def complete=(val)
+      raise "Batch cannot be modified once created" if bid
+      @success = to_callback(val)
+    end
+
+    def jobs(&block)
+      Faktory.server do |client|
+        if @bid.nil?
+          @bid = client.create_batch(self, &block)
+        else
+          client.reopen_batch(self, &block)
+        end
+      end
+    end
+
+    def to_h
+      raise ArgumentError, "Callback required" unless defined?(@success) || defined?(@complete)
+
+      hash = {}
+      hash["parent_bid"] = parent_bid if parent_bid
+      hash["description"] = description if description
+      hash["success"] = @success if defined?(@success)
+      hash["complete"] = @complete if defined?(@complete)
+      hash
+    end
+
+    private
+
+    def to_callback(val)
+      case val
+      when String
+        basic_job.merge({ "jobtype" => val })
+      when Class
+        basic_job.merge({ "jobtype" => val })
+      when Hash
+        basic_job.merge(val)
+      else
+        raise ArgumentError, "Unknown callback #{val}"
+      end
+    end
+
+    def basic_job
+      {
+        "jid"   => SecureRandom.hex(12),
+        "args"  => [],
+        "queue" => "default",
+      }
+    end
+  end
+
+  class BatchStatus
+    def initialize(bid)
+      @bid = bid
+    end
+
+    def hash
+      @hash ||= Faktory.server{|c| c.batch_status(@bid) }
+    end
+
+    def created_at
+      hash["created_at"]
+    end
+
+    def description
+      hash["description"]
+    end
+
+    def parent_bid
+      hash["parent_bid"]
+    end
+
+    def total
+      hash["total"]
+    end
+
+    def pending
+      hash["pending"]
+    end
+  end
+end