workhorse 1.0.0.beta0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5dd13b11d5f7488e58812f57ad2bb70da07b95f18b89653687bf9ec579bbcaa5
-  data.tar.gz: a0e7c53cf4de56b6109830decf16f6f70a8aead13ca703a795ce7709234c1b05
+  metadata.gz: d0994338972770f1cb6f44db74a01c54c7ebdb9aa88afd1e9938a1565997edc8
+  data.tar.gz: f48f3df959cca6b69a84f77a501989b6430cc04582e56345a7441af4258f62c0
 SHA512:
-  metadata.gz: 2c6e9162d5b2735cd1de678b56e4c4e4960e25776a64cc45dfe5fbfee4fe0e50b1b3a5d15e81ca65ebfe496cce6ac4d81cfe1ba47a65b591a54916807bbe5c68
-  data.tar.gz: a24094b51e7751010920e39ad6ac3924dbccdfb84f1728efec73ed1cc0e9376a873cdb2ff4a562f49d78b17ae08e5659fd596684b299fc35b8bdcd228be15f0e
+  metadata.gz: 2631da0ac85ffb054b9705cee07174a56c56d4b773fcd25b341ece72a6e92244eb10fb88f7748dfd905fb5d01f8fd17b64425b1d479b71b40ca850af607d6581
+  data.tar.gz: a4ac3e1f039fc3884e2ef0a066ff2d152b351d87926139a3098fba81ab7bedb8dd80c783e57ea0de79a6187296feee1aea73949207d0f8631af9b4415ab8cb68

data/.travis.yml

@@ -1,7 +1,7 @@
 language: ruby
 rvm:
- - 2.3.0
  - 2.5.0
+ - 2.6.0
 services:
  - mysql
 before_install:

data/CHANGELOG.md

@@ -1,5 +1,42 @@
 # Workhorse Changelog
 
+## 1.1.0 - 2020-12-24
+
+* Add `description` column to `DbJob`.
+
+If you're upgrading from a previous version, add the `description` column
+to your `DbJob` table, e.g. with such a migration:
+
+```ruby
+class AddDescriptionToWorkhorseDbJobs < ActiveRecord::Migration[6.0]
+  def change
+    add_column :db_jobs, :description, :string, after: :perform_at, null: true
+  end
+end
+```
+
+## 1.0.1 - 2020-12-15
+
+* Fix handling of empty pid files
+
+## 1.0.0 - 2020-09-21
+
+* Stable release, identical to 1.0.0.beta2 but now extensively battle-tested
+
+## 1.0.0.beta2 - 2020-08-27
+
+* Add option `config.silence_poller_exceptions` (default `false`)
+
+* Add option `config.silence_watcher` (default `false`)
+
+## 1.0.0.beta1 - 2020-08-20
+
+This is a stability release that is still experimental and has to be tested in
+battle before it can be considered stable.
+
+* Stop passing ActiveRecord job objects between polling and worker threads to
+  avoid AR race conditions. Now only IDs are passed between threads.
+
 ## 1.0.0.beta0 - 2020-08-19
 
 This is a stability release that is still experimental and has to be tested in

data/README.md

@@ -81,7 +81,7 @@ GRANT execute ON DBMS_LOCK TO <schema-name>;
 
 Workhorse can handle any jobs that support the `perform` method and are
 serializable. To queue a basic job, use the static method `Workhorse.enqueue`.
-You can optionally pass a queue name and a priority.
+You can optionally pass a queue name, a priority and a description (as a string).
 
 ```ruby
 class MyJob
@@ -94,7 +94,7 @@ class MyJob
   end
 end
 
-Workhorse.enqueue MyJob.new('John'), queue: :test, priority: 2
+Workhorse.enqueue MyJob.new('John'), queue: :test, priority: 2, description: 'Basic Job'
 ```
 
 ### RailsOps operations
@@ -290,6 +290,10 @@ Workhorse.setup do |config|
 end
 ```
 
+Using the settings `config.silence_poller_exceptions` and
+`config.silence_watcher`, you can silence certain exceptions / error outputs
+(both are disabled by default).
+
 ## Handling database jobs
 
 Jobs stored in the database can be accessed via the ActiveRecord model
@@ -377,7 +381,7 @@ configuration or else using `self.queue_adapter` in a job class inheriting from
 Per default, jobs remain in the database, no matter in which state. This can
 eventually lead to a very large jobs database. You are advised to clean your
 jobs database on a regular interval. Workhorse provides the job
-`Workhose::Jobs::CleanupSucceededJobs` for this purpose that cleans up all
+`Workhorse::Jobs::CleanupSucceededJobs` for this purpose that cleans up all
 succeeded jobs. You can run this using your scheduler in a specific interval.
 
 ## Caveats

data/VERSION

@@ -1 +1 @@
-1.0.0.beta0
+1.1.0

data/lib/generators/workhorse/templates/create_table_jobs.rb

@@ -17,6 +17,8 @@ class CreateTableJobs < ActiveRecord::Migration
       t.integer :priority, null: false
       t.datetime :perform_at, null: true
 
+      t.string :description, null: true
+
       t.timestamps null: false
     end
 

data/lib/workhorse.rb

@@ -1,7 +1,8 @@
-require 'socket'
-require 'active_support/all'
 require 'active_record'
+require 'active_support/all'
 require 'concurrent'
+require 'socket'
+require 'uri'
 
 require 'workhorse/enqueuer'
 require 'workhorse/scoped_env'
@@ -30,6 +31,17 @@ module Workhorse
     # ExceptionNotifier.notify_exception(exception)
   end
 
+  # If set to `true`, the defined `on_exception` will not be called when the
+  # poller encounters an exception and the worker has to be shut down. The
+  # exception will still be logged.
+  mattr_accessor :silence_poller_exceptions
+  self.silence_poller_exceptions = false
+
+  # If set to `true`, the `watch` command won't produce any output. This does
+  # not include warnings such as the "development mode" warning.
+  mattr_accessor :silence_watcher
+  self.silence_watcher = false
+
   mattr_accessor :perform_jobs_in_tx
   self.perform_jobs_in_tx = true
 

data/lib/workhorse/daemon.rb

@@ -38,21 +38,21 @@ module Workhorse
       @workers << Worker.new(@workers.size + 1, name, &block)
     end
 
-    def start
+    def start(quiet: false)
       code = 0
 
       for_each_worker do |worker|
         pid_file, pid = read_pid(worker)
 
         if pid_file && pid
-          warn "Worker ##{worker.id} (#{worker.name}): Already started (PID #{pid})"
+          warn "Worker ##{worker.id} (#{worker.name}): Already started (PID #{pid})" unless quiet
           code = 1
         elsif pid_file
           File.delete pid_file
-          puts "Worker ##{worker.id} (#{worker.name}): Starting (stale pid file)"
+          puts "Worker ##{worker.id} (#{worker.name}): Starting (stale pid file)" unless quiet
           start_worker worker
         else
-          warn "Worker ##{worker.id} (#{worker.name}): Starting"
+          warn "Worker ##{worker.id} (#{worker.name}): Starting" unless quiet
           start_worker worker
         end
       end
@@ -109,7 +109,7 @@ module Workhorse
       end
 
       if should_be_running && status(quiet: true) != 0
-        return start
+        return start(quiet: Workhorse.silence_watcher)
       else
         return 0
       end
@@ -177,7 +177,9 @@ module Workhorse
       file = pid_file_for(worker)
 
       if File.exist?(file)
-        pid = IO.read(file).to_i
+        raw_pid = IO.read(file)
+        return nil, nil if raw_pid.blank?
+        pid = Integer(raw_pid)
         return file, process?(pid) ? pid : nil
       else
         return nil, nil

data/lib/workhorse/db_job.rb

@@ -7,7 +7,7 @@ module Workhorse
     STATE_FAILED    = :failed
 
     if respond_to?(:attr_accessible)
-      attr_accessible :queue, :priority, :perform_at, :handler
+      attr_accessible :queue, :priority, :perform_at, :handler, :description
     end
 
     self.table_name = 'jobs'

data/lib/workhorse/enqueuer.rb

@@ -1,11 +1,12 @@
 module Workhorse
   module Enqueuer
     # Enqueue any object that is serializable and has a `perform` method
-    def enqueue(job, queue: nil, priority: 0, perform_at: Time.now)
+    def enqueue(job, queue: nil, priority: 0, perform_at: Time.now, description: nil)
       return DbJob.create!(
         queue: queue,
         priority: priority,
         perform_at: perform_at,
+        description: description,
         handler: Marshal.dump(job)
       )
     end

data/lib/workhorse/jobs/detect_stale_jobs_job.rb

@@ -22,7 +22,7 @@ module Workhorse::Jobs
         rel = rel.where('locked_at < ?', @locked_to_started_threshold.seconds.ago)
         ids = rel.pluck(:id)
 
-        if ids.size > 0
+        unless ids.empty?
           messages << "Detected #{ids.size} jobs that were locked more than "\
                       "#{@locked_to_started_threshold}s ago and might be stale: #{ids.inspect}."
         end
@@ -34,7 +34,7 @@ module Workhorse::Jobs
         rel = rel.where('started_at < ?', @run_time_threshold.seconds.ago)
         ids = rel.pluck(:id)
 
-        if ids.size > 0
+        unless ids.empty?
           messages << "Detected #{ids.size} jobs that are running for longer than "\
                       "#{@run_time_threshold}s ago and might be stale: #{ids.inspect}."
         end

data/lib/workhorse/performer.rb

@@ -2,8 +2,8 @@ module Workhorse
   class Performer
     attr_reader :worker
 
-    def initialize(db_job, worker)
-      @db_job = db_job
+    def initialize(db_job_id, worker)
+      @db_job = Workhorse::DbJob.find(db_job_id)
       @worker = worker
       @started = false
     end

data/lib/workhorse/poller.rb

@@ -3,8 +3,8 @@ module Workhorse
     MIN_LOCK_TIMEOUT = 0.1 # In seconds
     MAX_LOCK_TIMEOUT = 1.0 # In seconds
 
-    ORACLE_LOCK_MODE = 6           # X_MODE (exclusive)
-    ORACLE_LOCK_HANDLE = 478564848 # Randomly chosen number
+    ORACLE_LOCK_MODE   = 6           # X_MODE (exclusive)
+    ORACLE_LOCK_HANDLE = 478_564_848 # Randomly chosen number
 
     attr_reader :worker
     attr_reader :table
@@ -35,7 +35,7 @@ module Workhorse
           rescue Exception => e
             worker.log %(Poll encountered exception:\n#{e.message}\n#{e.backtrace.join("\n")})
             worker.log 'Worker shutting down...'
-            Workhorse.on_exception.call(e)
+            Workhorse.on_exception.call(e) unless Workhorse.silence_poller_exceptions
             @running = false
             worker.instance_variable_get(:@pool).shutdown
             break
@@ -72,7 +72,7 @@ module Workhorse
       end
     end
 
-    def with_global_lock(name: :workhorse, timeout: 2, &block)
+    def with_global_lock(name: :workhorse, timeout: 2, &_block)
       if @is_oracle
         result = Workhorse::DbJob.connection.select_all(
           "SELECT DBMS_LOCK.REQUEST(#{ORACLE_LOCK_HANDLE}, #{ORACLE_LOCK_MODE}, #{timeout}) FROM DUAL"
@@ -105,6 +105,8 @@ module Workhorse
       timeout = [MIN_LOCK_TIMEOUT, [MAX_LOCK_TIMEOUT, worker.polling_interval].min].max
 
       with_global_lock timeout: timeout do
+        job_ids = []
+
         Workhorse.tx_callback.call do
           # As we are the only thread posting into the worker pool, it is safe to
           # get the number of idle threads without mutex synchronization. The
@@ -119,10 +121,12 @@ module Workhorse
             jobs.each do |job|
               worker.log "Marking job #{job.id} as locked", :debug
               job.mark_locked!(worker.id)
-              worker.perform job
+              job_ids << job.id
             end
           end
         end
+
+        job_ids.each { |job_id| worker.perform(job_id) }
       end
     end
 

data/lib/workhorse/worker.rb

@@ -127,14 +127,14 @@ module Workhorse
       @pool.idle
     end
 
-    def perform(db_job)
+    def perform(db_job_id)
       mutex.synchronize do
         assert_state! :running
-        log "Posting job #{db_job.id} to thread pool"
+        log "Posting job #{db_job_id} to thread pool"
 
         @pool.post do
           begin
-            Workhorse::Performer.new(db_job, self).perform
+            Workhorse::Performer.new(db_job_id, self).perform
           rescue Exception => e
             log %(#{e.message}\n#{e.backtrace.join("\n")}), :error
           end

data/test/lib/db_schema.rb

@@ -18,6 +18,8 @@ ActiveRecord::Schema.define do
     t.integer :priority, null: false
     t.datetime :perform_at, null: true
 
+    t.string :description, null: true
+
     t.timestamps null: false
   end
 

data/test/workhorse/enqueuer_test.rb

@@ -33,6 +33,15 @@ class Workhorse::EnqueuerTest < WorkhorseTest
     assert_equal 1, Workhorse::DbJob.first.priority
   end
 
+  def test_with_description
+    assert_equal 0, Workhorse::DbJob.all.count
+    Workhorse.enqueue BasicJob.new, description: 'Lorem ipsum'
+    assert_equal 1, Workhorse::DbJob.all.count
+
+    db_job = Workhorse::DbJob.first
+    assert_equal 'Lorem ipsum', db_job.description
+  end
+
   def test_op
     Workhorse.enqueue_op DummyRailsOpsOp, { queue: :q1 }, foo: :bar
 

data/test/workhorse/poller_test.rb

@@ -48,7 +48,7 @@ class Workhorse::PollerTest < WorkhorseTest
     assert_equal %w[q1 q2], w.poller.send(:valid_queues)
   end
 
-  def test_valid_queues
+  def test_valid_queues_2
     w = Workhorse::Worker.new(polling_interval: 60)
 
     assert_equal [], w.poller.send(:valid_queues)
@@ -148,6 +148,7 @@ class Workhorse::PollerTest < WorkhorseTest
     assert_equal 25,  used_workers
   end
 
+  # rubocop: disable Style/GlobalVars
   def test_connection_loss
     $thread_conn = nil
 
@@ -174,6 +175,7 @@ class Workhorse::PollerTest < WorkhorseTest
 
     assert_equal 1, Workhorse::DbJob.succeeded.count
   end
+  # rubocop: enable Style/GlobalVars
 
   private
 

data/workhorse.gemspec

@@ -1,14 +1,14 @@
 # -*- encoding: utf-8 -*-
-# stub: workhorse 1.0.0.beta0 ruby lib
+# stub: workhorse 1.1.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "workhorse".freeze
-  s.version = "1.0.0.beta0"
+  s.version = "1.1.0"
 
-  s.required_rubygems_version = Gem::Requirement.new("> 1.3.1".freeze) if s.respond_to? :required_rubygems_version=
+  s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Sitrox".freeze]
-  s.date = "2020-08-19"
+  s.date = "2020-12-24"
   s.files = [".gitignore".freeze, ".releaser_config".freeze, ".rubocop.yml".freeze, ".travis.yml".freeze, "CHANGELOG.md".freeze, "FAQ.md".freeze, "Gemfile".freeze, "LICENSE".freeze, "README.md".freeze, "RUBY_VERSION".freeze, "Rakefile".freeze, "VERSION".freeze, "bin/rubocop".freeze, "lib/active_job/queue_adapters/workhorse_adapter.rb".freeze, "lib/generators/workhorse/install_generator.rb".freeze, "lib/generators/workhorse/templates/bin/workhorse.rb".freeze, "lib/generators/workhorse/templates/config/initializers/workhorse.rb".freeze, "lib/generators/workhorse/templates/create_table_jobs.rb".freeze, "lib/workhorse.rb".freeze, "lib/workhorse/daemon.rb".freeze, "lib/workhorse/daemon/shell_handler.rb".freeze, "lib/workhorse/db_job.rb".freeze, "lib/workhorse/enqueuer.rb".freeze, "lib/workhorse/jobs/cleanup_succeeded_jobs.rb".freeze, "lib/workhorse/jobs/detect_stale_jobs_job.rb".freeze, "lib/workhorse/jobs/run_active_job.rb".freeze, "lib/workhorse/jobs/run_rails_op.rb".freeze, "lib/workhorse/performer.rb".freeze, "lib/workhorse/poller.rb".freeze, "lib/workhorse/pool.rb".freeze, "lib/workhorse/scoped_env.rb".freeze, "lib/workhorse/worker.rb".freeze, "test/active_job/queue_adapters/workhorse_adapter_test.rb".freeze, "test/lib/db_schema.rb".freeze, "test/lib/jobs.rb".freeze, "test/lib/test_helper.rb".freeze, "test/workhorse/db_job_test.rb".freeze, "test/workhorse/enqueuer_test.rb".freeze, "test/workhorse/performer_test.rb".freeze, "test/workhorse/poller_test.rb".freeze, "test/workhorse/pool_test.rb".freeze, "test/workhorse/worker_test.rb".freeze, "workhorse.gemspec".freeze]
   s.rubygems_version = "3.0.3".freeze
   s.summary = "Multi-threaded job backend with database queuing for ruby.".freeze

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: workhorse
 version: !ruby/object:Gem::Version
-  version: 1.0.0.beta0
+  version: 1.1.0
 platform: ruby
 authors:
 - Sitrox
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-08-19 00:00:00.000000000 Z
+date: 2020-12-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -192,8 +192,8 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
-email: 
+description:
+email:
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -241,10 +241,10 @@ files:
 - test/workhorse/pool_test.rb
 - test/workhorse/worker_test.rb
 - workhorse.gemspec
-homepage: 
+homepage:
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -255,12 +255,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: Multi-threaded job backend with database queuing for ruby.
 test_files: