good_job 0.8.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ef13e088aa2f4e0d6671b796d01f9cf48588e4f5635c2600d6df922a7410bb63
-  data.tar.gz: 24773e7ab09699a3a4d0b88d87765e1bdeb4e1520410841bdd945ee7ee0b65c7
+  metadata.gz: 9e068306c80c1080f736520219cdf27230f9684bd69fa70cff0489ce37b1ed79
+  data.tar.gz: 8f10fb768248ca1ec1e14f44be2073f9c60069b2451c5858c571efbca84c0969
 SHA512:
-  metadata.gz: 02c14e07112294876b8d10cf1f2d3cd2c3a3de6eafdc23b77b913d48bd1a71bf8545b6b839871f3effd1cacac85c09cd2541d85d24f2c56a87251e55a1eef920
-  data.tar.gz: f51f8f509392bb432d47ff251877e8cacd1a5244099c6b9a556e4340c08e374d10138ec94239da973c575d548539467a11a2070c5166602165d37484e289a167
+  metadata.gz: 8ebe33709d71b2254fa2fc0dbc1c991df739a67ba6ee3649000bada6c4618b2d958c58372818502b57211fb4694b28b16be513227220b262da2125b6f87b61bb
+  data.tar.gz: 40db6accac1448cef4cfff7516e68d401b25711520d479cf3af4c20f03a31526f2e084aed5dabc64adceb9b23ae75ac41ae74e25bb80a4bb80c320a31fc9fb25

data/CHANGELOG.md

@@ -1,8 +1,54 @@
 # Changelog
 
+## [v1.0.1](https://github.com/bensheldon/good_job/tree/v1.0.1) (2020-07-21)
+
+[Full Changelog](https://github.com/bensheldon/good_job/compare/v1.0.0...v1.0.1)
+
+**Merged pull requests:**
+
+- Change threadpool idletime default to 60 seconds from 0 [\#49](https://github.com/bensheldon/good_job/pull/49) ([bensheldon](https://github.com/bensheldon))
+
+## [v1.0.0](https://github.com/bensheldon/good_job/tree/v1.0.0) (2020-07-20)
+
+[Full Changelog](https://github.com/bensheldon/good_job/compare/v0.9.0...v1.0.0)
+
+## [v0.9.0](https://github.com/bensheldon/good_job/tree/v0.9.0) (2020-07-20)
+
+[Full Changelog](https://github.com/bensheldon/good_job/compare/v0.8.2...v0.9.0)
+
+**Merged pull requests:**
+
+- Allow preservation of finished job records [\#46](https://github.com/bensheldon/good_job/pull/46) ([bensheldon](https://github.com/bensheldon))
+
+## [v0.8.2](https://github.com/bensheldon/good_job/tree/v0.8.2) (2020-07-18)
+
+[Full Changelog](https://github.com/bensheldon/good_job/compare/v0.8.1...v0.8.2)
+
+**Closed issues:**
+
+- Add a job timeout configuration to time out jobs that have run too long [\#19](https://github.com/bensheldon/good_job/issues/19)
+
+**Merged pull requests:**
+
+- Run Github Action tests on PRs from forks [\#44](https://github.com/bensheldon/good_job/pull/44) ([bensheldon](https://github.com/bensheldon))
+- Fix Rubygems homepage URL [\#43](https://github.com/bensheldon/good_job/pull/43) ([joshmn](https://github.com/joshmn))
+- Move where\(scheduled\_at: Time.current\) into dynamic part of GoodJob::Job::Performer [\#42](https://github.com/bensheldon/good_job/pull/42) ([bensheldon](https://github.com/bensheldon))
+
+## [v0.8.1](https://github.com/bensheldon/good_job/tree/v0.8.1) (2020-07-18)
+
+[Full Changelog](https://github.com/bensheldon/good_job/compare/v0.8.0...v0.8.1)
+
 ## [v0.8.0](https://github.com/bensheldon/good_job/tree/v0.8.0) (2020-07-17)
 
-[Full Changelog](https://github.com/bensheldon/good_job/compare/v0.6.0...v0.8.0)
+[Full Changelog](https://github.com/bensheldon/good_job/compare/v0.7.0...v0.8.0)
+
+**Merged pull requests:**
+
+- Replace Adapter inline boolean kwarg with execution\_mode instead [\#41](https://github.com/bensheldon/good_job/pull/41) ([bensheldon](https://github.com/bensheldon))
+
+## [v0.7.0](https://github.com/bensheldon/good_job/tree/v0.7.0) (2020-07-16)
+
+[Full Changelog](https://github.com/bensheldon/good_job/compare/v0.6.0...v0.7.0)
 
 **Closed issues:**
 
@@ -10,7 +56,6 @@
 
 **Merged pull requests:**
 
-- Replace Adapter inline boolean kwarg with execution\_mode instead [\#41](https://github.com/bensheldon/good_job/pull/41) ([bensheldon](https://github.com/bensheldon))
 - Add more examples to Readme [\#39](https://github.com/bensheldon/good_job/pull/39) ([bensheldon](https://github.com/bensheldon))
 - Add additional Rubocops and lint [\#38](https://github.com/bensheldon/good_job/pull/38) ([bensheldon](https://github.com/bensheldon))
 - Always store a default queue\_name, priority and scheduled\_at; index by queue\_name and scheduled\_at [\#37](https://github.com/bensheldon/good_job/pull/37) ([bensheldon](https://github.com/bensheldon))

data/README.md

@@ -43,9 +43,12 @@ $ bundle install
           t.integer :priority
           t.jsonb :serialized_params
           t.timestamp :scheduled_at
-   
-          t.index :scheduled_at
-          t.index [:queue_name, :scheduled_at]
+          t.timestamp :performed_at
+          t.timestamp :finished_at
+          t.text :error
+
+          add_index :good_jobs, :scheduled_at, where: "(finished_at IS NULL)"
+          add_index :good_jobs, [:queue_name, :scheduled_at], where: "(finished_at IS NULL)"
         end
       end
     end
@@ -152,6 +155,32 @@ If your application is already using an ActiveJob backend, you will need to inst
 
 1. Once you are confident that no unperformed jobs remain in the previous ActiveJob backend, code and configuration for that backend can be completely removed.
 
+### Monitoring and preserving worked jobs
+
+GoodJob is fully instrumented with [`ActiveSupport::Notifications`](https://edgeguides.rubyonrails.org/active_support_instrumentation.html#introduction-to-instrumentation).
+
+By default, GoodJob will delete job records after they are run, regardless of whether they succeed or not (raising a kind of `StandardError`), unless they are interrupted (raising a kind of `Exception`). 
+
+To preserve job records for later inspection, set an initializer:
+
+```ruby
+# config/initializers/good_job.rb
+GoodJob.preserve_job_records = true
+```
+
+It is also necessary to delete these preserved jobs from the database after a certain time period:
+
+- For example, in a Rake task:
+  
+    ```ruby
+    # GoodJob::Job.finished(1.day.ago).delete_all
+    ```
+- For example, using the `good_job` command-line utility:
+
+    ```bash
+    $ bundle exec good_job cleanup_preserved_jobs --before-seconds-ago=86400
+    ```
+
 ## Development
 
 To run tests:

data/lib/good_job.rb

@@ -4,13 +4,15 @@ require 'good_job/railtie'
 require 'good_job/logging'
 require 'good_job/lockable'
 require 'good_job/job'
-require "good_job/scheduler"
+require 'good_job/scheduler'
 require 'good_job/adapter'
 require 'good_job/pg_locks'
+require 'good_job/performer'
 
 require 'active_job/queue_adapters/good_job_adapter'
 
 module GoodJob
+  mattr_accessor :preserve_job_records, default: false
   include Logging
 
   ActiveSupport.run_load_hooks(:good_job, self)

data/lib/good_job/cli.rb

@@ -36,11 +36,16 @@ module GoodJob
         ENV['GOOD_JOB_POLL_INTERVAL']
       ).to_i
 
-      job_query = GoodJob::Job.all
+      job_query = GoodJob::Job.all.priority_ordered
       queue_names_without_all = queue_names.reject { |q| q == '*' }
       job_query = job_query.where(queue_name: queue_names_without_all) unless queue_names_without_all.size.zero?
 
-      job_performer = job_query.only_scheduled.priority_ordered.to_performer
+      performer_method = if GoodJob.preserve_job_records
+                           :perform_with_advisory_lock_and_preserve_job_records
+                         else
+                           :perform_with_advisory_lock_and_destroy_job_records
+                         end
+      job_performer = GoodJob::Performer.new(job_query, performer_method)
 
       $stdout.puts "GoodJob worker starting with max_threads=#{max_threads} on queues=#{queue_names.join(',')}"
 
@@ -68,6 +73,19 @@ module GoodJob
       $stdout.puts "GoodJob's jobs finished, exiting..."
     end
 
+    desc :cleanup_preserved_jobs, "Delete preserved job records"
+    method_option :before_seconds_ago,
+                  type: :numeric,
+                  default: 24 * 60 * 60,
+                  desc: "Delete records finished more than this many seconds ago"
+    def cleanup_preserved_jobs
+      require RAILS_ENVIRONMENT_RB
+
+      timestamp = Time.current - options[:before_seconds_ago]
+      result = GoodJob::Job.finished(timestamp).delete_all
+      $stdout.puts "Deleted #{result} preserved #{'job'.pluralize(result)} finished before #{timestamp}."
+    end
+
     default_task :start
   end
 end

data/lib/good_job/job.rb

@@ -2,32 +2,46 @@ module GoodJob
   class Job < ActiveRecord::Base
     include Lockable
 
+    PreviouslyPerformedError = Class.new(StandardError)
+
     DEFAULT_QUEUE_NAME = 'default'.freeze
     DEFAULT_PRIORITY = 0
 
     self.table_name = 'good_jobs'.freeze
 
+    scope :unfinished, (lambda do
+      if column_names.include?('finished_at')
+        where(finished_at: nil)
+      else
+        ActiveSupport::Deprecation.warn('GoodJob expects a good_jobs.finished_at column to exist. Please see the GoodJob README.md for migration instructions.')
+        nil
+      end
+    end)
     scope :only_scheduled, -> { where(arel_table['scheduled_at'].lteq(Time.current)).or(where(scheduled_at: nil)) }
     scope :priority_ordered, -> { order(priority: :desc) }
-    scope :to_performer, -> { Performer.new(self) }
+    scope :finished, ->(timestamp = nil) { timestamp ? where(arel_table['finished_at'].lteq(timestamp)) : where.not(finished_at: nil) }
 
-    class Performer
-      def initialize(query)
-        @query = query
-      end
+    def self.perform_with_advisory_lock(destroy_after: !GoodJob.preserve_job_records)
+      good_job = nil
+      result = nil
+      error = nil
+
+      unfinished.only_scheduled.limit(1).with_advisory_lock do |good_jobs|
+        good_job = good_jobs.first
+        break unless good_job
 
-      def next
-        good_job = nil
+        result, error = good_job.perform(destroy_after: destroy_after)
+      end
 
-        @query.limit(1).with_advisory_lock do |good_jobs|
-          good_job = good_jobs.first
-          break unless good_job
+      [good_job, result, error] if good_job
+    end
 
-          good_job.perform
-        end
+    def self.perform_with_advisory_lock_and_preserve_job_records
+      perform_with_advisory_lock(destroy_after: false)
+    end
 
-        good_job
-      end
+    def self.perform_with_advisory_lock_and_destroy_job_records
+      perform_with_advisory_lock(destroy_after: true)
     end
 
     def self.enqueue(active_job, scheduled_at: nil, create_with_advisory_lock: false)
@@ -50,16 +64,43 @@ module GoodJob
       good_job
     end
 
-    def perform
+    def perform(destroy_after: true)
+      raise PreviouslyPerformedError, 'Cannot perform a job that has already been performed' if finished_at
+
+      result = nil
+      error = nil
+
       ActiveSupport::Notifications.instrument("before_perform_job.good_job", { good_job: self })
+      self.performed_at = Time.current
+      save! unless destroy_after
+
       ActiveSupport::Notifications.instrument("perform_job.good_job", { good_job: self }) do
         params = serialized_params.merge(
           "provider_job_id" => id
         )
-        ActiveJob::Base.execute(params)
+        begin
+          result = ActiveJob::Base.execute(params)
+        rescue StandardError => e
+          error = e
+        end
+      end
 
+      if error.nil? && result.is_a?(Exception)
+        error = result
+        result = nil
+      end
+
+      error_message = "#{error.class}: #{error.message}" if error
+      self.error = error_message
+      self.finished_at = Time.current
+
+      if destroy_after
         destroy!
+      else
+        save!
       end
+
+      [result, error]
     end
   end
 end

data/lib/good_job/performer.rb

@@ -0,0 +1,12 @@
+module GoodJob
+  class Performer
+    def initialize(target, method_name)
+      @target = target
+      @method_name = method_name
+    end
+
+    def next
+      @target.public_send(@method_name)
+    end
+  end
+end

data/lib/good_job/scheduler.rb

@@ -15,7 +15,7 @@ module GoodJob
       min_threads: 0,
       max_threads: Concurrent.processor_count,
       auto_terminate: true,
-      idletime: 0,
+      idletime: 60,
       max_queue: 0,
       fallback_policy: :abort, # shouldn't matter -- 0 max queue
     }.freeze
@@ -59,21 +59,21 @@ module GoodJob
 
     def create_thread
       future = Concurrent::Future.new(args: [@performer], executor: @pool) do |performer|
-        result = nil
-        Rails.application.executor.wrap { result = performer.next }
-        result
+        output = nil
+        Rails.application.executor.wrap { output = performer.next }
+        output
       end
       future.add_observer(self, :task_observer)
       future.execute
     end
 
-    def timer_observer(time, executed_task, error)
-      ActiveSupport::Notifications.instrument("finished_timer_task.good_job", { result: executed_task, error: error, time: time })
+    def timer_observer(time, executed_task, thread_error)
+      ActiveSupport::Notifications.instrument("finished_timer_task.good_job", { result: executed_task, error: thread_error, time: time })
     end
 
-    def task_observer(time, result, error)
-      ActiveSupport::Notifications.instrument("finished_job_task.good_job", { result: result, error: error, time: time })
-      create_thread if result
+    def task_observer(time, output, thread_error)
+      ActiveSupport::Notifications.instrument("finished_job_task.good_job", { result: output, error: thread_error, time: time })
+      create_thread if output
     end
   end
 end

data/lib/good_job/version.rb

@@ -1,3 +1,3 @@
 module GoodJob
-  VERSION = '0.8.0'.freeze
+  VERSION = '1.0.1'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: good_job
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Ben Sheldon
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-07-17 00:00:00.000000000 Z
+date: 2020-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -242,18 +242,19 @@ files:
 - lib/good_job/job.rb
 - lib/good_job/lockable.rb
 - lib/good_job/logging.rb
+- lib/good_job/performer.rb
 - lib/good_job/pg_locks.rb
 - lib/good_job/railtie.rb
 - lib/good_job/scheduler.rb
 - lib/good_job/version.rb
-homepage: https://github.com/benheldon/good_job
+homepage: https://github.com/bensheldon/good_job
 licenses:
 - MIT
 metadata:
   bug_tracker_uri: https://github.com/bensheldon/good_job/issues
   changelog_uri: https://github.com/bensheldon/good_job/blob/master/CHANGELOG.md
   documentation_uri: https://rdoc.info/github/bensheldon/good_job
-  homepage_uri: https://github.com/benheldon/good_job
+  homepage_uri: https://github.com/bensheldon/good_job
   source_code_uri: https://github.com/bensheldon/good_job
 post_install_message: 
 rdoc_options: