good_job 0.8.2 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6c630bdd77afa672158924fa33dd200506354db2df4c1e795c85b3e02becc71b
-  data.tar.gz: 39ac5362e8832e62f50ba9703060af5907dcbacd64f8765b9cafaa98f10dbbb6
+  metadata.gz: bddc60fa297293af01b1334d4adf51981bfb8b314ba4577a7b3bff1d7385bd5a
+  data.tar.gz: 3533f21ffee4dba606558fa0fadd1f37a943aa285b4f6bebed4bb3b89045b61e
 SHA512:
-  metadata.gz: 86a6d5613d68428276244373b2c718a327538fe30c0461b1eaec0156ca51f72b2863d6e268e96e59f4c40e8096059c05758cc1ce83d02b680916f09a57bff428
-  data.tar.gz: 50749d69aa79ea96e3a9081b2a0cd02bdbeb4537d3da0bba748ffc3c43fb251149f84ea8cc1620fbc211794ab499fc713de8d57f8b74f49cead98fb1b8d4c1bd
+  metadata.gz: 74c0d3df2df08e78d06baa8982401d633af536277231a17d87687ff3c7ba1553d4123f7c1ce49ae23094bd781db5ee23eb9584eeba94aae49fb8cc83da0c68e0
+  data.tar.gz: 7df20f03621cb66b864a2c2950e4a11f60199cd4954ad5a622a1631254fa04bab1243ee376567600a2aed294a8c14fa9802fd1024855dc7d473a7ff8f7166515

data/CHANGELOG.md

@@ -1,8 +1,8 @@
 # Changelog
 
-## [v0.8.2](https://github.com/bensheldon/good_job/tree/v0.8.2) (2020-07-18)
+## [v0.9.0](https://github.com/bensheldon/good_job/tree/v0.9.0) (2020-07-20)
 
-[Full Changelog](https://github.com/bensheldon/good_job/compare/v0.6.0...v0.8.2)
+[Full Changelog](https://github.com/bensheldon/good_job/compare/v0.6.0...v0.9.0)
 
 **Closed issues:**
 
@@ -11,6 +11,7 @@
 
 **Merged pull requests:**
 
+- Allow preservation of finished job records [\#46](https://github.com/bensheldon/good_job/pull/46) ([bensheldon](https://github.com/bensheldon))
 - Run Github Action tests on PRs from forks [\#44](https://github.com/bensheldon/good_job/pull/44) ([bensheldon](https://github.com/bensheldon))
 - Fix Rubygems homepage URL [\#43](https://github.com/bensheldon/good_job/pull/43) ([joshmn](https://github.com/joshmn))
 - Move where\(scheduled\_at: Time.current\) into dynamic part of GoodJob::Job::Performer [\#42](https://github.com/bensheldon/good_job/pull/42) ([bensheldon](https://github.com/bensheldon))

data/README.md

@@ -43,9 +43,12 @@ $ bundle install
           t.integer :priority
           t.jsonb :serialized_params
           t.timestamp :scheduled_at
-   
-          t.index :scheduled_at
-          t.index [:queue_name, :scheduled_at]
+          t.timestamp :performed_at
+          t.timestamp :finished_at
+          t.text :error
+
+          add_index :good_jobs, :scheduled_at, where: "(finished_at IS NULL)"
+          add_index :good_jobs, [:queue_name, :scheduled_at], where: "(finished_at IS NULL)"
         end
       end
     end
@@ -152,6 +155,32 @@ If your application is already using an ActiveJob backend, you will need to inst
 
 1. Once you are confident that no unperformed jobs remain in the previous ActiveJob backend, code and configuration for that backend can be completely removed.
 
+### Monitoring and preserving worked jobs
+
+GoodJob is fully instrumented with [`ActiveSupport::Notifications`](https://edgeguides.rubyonrails.org/active_support_instrumentation.html#introduction-to-instrumentation).
+
+By default, GoodJob will delete job records after they are run, regardless of whether they succeed or not (raising a kind of `StandardError`), unless they are interrupted (raising a kind of `Exception`). 
+
+To preserve job records for later inspection, set an initializer:
+
+```ruby
+# config/initializers/good_job.rb
+GoodJob.preserve_job_records = true
+```
+
+It is also necessary to delete these preserved jobs from the database after a certain time period:
+
+- For example, in a Rake task:
+  
+    ```ruby
+    # GoodJob::Job.finished(1.day.ago).delete_all
+    ```
+- For example, using the `good_job` command-line utility:
+
+    ```bash
+    $ bundle exec good_job cleanup_preserved_jobs --before-seconds-ago=86400
+    ```
+
 ## Development
 
 To run tests:

data/lib/good_job.rb

@@ -4,13 +4,15 @@ require 'good_job/railtie'
 require 'good_job/logging'
 require 'good_job/lockable'
 require 'good_job/job'
-require "good_job/scheduler"
+require 'good_job/scheduler'
 require 'good_job/adapter'
 require 'good_job/pg_locks'
+require 'good_job/performer'
 
 require 'active_job/queue_adapters/good_job_adapter'
 
 module GoodJob
+  mattr_accessor :preserve_job_records, default: false
   include Logging
 
   ActiveSupport.run_load_hooks(:good_job, self)

data/lib/good_job/cli.rb

@@ -40,7 +40,12 @@ module GoodJob
       queue_names_without_all = queue_names.reject { |q| q == '*' }
       job_query = job_query.where(queue_name: queue_names_without_all) unless queue_names_without_all.size.zero?
 
-      job_performer = job_query.to_performer
+      performer_method = if GoodJob.preserve_job_records
+                           :perform_with_advisory_lock_and_preserve_job_records
+                         else
+                           :perform_with_advisory_lock_and_destroy_job_records
+                         end
+      job_performer = GoodJob::Performer.new(job_query, performer_method)
 
       $stdout.puts "GoodJob worker starting with max_threads=#{max_threads} on queues=#{queue_names.join(',')}"
 
@@ -68,6 +73,19 @@ module GoodJob
       $stdout.puts "GoodJob's jobs finished, exiting..."
     end
 
+    desc :cleanup_preserved_jobs, "Delete preserved job records"
+    method_option :before_seconds_ago,
+                  type: :numeric,
+                  default: 24 * 60 * 60,
+                  desc: "Delete records finished more than this many seconds ago"
+    def cleanup_preserved_jobs
+      require RAILS_ENVIRONMENT_RB
+
+      timestamp = Time.current - options[:before_seconds_ago]
+      result = GoodJob::Job.finished(timestamp).delete_all
+      $stdout.puts "Deleted #{result} preserved #{'job'.pluralize(result)} finished before #{timestamp}."
+    end
+
     default_task :start
   end
 end

data/lib/good_job/job.rb

@@ -2,32 +2,46 @@ module GoodJob
   class Job < ActiveRecord::Base
     include Lockable
 
+    PreviouslyPerformedError = Class.new(StandardError)
+
     DEFAULT_QUEUE_NAME = 'default'.freeze
     DEFAULT_PRIORITY = 0
 
     self.table_name = 'good_jobs'.freeze
 
+    scope :unfinished, (lambda do
+      if column_names.include?('finished_at')
+        where(finished_at: nil)
+      else
+        ActiveSupport::Deprecation.warn('GoodJob expects a good_jobs.finished_at column to exist. Please see the GoodJob README.md for migration instructions.')
+        nil
+      end
+    end)
     scope :only_scheduled, -> { where(arel_table['scheduled_at'].lteq(Time.current)).or(where(scheduled_at: nil)) }
     scope :priority_ordered, -> { order(priority: :desc) }
-    scope :to_performer, -> { Performer.new(self) }
+    scope :finished, ->(timestamp = nil) { timestamp ? where(arel_table['finished_at'].lteq(timestamp)) : where.not(finished_at: nil) }
 
-    class Performer
-      def initialize(query)
-        @query = query
-      end
+    def self.perform_with_advisory_lock(destroy_after: !GoodJob.preserve_job_records)
+      good_job = nil
+      result = nil
+      error = nil
+
+      unfinished.only_scheduled.limit(1).with_advisory_lock do |good_jobs|
+        good_job = good_jobs.first
+        break unless good_job
 
-      def next
-        good_job = nil
+        result, error = good_job.perform(destroy_after: destroy_after)
+      end
 
-        @query.only_scheduled.limit(1).with_advisory_lock do |good_jobs|
-          good_job = good_jobs.first
-          break unless good_job
+      [good_job, result, error] if good_job
+    end
 
-          good_job.perform
-        end
+    def self.perform_with_advisory_lock_and_preserve_job_records
+      perform_with_advisory_lock(destroy_after: false)
+    end
 
-        good_job
-      end
+    def self.perform_with_advisory_lock_and_destroy_job_records
+      perform_with_advisory_lock(destroy_after: true)
     end
 
     def self.enqueue(active_job, scheduled_at: nil, create_with_advisory_lock: false)
@@ -50,16 +64,43 @@ module GoodJob
       good_job
     end
 
-    def perform
+    def perform(destroy_after: true)
+      raise PreviouslyPerformedError, 'Cannot perform a job that has already been performed' if finished_at
+
+      result = nil
+      error = nil
+
       ActiveSupport::Notifications.instrument("before_perform_job.good_job", { good_job: self })
+      self.performed_at = Time.current
+      save! unless destroy_after
+
       ActiveSupport::Notifications.instrument("perform_job.good_job", { good_job: self }) do
         params = serialized_params.merge(
           "provider_job_id" => id
         )
-        ActiveJob::Base.execute(params)
+        begin
+          result = ActiveJob::Base.execute(params)
+        rescue StandardError => e
+          error = e
+        end
+      end
 
+      if error.nil? && result.is_a?(Exception)
+        error = result
+        result = nil
+      end
+
+      error_message = "#{error.class}: #{error.message}" if error
+      self.error = error_message
+      self.finished_at = Time.current
+
+      if destroy_after
         destroy!
+      else
+        save!
       end
+
+      [result, error]
     end
   end
 end

data/lib/good_job/performer.rb

@@ -0,0 +1,12 @@
+module GoodJob
+  class Performer
+    def initialize(target, method_name)
+      @target = target
+      @method_name = method_name
+    end
+
+    def next
+      @target.public_send(@method_name)
+    end
+  end
+end

data/lib/good_job/scheduler.rb

@@ -59,21 +59,21 @@ module GoodJob
 
     def create_thread
       future = Concurrent::Future.new(args: [@performer], executor: @pool) do |performer|
-        result = nil
-        Rails.application.executor.wrap { result = performer.next }
-        result
+        output = nil
+        Rails.application.executor.wrap { output = performer.next }
+        output
       end
       future.add_observer(self, :task_observer)
       future.execute
     end
 
-    def timer_observer(time, executed_task, error)
-      ActiveSupport::Notifications.instrument("finished_timer_task.good_job", { result: executed_task, error: error, time: time })
+    def timer_observer(time, executed_task, thread_error)
+      ActiveSupport::Notifications.instrument("finished_timer_task.good_job", { result: executed_task, error: thread_error, time: time })
     end
 
-    def task_observer(time, result, error)
-      ActiveSupport::Notifications.instrument("finished_job_task.good_job", { result: result, error: error, time: time })
-      create_thread if result
+    def task_observer(time, output, thread_error)
+      ActiveSupport::Notifications.instrument("finished_job_task.good_job", { result: output, error: thread_error, time: time })
+      create_thread if output
     end
   end
 end

data/lib/good_job/version.rb

@@ -1,3 +1,3 @@
 module GoodJob
-  VERSION = '0.8.2'.freeze
+  VERSION = '0.9.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: good_job
 version: !ruby/object:Gem::Version
-  version: 0.8.2
+  version: 0.9.0
 platform: ruby
 authors:
 - Ben Sheldon
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-07-18 00:00:00.000000000 Z
+date: 2020-07-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -242,6 +242,7 @@ files:
 - lib/good_job/job.rb
 - lib/good_job/lockable.rb
 - lib/good_job/logging.rb
+- lib/good_job/performer.rb
 - lib/good_job/pg_locks.rb
 - lib/good_job/railtie.rb
 - lib/good_job/scheduler.rb