good_job 3.15.3 → 3.15.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 85c6903e5f8838cb72f4687e364fa20cf9c461f0c4b79412bb8b61549bbf6333
-  data.tar.gz: d00fecc5a4fe561bc1a7f21a79e46245cb5ab603fafbbb4369344395c9c0fac0
+  metadata.gz: ee08a99ae7d26df57e3f0a7a64b737e987041815805117ac6d4e6119df229932
+  data.tar.gz: fd747fed38d2227bde7898438b6d597d55c5cb72f6f7125171b5637aff168d5a
 SHA512:
-  metadata.gz: a1116f1e94e07b62bc316a3ec76a1b0768d22b5a511a65cf18e40bc13d4e8c25290c7c879d5cdfea1f1a988e3809b130462b57bd25a7c2405b0152cbe9434f51
-  data.tar.gz: 227c89386f14a783260fff68c55f0f4b0d10db331ab705b922f02fdce79a8ec4cf0f969e554f02e55f030dbcda6efbd9439bcaafd7bc471028c35375a7b58ccf
+  metadata.gz: e32e66f000d6d7cd0a84fd6ab5a7bdf08f2a5f996bd5533ffc11f68ba33ee96e9d4ddc90a64dea1dbfcad43b07b75fcb34ed66c38546cda52b2cd5c0da90a329
+  data.tar.gz: 8f2a8c72500ee21f634ae6436d12ddc0641e730281b8b9534189f5892b795c6dc3a214e30e4a348878285a8912342dea3fbc74a6ab615a4de5b38f025767db3f

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## [v3.15.4](https://github.com/bensheldon/good_job/tree/v3.15.4) (2023-04-22)
+
+[Full Changelog](https://github.com/bensheldon/good_job/compare/v3.15.3...v3.15.4)
+
+**Merged pull requests:**
+
+- Create "discrete" `good_job_executions` table to separate Job records from Execution records and have a 1-to-1 correspondence between `good_jobs` records and Active Job jobs [\#928](https://github.com/bensheldon/good_job/pull/928) ([bensheldon](https://github.com/bensheldon))
+
 ## [v3.15.3](https://github.com/bensheldon/good_job/tree/v3.15.3) (2023-04-22)
 
 [Full Changelog](https://github.com/bensheldon/good_job/compare/v3.15.2...v3.15.3)

data/app/models/good_job/base_execution.rb

@@ -33,12 +33,25 @@ module GoodJob
       def coalesce_scheduled_at_created_at
         arel_table.coalesce(arel_table['scheduled_at'], arel_table['created_at'])
       end
+
+      def discrete_support?
+        if connection.table_exists?('good_job_executions')
+          true
+        else
+          migration_pending_warning!
+          false
+        end
+      end
     end
 
     # The ActiveJob job class, as a string
     # @return [String]
     def job_class
-      serialized_params['job_class']
+      discrete? ? attributes['job_class'] : serialized_params['job_class']
+    end
+
+    def discrete?
+      self.class.discrete_support? && is_discrete?
     end
   end
 end

data/app/models/good_job/discrete_execution.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module GoodJob # :nodoc:
+  class DiscreteExecution < BaseRecord
+    self.table_name = 'good_job_executions'
+
+    belongs_to :execution, class_name: 'GoodJob::Execution', foreign_key: 'active_job_id', primary_key: 'active_job_id', inverse_of: :discrete_executions, optional: true
+    belongs_to :job, class_name: 'GoodJob::Job', foreign_key: 'active_job_id', primary_key: 'active_job_id', inverse_of: :discrete_executions, optional: true
+
+    scope :finished, -> { where.not(finished_at: nil) }
+
+    alias_attribute :performed_at, :created_at
+
+    def number
+      serialized_params.fetch('executions', 0) + 1
+    end
+
+    # Time between when this job was expected to run and when it started running
+    def queue_latency
+      created_at - scheduled_at
+    end
+
+    # Time between when this job started and finished
+    def runtime_latency
+      (finished_at || Time.current) - performed_at if performed_at
+    end
+
+    def last_status_at
+      finished_at || created_at
+    end
+
+    def status
+      if finished_at.present?
+        if error.present?
+          :retried
+        elsif error.present? && job.finished_at.present?
+          :discarded
+        else
+          :succeeded
+        end
+      else
+        :running
+      end
+    end
+
+    def display_serialized_params
+      serialized_params.merge({
+                                _good_job_execution: attributes.except('serialized_params'),
+                              })
+    end
+  end
+end

data/app/models/good_job/execution.rb

@@ -70,9 +70,13 @@ module GoodJob
     end
 
     belongs_to :batch, class_name: 'GoodJob::BatchRecord', optional: true, inverse_of: :executions
-
     belongs_to :job, class_name: 'GoodJob::Job', foreign_key: 'active_job_id', primary_key: 'active_job_id', optional: true, inverse_of: :executions
-    after_destroy -> { self.class.active_job_id(active_job_id).delete_all }, if: -> { @_destroy_job }
+    has_many :discrete_executions, class_name: 'GoodJob::DiscreteExecution', foreign_key: 'active_job_id', primary_key: 'active_job_id', inverse_of: :execution # rubocop:disable Rails/HasManyOrHasOneDependent
+
+    after_destroy lambda {
+      GoodJob::DiscreteExecution.where(active_job_id: active_job_id).delete_all if discrete? # TODO: move into association `dependent: :delete_all` after v4
+      self.class.active_job_id(active_job_id).delete_all
+    }, if: -> { @_destroy_job }
 
     # Get executions with given ActiveJob ID
     # @!method active_job_id
@@ -201,8 +205,12 @@ module GoodJob
       end
     end)
 
-    # Construct a GoodJob::Execution from an ActiveJob instance.
     def self.build_for_enqueue(active_job, overrides = {})
+      new(**enqueue_args(active_job, overrides))
+    end
+
+    # Construct arguments for GoodJob::Execution from an ActiveJob instance.
+    def self.enqueue_args(active_job, overrides = {})
       if active_job.priority && GoodJob.configuration.smaller_number_is_higher_priority.nil?
         ActiveSupport::Deprecation.warn(<<~DEPRECATION)
           The next major version of GoodJob (v4.0) will change job `priority` to give smaller numbers higher priority (default: `0`), in accordance with Active Job's definition of priority.
@@ -218,6 +226,7 @@ module GoodJob
         serialized_params: active_job.serialize,
         scheduled_at: active_job.scheduled_at,
       }
+
       execution_args[:concurrency_key] = active_job.good_job_concurrency_key if active_job.respond_to?(:good_job_concurrency_key)
 
       reenqueued_current_execution = CurrentThread.active_job_id && CurrentThread.active_job_id == active_job.job_id
@@ -238,7 +247,7 @@ module GoodJob
         execution_args[:cron_at] = CurrentThread.cron_at
       end
 
-      new(**execution_args.merge(overrides))
+      execution_args.merge(overrides)
     end
 
     # Finds the next eligible Execution, acquire an advisory lock related to it, and
@@ -298,19 +307,47 @@ module GoodJob
     #   The new {Execution} instance representing the queued ActiveJob job.
     def self.enqueue(active_job, scheduled_at: nil, create_with_advisory_lock: false)
       ActiveSupport::Notifications.instrument("enqueue_job.good_job", { active_job: active_job, scheduled_at: scheduled_at, create_with_advisory_lock: create_with_advisory_lock }) do |instrument_payload|
-        execution = build_for_enqueue(active_job, { scheduled_at: scheduled_at })
+        current_execution = CurrentThread.execution
+
+        retried = current_execution && current_execution.active_job_id == active_job.job_id
+        if retried
+          if current_execution.discrete?
+            execution = current_execution
+            execution.assign_attributes(enqueue_args(active_job, { scheduled_at: scheduled_at }))
+            execution.scheduled_at ||= Time.current
+            execution.performed_at = nil
+            execution.finished_at = nil
+          else
+            execution = build_for_enqueue(active_job, { scheduled_at: scheduled_at })
+          end
+        else
+          execution = build_for_enqueue(active_job, { scheduled_at: scheduled_at })
+          execution.make_discrete if discrete_support?
+        end
 
-        execution.create_with_advisory_lock = create_with_advisory_lock
-        instrument_payload[:execution] = execution
+        if create_with_advisory_lock
+          if execution.persisted?
+            execution.advisory_lock
+          else
+            execution.create_with_advisory_lock = true
+          end
+        end
 
+        instrument_payload[:execution] = execution
         execution.save!
-        active_job.provider_job_id = execution.id
-        CurrentThread.execution.retried_good_job_id = execution.id if CurrentThread.active_job_id && CurrentThread.active_job_id == active_job.job_id
 
+        CurrentThread.execution.retried_good_job_id = execution.id if retried && !CurrentThread.execution.discrete?
+        active_job.provider_job_id = execution.id
         execution
       end
     end
 
+    def self.format_error(error)
+      raise ArgumentError unless error.is_a?(Exception)
+
+      [error.class.to_s, ERROR_MESSAGE_SEPARATOR, error.message].join
+    end
+
     # Execute the ActiveJob job this {Execution} represents.
     # @return [ExecutionResult]
     #   An array of the return value of the job's +#perform+ method and the
@@ -320,12 +357,39 @@ module GoodJob
       run_callbacks(:perform) do
         raise PreviouslyPerformedError, 'Cannot perform a job that has already been performed' if finished_at
 
+        discrete_execution = nil
         result = GoodJob::CurrentThread.within do |current_thread|
           current_thread.reset
           current_thread.execution = self
 
-          current_thread.execution_interrupted = performed_at if performed_at
-          update!(performed_at: Time.current)
+          if performed_at
+            current_thread.execution_interrupted = performed_at
+
+            if discrete?
+              interrupt_error_string = self.class.format_error(GoodJob::InterruptError.new("Interrupted after starting perform at '#{performed_at}'"))
+              self.error = interrupt_error_string
+              discrete_executions.where(finished_at: nil).where.not(performed_at: nil).update_all( # rubocop:disable Rails/SkipsModelValidations
+                error: interrupt_error_string,
+                finished_at: Time.current
+              )
+            end
+          end
+
+          if discrete?
+            transaction do
+              now = Time.current
+              discrete_execution = discrete_executions.create!(
+                job_class: job_class,
+                queue_name: queue_name,
+                serialized_params: serialized_params,
+                scheduled_at: (scheduled_at || created_at),
+                created_at: now
+              )
+              update!(performed_at: now, executions_count: ((executions_count || 0) + 1))
+            end
+          else
+            update!(performed_at: Time.current)
+          end
 
           ActiveSupport::Notifications.instrument("perform_job.good_job", { execution: self, process_id: current_thread.process_id, thread_name: current_thread.thread_name }) do |instrument_payload|
             value = ActiveJob::Base.execute(active_job_data)
@@ -349,14 +413,42 @@ module GoodJob
         end
 
         job_error = result.handled_error || result.unhandled_error
-        self.error = [job_error.class, ERROR_MESSAGE_SEPARATOR, job_error.message].join if job_error
+
+        if job_error
+          error_string = self.class.format_error(job_error)
+          self.error = error_string
+          discrete_execution.error = error_string if discrete_execution
+        else
+          self.error = nil
+        end
 
         reenqueued = result.retried? || retried_good_job_id.present?
         if result.unhandled_error && GoodJob.retry_on_unhandled_error
-          save!
+          if discrete_execution
+            transaction do
+              discrete_execution.update!(finished_at: Time.current)
+              update!(performed_at: nil, finished_at: nil, retried_good_job_id: nil)
+            end
+          else
+            save!
+          end
         elsif GoodJob.preserve_job_records == true || reenqueued || (result.unhandled_error && GoodJob.preserve_job_records == :on_unhandled_error) || cron_key.present?
-          self.finished_at = Time.current
-          save!
+          now = Time.current
+          if discrete_execution
+            if reenqueued
+              self.performed_at = nil
+            else
+              self.finished_at = now
+            end
+            discrete_execution.finished_at = now
+            transaction do
+              discrete_execution.save!
+              save!
+            end
+          else
+            self.finished_at = now
+            save!
+          end
         else
           destroy_job
         end
@@ -371,6 +463,17 @@ module GoodJob
       self.class.unscoped.unfinished.owns_advisory_locked.exists?(id: id)
     end
 
+    def make_discrete
+      self.is_discrete = true
+      self.id = active_job_id
+      self.job_class = serialized_params['job_class']
+      self.executions_count ||= 0
+
+      current_time = Time.current
+      self.created_at ||= current_time
+      self.scheduled_at ||= current_time
+    end
+
     # Build an ActiveJob instance and deserialize the arguments, using `#active_job_data`.
     #
     # @param ignore_deserialization_errors [Boolean]

data/app/models/good_job/job.rb

@@ -30,6 +30,11 @@ module GoodJob
 
     belongs_to :batch, class_name: 'GoodJob::BatchRecord', inverse_of: :jobs, optional: true
     has_many :executions, -> { order(created_at: :asc) }, class_name: 'GoodJob::Execution', foreign_key: 'active_job_id', inverse_of: :job # rubocop:disable Rails/HasManyOrHasOneDependent
+    has_many :discrete_executions, -> { order(created_at: :asc) }, class_name: 'GoodJob::DiscreteExecution', foreign_key: 'active_job_id', primary_key: :active_job_id, inverse_of: :job # rubocop:disable Rails/HasManyOrHasOneDependent
+
+    after_destroy lambda {
+      GoodJob::DiscreteExecution.where(active_job_id: active_job_id).delete_all if discrete? # TODO: move into association `dependent: :delete_all` after v4
+    }
 
     # Only the most-recent unretried execution represents a "Job"
     default_scope { where(retried_good_job_id: nil) }
@@ -56,6 +61,8 @@ module GoodJob
     # Errored but will not be retried
     scope :discarded, -> { finished.where.not(error: nil) }
 
+    scope :unfinished_undiscrete, -> { where(finished_at: nil, retried_good_job_id: nil, is_discrete: [nil, false]) }
+
     # The job's ActiveJob UUID
     # @return [String]
     def id
@@ -191,9 +198,10 @@ module GoodJob
 
           execution.class.transaction(joinable: false, requires_new: true) do
             new_active_job = active_job.retry_job(wait: 0, error: execution.error)
-            execution.save
+            execution.save!
           end
         end
+
         new_active_job
       end
     end
@@ -213,7 +221,7 @@ module GoodJob
         update_execution = proc do
           execution.update(
             finished_at: Time.current,
-            error: [job_error.class, GoodJob::Execution::ERROR_MESSAGE_SEPARATOR, job_error.message].join
+            error: GoodJob::Execution.format_error(job_error)
           )
         end
 

data/app/views/good_job/jobs/show.html.erb

@@ -3,7 +3,12 @@
     <nav aria-label="breadcrumb">
       <ol class="breadcrumb small mb-0">
         <li class="breadcrumb-item"><%= link_to t(".jobs"), jobs_path %></li>
-        <li class="breadcrumb-item active" aria-current="page"><%= tag.code @job.id, class: "text-muted" %></li>
+        <li class="breadcrumb-item active" aria-current="page">
+          <%= tag.code @job.id, class: "text-muted" %>
+          <% if @job.discrete? %>
+            <span class="badge bg-info text-dark">Discrete</span>
+          <% end %>
+        </li>
       </ol>
     </nav>
     <div class="row align-items-center">
@@ -21,6 +26,10 @@
         <div class="font-monospace fw-bold small my-2"><%= tag.strong @job.priority %></div>
       </div>
       <div class="col text-end">
+        <div class="mb-2">
+          <%= tag.span relative_time(@job.last_status_at), class: "small" %>
+          <%= status_badge @job.status %>
+        </div>
         <% if @job.status.in? [:scheduled, :retried, :queued] %>
           <%= button_to reschedule_job_path(@job.id), method: :put,
               class: "btn btn-sm btn-outline-primary",
@@ -59,8 +68,25 @@
 </div>
 
 <div class="my-4">
-  <h5><%= t "good_job.models.job.arguments" %></h5>
-  <%= tag.pre @job.serialized_params["arguments"].map(&:inspect).join(', '), class: 'text-wrap text-break' %>
+  <h5>
+    <%= t "good_job.models.job.arguments" %>
+    <%= tag.button type: "button", class: "btn btn-sm text-muted", role: "button",
+                   title: t("good_job.actions.inspect"),
+                   data: { bs_toggle: "collapse", bs_target: "##{dom_id(@job, 'params')}" },
+                   aria: { expanded: false, controls: dom_id(@job, "params") } do %>
+      <%= render_icon "info" %>
+      <span class="visually-hidden"><%= t "good_job.actions.inspect" %></span>
+    <% end %>
+  </h5>
 </div>
+<%= tag.pre @job.serialized_params["arguments"].map(&:inspect).join(', '), class: 'text-wrap text-break' %>
+
+<%= tag.div id: dom_id(@job, "params"), class: "list-group-item collapse small bg-dark text-light" do %>
+  <%= tag.pre JSON.pretty_generate(@job.display_serialized_params) %>
+<% end %>
 
-<%= render 'executions', executions: @job.executions.includes_advisory_locks.reverse %>
+<% if @job.discrete? %>
+  <%= render 'executions', executions: @job.discrete_executions.reverse %>
+<% else %>
+  <%= render 'executions', executions: @job.executions.includes_advisory_locks.reverse %>
+<% end %>

data/lib/generators/good_job/templates/install/migrations/create_good_jobs.rb.erb

@@ -22,6 +22,10 @@ class CreateGoodJobs < ActiveRecord::Migration<%= migration_version %>
 
       t.uuid :batch_id
       t.uuid :batch_callback_id
+
+      t.boolean :is_discrete
+      t.integer :executions_count
+      t.text :job_class
     end
 
     create_table :good_job_batches, id: :uuid do |t|
@@ -38,6 +42,18 @@ class CreateGoodJobs < ActiveRecord::Migration<%= migration_version %>
       t.datetime :finished_at
     end
 
+    create_table :good_job_executions, id: :uuid do |t|
+      t.timestamps
+
+      t.uuid :active_job_id, null: false
+      t.text :job_class
+      t.text :queue_name
+      t.jsonb :serialized_params
+      t.datetime :scheduled_at
+      t.datetime :finished_at
+      t.text :error
+    end
+
     create_table :good_job_processes, id: :uuid do |t|
       t.timestamps
       t.jsonb :state
@@ -62,5 +78,7 @@ class CreateGoodJobs < ActiveRecord::Migration<%= migration_version %>
       where: "finished_at IS NULL", name: :index_good_jobs_jobs_on_priority_created_at_when_unfinished
     add_index :good_jobs, [:batch_id], where: "batch_id IS NOT NULL"
     add_index :good_jobs, [:batch_callback_id], where: "batch_callback_id IS NOT NULL"
+
+    add_index :good_job_executions, [:active_job_id, :created_at], name: :index_good_job_executions_on_active_job_id_and_created_at
   end
 end

data/lib/generators/good_job/templates/update/migrations/05_create_good_job_executions.rb.erb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+class CreateGoodJobExecutions < ActiveRecord::Migration<%= migration_version %>
+  def change
+    reversible do |dir|
+      dir.up do
+        # Ensure this incremental update migration is idempotent
+        # with monolithic install migration.
+        return if connection.table_exists?(:good_job_executions)
+      end
+    end
+
+    create_table :good_job_executions, id: :uuid do |t|
+      t.timestamps
+
+      t.uuid :active_job_id, null: false
+      t.text :job_class
+      t.text :queue_name
+      t.jsonb :serialized_params
+      t.datetime :scheduled_at
+      t.datetime :finished_at
+      t.text :error
+
+      t.index [:active_job_id, :created_at], name: :index_good_job_executions_on_active_job_id_and_created_at
+    end
+
+    change_table :good_jobs do |t|
+      t.boolean :is_discrete
+      t.integer :executions_count
+      t.text :job_class
+    end
+  end
+end

data/lib/good_job/adapter.rb

@@ -50,11 +50,15 @@ module GoodJob
 
       current_time = Time.current
       executions = active_jobs.map do |active_job|
-        GoodJob::Execution.build_for_enqueue(active_job, {
-                                               id: SecureRandom.uuid,
-                                               created_at: current_time,
-                                               updated_at: current_time,
-                                             })
+        GoodJob::Execution.build_for_enqueue(active_job).tap do |execution|
+          if GoodJob::Execution.discrete_support?
+            execution.make_discrete
+            execution.scheduled_at = current_time if execution.scheduled_at == execution.created_at
+          end
+
+          execution.created_at = current_time
+          execution.updated_at = current_time
+        end
       end
 
       inline_executions = []

data/lib/good_job/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 module GoodJob
   # GoodJob gem version.
-  VERSION = '3.15.3'
+  VERSION = '3.15.4'
 end

data/lib/good_job.rb

@@ -170,14 +170,19 @@ module GoodJob
     ActiveSupport::Notifications.instrument("cleanup_preserved_jobs.good_job", { older_than: older_than, timestamp: timestamp }) do |payload|
       deleted_executions_count = 0
       deleted_batches_count = 0
+      deleted_discrete_executions_count = 0
 
       jobs_query = GoodJob::Job.where('finished_at <= ?', timestamp).order(finished_at: :asc).limit(in_batches_of)
       jobs_query = jobs_query.succeeded unless include_discarded
       loop do
-        deleted = GoodJob::Execution.where(job: jobs_query).delete_all
-        break if deleted.zero?
+        active_job_ids = jobs_query.pluck(:active_job_id)
+        break if active_job_ids.empty?
 
-        deleted_executions_count += deleted
+        deleted_discrete_executions = GoodJob::DiscreteExecution.where(active_job_id: active_job_ids).delete_all
+        deleted_discrete_executions_count += deleted_discrete_executions
+
+        deleted_executions = GoodJob::Execution.where(active_job_id: active_job_ids).delete_all
+        deleted_executions_count += deleted_executions
       end
 
       if GoodJob::BatchRecord.migrated?
@@ -191,9 +196,14 @@ module GoodJob
         end
       end
 
-      payload[:destroyed_executions_count] = deleted_executions_count
       payload[:destroyed_batches_count] = deleted_batches_count
-      payload[:destroyed_records_count] = deleted_executions_count + deleted_batches_count
+      payload[:destroyed_discrete_executions_count] = deleted_discrete_executions_count
+      payload[:destroyed_executions_count] = deleted_executions_count
+
+      destroyed_records_count = deleted_batches_count + deleted_discrete_executions_count + deleted_executions_count
+      payload[:destroyed_records_count] = destroyed_records_count
+
+      destroyed_records_count
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: good_job
 version: !ruby/object:Gem::Version
-  version: 3.15.3
+  version: 3.15.4
 platform: ruby
 authors:
 - Ben Sheldon
@@ -350,6 +350,7 @@ files:
 - app/models/good_job/batch.rb
 - app/models/good_job/batch_record.rb
 - app/models/good_job/cron_entry.rb
+- app/models/good_job/discrete_execution.rb
 - app/models/good_job/execution.rb
 - app/models/good_job/execution_result.rb
 - app/models/good_job/job.rb
@@ -401,6 +402,7 @@ files:
 - lib/generators/good_job/templates/update/migrations/02_create_good_job_settings.rb.erb
 - lib/generators/good_job/templates/update/migrations/03_create_index_good_jobs_jobs_on_priority_created_at_when_unfinished.rb.erb
 - lib/generators/good_job/templates/update/migrations/04_create_good_job_batches.rb.erb
+- lib/generators/good_job/templates/update/migrations/05_create_good_job_executions.rb.erb
 - lib/generators/good_job/update_generator.rb
 - lib/good_job.rb
 - lib/good_job/active_job_extensions/batches.rb