marj 1.0.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5fd21edb773f1e25cc23e4099e7b0e6a3e2bb58485a988b063c37fdb71f31b82
-  data.tar.gz: aadc1963f4283a092a000bf0c8f732cd57a53f0ed69ad4c83c8c4dbd053907a3
+  metadata.gz: c4feb80230bafd2d67bdb0316598f4c46eabd01ca86c35fb8b01e04c865f91a8
+  data.tar.gz: be461234b8359c9f974dd4160d896cc048d0dcc1f4b7d74b57042d62f9950a18
 SHA512:
-  metadata.gz: 7ddc1d33db20fb9d9ee3a587a67e66c54dcffb8d7f04dc366f53d40605b3239642c5466196e27debe86b1a8c74e10138557f2797232324c7f7a4d49b5df0134c
-  data.tar.gz: 04a4170a8e39b64648bf125da0f1fe1eae1e5888bd33412ff896dc0358f1a6c5b37bd4df1305fb9b2be715812318a49354b0eae1ccc756ceff65bf96879060cc
+  metadata.gz: 31735c1c143a47ab490ed08eca12af6d8fce73b551637ed92e37bcb42e8b2560135ad674dfc832353988f4246c10f9ff2822f624bb10ba256070632dd35c209c
+  data.tar.gz: 7f19dee3c8b0510d27ca10eb8f9d64ecfc2e1174ae1cea71e17abf42f0ce8ef07be4f84e1eff9cf0150e6162bb3a790e066e36f17db131056e30ebbeada1bdc3

data/README.md

@@ -5,12 +5,14 @@ Marj is a Minimal ActiveRecord-based Jobs library.
 API docs: https://www.rubydoc.info/github/nicholasdower/marj <br>
 RubyGems: https://rubygems.org/gems/marj <br>
 Changelog: https://github.com/nicholasdower/marj/releases <br>
-Issues: https://github.com/nicholasdower/marj/issues
+Issues: https://github.com/nicholasdower/marj/issues <br>
+Development: https://github.com/nicholasdower/marj/blob/master/CONTRIBUTING.md
 
 For more information on ActiveJob, see:
 
 - https://edgeguides.rubyonrails.org/active_job_basics.html
 - https://www.rubydoc.info/gems/activejob
+- https://github.com/nicholasdower/marj/blob/master/README.md#activejob-cheatsheet
 
 ## Setup
 
@@ -22,9 +24,11 @@ Add the following to your Gemfile:
 gem 'marj', '~> 1.0'
 ```
 
-### 2. Create the jobs table
+```shell
+bundle install
+```
 
-Apply a database migration:
+### 2. Create the jobs table
 
 ```ruby
 class CreateJobs < ActiveRecord::Migration[7.1]
@@ -53,34 +57,28 @@ class CreateJobs < ActiveRecord::Migration[7.1]
 end
 ```
 
-### 3. Configure the queue adapter
-
-If using Rails, configure the queue adapter via `Rails::Application`:
+To use a different table name:
 
 ```ruby
 require 'marj'
 
-# Configure via Rails::Application:
-class MyApplication < Rails::Application
-  config.active_job.queue_adapter = :marj
-end
-
-# Or for specific jobs:
-class SomeJob < ActiveJob::Base
-  self.queue_adapter = :marj
-end
+MarjConfig.table_name = 'some_name'
 ```
 
-If not using Rails:
+### 3. Configure the queue adapter
 
 ```ruby
 require 'marj'
-require 'marj/record' # Loads ActiveRecord
 
-# Configure via ActiveJob::Base:
+# With rails:
+class MyApplication < Rails::Application
+  config.active_job.queue_adapter = :marj
+end
+
+# Without Rails:
 ActiveJob::Base.queue_adapter = :marj
 
-# Or for specific jobs:
+# Or for specific jobs (with or without Rails):
 class SomeJob < ActiveJob::Base
   self.queue_adapter = :marj
 end
@@ -95,18 +93,87 @@ job.perform_now
 
 # Enqueue, retrieve and manually run a job:
 SomeJob.perform_later('foo')
-Marj.first.execute
+record = Marj.first
+record.execute
 
-# Run all available jobs:
-Marj.work_off
+# Run all ready jobs:
+while (record = Marj.ready.first)
+  record.execute
+end
 
-# Run jobs as they become available:
+# Run all ready jobs in a specific queue:
+while (record = Marj.where(queue_name: 'foo').ready.first)
+  record.execute
+end
+
+# Run jobs as they become ready:
 loop do
-  Marj.work_off
+  while (record = Marj.ready.first)
+    record.execute
+  end
   sleep 5.seconds
 end
 ```
 
+## Extension Examples
+
+### Timeouts
+
+```ruby
+class ApplicationJob < ActiveJob::Base
+  def self.timeout_after(duration)
+    @timeout = duration
+  end
+
+  around_perform do |job, block|
+    if (timeout = job.class.instance_variable_get(:@timeout))
+      ::Timeout.timeout(timeout, StandardError, 'execution expired') { block.call }
+    else
+      block.call
+    end
+  end
+end
+```
+
+### Last Error
+
+```ruby
+class AddLastErrorToJobs < ActiveRecord::Migration[7.1]
+  def self.up
+    add_column :jobs, :last_error, :text
+  end
+
+  def self.down
+    remove_column :jobs, :last_error
+  end
+end
+
+class ApplicationJob < ActiveJob::Base
+  attr_reader :last_error
+
+  def last_error=(error)
+    if error.is_a?(Exception)
+      backtrace = error.backtrace&.map { |line| "\t#{line}" }&.join("\n")
+      error = backtrace ? "#{error.class}: #{error.message}\n#{backtrace}" : "#{error.class}: #{error.message}"
+    end
+
+    @last_error = error&.truncate(10_000, omission: '… (truncated)')
+  end
+
+  def set(options = {})
+    super.tap { self.last_error = options[:error] if options[:error] }
+  end
+
+  def serialize
+    super.merge('last_error' => @last_error)
+  end
+
+  def deserialize(job_data)
+    super.tap { self.last_error = job_data['last_error'] }
+  end
+end
+```
+
 ## ActiveJob Cheatsheet
 
 ### Configuring a Queue Adapter
@@ -127,7 +194,7 @@ SomeJob.queue_adapter = :foo                       # Instantiates FooAdapter
 SomeJob.queue_adapter = FooAdapter.new             # Uses FooAdapter directly
 ```
 
-## Configuration
+### Configuration
 
 - `config.active_job.default_queue_name`
 - `config.active_job.queue_name_prefix`
@@ -158,7 +225,7 @@ SomeJob.queue_adapter = FooAdapter.new             # Uses FooAdapter directly
 - `ActiveJob::Callbacks.singleton_class.set_callback(:execute, :after, &block)`
 - `ActiveJob::Callbacks.singleton_class.set_callback(:execute, :around, &block)`
 
-## Handling Exceptions
+### Handling Exceptions
 
 - `SomeJob.retry_on`
 - `SomeJob.discard_on`
@@ -207,9 +274,9 @@ SomeJob.set(options).perform_all_later(SomeJob.new, SomeJob.new, options:)
 # Executed without enqueueing, enqueued on failure if retries configured
 SomeJob.new(args).perform_now
 SomeJob.perform_now(args)
-ActiveJob::Base.exeucute(SomeJob.new(args).serialize)
+ActiveJob::Base.execute(SomeJob.new(args).serialize)
 
 # Executed after enqueueing
 SomeJob.perform_later(args).perform_now
-ActiveJob::Base.exeucute(SomeJob.perform_later(args).serialize)
+ActiveJob::Base.execute(SomeJob.perform_later(args).serialize)
 ```

data/app/models/marj.rb

@@ -1,18 +1,70 @@
 # frozen_string_literal: true
 
+require 'active_job'
 require 'active_record'
+require_relative '../../lib/marj_config'
 
 # Marj is a Minimal ActiveRecord-based Jobs library.
 #
 # See https://github.com/nicholasdower/marj
 class Marj < ActiveRecord::Base
   # The Marj version.
-  VERSION = '1.0.0'
+  VERSION = '2.0.0'
 
-  self.table_name = 'jobs'
-  self.implicit_order_column = 'enqueued_at' # Order by +enqueued_at+ rather than +job_id+ (the default)
+  # Executes the job associated with this record and returns the result.
+  def execute
+    # Normally we would call ActiveJob::Base#execute which has the following implementation:
+    #   ActiveJob::Callbacks.run_callbacks(:execute) do
+    #     job = deserialize(job_data)
+    #     job.perform_now
+    #   end
+    # However, we need to instantiate the job ourselves in order to register callbacks before execution.
+    ActiveJob::Callbacks.run_callbacks(:execute) do
+      # See register_callbacks for details on how callbacks are used.
+      job = job_class.new.tap { Marj.send(:register_callbacks, _1, self) }
+
+      # ActiveJob::Base#deserialize expects serialized arguments. But the record arguments have already been
+      # deserialized by a custom ActiveRecord serializer (see below). So instead we use the raw arguments string.
+      job_data = attributes.merge('arguments' => JSON.parse(read_attribute_before_type_cast(:arguments)))
+
+      # ActiveJob::Base#deserialize expects dates to be strings rather than Time objects.
+      job_data = job_data.to_h { |k, v| [k, %w[enqueued_at scheduled_at].include?(k) ? v&.iso8601 : v] }
+      job.deserialize(job_data)
+
+      new_executions = executions + 1
+      job.perform_now.tap do
+        # If no error was raised, the job should either be destroyed (success) or updated (retryable failure).
+        raise "job #{job_id} not destroyed or updated" unless destroyed? || (executions == new_executions && !changed?)
+      end
+    end
+  end
+
+  # Returns an ActiveRecord::Relation scope for enqueued jobs with a +scheduled_at+ that is either +null+ or in the
+  # past. Jobs are ordered by +priority+ (+null+ last), then +scheduled_at+ (+null+ last), then +enqueued_at+.
+  #
+  # @return [ActiveRecord::Relation]
+  def self.ready
+    where('scheduled_at is null or scheduled_at <= ?', Time.now.utc).order(
+      Arel.sql(<<~SQL.squish)
+        CASE WHEN priority IS NULL THEN 1 ELSE 0 END, priority,
+        CASE WHEN scheduled_at IS NULL THEN 1 ELSE 0 END, scheduled_at,
+        enqueued_at
+    SQL
+    )
+  end
+
+  self.table_name = MarjConfig.table_name
 
+  # Order by +enqueued_at+ rather than +job_id+ (the default)
+  self.implicit_order_column = 'enqueued_at'
+
+  # Using a custom serializer for exception_executions so that we can interact with it as a hash rather than a string.
   serialize(:exception_executions, coder: JSON)
+
+  # Using a custom serializer for arguments so that we can interact with as an array rather than a string.
+  # This enables code like:
+  #   Marj.first.arguments.first
+  #   Marj.first.update!(arguments: ['foo', 1, Time.now])
   serialize(:arguments, coder: Class.new do
     def self.dump(arguments)
       return ActiveJob::Arguments.serialize(arguments).to_json if arguments.is_a?(Array)
@@ -25,6 +77,8 @@ class Marj < ActiveRecord::Base
       arguments ? ActiveJob::Arguments.deserialize(JSON.parse(arguments)) : nil
     end
   end)
+
+  # Using a custom serializer for job_class so that we can interact with it as a class rather than a string.
   serialize(:job_class, coder: Class.new do
     def self.dump(clazz)
       return clazz.name if clazz.is_a?(Class)
@@ -38,43 +92,17 @@ class Marj < ActiveRecord::Base
     end
   end)
 
-  # Returns an ActiveRecord::Relation scope for enqueued jobs with a +scheduled_at+ that is either +null+ or in the
-  # past. Jobs are ordered by +priority+ (+null+ last), then +scheduled_at+ (+null+ last), then +enqueued_at+.
-  #
-  # @return [ActiveRecord::Relation]
-  def self.available
-    where('scheduled_at is null or scheduled_at <= ?', Time.now.utc).order(
-      Arel.sql(<<~SQL.squish)
-        CASE WHEN priority IS NULL THEN 1 ELSE 0 END, priority,
-        CASE WHEN scheduled_at IS NULL THEN 1 ELSE 0 END, scheduled_at,
-        enqueued_at
-      SQL
-    )
-  end
-
-  # Executes any available jobs from the specified source.
-  #
-  # @param source [Proc] a job source
-  # @return [NilClass]
-  def self.work_off(source = -> { Marj.available.first })
-    while (record = source.call)
-      executions = record.executions
-      begin
-        record.execute
-      rescue Exception
-        # The job should either be discarded or updated. Otherwise, something went wrong.
-        raise unless record.destroyed? || (record.executions == (executions + 1) && !record.changed?)
-      end
-    end
-  end
-
   # Registers job callbacks used to keep the database record for the specified job in sync.
   #
   # @param job [ActiveJob::Base]
   # @return [ActiveJob::Base]
   def self.register_callbacks(job, record)
-    return if job.singleton_class.instance_variable_get(:@__marj)
+    raise 'callbacks already registered' if job.singleton_class.instance_variable_get(:@__marj)
 
+    # We need to detect three cases:
+    #  - If a job succeeds, after_perform will be called.
+    #  - If a job fails and should be retried, enqueue will be called. This is handled by the enqueue method.
+    #  - If a job exceeds its max attempts, after_discard will be called.
     job.singleton_class.after_perform { |_j| record.destroy! }
     job.singleton_class.after_discard { |_j, _exception| record.destroy! }
     job.singleton_class.instance_variable_set(:@__marj, record)
@@ -89,22 +117,31 @@ class Marj < ActiveRecord::Base
   # @return [ActiveJob::Base] the enqueued job
   def self.enqueue(job, time = nil)
     job.scheduled_at = time
+    # Argument serialization is done by ActiveJob. ActiveRecord expects deserialized arguments.
     serialized = job.serialize.symbolize_keys!.without(:provider_job_id).merge(arguments: job.arguments)
+
+    # When a job is enqueued, we must create/update the corresponding database record. We also must ensure callbacks are
+    # registered on the job instance so that when the job is executed, the database record is deleted or updated
+    # (depending on the result).
+    #
+    # There are three cases:
+    #  - The first time a job is enqueued, we need to create the record and register callbacks.
+    #  - If a previously enqueued job instance is re-enqueued, for instance after execution fails, callbacks have
+    #    already been registered. In this case we only need to update the record.
+    #  - It is also possible for new job instance to be created for a job that is already in the database. In this case
+    #    we need to update the record and register callbacks.
+    #
+    # We keep track of whether callbacks have been registered by setting the @__marj instance variable on the job's
+    # singleton class. This holds a reference to the record. This allows us to update the record without re-fetching it
+    # and also ensures that if execute is called on a record any updates to the database are reflected on that record
+    # instance.
     if (record = job.singleton_class.instance_variable_get(:@__marj))
       record.update!(serialized)
     else
-      record = Marj.find_by(job_id: job.job_id)&.update!(serialized) || Marj.create!(serialized)
+      record = Marj.find_or_create_by!(job_id: job.job_id) { _1.assign_attributes(serialized) }
+      register_callbacks(job, record)
     end
-    register_callbacks(job, record)
+    job
   end
   private_class_method :enqueue
-
-  # Executes the job associated with this record and returns the result.
-  def execute
-    job = Marj.send(:register_callbacks, job_class.new, self)
-    job_data = attributes.merge('arguments' => JSON.parse(read_attribute_before_type_cast(:arguments)))
-    job_data = job_data.to_h { |k, v| [k, %w[enqueued_at scheduled_at].include?(k) ? v&.iso8601 : v] }
-    job.deserialize(job_data)
-    ActiveJob::Callbacks.run_callbacks(:execute) { job.perform_now }
-  end
 end

data/lib/marj.rb

@@ -1,24 +1,6 @@
 # frozen_string_literal: true
 
-# ActiveJob queue adapter for Marj.
-class MarjAdapter
-  # Enqueue a job for immediate execution.
-  #
-  # @param job [ActiveJob::Base] the job to enqueue
-  # @return [ActiveJob::Base] the enqueued job
-  def enqueue(job)
-    Marj.send(:enqueue, job)
-  end
+require_relative 'marj_adapter'
+require_relative 'marj_config'
 
-  # Enqueue a job for execution at the specified time.
-  #
-  # @param job [ActiveJob::Base] the job to enqueue
-  # @param timestamp [Numeric, NilClass] optional number of seconds since Unix epoch at which to execute the job
-  # @return [ActiveJob::Base] the enqueued job
-  def enqueue_at(job, timestamp)
-    Marj.send(:enqueue, job, timestamp ? Time.at(timestamp).utc : nil)
-  end
-end
-
-# Enable auto-loading when running in Rails.
-class MarjEngine < Rails::Engine; end if defined?(Rails)
+Kernel.autoload(:Marj, File.expand_path('../app/models/marj.rb', __dir__))

data/lib/marj_adapter.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# ActiveJob queue adapter for Marj.
+class MarjAdapter
+  # Enqueue a job for immediate execution.
+  #
+  # @param job [ActiveJob::Base] the job to enqueue
+  # @return [ActiveJob::Base] the enqueued job
+  def enqueue(job)
+    Marj.send(:enqueue, job)
+  end
+
+  # Enqueue a job for execution at the specified time.
+  #
+  # @param job [ActiveJob::Base] the job to enqueue
+  # @param timestamp [Numeric, NilClass] optional number of seconds since Unix epoch at which to execute the job
+  # @return [ActiveJob::Base] the enqueued job
+  def enqueue_at(job, timestamp)
+    Marj.send(:enqueue, job, timestamp ? Time.at(timestamp).utc : nil)
+  end
+end

data/lib/marj_config.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+# Marj configuration.
+class MarjConfig
+  @table_name = 'jobs'
+
+  class << self
+    # The name of the database table. Defaults to "jobs".
+    #
+    # @return [String]
+    attr_accessor :table_name
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: marj
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Nick Dower
@@ -48,14 +48,15 @@ files:
 - README.md
 - app/models/marj.rb
 - lib/marj.rb
-- lib/marj/record.rb
+- lib/marj_adapter.rb
+- lib/marj_config.rb
 homepage: https://github.com/nicholasdower/marj
 licenses:
 - MIT
 metadata:
   bug_tracker_uri: https://github.com/nicholasdower/marj/issues
-  changelog_uri: https://github.com/nicholasdower/marj/releases/tag/v1.0.0
-  documentation_uri: https://www.rubydoc.info/github/nicholasdower/marj/v1.0.0
+  changelog_uri: https://github.com/nicholasdower/marj/releases/tag/v2.0.0
+  documentation_uri: https://www.rubydoc.info/github/nicholasdower/marj/v2.0.0
   homepage_uri: https://github.com/nicholasdower/marj
   rubygems_mfa_required: 'true'
   source_code_uri: https://github.com/nicholasdower/marj

data/lib/marj/record.rb

@@ -1,3 +0,0 @@
-# frozen_string_literal: true
-
-require_relative '../../app/models/marj'