marj 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5fd21edb773f1e25cc23e4099e7b0e6a3e2bb58485a988b063c37fdb71f31b82
-  data.tar.gz: aadc1963f4283a092a000bf0c8f732cd57a53f0ed69ad4c83c8c4dbd053907a3
+  metadata.gz: f5fe8134484360b68c0cf69356f6a037897c6896be81bc393a9dfc0fb0684055
+  data.tar.gz: 4b7575b721d05681876846f3f06d47440d366bc875e02cadeae7de4582270114
 SHA512:
-  metadata.gz: 7ddc1d33db20fb9d9ee3a587a67e66c54dcffb8d7f04dc366f53d40605b3239642c5466196e27debe86b1a8c74e10138557f2797232324c7f7a4d49b5df0134c
-  data.tar.gz: 04a4170a8e39b64648bf125da0f1fe1eae1e5888bd33412ff896dc0358f1a6c5b37bd4df1305fb9b2be715812318a49354b0eae1ccc756ceff65bf96879060cc
+  metadata.gz: dbfa1cd4cfb90ca4c47b039f0f320e5cc847aa9863b640cd52b445f5074c13aaa0f2679f4ec950001a086f0b981f58f6aa180a250a4790753691b249dc4fa19f
+  data.tar.gz: d1903bcb55c22522d270d950b316ee99ce05edca0c279cc055cb39a71ecfd389d02acd5d1bcc029e931636e26c4708bfd872a20ab71ecc9328e69928a3b535c2

data/README.md

@@ -55,32 +55,18 @@ end
 
 ### 3. Configure the queue adapter
 
-If using Rails, configure the queue adapter via `Rails::Application`:
-
 ```ruby
 require 'marj'
 
-# Configure via Rails::Application:
+# With rails:
 class MyApplication < Rails::Application
   config.active_job.queue_adapter = :marj
 end
 
-# Or for specific jobs:
-class SomeJob < ActiveJob::Base
-  self.queue_adapter = :marj
-end
-```
-
-If not using Rails:
-
-```ruby
-require 'marj'
-require 'marj/record' # Loads ActiveRecord
-
-# Configure via ActiveJob::Base:
+# Without Rails:
 ActiveJob::Base.queue_adapter = :marj
 
-# Or for specific jobs:
+# Or for specific jobs (with or without Rails):
 class SomeJob < ActiveJob::Base
   self.queue_adapter = :marj
 end
@@ -95,11 +81,15 @@ job.perform_now
 
 # Enqueue, retrieve and manually run a job:
 SomeJob.perform_later('foo')
-Marj.first.execute
+record = Marj.first
+record.execute
 
 # Run all available jobs:
 Marj.work_off
 
+# Run all available jobs in a specific queue:
+Marj.work_off(source = -> { Marj.where(queue_name: 'foo').available.first })
+
 # Run jobs as they become available:
 loop do
   Marj.work_off
@@ -127,7 +117,7 @@ SomeJob.queue_adapter = :foo                       # Instantiates FooAdapter
 SomeJob.queue_adapter = FooAdapter.new             # Uses FooAdapter directly
 ```
 
-## Configuration
+### Configuration
 
 - `config.active_job.default_queue_name`
 - `config.active_job.queue_name_prefix`
@@ -158,7 +148,7 @@ SomeJob.queue_adapter = FooAdapter.new             # Uses FooAdapter directly
 - `ActiveJob::Callbacks.singleton_class.set_callback(:execute, :after, &block)`
 - `ActiveJob::Callbacks.singleton_class.set_callback(:execute, :around, &block)`
 
-## Handling Exceptions
+### Handling Exceptions
 
 - `SomeJob.retry_on`
 - `SomeJob.discard_on`
@@ -207,9 +197,9 @@ SomeJob.set(options).perform_all_later(SomeJob.new, SomeJob.new, options:)
 # Executed without enqueueing, enqueued on failure if retries configured
 SomeJob.new(args).perform_now
 SomeJob.perform_now(args)
-ActiveJob::Base.exeucute(SomeJob.new(args).serialize)
+ActiveJob::Base.execute(SomeJob.new(args).serialize)
 
 # Executed after enqueueing
 SomeJob.perform_later(args).perform_now
-ActiveJob::Base.exeucute(SomeJob.perform_later(args).serialize)
+ActiveJob::Base.execute(SomeJob.perform_later(args).serialize)
 ```

data/app/models/marj.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'active_job'
 require 'active_record'
 
 # Marj is a Minimal ActiveRecord-based Jobs library.
@@ -7,36 +8,31 @@ require 'active_record'
 # See https://github.com/nicholasdower/marj
 class Marj < ActiveRecord::Base
   # The Marj version.
-  VERSION = '1.0.0'
+  VERSION = '1.1.0'
 
-  self.table_name = 'jobs'
-  self.implicit_order_column = 'enqueued_at' # Order by +enqueued_at+ rather than +job_id+ (the default)
-
-  serialize(:exception_executions, coder: JSON)
-  serialize(:arguments, coder: Class.new do
-    def self.dump(arguments)
-      return ActiveJob::Arguments.serialize(arguments).to_json if arguments.is_a?(Array)
-      return arguments if arguments.is_a?(String) || arguments.nil?
-
-      raise "invalid arguments: #{arguments}"
-    end
+  # Executes the job associated with this record and returns the result.
+  def execute
+    # Normally we would call ActiveJob::Base#execute which has the following implemenation:
+    #   ActiveJob::Callbacks.run_callbacks(:execute) do
+    #     job = deserialize(job_data)
+    #     job.perform_now
+    #   end
+    # However, we need to instantiate the job ourselves in order to register callbacks before execution.
+    ActiveJob::Callbacks.run_callbacks(:execute) do
+      # See register_callbacks for details on how callbacks are used.
+      job = job_class.new.tap { Marj.send(:register_callbacks, _1, self) }
 
-    def self.load(arguments)
-      arguments ? ActiveJob::Arguments.deserialize(JSON.parse(arguments)) : nil
-    end
-  end)
-  serialize(:job_class, coder: Class.new do
-    def self.dump(clazz)
-      return clazz.name if clazz.is_a?(Class)
-      return clazz if clazz.is_a?(String) || clazz.nil?
+      # ActiveJob::Base#deserialize expects serialized arguments. But the record arguments have already been
+      # deserialized by a custom ActiveRecord serializer (see below). So instead we use the raw arguments string.
+      job_data = attributes.merge('arguments' => JSON.parse(read_attribute_before_type_cast(:arguments)))
 
-      raise "invalid class: #{clazz}"
-    end
+      # ActiveJob::Base#deserialize expects dates to be strings rather than Time objects.
+      job_data = job_data.to_h { |k, v| [k, %w[enqueued_at scheduled_at].include?(k) ? v&.iso8601 : v] }
+      job.deserialize(job_data)
 
-    def self.load(str)
-      str&.constantize
+      job.perform_now
     end
-  end)
+  end
 
   # Returns an ActiveRecord::Relation scope for enqueued jobs with a +scheduled_at+ that is either +null+ or in the
   # past. Jobs are ordered by +priority+ (+null+ last), then +scheduled_at+ (+null+ last), then +enqueued_at+.
@@ -48,7 +44,7 @@ class Marj < ActiveRecord::Base
         CASE WHEN priority IS NULL THEN 1 ELSE 0 END, priority,
         CASE WHEN scheduled_at IS NULL THEN 1 ELSE 0 END, scheduled_at,
         enqueued_at
-      SQL
+    SQL
     )
   end
 
@@ -68,13 +64,56 @@ class Marj < ActiveRecord::Base
     end
   end
 
+  self.table_name = 'jobs'
+
+  # Order by +enqueued_at+ rather than +job_id+ (the default)
+  self.implicit_order_column = 'enqueued_at'
+
+  # Using a custom serializer for exception_executions so that we can interact with it as a hash rather than a string.
+  serialize(:exception_executions, coder: JSON)
+
+  # Using a custom serializer for arguments so that we can interact with as an array rather than a string.
+  # This enables code like:
+  #   Marj.first.arguments.first
+  #   Marj.first.update!(arguments: ['foo', 1, Time.now])
+  serialize(:arguments, coder: Class.new do
+    def self.dump(arguments)
+      return ActiveJob::Arguments.serialize(arguments).to_json if arguments.is_a?(Array)
+      return arguments if arguments.is_a?(String) || arguments.nil?
+
+      raise "invalid arguments: #{arguments}"
+    end
+
+    def self.load(arguments)
+      arguments ? ActiveJob::Arguments.deserialize(JSON.parse(arguments)) : nil
+    end
+  end)
+
+  # Using a custom serializer for job_class so that we can interact with it as a class rather than a string.
+  serialize(:job_class, coder: Class.new do
+    def self.dump(clazz)
+      return clazz.name if clazz.is_a?(Class)
+      return clazz if clazz.is_a?(String) || clazz.nil?
+
+      raise "invalid class: #{clazz}"
+    end
+
+    def self.load(str)
+      str&.constantize
+    end
+  end)
+
   # Registers job callbacks used to keep the database record for the specified job in sync.
   #
   # @param job [ActiveJob::Base]
   # @return [ActiveJob::Base]
   def self.register_callbacks(job, record)
-    return if job.singleton_class.instance_variable_get(:@__marj)
+    raise 'callbacks already registered' if job.singleton_class.instance_variable_get(:@__marj)
 
+    # We need to detect three cases:
+    #  - If a job succeeds, after_perform will be called.
+    #  - If a job fails and should be retried, enqueue will be called. This is handled by the enqueue method.
+    #  - If a job exceeds its max attempts, after_discard will be called.
     job.singleton_class.after_perform { |_j| record.destroy! }
     job.singleton_class.after_discard { |_j, _exception| record.destroy! }
     job.singleton_class.instance_variable_set(:@__marj, record)
@@ -89,22 +128,31 @@ class Marj < ActiveRecord::Base
   # @return [ActiveJob::Base] the enqueued job
   def self.enqueue(job, time = nil)
     job.scheduled_at = time
+    # Argument serialization is done by ActiveJob. ActiveRecord expects deserialized arguments.
     serialized = job.serialize.symbolize_keys!.without(:provider_job_id).merge(arguments: job.arguments)
+
+    # When a job is enqueued, we must create/update the corresponding database record. We also must ensure callbacks are
+    # registered on the job instance so that when the job is executed, the database record is deleted or updated
+    # (depending on the result).
+    #
+    # There are three cases:
+    #  - The first time a job is enqueued, we need to create the record and register callbacks.
+    #  - If a previously enqueued job instance is re-enqueued, for instance after execution fails, callbacks have
+    #    already been registered. In this case we only need to update the record.
+    #  - It is also possible for new job instance to be created for a job that is already in the database. In this case
+    #    we need to update the record and register callbacks.
+    #
+    # We keep track of whether callbacks have been registered by setting the @__marj instance variable on the job's
+    # singleton class. This holds a reference to the record. This allows us to update the record without re-fetching it
+    # and also ensures that if execute is called on a record any updates to the database are reflected on that record
+    # instance.
     if (record = job.singleton_class.instance_variable_get(:@__marj))
       record.update!(serialized)
     else
-      record = Marj.find_by(job_id: job.job_id)&.update!(serialized) || Marj.create!(serialized)
+      record = Marj.find_or_create_by!(job_id: job.job_id) { _1.assign_attributes(serialized) }
+      register_callbacks(job, record)
     end
-    register_callbacks(job, record)
+    job
   end
   private_class_method :enqueue
-
-  # Executes the job associated with this record and returns the result.
-  def execute
-    job = Marj.send(:register_callbacks, job_class.new, self)
-    job_data = attributes.merge('arguments' => JSON.parse(read_attribute_before_type_cast(:arguments)))
-    job_data = job_data.to_h { |k, v| [k, %w[enqueued_at scheduled_at].include?(k) ? v&.iso8601 : v] }
-    job.deserialize(job_data)
-    ActiveJob::Callbacks.run_callbacks(:execute) { job.perform_now }
-  end
 end

data/lib/marj.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+Kernel.autoload(:Marj, File.expand_path('../app/models/marj.rb', __dir__))
+
 # ActiveJob queue adapter for Marj.
 class MarjAdapter
   # Enqueue a job for immediate execution.
@@ -19,6 +21,3 @@ class MarjAdapter
     Marj.send(:enqueue, job, timestamp ? Time.at(timestamp).utc : nil)
   end
 end
-
-# Enable auto-loading when running in Rails.
-class MarjEngine < Rails::Engine; end if defined?(Rails)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: marj
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Nick Dower
@@ -48,14 +48,13 @@ files:
 - README.md
 - app/models/marj.rb
 - lib/marj.rb
-- lib/marj/record.rb
 homepage: https://github.com/nicholasdower/marj
 licenses:
 - MIT
 metadata:
   bug_tracker_uri: https://github.com/nicholasdower/marj/issues
-  changelog_uri: https://github.com/nicholasdower/marj/releases/tag/v1.0.0
-  documentation_uri: https://www.rubydoc.info/github/nicholasdower/marj/v1.0.0
+  changelog_uri: https://github.com/nicholasdower/marj/releases/tag/v1.1.0
+  documentation_uri: https://www.rubydoc.info/github/nicholasdower/marj/v1.1.0
   homepage_uri: https://github.com/nicholasdower/marj
   rubygems_mfa_required: 'true'
   source_code_uri: https://github.com/nicholasdower/marj

data/lib/marj/record.rb

@@ -1,3 +0,0 @@
-# frozen_string_literal: true
-
-require_relative '../../app/models/marj'