chained_job 0.5.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dd4890f548ef2ee47ff48180b593134f61b83fedc9017733346ab15d518d2c8d
-  data.tar.gz: c315bdee23b062ba3834bd23f8fcf18f07b8c7ca111dff9082d1bd251a5071e7
+  metadata.gz: 627fdbd88729785e5f036aa8eead67eba9e829ab8fff3fa7d7792249a9dcea0e
+  data.tar.gz: d3e9721bfae1f4676428d73a3efba5c417c89527f2750de10ef859c0846e567e
 SHA512:
-  metadata.gz: 0205f904ea84911f3258ac4798f52077c9e61e6850165db93b01b461ecbbf734d385093741211583aabddffcdda4e2cd4bc560ec10033dde9d5467f0232a28f5
-  data.tar.gz: 1e65c4a589d8db54ab53a76d80da2049a619945ed4a75428edf8aba28309a3e90b9ec222bb916fcd43842c945c38bf7fa9b881e9a5e2adefc90be92562fcf4eb
+  metadata.gz: cadabdd000cb69d656b930d0ad960af23901634c02725e6299669aa963ec9e0d732b98d6adce7785d1b1f37b8e0bab4453fa7b6ed558dbe208d7bbe07d3a7d95
+  data.tar.gz: c1b8e3ed98fe248078ac19bcc07db5c0a04bd2b747fb6b88f43698835a6da805db33b7b36685d5a38432fc42613e329418080035e564b1b95c56c946b9cafc5f

data/lib/chained_job/clean_up_queue.rb

@@ -4,16 +4,16 @@ require 'chained_job/helpers'
 
 module ChainedJob
   class CleanUpQueue
-    def self.run(job_class)
-      new(job_class).run
+    def self.run(job_arguments_key)
+      new(job_arguments_key).run
     end
 
     TRIM_STEP_SIZE = 1_000
 
-    attr_reader :job_class
+    attr_reader :job_arguments_key
 
-    def initialize(job_class)
-      @job_class = job_class
+    def initialize(job_arguments_key)
+      @job_arguments_key = job_arguments_key
     end
 
     # rubocop:disable Metrics/AbcSize
@@ -39,7 +39,7 @@ module ChainedJob
     end
 
     def job_key
-      @job_key ||= Helpers.job_key(job_class)
+      @job_key ||= Helpers.job_key(job_arguments_key)
     end
   end
 end

data/lib/chained_job/helpers.rb

@@ -4,8 +4,8 @@ module ChainedJob
   module Helpers
     module_function
 
-    def job_key(job_class)
-      "chained_job:#{job_class}"
+    def job_key(job_arguments_key)
+      "chained_job:#{job_arguments_key}"
     end
 
     def redis_key(job_key, tag)
@@ -15,5 +15,9 @@ module ChainedJob
     def tag_list(prefix)
       "#{prefix}:tags"
     end
+
+    def serialize(arguments)
+      arguments.map { |argument| Marshal.dump(argument) }
+    end
   end
 end

data/lib/chained_job/middleware.rb

@@ -9,16 +9,20 @@ module ChainedJob
       base.queue_as ChainedJob.config.queue if ChainedJob.config.queue
     end
 
-    def perform(worker_id = nil, tag = nil)
+    def perform(args = {}, worker_id = nil, tag = nil)
+      unless Hash === args
+        # backward compatibility
+        args, worker_id, tag = {}, args, worker_id
+      end
       if worker_id
-        ChainedJob::Process.run(self, worker_id, tag)
+        ChainedJob::Process.run(args, self, job_arguments_key, worker_id, tag)
       else
-        ChainedJob::StartChains.run(self.class, arguments_array, parallelism)
+        ChainedJob::StartChains.run(args, self.class, job_arguments_key, arguments_array(args), parallelism)
       end
     end
 
-    def arguments_array
-      options = { job_class: self.class }
+    def arguments_array(args)
+      options = { job_class: self.class, args: args }
       ChainedJob.config.around_array_of_job_arguments.call(options) { array_of_job_arguments }
     end
 
@@ -29,5 +33,9 @@ module ChainedJob
     def parallelism
       raise NoMethodError, 'undefined method parallelism'
     end
+
+    def job_arguments_key
+      self.class
+    end
   end
 end

data/lib/chained_job/process.rb

@@ -4,14 +4,16 @@ require 'chained_job/helpers'
 
 module ChainedJob
   class Process
-    def self.run(job_instance, worker_id, job_tag)
-      new(job_instance, worker_id, job_tag).run
+    def self.run(args, job_instance, job_arguments_key, worker_id, job_tag)
+      new(args, job_instance, job_arguments_key, worker_id, job_tag).run
     end
 
-    attr_reader :job_instance, :worker_id, :job_tag
+    attr_reader :args, :job_instance, :job_arguments_key, :worker_id, :job_tag
 
-    def initialize(job_instance, worker_id, job_tag)
+    def initialize(args, job_instance, job_arguments_key, worker_id, job_tag)
+      @args = args
       @job_instance = job_instance
+      @job_arguments_key = job_arguments_key
       @worker_id = worker_id
       @job_tag = job_tag
     end
@@ -20,19 +22,28 @@ module ChainedJob
       with_hooks do
         return finished_worker unless argument
 
-        job_instance.process(argument)
-        job_instance.class.perform_later(worker_id, job_tag)
+        begin
+          job_instance.process(argument)
+        rescue StandardError => e
+          push_job_arguments_back if handle_retry?
+          raise e
+        end
+        job_instance.class.perform_later(args, worker_id, job_tag)
       end
     end
 
     private
 
+    def handle_retry?
+      job_instance.try(:handle_retry?)
+    end
+
     def with_hooks
       ChainedJob.config.around_chain_process.call(options) { yield }
     end
 
     def options
-      @options ||= { job_class: job_instance.class, worker_id: worker_id }
+      @options ||= { job_class: job_instance.class, worker_id: worker_id, args: args }
     end
 
     def finished_worker
@@ -52,10 +63,9 @@ module ChainedJob
     end
 
     def deserialized_argument
+      return unless serialized_argument
+
       Marshal.load(serialized_argument)
-    rescue TypeError, ArgumentError
-      # backward compatibility
-      serialized_argument
     end
 
     def serialized_argument
@@ -69,7 +79,11 @@ module ChainedJob
     end
 
     def job_key
-      Helpers.job_key(job_instance.class)
+      Helpers.job_key(job_arguments_key)
+    end
+
+    def push_job_arguments_back
+      ChainedJob.redis.rpush(redis_key, Helpers.serialize([argument]))
     end
   end
 end

data/lib/chained_job/start_chains.rb

@@ -6,14 +6,16 @@ require 'chained_job/store_job_arguments'
 
 module ChainedJob
   class StartChains
-    def self.run(job_class, array_of_job_arguments, parallelism)
-      new(job_class, array_of_job_arguments, parallelism).run
+    def self.run(args, job_class, job_arguments_key, array_of_job_arguments, parallelism)
+      new(args, job_class, job_arguments_key, array_of_job_arguments, parallelism).run
     end
 
-    attr_reader :job_class, :array_of_job_arguments, :parallelism
+    attr_reader :args, :job_class, :job_arguments_key, :array_of_job_arguments, :parallelism
 
-    def initialize(job_class, array_of_job_arguments, parallelism)
+    def initialize(args, job_class, job_arguments_key, array_of_job_arguments, parallelism)
+      @args = args
       @job_class = job_class
+      @job_arguments_key = job_arguments_key
       @array_of_job_arguments = array_of_job_arguments
       @parallelism = parallelism
     end
@@ -23,15 +25,15 @@ module ChainedJob
       with_hooks do
         log_chained_job_cleanup
 
-        ChainedJob::CleanUpQueue.run(job_class)
+        ChainedJob::CleanUpQueue.run(job_arguments_key)
 
         next unless array_of_job_arguments.count.positive?
 
-        ChainedJob::StoreJobArguments.run(job_class, job_tag, array_of_job_arguments)
+        ChainedJob::StoreJobArguments.run(job_arguments_key, job_tag, array_of_job_arguments)
 
         log_chained_job_start
 
-        parallelism.times { |worked_id| job_class.perform_later(worked_id, job_tag) }
+        parallelism.times { |worked_id| job_class.perform_later(args, worked_id, job_tag) }
       end
     end
     # rubocop:enable Metrics/AbcSize
@@ -47,6 +49,7 @@ module ChainedJob
         job_class: job_class,
         array_of_job_arguments: array_of_job_arguments,
         parallelism: parallelism,
+        args: args,
       }
     end
 

data/lib/chained_job/store_job_arguments.rb

@@ -4,14 +4,14 @@ require 'chained_job/helpers'
 
 module ChainedJob
   class StoreJobArguments
-    def self.run(job_class, job_tag, array_of_job_arguments)
-      new(job_class, job_tag, array_of_job_arguments).run
+    def self.run(job_arguments_key, job_tag, array_of_job_arguments)
+      new(job_arguments_key, job_tag, array_of_job_arguments).run
     end
 
-    attr_reader :job_class, :job_tag, :array_of_job_arguments
+    attr_reader :job_arguments_key, :job_tag, :array_of_job_arguments
 
-    def initialize(job_class, job_tag, array_of_job_arguments)
-      @job_class = job_class
+    def initialize(job_arguments_key, job_tag, array_of_job_arguments)
+      @job_arguments_key = job_arguments_key
       @job_tag = job_tag
       @array_of_job_arguments = array_of_job_arguments
     end
@@ -20,7 +20,7 @@ module ChainedJob
       set_tag_list
 
       array_of_job_arguments.each_slice(config.arguments_batch_size) do |sublist|
-        ChainedJob.redis.rpush(redis_key, serialize(sublist))
+        ChainedJob.redis.rpush(redis_key, Helpers.serialize(sublist))
       end
 
       ChainedJob.redis.expire(redis_key, config.arguments_queue_expiration)
@@ -41,11 +41,7 @@ module ChainedJob
     end
 
     def job_key
-      @job_key ||= Helpers.job_key(job_class)
-    end
-
-    def serialize(arguments)
-      arguments.map { |argument| Marshal.dump(argument) }
+      @job_key ||= Helpers.job_key(job_arguments_key)
     end
 
     def config

data/lib/chained_job/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ChainedJob
-  VERSION = '0.5.0'
+  VERSION = '0.7.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: chained_job
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Mantas Kūjalis
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-04-22 00:00:00.000000000 Z
+date: 2021-08-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler