chained_job 0.3.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b302104a73850add4edc180a181961885c7c5a5c8bc520fc26b3dfa1a82c6d39
-  data.tar.gz: 25c67ef1d56381c0704ef70e182937412c120ff995b01ef6036e2fd840f5ab29
+  metadata.gz: 4699f818627cf6babf70086a5c5d4ac5016b17bedfb48670fd262f28b2da9350
+  data.tar.gz: 3d2f5a59e1fedaa8b11894f819739d254aa75620da658dafbb4435d1796412f1
 SHA512:
-  metadata.gz: 46c772e243c573a1cb4dd16b21d2cd34127665e174eab7c90946dd3065941f509b35a05b5c6fb498f634f5b03d4b926ccd3747626f66fbe2591f53423ace47a3
-  data.tar.gz: fc1881df18aa578066ca959c22e21617dc089b0a7615fc4c340c501bb06a35e52e99f587084b50729e6ab3c62b7aafb6bec7e60de4a2663968423db88b267fc9
+  metadata.gz: c4d6af80816570f45cc5b3e02aadb8d53f63fa10c009278401318a0a608d93d9e789038c594dedd4acd7cc54d6fd9459b13169f70d29d1687b1c82af2546e05e
+  data.tar.gz: c4f945a8a699eeda1fa4524ba526442efa91dc83c7a5aab1485bcdda5f79c65e94d085bb706791b65262012770dd8ea85f97b6979631576ee30c8dabea8c2257

data/lib/chained_job/clean_up_queue.rb

@@ -4,16 +4,16 @@ require 'chained_job/helpers'
 
 module ChainedJob
   class CleanUpQueue
-    def self.run(job_class)
-      new(job_class).run
+    def self.run(job_arguments_key)
+      new(job_arguments_key).run
     end
 
     TRIM_STEP_SIZE = 1_000
 
-    attr_reader :job_class
+    attr_reader :job_arguments_key
 
-    def initialize(job_class)
-      @job_class = job_class
+    def initialize(job_arguments_key)
+      @job_arguments_key = job_arguments_key
     end
 
     # rubocop:disable Metrics/AbcSize
@@ -39,7 +39,7 @@ module ChainedJob
     end
 
     def job_key
-      @job_key ||= Helpers.job_key(job_class)
+      @job_key ||= Helpers.job_key(job_arguments_key)
     end
   end
 end

data/lib/chained_job/config.rb

@@ -13,6 +13,7 @@ module ChainedJob
       :around_start_chains,
       :around_chain_process,
       :around_array_of_job_arguments,
+      :after_worker_finished,
       :debug,
       :logger,
       :redis,
@@ -28,6 +29,7 @@ module ChainedJob
       self.around_start_chains = ->(_options, &block) { block.call }
       self.around_chain_process = ->(_options, &block) { block.call }
       self.around_array_of_job_arguments = ->(_options, &block) { block.call }
+      self.after_worker_finished = ->(_options) {}
 
       self.debug = true
     end

data/lib/chained_job/helpers.rb

@@ -4,8 +4,8 @@ module ChainedJob
   module Helpers
     module_function
 
-    def job_key(job_class)
-      "chained_job:#{job_class}"
+    def job_key(job_arguments_key)
+      "chained_job:#{job_arguments_key}"
     end
 
     def redis_key(job_key, tag)

data/lib/chained_job/middleware.rb

@@ -9,11 +9,15 @@ module ChainedJob
       base.queue_as ChainedJob.config.queue if ChainedJob.config.queue
     end
 
-    def perform(worker_id = nil, tag = nil)
+    def perform(args = {}, worker_id = nil, tag = nil)
+      unless Hash === args
+        # backward compatibility
+        args, worker_id, tag = {}, args, worker_id
+      end
       if worker_id
-        ChainedJob::Process.run(self, worker_id, tag)
+        ChainedJob::Process.run(args, self, job_arguments_key, worker_id, tag)
       else
-        ChainedJob::StartChains.run(self.class, arguments_array, parallelism)
+        ChainedJob::StartChains.run(args, self.class, job_arguments_key, arguments_array, parallelism)
       end
     end
 
@@ -29,5 +33,9 @@ module ChainedJob
     def parallelism
       raise NoMethodError, 'undefined method parallelism'
     end
+
+    def job_arguments_key
+      self.class
+    end
   end
 end

data/lib/chained_job/process.rb

@@ -4,24 +4,26 @@ require 'chained_job/helpers'
 
 module ChainedJob
   class Process
-    def self.run(job_instance, worker_id, job_tag)
-      new(job_instance, worker_id, job_tag).run
+    def self.run(args, job_instance, job_arguments_key, worker_id, job_tag)
+      new(args, job_instance, job_arguments_key, worker_id, job_tag).run
     end
 
-    attr_reader :job_instance, :worker_id, :job_tag
+    attr_reader :args, :job_instance, :job_arguments_key, :worker_id, :job_tag
 
-    def initialize(job_instance, worker_id, job_tag)
+    def initialize(args, job_instance, job_arguments_key, worker_id, job_tag)
+      @args = args
       @job_instance = job_instance
+      @job_arguments_key = job_arguments_key
       @worker_id = worker_id
       @job_tag = job_tag
     end
 
     def run
       with_hooks do
-        return log_finished_worker unless argument
+        return finished_worker unless argument
 
         job_instance.process(argument)
-        job_instance.class.perform_later(worker_id, job_tag)
+        job_instance.class.perform_later(args, worker_id, job_tag)
       end
     end
 
@@ -32,7 +34,13 @@ module ChainedJob
     end
 
     def options
-      { job_class: job_instance.class, worker_id: worker_id }
+      @options ||= { job_class: job_instance.class, worker_id: worker_id }
+    end
+
+    def finished_worker
+      log_finished_worker
+
+      ChainedJob.config.after_worker_finished&.call(options)
     end
 
     def log_finished_worker
@@ -42,7 +50,19 @@ module ChainedJob
     end
 
     def argument
-      @argument ||= ChainedJob.redis.lpop(redis_key)
+      @argument ||= deserialized_argument
+    end
+
+    def deserialized_argument
+      return unless serialized_argument
+
+      Marshal.load(serialized_argument)
+    end
+
+    def serialized_argument
+      return @serialized_argument if defined?(@serialized_argument)
+
+      @serialized_argument = ChainedJob.redis.lpop(redis_key)
     end
 
     def redis_key
@@ -50,7 +70,7 @@ module ChainedJob
     end
 
     def job_key
-      Helpers.job_key(job_instance.class)
+      Helpers.job_key(job_arguments_key)
     end
   end
 end

data/lib/chained_job/start_chains.rb

@@ -6,14 +6,16 @@ require 'chained_job/store_job_arguments'
 
 module ChainedJob
   class StartChains
-    def self.run(job_class, array_of_job_arguments, parallelism)
-      new(job_class, array_of_job_arguments, parallelism).run
+    def self.run(args, job_class, job_arguments_key, array_of_job_arguments, parallelism)
+      new(args, job_class, job_arguments_key, array_of_job_arguments, parallelism).run
     end
 
-    attr_reader :job_class, :array_of_job_arguments, :parallelism
+    attr_reader :args, :job_class, :job_arguments_key, :array_of_job_arguments, :parallelism
 
-    def initialize(job_class, array_of_job_arguments, parallelism)
+    def initialize(args, job_class, job_arguments_key, array_of_job_arguments, parallelism)
+      @args = args
       @job_class = job_class
+      @job_arguments_key = job_arguments_key
       @array_of_job_arguments = array_of_job_arguments
       @parallelism = parallelism
     end
@@ -23,15 +25,15 @@ module ChainedJob
       with_hooks do
         log_chained_job_cleanup
 
-        ChainedJob::CleanUpQueue.run(job_class)
+        ChainedJob::CleanUpQueue.run(job_arguments_key)
 
         next unless array_of_job_arguments.count.positive?
 
-        ChainedJob::StoreJobArguments.run(job_class, job_tag, array_of_job_arguments)
+        ChainedJob::StoreJobArguments.run(job_arguments_key, job_tag, array_of_job_arguments)
 
         log_chained_job_start
 
-        parallelism.times { |worked_id| job_class.perform_later(worked_id, job_tag) }
+        parallelism.times { |worked_id| job_class.perform_later(args, worked_id, job_tag) }
       end
     end
     # rubocop:enable Metrics/AbcSize

data/lib/chained_job/store_job_arguments.rb

@@ -4,14 +4,14 @@ require 'chained_job/helpers'
 
 module ChainedJob
   class StoreJobArguments
-    def self.run(job_class, job_tag, array_of_job_arguments)
-      new(job_class, job_tag, array_of_job_arguments).run
+    def self.run(job_arguments_key, job_tag, array_of_job_arguments)
+      new(job_arguments_key, job_tag, array_of_job_arguments).run
     end
 
-    attr_reader :job_class, :job_tag, :array_of_job_arguments
+    attr_reader :job_arguments_key, :job_tag, :array_of_job_arguments
 
-    def initialize(job_class, job_tag, array_of_job_arguments)
-      @job_class = job_class
+    def initialize(job_arguments_key, job_tag, array_of_job_arguments)
+      @job_arguments_key = job_arguments_key
       @job_tag = job_tag
       @array_of_job_arguments = array_of_job_arguments
     end
@@ -20,7 +20,7 @@ module ChainedJob
       set_tag_list
 
       array_of_job_arguments.each_slice(config.arguments_batch_size) do |sublist|
-        ChainedJob.redis.rpush(redis_key, sublist)
+        ChainedJob.redis.rpush(redis_key, serialize(sublist))
       end
 
       ChainedJob.redis.expire(redis_key, config.arguments_queue_expiration)
@@ -41,7 +41,11 @@ module ChainedJob
     end
 
     def job_key
-      @job_key ||= Helpers.job_key(job_class)
+      @job_key ||= Helpers.job_key(job_arguments_key)
+    end
+
+    def serialize(arguments)
+      arguments.map { |argument| Marshal.dump(argument) }
     end
 
     def config

data/lib/chained_job/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ChainedJob
-  VERSION = '0.3.0'
+  VERSION = '0.6.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: chained_job
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Mantas Kūjalis
@@ -9,22 +9,22 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-19 00:00:00.000000000 Z
+date: 2021-05-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.17'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.17'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -53,25 +53,10 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '12.0'
-- !ruby/object:Gem::Dependency
-  name: rubocop-vinted
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.3'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.3'
 description: Chained job allows you to define an array of queued jobs that should
   be run in sequence after the main job has been executed successfully.
 email:
-- mantas.kujalis@vinted.com
-- titas@vinted.com
+- backend@vinted.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -104,7 +89,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Chained job helper