chained_job 0.1.2 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0cf0d92c19a3955df5206dc54e7ca3883120a73fdcc1fd8743495f9c37690308
-  data.tar.gz: '09617e3e0873985b586da6903f604e89f1d429bab84bf99e88d652d807408d08'
+  metadata.gz: 84c8c12fdb15c63dfa9fe2cb031382284e0eedcb2fdfad5f5af4d5dc8f01370a
+  data.tar.gz: 467051fa730945c836e93222c8057b9c8e2a9d635a46e550db299cc99b36f380
 SHA512:
-  metadata.gz: 37ca1b1aa5dc58b495a0dac015b024919713ecf4a6ccfc63b252f81f1adfb9090b702e6175f3f9aaa5da85a0293e043e398226a67c684138bffceb28c819db63
-  data.tar.gz: dbe7e4ce4dd1c92f955e62f0e0cf806df9a580fa54658960ee9889c92adb02eb30f8de3b61c57b9e652a1fff1167f86cb8c0caf613e43ca2ac07fc7b77bf2dfb
+  metadata.gz: b590497db0ae0fdb36f2e928da3bcb72c5c4ae7eabe5225a863d301ec27f462dda2f13c0a57f941eec07cf96b347935eed995835c67a89d3617d184516709f7a
+  data.tar.gz: 98146d3133f14943b3a262014e6dba6e729cc019e945af887de311e416995b5dc52f4ba8f236c0d8df941cfec208de72712b19f1ad5ff8aeb89ceba2e4dd0d6b

data/lib/chained_job/clean_up_queue.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require 'chained_job/helpers'
+
+module ChainedJob
+  class CleanUpQueue
+    def self.run(job_class)
+      new(job_class).run
+    end
+
+    TRIM_STEP_SIZE = 1_000
+
+    attr_reader :job_class
+
+    def initialize(job_class)
+      @job_class = job_class
+    end
+
+    # rubocop:disable Metrics/AbcSize
+    def run
+      loop do
+        tag = ChainedJob.redis.spop(tag_list)
+
+        break unless tag
+
+        redis_key = Helpers.redis_key(job_key, tag)
+        size = ChainedJob.redis.llen(redis_key)
+        (size / TRIM_STEP_SIZE).times { ChainedJob.redis.ltrim(redis_key, 0, -TRIM_STEP_SIZE) }
+
+        ChainedJob.redis.del(redis_key)
+      end
+    end
+    # rubocop:enable Metrics/AbcSize
+
+    private
+
+    def tag_list
+      @tag_list ||= Helpers.tag_list(job_key)
+    end
+
+    def job_key
+      @job_key ||= Helpers.job_key(job_class)
+    end
+  end
+end

data/lib/chained_job/config.rb

@@ -12,9 +12,12 @@ module ChainedJob
       :arguments_queue_expiration,
       :around_start_chains,
       :around_chain_process,
+      :around_array_of_job_arguments,
+      :after_worker_finished,
       :debug,
       :logger,
       :redis,
+      :queue,
     )
 
     def initialize
@@ -25,6 +28,8 @@ module ChainedJob
 
       self.around_start_chains = ->(_options, &block) { block.call }
       self.around_chain_process = ->(_options, &block) { block.call }
+      self.around_array_of_job_arguments = ->(_options, &block) { block.call }
+      self.after_worker_finished = ->(_options) {}
 
       self.debug = true
     end

data/lib/chained_job/helpers.rb

@@ -4,8 +4,16 @@ module ChainedJob
   module Helpers
     module_function
 
-    def redis_key(job_class)
+    def job_key(job_class)
       "chained_job:#{job_class}"
     end
+
+    def redis_key(job_key, tag)
+      "#{job_key}:#{tag}"
+    end
+
+    def tag_list(prefix)
+      "#{prefix}:tags"
+    end
   end
 end

data/lib/chained_job/middleware.rb

@@ -5,14 +5,23 @@ require 'chained_job/process'
 
 module ChainedJob
   module Middleware
-    def perform(worker_id = nil)
+    def self.included(base)
+      base.queue_as ChainedJob.config.queue if ChainedJob.config.queue
+    end
+
+    def perform(worker_id = nil, tag = nil)
       if worker_id
-        ChainedJob::Process.run(self, worker_id)
+        ChainedJob::Process.run(self, worker_id, tag)
       else
-        ChainedJob::StartChains.run(self.class, array_of_job_arguments, parallelism)
+        ChainedJob::StartChains.run(self.class, arguments_array, parallelism)
       end
     end
 
+    def arguments_array
+      options = { job_class: self.class }
+      ChainedJob.config.around_array_of_job_arguments.call(options) { array_of_job_arguments }
+    end
+
     def array_of_job_arguments
       raise NoMethodError, 'undefined method array_of_job_arguments'
     end

data/lib/chained_job/process.rb

@@ -4,23 +4,24 @@ require 'chained_job/helpers'
 
 module ChainedJob
   class Process
-    def self.run(job_instance, worker_id)
-      new(job_instance, worker_id).run
+    def self.run(job_instance, worker_id, job_tag)
+      new(job_instance, worker_id, job_tag).run
     end
 
-    attr_reader :job_instance, :worker_id
+    attr_reader :job_instance, :worker_id, :job_tag
 
-    def initialize(job_instance, worker_id)
+    def initialize(job_instance, worker_id, job_tag)
       @job_instance = job_instance
       @worker_id = worker_id
+      @job_tag = job_tag
     end
 
     def run
       with_hooks do
-        return log_finished_worker unless argument
+        return finished_worker unless argument
 
         job_instance.process(argument)
-        job_instance.class.perform_later(worker_id)
+        job_instance.class.perform_later(worker_id, job_tag)
       end
     end
 
@@ -31,12 +32,18 @@ module ChainedJob
     end
 
     def options
-      { job_class: job_instance.class, worker_id: worker_id }
+      @options ||= { job_class: job_instance.class, worker_id: worker_id }
+    end
+
+    def finished_worker
+      log_finished_worker
+
+      ChainedJob.config.after_worker_finished&.call(options)
     end
 
     def log_finished_worker
       ChainedJob.logger.info(
-        "#{job_instance.class} worker #{worker_id} finished"
+        "#{job_instance.class}:#{job_tag} worker #{worker_id} finished"
       )
     end
 
@@ -45,7 +52,11 @@ module ChainedJob
     end
 
     def redis_key
-      Helpers.redis_key(job_instance.class)
+      Helpers.redis_key(job_key, job_tag)
+    end
+
+    def job_key
+      Helpers.job_key(job_instance.class)
     end
   end
 end

data/lib/chained_job/start_chains.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require 'chained_job/helpers'
+require 'chained_job/clean_up_queue'
+require 'chained_job/store_job_arguments'
 
 module ChainedJob
   class StartChains
@@ -16,19 +18,23 @@ module ChainedJob
       @parallelism = parallelism
     end
 
+    # rubocop:disable Metrics/AbcSize
     def run
       with_hooks do
-        redis.del(redis_key)
+        log_chained_job_cleanup
+
+        ChainedJob::CleanUpQueue.run(job_class)
 
         next unless array_of_job_arguments.count.positive?
 
-        store_job_arguments
+        ChainedJob::StoreJobArguments.run(job_class, job_tag, array_of_job_arguments)
 
         log_chained_job_start
 
-        parallelism.times { |worked_id| job_class.perform_later(worked_id) }
+        parallelism.times { |worked_id| job_class.perform_later(worked_id, job_tag) }
       end
     end
+    # rubocop:enable Metrics/AbcSize
 
     private
 
@@ -44,31 +50,21 @@ module ChainedJob
       }
     end
 
-    def store_job_arguments
-      array_of_job_arguments.each_slice(config.arguments_batch_size) do |sublist|
-        redis.rpush(redis_key, sublist)
-      end
-
-      redis.expire(redis_key, config.arguments_queue_expiration)
+    def job_tag
+      @job_tag ||= Time.now.to_f.to_s
     end
 
     def log_chained_job_start
       ChainedJob.logger.info(
-        "#{job_class} starting #{parallelism} workers "\
+        "#{job_class}:#{job_tag} starting #{parallelism} workers "\
         "processing #{array_of_job_arguments.count} items"
       )
     end
 
-    def redis
-      ChainedJob.redis
-    end
-
-    def redis_key
-      Helpers.redis_key(job_class)
-    end
-
-    def config
-      ChainedJob.config
+    def log_chained_job_cleanup
+      ChainedJob.logger.info(
+        "#{job_class}:#{job_tag} cleanup"
+      )
     end
   end
 end

data/lib/chained_job/store_job_arguments.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'chained_job/helpers'
+
+module ChainedJob
+  class StoreJobArguments
+    def self.run(job_class, job_tag, array_of_job_arguments)
+      new(job_class, job_tag, array_of_job_arguments).run
+    end
+
+    attr_reader :job_class, :job_tag, :array_of_job_arguments
+
+    def initialize(job_class, job_tag, array_of_job_arguments)
+      @job_class = job_class
+      @job_tag = job_tag
+      @array_of_job_arguments = array_of_job_arguments
+    end
+
+    def run
+      set_tag_list
+
+      array_of_job_arguments.each_slice(config.arguments_batch_size) do |sublist|
+        ChainedJob.redis.rpush(redis_key, sublist)
+      end
+
+      ChainedJob.redis.expire(redis_key, config.arguments_queue_expiration)
+    end
+
+    private
+
+    def set_tag_list
+      ChainedJob.redis.sadd(tag_list, job_tag)
+    end
+
+    def tag_list
+      Helpers.tag_list(job_key)
+    end
+
+    def redis_key
+      @redis_key ||= Helpers.redis_key(job_key, job_tag)
+    end
+
+    def job_key
+      @job_key ||= Helpers.job_key(job_class)
+    end
+
+    def config
+      ChainedJob.config
+    end
+  end
+end

data/lib/chained_job/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ChainedJob
-  VERSION = '0.1.2'
+  VERSION = '0.4.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: chained_job
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.4.1
 platform: ruby
 authors:
 - Mantas Kūjalis
@@ -9,22 +9,22 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-29 00:00:00.000000000 Z
+date: 2021-04-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.17'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.17'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -53,35 +53,22 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '12.0'
-- !ruby/object:Gem::Dependency
-  name: rubocop-vinted
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.3'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.3'
 description: Chained job allows you to define an array of queued jobs that should
   be run in sequence after the main job has been executed successfully.
 email:
-- mantas.kujalis@vinted.com
-- titas@vinted.com
+- backend@vinted.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - lib/chained_job.rb
+- lib/chained_job/clean_up_queue.rb
 - lib/chained_job/config.rb
 - lib/chained_job/helpers.rb
 - lib/chained_job/middleware.rb
 - lib/chained_job/process.rb
 - lib/chained_job/start_chains.rb
+- lib/chained_job/store_job_arguments.rb
 - lib/chained_job/version.rb
 homepage: https://github.com/vinted/chained_job
 licenses:
@@ -102,7 +89,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Chained job helper