chained_job 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 21a653ae3a0eaaf596d741a3208b4aee1a7d297700e1d506c7005bf7cd529618
+  data.tar.gz: 03ab75c95d0610a9f192d3f6f7bfc7eeac516f641b4bbc32730ac45d1710f3bc
+SHA512:
+  metadata.gz: fb6be468a09e6824fb2dd21fd8aa8029790a63c145d9f5048b5c29dea2632e85e6b6ebcfb29558e7504982a4ad48261140e6b96b3b8e4ef55726d58a18ed58c0
+  data.tar.gz: 75a1aefe58b764b69d05514bfa47209ec4ff9d3f743379301841d132e15fc60c34f6254e0fae7dfacbca393c973f4ad2809dffcfb5a7041a42e62d3155a95857

data/lib/chained_job.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'chained_job/config'
+require 'chained_job/middleware'
+require 'chained_job/version'
+
+module ChainedJob
+  class Error < StandardError; end
+  class ConfigurationError < Error; end
+
+  module_function
+
+  def redis
+    config.redis || raise(ConfigurationError, 'Redis is not configured')
+  end
+
+  def logger
+    config.logger
+  end
+
+  def config
+    @config ||= ChainedJob::Config.new
+  end
+
+  def configure
+    yield(config)
+  end
+end

data/lib/chained_job/clean_up_queue.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require 'chained_job/helpers'
+
+module ChainedJob
+  class CleanUpQueue
+    def self.run(job_class)
+      new(job_class).run
+    end
+
+    TRIM_STEP_SIZE = 1_000
+
+    attr_reader :job_class
+
+    def initialize(job_class)
+      @job_class = job_class
+    end
+
+    # rubocop:disable Metrics/AbcSize
+    def run
+      loop do
+        tag = ChainedJob.redis.spop(tag_list)
+
+        break unless tag
+
+        redis_key = Helpers.redis_key(job_key, tag)
+        size = ChainedJob.redis.llen(redis_key)
+        (size / TRIM_STEP_SIZE).times { ChainedJob.redis.ltrim(redis_key, 0, -TRIM_STEP_SIZE) }
+
+        ChainedJob.redis.del(redis_key)
+      end
+    end
+    # rubocop:enable Metrics/AbcSize
+
+    private
+
+    def tag_list
+      @tag_list ||= Helpers.tag_list(job_key)
+    end
+
+    def job_key
+      @job_key ||= Helpers.job_key(job_class)
+    end
+  end
+end

data/lib/chained_job/config.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'logger'
+
+module ChainedJob
+  class Config
+    DEFAULT_ARGUMENTS_BATCH_SIZE = 1_000
+    DEFAULT_ARGUMENTS_QUEUE_EXPIRATION = 7 * 24 * 60 * 60 # 7 days
+
+    attr_accessor(
+      :arguments_batch_size,
+      :arguments_queue_expiration,
+      :around_start_chains,
+      :around_chain_process,
+      :debug,
+      :logger,
+      :redis,
+      :queue,
+    )
+
+    def initialize
+      self.arguments_batch_size = DEFAULT_ARGUMENTS_BATCH_SIZE
+      self.arguments_queue_expiration = DEFAULT_ARGUMENTS_QUEUE_EXPIRATION
+
+      self.logger = ::Logger.new(STDOUT)
+
+      self.around_start_chains = ->(_options, &block) { block.call }
+      self.around_chain_process = ->(_options, &block) { block.call }
+
+      self.debug = true
+    end
+  end
+end

data/lib/chained_job/helpers.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module ChainedJob
+  module Helpers
+    module_function
+
+    def job_key(job_class)
+      "chained_job:#{job_class}"
+    end
+
+    def redis_key(job_key, tag)
+      "#{job_key}:#{tag}"
+    end
+
+    def tag_list(prefix)
+      "#{prefix}:tags"
+    end
+  end
+end

data/lib/chained_job/middleware.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'chained_job/start_chains'
+require 'chained_job/process'
+
+module ChainedJob
+  module Middleware
+    def self.included(base)
+      base.queue_as ChainedJob.config.queue if ChainedJob.config.queue
+    end
+
+    def perform(worker_id = nil, tag = nil)
+      if worker_id
+        ChainedJob::Process.run(self, worker_id, tag)
+      else
+        ChainedJob::StartChains.run(self.class, array_of_job_arguments, parallelism)
+      end
+    end
+
+    def array_of_job_arguments
+      raise NoMethodError, 'undefined method array_of_job_arguments'
+    end
+
+    def parallelism
+      raise NoMethodError, 'undefined method parallelism'
+    end
+  end
+end

data/lib/chained_job/process.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'chained_job/helpers'
+
+module ChainedJob
+  class Process
+    def self.run(job_instance, worker_id, job_tag)
+      new(job_instance, worker_id, job_tag).run
+    end
+
+    attr_reader :job_instance, :worker_id, :job_tag
+
+    def initialize(job_instance, worker_id, job_tag)
+      @job_instance = job_instance
+      @worker_id = worker_id
+      @job_tag = job_tag
+    end
+
+    def run
+      with_hooks do
+        return log_finished_worker unless argument
+
+        job_instance.process(argument)
+        job_instance.class.perform_later(worker_id, job_tag)
+      end
+    end
+
+    private
+
+    def with_hooks
+      ChainedJob.config.around_chain_process.call(options) { yield }
+    end
+
+    def options
+      { job_class: job_instance.class, worker_id: worker_id }
+    end
+
+    def log_finished_worker
+      ChainedJob.logger.info(
+        "#{job_instance.class}:#{job_tag} worker #{worker_id} finished"
+      )
+    end
+
+    def argument
+      @argument ||= ChainedJob.redis.lpop(redis_key)
+    end
+
+    def redis_key
+      Helpers.redis_key(job_key, job_tag)
+    end
+
+    def job_key
+      Helpers.job_key(job_instance.class)
+    end
+  end
+end

data/lib/chained_job/start_chains.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'chained_job/helpers'
+require 'chained_job/clean_up_queue'
+require 'chained_job/store_job_arguments'
+
+module ChainedJob
+  class StartChains
+    def self.run(job_class, array_of_job_arguments, parallelism)
+      new(job_class, array_of_job_arguments, parallelism).run
+    end
+
+    attr_reader :job_class, :array_of_job_arguments, :parallelism
+
+    def initialize(job_class, array_of_job_arguments, parallelism)
+      @job_class = job_class
+      @array_of_job_arguments = array_of_job_arguments
+      @parallelism = parallelism
+    end
+
+    # rubocop:disable Metrics/AbcSize
+    def run
+      with_hooks do
+        ChainedJob::CleanUpQueue.run(job_class)
+
+        next unless array_of_job_arguments.count.positive?
+
+        ChainedJob::StoreJobArguments.run(job_class, job_tag, array_of_job_arguments)
+
+        log_chained_job_start
+
+        parallelism.times { |worked_id| job_class.perform_later(worked_id, job_tag) }
+      end
+    end
+    # rubocop:enable Metrics/AbcSize
+
+    private
+
+    def with_hooks
+      ChainedJob.config.around_start_chains.call(options) { yield }
+    end
+
+    def options
+      {
+        job_class: job_class,
+        array_of_job_arguments: array_of_job_arguments,
+        parallelism: parallelism,
+      }
+    end
+
+    def job_tag
+      @job_tag ||= Time.now.to_f.to_s
+    end
+
+    def log_chained_job_start
+      ChainedJob.logger.info(
+        "#{job_class}:#{job_tag} starting #{parallelism} workers "\
+        "processing #{array_of_job_arguments.count} items"
+      )
+    end
+  end
+end

data/lib/chained_job/store_job_arguments.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'chained_job/helpers'
+
+module ChainedJob
+  class StoreJobArguments
+    def self.run(job_class, job_tag, array_of_job_arguments)
+      new(job_class, job_tag, array_of_job_arguments).run
+    end
+
+    attr_reader :job_class, :job_tag, :array_of_job_arguments
+
+    def initialize(job_class, job_tag, array_of_job_arguments)
+      @job_class = job_class
+      @job_tag = job_tag
+      @array_of_job_arguments = array_of_job_arguments
+    end
+
+    def run
+      set_tag_list
+
+      array_of_job_arguments.each_slice(config.arguments_batch_size) do |sublist|
+        ChainedJob.redis.rpush(redis_key, sublist)
+      end
+
+      ChainedJob.redis.expire(redis_key, config.arguments_queue_expiration)
+    end
+
+    private
+
+    def set_tag_list
+      ChainedJob.redis.sadd(tag_list, job_tag)
+    end
+
+    def tag_list
+      Helpers.tag_list(job_key)
+    end
+
+    def redis_key
+      @redis_key ||= Helpers.redis_key(job_key, job_tag)
+    end
+
+    def job_key
+      @job_key ||= Helpers.job_key(job_class)
+    end
+
+    def config
+      ChainedJob.config
+    end
+  end
+end

data/lib/chained_job/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module ChainedJob
+  VERSION = '0.2.0'
+end

metadata

@@ -0,0 +1,111 @@
+--- !ruby/object:Gem::Specification
+name: chained_job
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Mantas Kūjalis
+- Titas Norkūnas
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-07-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-vinted
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+description: Chained job allows you to define an array of queued jobs that should
+  be run in sequence after the main job has been executed successfully.
+email:
+- mantas.kujalis@vinted.com
+- titas@vinted.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/chained_job.rb
+- lib/chained_job/clean_up_queue.rb
+- lib/chained_job/config.rb
+- lib/chained_job/helpers.rb
+- lib/chained_job/middleware.rb
+- lib/chained_job/process.rb
+- lib/chained_job/start_chains.rb
+- lib/chained_job/store_job_arguments.rb
+- lib/chained_job/version.rb
+homepage: https://github.com/vinted/chained_job
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Chained job helper
+test_files: []