chained_job 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 7a8aa04f7ae42cb12af32784e969ce63952884ff5db3e6e435943bcf1112f50b
+  data.tar.gz: db0cc9308feec6d98a8d8444f7c96da901b204f14d265cc1f6b20df9eab6a72a
+SHA512:
+  metadata.gz: b865a3f752707bea1255f0bbc76bbb29ed901ab83b05092ef4c03274651d7b0eadee5307f6f28610a0cbfe374ede0e141a482c837b9677ed2defb52923b03190
+  data.tar.gz: 6b3b01a064ed9a76948196c16715c24226382c6fed5925a5fa3f53455a0b6b7d7976a066721de58746b0cb1bf6d3ed28899dd97901bfbf510f28b38f40a033dd

data/bin/console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'irb'
+
+require 'chained_job'
+
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/chained_job.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'chained_job/config'
+require 'chained_job/middleware'
+require 'chained_job/version'
+
+module ChainedJob
+  class Error < StandardError; end
+  class ConfigurationError < Error; end
+
+  module_function
+
+  def redis
+    config.redis || raise(ConfigurationError, 'Redis is not configured')
+  end
+
+  def logger
+    config.logger
+  end
+
+  def config
+    @config ||= ChainedJob::Config.new
+  end
+
+  def configure
+    yield(config)
+  end
+end

data/lib/chained_job/config.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require 'logger'
+
+module ChainedJob
+  class Config
+    DEFAULT_ARGUMENTS_BATCH_SIZE = 1_000
+    DEFAULT_ARGUMENTS_QUEUE_EXPIRATION = 7 * 24 * 60 * 60 # 7 days
+
+    attr_accessor(
+      :arguments_batch_size,
+      :arguments_queue_expiration,
+      :around_start_chains,
+      :around_chain_process,
+      :debug,
+      :logger,
+      :redis,
+    )
+
+    def initialize
+      self.arguments_batch_size = DEFAULT_ARGUMENTS_BATCH_SIZE
+      self.arguments_queue_expiration = DEFAULT_ARGUMENTS_QUEUE_EXPIRATION
+
+      self.logger = ::Logger.new(STDOUT)
+
+      self.around_start_chains = ->(_options, &block) { block.call }
+      self.around_chain_process = ->(_options, &block) { block.call }
+
+      self.debug = true
+    end
+  end
+end

data/lib/chained_job/helpers.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module ChainedJob
+  module Helpers
+    module_function
+
+    def redis_key(job_class)
+      "chained_job:#{job_class}"
+    end
+  end
+end

data/lib/chained_job/middleware.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'chained_job/start_chains'
+require 'chained_job/process'
+
+module ChainedJob
+  module Middleware
+    def perform(worker_id = nil)
+      if worker_id
+        ChainedJob::Process.run(self, worker_id)
+      else
+        ChainedJob::StartChains.run(self.class, array_of_job_arguments, parallelism)
+      end
+    end
+
+    def array_of_job_arguments
+      raise NoMethodError, 'undefined method array_of_job_arguments'
+    end
+
+    def parallelism
+      raise NoMethodError, 'undefined method parallelism'
+    end
+  end
+end

data/lib/chained_job/process.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'chained_job/helpers'
+
+module ChainedJob
+  class Process
+    def self.run(job_instance, worker_id)
+      new(job_instance, worker_id).run
+    end
+
+    attr_reader :job_instance, :worker_id
+
+    def initialize(job_instance, worker_id)
+      @job_instance = job_instance
+      @worker_id = worker_id
+    end
+
+    def run
+      with_hooks do
+        return log_finished_worker unless argument
+
+        job_instance.process(argument)
+        job_instance.class.perform_later(worker_id)
+      end
+    end
+
+    private
+
+    def with_hooks
+      ChainedJob.config.around_chain_process.call(options) { yield }
+    end
+
+    def options
+      { job_class: job_instance.class, worker_id: worker_id }
+    end
+
+    def log_finished_worker
+      ChainedJob.logger.info(
+        "#{job_instance.class} worker #{worker_id} finished"
+      )
+    end
+
+    def argument
+      @argument ||= ChainedJob.redis.lpop(redis_key)
+    end
+
+    def redis_key
+      Helpers.redis_key(job_instance.class)
+    end
+  end
+end

data/lib/chained_job/start_chains.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require 'chained_job/helpers'
+
+module ChainedJob
+  class StartChains
+    def self.run(job_class, array_of_job_arguments, parallelism)
+      new(job_class, array_of_job_arguments, parallelism).run
+    end
+
+    attr_reader :job_class, :array_of_job_arguments, :parallelism
+
+    def initialize(job_class, array_of_job_arguments, parallelism)
+      @job_class = job_class
+      @array_of_job_arguments = array_of_job_arguments
+      @parallelism = parallelism
+    end
+
+    def run
+      with_hooks do
+        redis.del(redis_key)
+
+        return unless array_of_job_arguments.count.positive?
+
+        store_job_arguments
+
+        log_chained_job_start
+
+        parallelism.times { |worked_id| job_class.perform_later(worked_id) }
+      end
+    end
+
+    private
+
+    def with_hooks
+      ChainedJob.config.around_start_chains.call(options) { yield }
+    end
+
+    def options
+      {
+        job_class: job_class,
+        array_of_job_arguments: array_of_job_arguments,
+        parallelism: parallelism,
+      }
+    end
+
+    def store_job_arguments
+      array_of_job_arguments.each_slice(config.arguments_batch_size) do |sublist|
+        redis.rpush(redis_key, sublist)
+      end
+
+      redis.expire(redis_key, config.arguments_queue_expiration)
+    end
+
+    def log_chained_job_start
+      ChainedJob.logger.info(
+        "#{job_class} starting #{parallelism} workers "\
+        "processing #{array_of_job_arguments.count} items"
+      )
+    end
+
+    def redis
+      ChainedJob.redis
+    end
+
+    def redis_key
+      Helpers.redis_key(job_class)
+    end
+
+    def config
+      ChainedJob.config
+    end
+  end
+end

data/lib/chained_job/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module ChainedJob
+  VERSION = '0.1.0'
+end

metadata

@@ -0,0 +1,113 @@
+--- !ruby/object:Gem::Specification
+name: chained_job
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Mantas Kūjalis
+- Titas Norkūnas
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-04-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-vinted
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+description: Chained job allows you to define an array of queued jobs that should
+  be run in sequence after the main job has been executed successfully.
+email:
+- mantas.kujalis@vinted.com
+- titas@vinted.com
+executables:
+- setup
+- console
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/console
+- bin/setup
+- lib/chained_job.rb
+- lib/chained_job/config.rb
+- lib/chained_job/helpers.rb
+- lib/chained_job/middleware.rb
+- lib/chained_job/process.rb
+- lib/chained_job/start_chains.rb
+- lib/chained_job/version.rb
+homepage: https://github.com/vinted/chained_job
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Chained job helper
+test_files: []