distributed_job 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e8aa1822c85ccb44521e470a312b430b9409dd537add85122045f7cd71d075ed
+  data.tar.gz: a781158c7699303544cb78185a200059da25013830f677a44a704b6a0c115af8
+SHA512:
+  metadata.gz: 6885d739ae86c615588dfd85ca1b4d0b0b3e4bcdf8cca1ddf1aebcb806a703c1980ca2d1e6f758d22b38a24a3859d5673fa47e7b2438db4635a4e24351bff4eb
+  data.tar.gz: ffbea863763a62d8f22a2e8c2f2b8555a910fc908801e9a93157703a3a2e7e6fbd9bee3b90e6c35971d430b8b720781b02c1d116159e1ea2c3cb00b9fa57b7d4

data/.github/workflows/test.yml

@@ -0,0 +1,30 @@
+on: push
+name: test
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        redis:
+          - redis:5.0
+          - redis:6.0
+        ruby:
+          - 2.6
+          - 2.7
+          - 3.0
+    services:
+      elasticsearch:
+        image: ${{ matrix.redis }}
+        ports:
+          - 6379:6379
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - run: gem install bundler
+      - run: bundle
+      - run: sleep 3
+      - run: bundle exec rspec
+      - run: bundle exec rubocop

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,19 @@
+AllCops:
+  NewCops: enable
+  TargetRubyVersion: 2.5 
+  SuggestExtensions: false
+
+Metrics/MethodLength:
+  Enabled: false
+
+Metrics/BlockLength:
+  Enabled: false
+
+Layout/LineLength:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/NumericPredicate:
+  Enabled: false

data/.travis.yml

@@ -0,0 +1,6 @@
+---
+language: ruby
+cache: bundler
+rvm:
+  - 2.6.6
+before_install: gem install bundler -v 2.1.4

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+# CHANGELOG
+
+## v1.0.0
+
+* Initial release

data/Gemfile

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in distributed_job.gemspec
+gemspec
+
+gem 'rake', '~> 12.0'
+gem 'rspec', '~> 3.0'

data/Gemfile.lock

@@ -0,0 +1,57 @@
+PATH
+  remote: .
+  specs:
+    distributed_job (1.0.0)
+      redis
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.2)
+    diff-lcs (1.4.4)
+    parallel (1.21.0)
+    parser (3.0.2.0)
+      ast (~> 2.4.1)
+    rainbow (3.0.0)
+    rake (12.3.3)
+    redis (4.5.1)
+    regexp_parser (2.1.1)
+    rexml (3.2.5)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.2)
+    rubocop (1.22.1)
+      parallel (~> 1.10)
+      parser (>= 3.0.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.12.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.12.0)
+      parser (>= 3.0.1.1)
+    ruby-progressbar (1.11.0)
+    unicode-display_width (2.1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  distributed_job!
+  rake (~> 12.0)
+  rspec (~> 3.0)
+  rubocop
+
+BUNDLED WITH
+   2.1.4

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2021 Benjamin Vetter
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,100 @@
+# DistributedJob
+
+Easily keep track of distributed jobs consisting of an arbitrary number of
+parts spanning multiple workers using redis. Can be used with any kind of
+backround job processing queue.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'distributed_job'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install distributed_job
+
+## Usage
+
+Getting started is very easy. A `DistributedJob` allows to keep track of a
+distributed job, i.e. a job which is split into multiple units running in
+parallel and in multiple workers.
+
+To create a distributed job and add parts, i.e. units of work, to it, simply:
+
+```ruby
+  distributed_job = DistributedJob.new(redis: Redis.new, token: SecureRandom.hex)
+
+  distributed_job.push_each(Date.parse('2021-01-01')..Date.today) do |date, part|
+    SomeBackgroundJob.perform_async(date, distributed_job.token, part)
+  end
+
+  distributed_job.token # can be used to query the status of the distributed job
+```
+
+The `token` can be used to keep query the status of the distributed job, e.g.
+on a job summary page or similar. You can show some progress bar in the browser
+or in the terminal, etc:
+
+```ruby
+# token is given via URL or via some other means
+distributed_job = Distributed.new(redis: Redis.new, token: params[:token])
+
+distributed_job.total # total number of parts
+distributed_job.count # number of unfinished parts
+distributed_job.finished?
+```
+
+Within the background job, you use the passed token and part to query and
+update the status of the distributed job and part accordingly. Please note
+that you can use whatever background job processing tool you like most.
+
+```ruby
+class SomeBackgroundJob
+  def perform(whatever, token, part)
+    distributed_job = DistributedJob.new(redis: Redis.new, token: token)
+
+    return if distributed_job.stopped?
+
+    # ...
+
+    distributed_job.done(part)
+
+    if distributed_job.finished?
+      # perform e.g. cleanup or the some other job
+    end
+  rescue
+    distributed_job.stop
+
+    raise
+  end
+end
+```
+
+The `#stop` and `#stopped?` methods can be used to globally stop a distributed
+job in case of errors. Contrary, the `#done` method tells the `DistributedJob` that the
+specified part has successfully finished. Finally, the `#finished?` method
+returns true when all parts of the distributed job are finished, which is useful
+to start cleanup jobs or to even start another subsequent distributed job.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run
+`bundle exec rspec` to run the tests. You can also run `bin/console` for an
+interactive prompt that will allow you to experiment.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at
+https://github.com/mrkamel/distributed_job.
+
+## License
+
+The gem is available as open source under the terms of the [MIT
+License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'distributed_job'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/distributed_job.gemspec

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative 'lib/distributed_job/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'distributed_job'
+  spec.version       = DistributedJob::VERSION
+  spec.authors       = ['Benjamin Vetter']
+  spec.email         = ['benjamin.vetter@wlw.de']
+
+  spec.summary       = 'Keep track of distributed jobs using redis'
+  spec.description   = 'Keep track of distributed jobs spanning multiple workers using redis'
+  spec.homepage      = 'https://github.com/mrkamel/distributed_job'
+  spec.license       = 'MIT'
+  spec.required_ruby_version = Gem::Requirement.new('>= 2.5.0')
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = 'https://github.com/mrkamel/distributed_job'
+  spec.metadata['changelog_uri'] = 'https://github.com/mrkamel/distributed_job/blob/master/CHANGELOG.md'
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'rubocop'
+  spec.add_dependency 'redis'
+end

data/docker-compose.yml

@@ -0,0 +1,6 @@
+version: '2' 
+services:
+  elasticsearch:
+    image: redis
+    ports:
+      - 6379:6379

data/lib/distributed_job/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class DistributedJob
+  VERSION = '1.0.0'
+end

data/lib/distributed_job.rb

@@ -0,0 +1,275 @@
+# frozen_string_literal: true
+
+require 'distributed_job/version'
+require 'redis'
+
+# A distributed job instance allows to keep track of distributed jobs, i.e.
+# jobs which are split into multiple units running in parallel and in multiple
+# workers using redis.
+#
+# @example Creating a distributed job
+#   distributed_job = DistributedJob.new(redis: Redis.new, token: SecureRandom.hex)
+#
+#   distributed_job.push_each(Date.parse('2021-01-01')..Date.today) do |date, part|
+#     SomeBackgroundJob.perform_async(date, distributed_job.token, part)
+#   end
+#
+#   distributed_job.token # can be used to query the status of the distributed job
+#
+# @example Processing a distributed job part
+#   class SomeBackgroundJob
+#     def perform(whatever, token, part)
+#       distributed_job = DistributedJob.new(redis: Redis.new, token: token)
+#
+#       return if distributed_job.stopped?
+#
+#       # ...
+#
+#       distributed_job.done(part)
+#
+#       if distributed_job.finished?
+#         # perform e.g. cleanup or the some other job
+#       end
+#     rescue
+#       distributed_job.stop
+#
+#       raise
+#     end
+#   end
+
+class DistributedJob
+  attr_reader :redis, :token, :ttl
+
+  # Initializes a new distributed job.
+  #
+  # @param redis [Redis] The redis connection instance
+  # @param token [String] Some token to be used to identify the job. You can
+  #   e.g. use SecureRandom.hex to generate one.
+  # @param ttl [Integer] The number of seconds this job will stay available
+  #   in redis. This value is used to automatically expire and clean up the
+  #   job in redis. Default is 86400, i.e. one day. The ttl is used everytime
+  #   the job is modified in redis.
+  #
+  # @example
+  #   distributed_job = DistributedJob.new(redis: Redis.new, token: SecureRandom.hex)
+
+  def initialize(redis:, token:, ttl: 86_400)
+    @redis = redis
+    @token = token
+    @ttl = ttl
+  end
+
+  # Pass an enum to be used to iterate all the units of work of the distributed
+  # job. The distributed job needs to know all of them to keep track of the
+  # overall number and status of the parts. Passing an enum is much better
+  # compared to pushing the parts manually, because the distributed job needs
+  # to be closed before the last part of the distributed job is enqueued into
+  # some job queue. Otherwise it could potentially happen that the last part is
+  # already processed in the job queue before it is pushed to redis, such that
+  # the last job doesn't know that the distributed job is finished.
+  #
+  # @param enum [#each_with_index] The enum which can be iterated to get all
+  #   job parts
+  #
+  # @example
+  #   distributed_job.push_each(Date.parse('2021-01-01')..Date.today) do |date, part|
+  #     # e.g. SomeBackgroundJob.perform_async(date, distributed_job.token, part)
+  #   end
+  #
+  # @example ActiveRecord
+  #   distributed_job.push_each(User.select(:id).find_in_batches) do |batch, part|
+  #     # e.g. SomeBackgroundJob.perform_async(batch.first.id, batch.last.id, distributed_job.token, part)
+  #   end
+
+  def push_each(enum)
+    return enum_for(:push_each, enum) unless block_given?
+
+    previous_object = nil
+    previous_index = nil
+
+    enum.each_with_index do |current_object, current_index|
+      push(current_index)
+
+      yield(previous_object, previous_index.to_s) if previous_index
+
+      previous_object = current_object
+      previous_index = current_index
+    end
+
+    close
+
+    yield(previous_object, previous_index.to_s) if previous_index
+  end
+
+  # Returns all pushed parts of the distributed job
+  #
+  # @return [Enumerator] The enum which allows to iterate all parts
+
+  def parts
+    redis.sscan_each("#{redis_key}:parts")
+  end
+
+  # Removes the specified part from the distributed job, i.e. from the set of
+  # unfinished parts. Use this method when the respective job part has been
+  # successfully processed, i.e. finished.
+  #
+  # @param part [String] The job part
+  # @returns [Boolean] Returns true when there are no more unfinished parts
+  #   left or false otherwise
+  #
+  # @example
+  #   class SomeBackgroundJob
+  #     def perform(whatever, token, part)
+  #       distributed_job = DistributedJob.new(redis: Redis.new, token: token)
+  #
+  #       # ...
+  #
+  #       distributed_job.done(part)
+  #     end
+  #   end
+
+  def done(part)
+    @done_script ||= <<~SCRIPT
+      local key, part, ttl = ARGV[1], ARGV[2], tonumber(ARGV[3])
+
+      if redis.call('srem', key .. ':parts', part) == 0 then return end
+
+      redis.call('expire', key .. ':parts', ttl)
+      redis.call('expire', key .. ':state', ttl)
+
+      return redis.call('scard', key .. ':parts')
+    SCRIPT
+
+    redis.eval(@done_script, argv: [redis_script_key, part.to_s, ttl]) == 0 && closed?
+  end
+
+  # Returns the total number of pushed parts, no matter if finished or not.
+  #
+  # @example
+  #   distributed_job.total # => e.g. 13
+
+  def total
+    redis.hget("#{redis_key}:state", 'total').to_i
+  end
+
+  # Returns the number of pushed parts which are not finished.
+  #
+  # @example
+  #   distributed_job.count # => e.g. 8
+
+  def count
+    redis.scard("#{redis_key}:parts")
+  end
+
+  # Returns true if there are no more unfinished parts.
+  #
+  # @example
+  #   distributed_job.finished? #=> true/false
+
+  def finished?
+    closed? && count.zero?
+  end
+
+  # Allows to stop a distributed job. This is useful if some error occurred in
+  # some part, i.e. background job, of the distributed job and you then want to
+  # stop all other not yet finished parts. Please note that only jobs can be
+  # stopped which ask the distributed job actively whether or not it was
+  # stopped.
+  #
+  # @returns [Boolean] Always returns true
+  #
+  # @example
+  #   class SomeBackgroundJob
+  #     def perform(whatever, token, part)
+  #       distributed_job = DistributedJob.new(redis: Redis.new, token: token)
+  #
+  #       return if distributed_job.stopped?
+  #
+  #       # ...
+  #
+  #       distributed_job.done(part)
+  #     rescue
+  #       distributed_job.stop
+  #
+  #       raise
+  #     end
+  #   end
+
+  def stop
+    redis.multi do
+      redis.hset("#{redis_key}:state", 'stopped', 1)
+
+      redis.expire("#{redis_key}:state", ttl)
+      redis.expire("#{redis_key}:parts", ttl)
+    end
+
+    true
+  end
+
+  # Returns true when the distributed job was stopped or false otherwise.
+  #
+  # @returns [Boolean] Returns true or false
+  #
+  # @example
+  #   class SomeBackgroundJob
+  #     def perform(whatever, token, part)
+  #       distributed_job = DistributedJob.new(redis: Redis.new, token: token)
+  #
+  #       return if distributed_job.stopped?
+  #
+  #       # ...
+  #
+  #       distributed_job.done(part)
+  #     rescue
+  #       distributed_job.stop
+  #
+  #       raise
+  #     end
+  #   end
+
+  def stopped?
+    redis.hget("#{redis_key}:state", 'stopped') == '1'
+  end
+
+  private
+
+  def close
+    redis.multi do
+      redis.hset("#{redis_key}:state", 'closed', 1)
+
+      redis.expire("#{redis_key}:state", ttl)
+      redis.expire("#{redis_key}:parts", ttl)
+    end
+
+    true
+  end
+
+  def closed?
+    redis.hget("#{redis_key}:state", 'closed') == '1'
+  end
+
+  def push(part)
+    @push_script ||= <<~SCRIPT
+      local key, part, ttl = ARGV[1], ARGV[2], tonumber(ARGV[3])
+
+      if redis.call('sadd', key .. ':parts', part) == 1 then
+        redis.call('hincrby', key .. ':state', 'total', 1)
+      end
+
+      redis.call('expire', key .. ':parts', ttl)
+      redis.call('expire', key .. ':state', ttl)
+    SCRIPT
+
+    redis.eval(@push_script, argv: [redis_script_key, part.to_s, ttl])
+  end
+
+  def redis_key
+    @redis_key ||= "distributed_jobs:#{token}"
+  end
+
+  def redis_script_key
+    return "#{redis.namespace}:#{redis_key}" if redis.respond_to?(:namespace)
+
+    redis_key
+  end
+end

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification
+name: distributed_job
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Benjamin Vetter
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2021-10-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Keep track of distributed jobs spanning multiple workers using redis
+email:
+- benjamin.vetter@wlw.de
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/test.yml"
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- CHANGELOG.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- distributed_job.gemspec
+- docker-compose.yml
+- lib/distributed_job.rb
+- lib/distributed_job/version.rb
+homepage: https://github.com/mrkamel/distributed_job
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/mrkamel/distributed_job
+  source_code_uri: https://github.com/mrkamel/distributed_job
+  changelog_uri: https://github.com/mrkamel/distributed_job/blob/master/CHANGELOG.md
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.5.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Keep track of distributed jobs using redis
+test_files: []