multi-background-job 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 102602d0d8739d98e0baa6a10f9581bb91c4e486ce03ff1d0b3a1448c21756c3
+  data.tar.gz: 8079bfc60fc76222ae8ec886dd8dd076ee17d6903a218afe79c1b6d9cc9c95ac
+SHA512:
+  metadata.gz: 5a62442c88fa86368af365c8df9299390ff059994f91bdacfeee9bf8492a495e77951ecb305566d838b50f14d385ed94ee24979d31fb08e2a851ad56578b9f3c
+  data.tar.gz: fb87cf4bcf49808a43766e7c6340d9f24d9eeb4ba1ca15b600a922a80ba89fbab8e62dbc4c48052bb12aed137e7557de40cec5888aabf1480cebc6da23d595ef

data/.github/workflows/specs.yml

@@ -0,0 +1,35 @@
+name: Specs
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    services:
+      redis:
+        image: redis
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+    # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
+    # change this to (see https://github.com/ruby/setup-ruby#versioning):
+    # uses: ruby/setup-ruby@v1
+      uses: ruby/setup-ruby@ec106b438a1ff6ff109590de34ddc62c540232e0
+      with:
+        ruby-version: 2.6
+    - name: Install dependencies
+      run: bundle install
+    - name: Run tests
+      run: bundle exec rspec
+      env:
+        REDIS_URL: redis://localhost:6379

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.env
+/.rspec_status
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/.tool-versions

@@ -0,0 +1 @@
+ruby 2.6.3

data/Gemfile

@@ -0,0 +1,12 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in multi-background-job.gemspec
+gemspec
+
+gem 'rake', '~> 12.0'
+gem 'pry'
+gem 'awesome_print'
+gem 'dotenv'
+gem 'rspec'
+gem 'timecop'
+gem 'faktory_worker_ruby', require: false

data/Gemfile.lock

@@ -0,0 +1,55 @@
+PATH
+  remote: .
+  specs:
+    multi-background-job (0.1.0)
+      connection_pool
+      multi_json
+      redis
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    awesome_print (1.8.0)
+    coderay (1.1.3)
+    connection_pool (2.2.3)
+    diff-lcs (1.4.4)
+    dotenv (2.7.6)
+    faktory_worker_ruby (1.0.0)
+      connection_pool (~> 2.2, >= 2.2.2)
+    method_source (1.0.0)
+    multi_json (1.15.0)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rake (12.3.3)
+    redis (4.2.2)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.3)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+    timecop (0.9.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  awesome_print
+  dotenv
+  faktory_worker_ruby
+  multi-background-job!
+  pry
+  rake (~> 12.0)
+  rspec
+  timecop
+
+BUNDLED WITH
+   2.1.4

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Marcos G. Zimmermann
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,137 @@
+# MultiBackgroundJob
+
+This library provices an centralized interface to push jobs to a variety of queuing backends. Thus allowing to send jobs to multiple external services. If you are running a [Ruby on Rails](https://github.com/rails/rails) application consider using [Active Jobs](https://github.com/rails/rails/tree/master/activejob). ActiveJobs integrates with a wider range of services and builtin support.
+
+Supported Services:
+* Faktory (Faktory::Client is used as depency to push jobs)
+* Sidekiq (Sidekiq gem is not a depenency. We are using redis connection to push jobs)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'multi-background-job'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install multi-background-job
+
+## Usage
+
+
+This gem should work with default configurations if you have `REDIS_URL` and/or `FAKTORY_URL` environment variables correctly defined. But you can use the `MultiBackgroundJob#configure` method to customize default settings.
+
+```ruby
+MultiBackgroundJob.configure do |config|
+  config.config_path = 'config/background_jobs.yml' # You can use an YAML file. (Default to nil)
+  config.redis_pool_size = 10                       # Connection Pool size for redis. (Default to 5)
+  config.redis_pool_timeout = 10                    # Connection Pool timeouf for redis. (Default to 5)
+  config.redis_namespace = 'app-name'               # The prefix of redis storage keys. (Default to multi-bg)
+  config.redis_config = { path: "/tmp/redis.sock" } # List of configurations to be passed along to the Redis.new. (Default to {})
+  config.workers = {                                # List of workers and its configurations. (Default to {})
+    'Accounts::ConfirmationEmailWorker' => { retry: 5, queue: 'mailing' },
+  }
+  config.strict = false                             # Only allow to push jobs to known workers. See `config.workers`. (Default to true)
+end
+```
+
+### Client Setup
+
+You can use the DSL to start building worker and push to background job services.
+
+```ruby
+# Enqueue the 'Accounts::ConfirmationEmailWorker' job with 'User', 1 arguments
+# to the sidekiq "other_mailing" queue
+MultiBackgroundJob['Accounts::ConfirmationEmailWorker', queue: 'other_mailing' ]
+  .with_args('User', 1)
+  .push(to: :sidekiq)
+
+# Schedule the 'Accounts::ConfirmationEmailWorker' job with 'User', 1 arguments
+# to the sidekiq "other_mailing" queue to be executed in one hour.
+MultiBackgroundJob['Accounts::ConfirmationEmailWorker', queue: 'other_mailing' ]
+  .with_args('User', 1)
+  .in(1.hour)
+  .push(to: :sidekiq)
+
+# Enqueue the 'Accounts::ConfirmationEmailWorker' job with 'User', 2 arguments
+# to the faktory "mailing" queue(Using :queu from global config.workers definition)
+MultiBackgroundJob['Accounts::ConfirmationEmailWorker']
+  .with_args('User', 2)
+  .push(to: :faktory)
+```
+
+MultiBackgroundJob is not required as a dependency of backend servers if you are only use it to push jobs(** Except when you are using middleware like **UniqueJobs Middleware** of next section)
+
+### Server Setup
+
+This is only a necessary step in the case of using the **UniqueJobs Middleware** of next section.
+
+Example of sidekiq worker:
+
+```diff
+class Accounts::ConfirmationEmailWorker
+  + extend MultiBackgroundJob.for(:sidekiq, queue: :mailing)
+  - include Sidekiq::Worker
+  - sidekiq_options queue: :mailing
+
+  def perform(resource_type, resource_id); end;
+end
+```
+
+Example of faktory worker:
+
+```diff
+class Accounts::ConfirmationEmailWorker
+  + extend MultiBackgroundJob.for(:sidekiq, queue: :mailing)
+  - include Faktory::Job
+  - faktory_options queue: :mailing
+
+  def perform(resource_type, resource_id); end;
+end
+```
+
+Now when you call `Accounts::ConfirmationEmailWorker.perform_async` or `Accounts::ConfirmationEmailWorker.perform_in` it will use this gem to push jobs to the backend server.
+
+Note that settings defined througth the worker class have greater weight then the ones defined from global `MultiBackgroundJob.config.workers`. And the `MultiBackgroundJob.config.workers` have greater weight then both `Sidekiq.default_worker_options` or `Faktory.default_job_options`.
+
+### Unique Jobs
+
+This library provides one experimental technology to avoid enqueue duplicated jobs. Pro versions of sidekiq and faktory provides this functionality. But this project exposes a mechanism to make this control using `Redis`. It's not required by default. You can load this function by require and initialize the `UniqueJob` middleware according to the service(`:faktory` or `:sidekiq`).
+
+```ruby
+require 'multi_background_job/middleware/unique_job'
+MultiBackgroundJob::Middleware::UniqueJob::bootstrap(service: :sidekiq)
+# Or
+MultiBackgroundJob::Middleware::UniqueJob::bootstrap(service: :faktory)
+```
+
+After that just define the `:uniq` settings by worker
+
+```ruby
+MultiBackgroundJob['Mailing::SignUpWorker', uniq: { across: :queue, timeout: 120 }]
+  .with_args('User', 1)
+  .push(to: :sidekiq)
+```
+
+You can globally disable/enable this function with the `MultiBackgroundJob.config.unique_job_active = <true|false>`
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/marcosgz/multi-background-job.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'dotenv/load'
+require 'pry'
+require 'awesome_print'
+require 'multi_background_job'
+
+MultiBackgroundJob.configure do |config|
+  config.strict = false
+end
+
+Pry.start

data/bin/setup

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install

data/lib/multi-background-job.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative './multi_background_job'

data/lib/multi_background_job.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require 'yaml'
+require 'time'
+require 'securerandom'
+require 'multi_json'
+
+require_relative './multi_background_job/version'
+require_relative 'multi_background_job/errors'
+require_relative 'multi_background_job/config'
+require_relative 'multi_background_job/worker'
+require_relative 'multi_background_job/adapters/adapter'
+require_relative 'multi_background_job/adapters/sidekiq'
+require_relative 'multi_background_job/adapters/faktory'
+
+# This is a central point of our background job queue system.
+# We have more external services like API and Lme that queue jobs for pipeline processing.
+# So that way all services can share the same codebase and avoid incompatibility issues
+#
+# Example:
+#
+# Standard job.
+#   MultiBackgroundJob['UserWorker', queue: 'default']
+#     .with_args(1)
+#     .push(to: :sidekiq)
+#
+# Schedule the time when a job will be executed.
+#   MultiBackgroundJob['UserWorker']
+#     .with_args(1)
+#     .at(timestamp)
+#     .push(to: :sidekiq)
+#   MultiBackgroundJob['UserWorker']
+#     .with_args(1)
+#     .in(10.minutes)
+#     .push(to: :sidekiq)
+#
+# Unique jobs.
+#   MultiBackgroundJob['UserWorker', uniq: { across: :queue, timeout: 1.minute, unlock_policy: :start }]
+#     .with_args(1)
+#     .push(to: :sidekiq)
+module MultiBackgroundJob
+  SERVICES = {
+    sidekiq: Adapters::Sidekiq,
+    faktory: Adapters::Faktory,
+  }
+
+  # @param worker_class [String] The worker class name
+  # @param options [Hash] Options that will be passed along to the worker instance
+  # @return [MultiBackgroundJob::Worker] An instance of worker
+  def self.[](worker_class, **options)
+    Worker.new(worker_class, **config.worker_options(worker_class).merge(options))
+  end
+
+  def self.jid
+    SecureRandom.hex(12)
+  end
+
+  def self.for(service, **options)
+    require_relative "multi_background_job/workers/#{service}"
+    service = service.to_sym
+    worker_options = options.merge(service: service)
+    module_name = service.to_s.split(/_/i).collect!{ |w| w.capitalize }.join
+    mod = Workers.const_get(module_name)
+    mod.module_eval do
+      define_method(:bg_worker_options) do
+        worker_options
+      end
+    end
+    mod
+  end
+
+  def self.config
+    @config ||= Config.new
+  end
+
+  def self.configure(&block)
+    return unless block_given?
+
+    config.instance_eval(&block)
+    @redis_pool = nil
+    config
+  end
+
+  def self.redis_pool
+    @redis_pool ||= ConnectionPool.new(config.redis_pool) do
+      Redis.new(config.redis_config)
+    end
+  end
+end

data/lib/multi_background_job/adapters/adapter.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module MultiBackgroundJob
+  module Adapters
+    class Adapter
+      # Push the worker job to the service
+      # @param _worker [MultiBackgroundJob::Worker] An instance of background worker
+      # @abstract Child classes should override this method
+      def self.push(_worker)
+        raise NotImplemented
+      end
+
+      # Coerces the raw payload into an instance of Worker
+      # @param payload [Object] the object that should be coerced to a Worker
+      # @options options [Hash] list of options that will be passed along to the Worker instance
+      # @return [MultiBackgroundJob::Worker] and instance of MultiBackgroundJob::Worker
+      # @abstract Child classes should override this method
+      def self.coerce_to_worker(payload, **options)
+        raise NotImplemented
+      end
+    end
+  end
+end

data/lib/multi_background_job/adapters/faktory.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module MultiBackgroundJob
+  module Adapters
+    # This is a Faktory adapter that converts MultiBackgroundJob::Worker object into a faktory readable format
+    # and then push the jobs into the service.
+    class Faktory < Adapter
+      attr_reader :worker, :queue
+
+      def initialize(worker)
+        @worker = worker
+        @queue = worker.options.fetch(:queue, 'default')
+
+        @payload = worker.payload.merge(
+          'jobtype' => worker.worker_class,
+          'queue'   => @queue,
+          'retry'   => parse_retry(worker.options[:retry]),
+        )
+        @payload['created_at'] ||= Time.now.to_f
+      end
+
+      # Coerces the raw payload into an instance of Worker
+      # @param payload [Hash] The job as json from redis
+      # @options options [Hash] list of options that will be passed along to the Worker instance
+      # @return [MultiBackgroundJob::Worker] and instance of MultiBackgroundJob::Worker
+      def self.coerce_to_worker(payload, **options)
+        raise(Error, 'invalid payload') unless payload.is_a?(Hash)
+        raise(Error, 'invalid payload') unless payload['jobtype'].is_a?(String)
+
+        options[:retry] ||= payload['retry'] if payload.key?('retry')
+        options[:queue] ||= payload['queue'] if payload.key?('queue')
+
+        MultiBackgroundJob[payload['jobtype'], **options].tap do |worker|
+          worker.with_args(*Array(payload['args'])) if payload.key?('args')
+          worker.with_job_jid(payload['jid']) if payload.key?('jid')
+          worker.created_at(payload['created_at']) if payload.key?('created_at')
+          worker.enqueued_at(payload['enqueued_at']) if payload.key?('enqueued_at')
+          worker.at(payload['at']) if payload.key?('at')
+          worker.unique(payload['uniq']) if payload.key?('uniq')
+        end
+      end
+
+      # Initializes adapter and push job into the faktory service
+      #
+      # @param worker [MultiBackgroundJob::Worker] An instance of MultiBackgroundJob::Worker
+      # @return [Hash] Job payload
+      # @see push method for more details
+      def self.push(worker)
+        new(worker).push
+      end
+
+      # Push job to Faktory
+      #   * If job has the 'at' key. Then schedule it
+      #   * Otherwise enqueue for immediate execution
+      #
+      # @raise [MultiBackgroundJob::Error] raise and error when faktory dependency is not loaded
+      # @return [Hash] Payload that was sent to server
+      def push
+        unless Object.const_defined?(:Faktory)
+          raise MultiBackgroundJob::Error, <<~ERR
+          Faktory client for ruby is not loaded. You must install and require https://github.com/contribsys/faktory_worker_ruby.
+          ERR
+        end
+        @payload['enqueued_at'] ||= Time.now.to_f
+        {'created_at' => false, 'enqueued_at' => false, 'at' => true}.each do |field, past_remove|
+          # Optimization to enqueue something now that is scheduled to go out now or in the past
+          if (time = @payload.delete(field)) &&
+              (!past_remove || (past_remove && time > Time.now.to_f))
+            @payload[field] = parse_time(time)
+          end
+        end
+
+        pool = Thread.current[:faktory_via_pool] || ::Faktory.server_pool
+        ::Faktory.client_middleware.invoke(@payload, pool) do
+          pool.with do |c|
+            c.push(@payload)
+          end
+        end
+        @payload
+      end
+
+      protected
+
+      # Convert worker retry value acording to the Go struct datatype.
+      #
+      # * 25 is the default.
+      # * 0 means the job is completely ephemeral. No matter if it fails or succeeds, it will be discarded.
+      # * -1 means the job will go straight to the Dead set if it fails, no retries.
+      def parse_retry(value)
+        case value
+        when Numeric then value.to_i
+        when false then -1
+        else
+          25
+        end
+      end
+
+      def parse_time(value)
+        case value
+        when Numeric then Time.at(value).to_datetime.rfc3339(9)
+        when Time then value.to_datetime.rfc3339(9)
+        when DateTime then value.rfc3339(9)
+        end
+      end
+
+      def to_json(value)
+        MultiJson.dump(value, mode: :compat)
+      end
+    end
+  end
+end