background_job 0.0.1.rc1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 7073c14e1fd364a96f5f4ddc812c5d67921905307af97f85bc3d8a0173304093
+  data.tar.gz: 95f18748a479ffa52bf39ac94830080564a95223193c8655c59a36083b6eebc7
+SHA512:
+  metadata.gz: aa43df351a913191022dc34d5773ec51ab80bfbdf575cad58c1d33845150fa75a443f3e394e00b46da3283f7e0e69a7ed86b4637ef6e2e960ff797021172758f
+  data.tar.gz: c78f2257f401f3d9585f210007cb314da943aaa971dd396cccee703e142d1c949e1c037a8fcbb20bb3e8a623b7db0e36b6feb4fd25f1b42338669913465dfe4b

data/.github/workflows/specs.yml

@@ -0,0 +1,34 @@
+name: Specs
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    services:
+      redis:
+        image: redis
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 2.7
+        bundler-cache: true
+    - name: Install dependencies
+      run: bundle install
+    - name: Run tests
+      run: bundle exec rspec
+      env:
+        REDIS_URL: redis://localhost:6379

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.env
+/.rspec_status
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.gem

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/.tool-versions

@@ -0,0 +1 @@
+ruby 2.7.8

data/CHANGELOG.md

@@ -0,0 +1,10 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## 0.0.1 - 2024-08-01
+The first release of the gem
+* Added: Initial implementation with support to Sidekiq and Faktory backends
+* UniqueJob middleware to avoid duplicated jobs

data/Gemfile

@@ -0,0 +1,12 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in background_job.gemspec
+gemspec
+
+gem 'rake', '~> 12.0'
+gem 'pry'
+gem 'awesome_print'
+gem 'dotenv'
+gem 'rspec'
+gem 'timecop'
+gem 'faktory_worker_ruby', require: false

data/Gemfile.lock

@@ -0,0 +1,58 @@
+PATH
+  remote: .
+  specs:
+    background_job (0.0.1.rc1)
+      connection_pool
+      multi_json
+      redis
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    awesome_print (1.9.2)
+    coderay (1.1.3)
+    connection_pool (2.4.1)
+    diff-lcs (1.5.1)
+    dotenv (2.8.1)
+    faktory_worker_ruby (2.0.0)
+      connection_pool (~> 2.2, >= 2.2.2)
+    method_source (1.1.0)
+    multi_json (1.15.0)
+    pry (0.14.2)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rake (12.3.3)
+    redis (5.2.0)
+      redis-client (>= 0.22.0)
+    redis-client (0.22.2)
+      connection_pool
+    rspec (3.13.0)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.0)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.1)
+    timecop (0.9.10)
+
+PLATFORMS
+  x86_64-linux
+
+DEPENDENCIES
+  awesome_print
+  background_job!
+  dotenv
+  faktory_worker_ruby
+  pry
+  rake (~> 12.0)
+  rspec
+  timecop
+
+BUNDLED WITH
+   2.3.22

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Marcos G. Zimmermann
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,247 @@
+# BackgroundJob
+
+The purpose of this gem is to provide a simple way to enqueue background jobs in different background job clientes like Sidekiq, Faktory. (More to come). You can push jobs to the clients without actually have the client installed in your system. This is useful for distributed system or data pipelines where you want to use jobs to communicate between different services.
+
+If you are using a monolithic application, you should use the client directly. Or in a [Ruby on Rails](https://github.com/rails/rails) application consider using [Active Jobs](https://github.com/rails/rails/tree/master/activejob). ActiveJobs integrates with a wider range of services and builtin support.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'background_job'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install background_job
+
+## Usage
+
+Right now the gem supports Sidekiq and Faktory. Client configurations will be covered in the next section.
+
+You can build and push jobs using a simple DSL. The DSL is the same for all clients, but the configurations may vary.
+
+```ruby
+# Enqueue the 'Accounts::ConfirmationEmailWorker' job
+#   with 'User', 1 arguments to the sidekiq "high_priority_mailing" queue
+#   to be executed as soon as possible.
+BackgroundJob.sidekiq("Accounts::ConfirmationEmailWorker", queue: 'high_priority_mailing')
+  .with_args("User", 1)
+  .push
+```
+
+```ruby
+# Schedule the 'Accounts::ConfirmationEmailWorker' job
+#   with 'User', 1 arguments to the sidekiq "high_priority_mailing" queue
+#   to be executed in one hour.
+BackgroundJob.sidekiq("Accounts::ConfirmationEmailWorker", queue: 'high_priority_mailing')
+  .with_args("User", 1)
+  .in(1.hour)
+  .push
+```
+
+```ruby
+# Enqueue the 'Accounts::ConfirmationEmailWorker' job
+#   with 'User', 1 arguments to the faktory "mailing" queue
+#   to be executed as soon as possible.
+BackgroundJob.faktory("Accounts::ConfirmationEmailWorker", queue: 'mailing')
+  .with_args("User", 1)
+  .push
+```
+
+DSL Methods:
+* `with_args(*args)`: Pass the arguments to the job
+* `in(time)`: Schedule the job to be executed in the future. The time can be a `Time`, `DateTime`, `ActiveSupport::Duration` or a number of seconds.
+* `with_job_jid(jid)`: Set the job JID. This is useful to track the job status or cancel it. Optional, it will be generated automatically.
+* `created_at(time)`: Set the job creation time. Optional, it will be generated automatically.
+* `enqueue_at(time)`: Set the job enqueue time.  Optional, it will be generated automatically.
+* `push`: Push the job to the client.
+
+### Sidekiq
+
+Sidekiq configurations are under a `BackgroundJob.config.sidekiq` config. You must set the `redis` connection where Sidekiq is running.
+
+```ruby
+BackgroundJob.configure do |conf|
+  conf.sidekiq.redis = { url: 'redis://localhost:6379/0' } # You can pass the Redis instance directly as well
+  # Or using a connection pool
+  conf.sidekiq.redis = ConnectionPool.new(size: 5, timeout: 5) do
+    Redis.new(url: 'redis://localhost:6379/0')
+  end
+  # config.sidekiq.namespace = 'sidekiq' # Optional, default is nil in favor of number of databases of Redis
+end
+```
+
+From an YAML file
+```yaml
+redis:
+  url: 'redis://localhost:6379/0'
+jobs:
+  UsesJob:
+    queue: 'default'
+    retry: 3
+  BatchImportJob:
+    queue: 'import'
+    retry: 0
+```
+
+```ruby
+BackgroundJob.config_for(:sidekiq) do |config|
+  config.config_path = 'config/background_job.yml'
+end
+```
+
+#### Client DSL for Sidekiq to enqueue jobs
+
+If your are using Sidekiq in a service that does not have a jobs/worker defined, you may want to specify the list of jobs and their configurations like `queue` and `retry` in the `BackgroundJob.config.sidekiq.jobs` configuration.
+
+```ruby
+BackgroundJob.configure do |conf|
+  conf.sidekiq.redis = { url: 'redis://localhost:6379/0' }
+  conf.sidekiq.jobs = {
+    "UsesJob" => { queue: 'default', retry: 3 },
+    "BatchImportJob" => { queue: 'import', retry: 0 }
+  }
+  # Default is true, it means that will raise an error if the job is
+  # not specified in the jobs configuration
+  #
+  # conf.sidekiq.strict = false
+end
+```
+
+#### Backend Mixins for Sidekiq
+
+This are optional, you can keep your backend implementation in your application according to the client documentation. But if you want to get benefit of global configurations, or custom middlewares, you can use the provided mixins.
+
+```diff
+class Accounts::ConfirmationEmailWorker
++  extend BackgroundJob.mixin(:sidekiq, queue: :mailing)
+-  include Sidekiq::Worker
+-  sidekiq_options queue: :mailing
+
+  def perform(resource_type, resource_id)
+    # Do something
+  end
+end
+```
+
+Now when you call `Accounts::ConfirmationEmailWorker.perform_async` or `Accounts::ConfirmationEmailWorker.perform_in` it will use this gem to push jobs to the backend server with the configurations defined in the mixin and the global configurations.
+
+### Faktory
+
+Faktory configurations are under a `BackgroundJob.config.faktory` config. This is in an erly stage. It means that the [faktory_worker_ruby](https://github.com/contribsys/faktory_worker_ruby) gem must be installed in your system.
+
+```ruby
+require 'faktory'
+BackgroundJob.configure do |conf|
+  conf.faktory # Just call it to enable the Faktory client
+  # Default is true, it means that will raise an error if the job is
+  # not specified in the jobs configuration
+  #
+  # conf.faktory.strict = false
+end
+```
+
+#### Client DSL for Faktory to enqueue jobs
+
+If your are using Faktory in a service that does not have a jobs/worker defined, you may want to specify the list of jobs and their configurations like `queue` and `retry` in the `BackgroundJob.config.faktory.jobs` configuration.
+
+```ruby
+BackgroundJob.configure do |conf|
+  conf.faktory
+  conf.faktory.jobs = {
+    "UsesJob" => { queue: 'default', retry: 3 },
+    "BatchImportJob" => { queue: 'import', retry: 0 }
+  }
+end
+```
+
+#### Backend Mixins for Faktory
+
+This are optional, you can keep your backend implementation in your application according to the client documentation. But if you want to get benefit of global configurations, or custom middlewares, you can use the provided mixins.
+
+```diff
+class Accounts::ConfirmationEmailWorker
++  extend BackgroundJob.mixin(:faktory, queue: :mailing)
+-  include Faktory::Job
+
+  def perform(resource_type, resource_id)
+    # Do something
+  end
+end
+```
+
+Now when you call `Accounts::ConfirmationEmailWorker.perform_async` or `Accounts::ConfirmationEmailWorker.perform_in` it will use this gem to push jobs to the backend server with the configurations defined in the mixin and the global configurations.
+
+## Middleware
+
+You can define middlewares to run before and after the job execution. This is useful to add custom logging, error handling, or any other custom behavior. The current version implements a UniqueJob middleware that prevents the job to be enqueued if it is already in the queue. More details in the next section.
+
+This is an example of a minimal middleware, note the method must return the result or the job will not push the server
+
+```ruby
+class MyMiddleware
+  def call(job, conn_pool)
+    puts "Before push"
+    result = yield
+    puts "After push"
+    result
+  end
+end
+
+BackgroundJob.config_for(:sidekiq) do |config|
+  config.middleware do |chain|
+    chain.add MyMiddleware
+  end
+end
+```
+
+### Unique Jobs
+
+This library provides one experimental technology to avoid enqueue duplicated jobs. Pro versions of sidekiq and faktory provides this functionality. But this project exposes a mechanism to make this control using `Redis`. It's not loaded by default. You can load this function by require and initialize the `UniqueJob` middleware according to the service(`:faktory` or `:sidekiq`).
+
+```ruby
+require 'background_job/middleware/unique_job'
+BackgroundJob::Middleware::UniqueJob::bootstrap(service: :sidekiq)
+# Or
+BackgroundJob::Middleware::UniqueJob::bootstrap(service: :faktory)
+
+# Make sure to add a redis connection to the configuration
+BackgroundJob.configure do |conf|
+  conf.redis = { url: 'redis://localhost:6379/0' }
+  # Or using a connection pool
+  conf.redis = ConnectionPool.new(size: 5, timeout: 5) do
+    Redis.new(url: 'redis://localhost:6379/0')
+  end
+end
+```
+
+After that just define the `:uniq` settings by worker
+
+```ruby
+BackgroundJob.sidekiq('Mailing::SignUpWorker', uniq: { across: :queue, timeout: 120 })
+  .with_args('User', 1)
+  .push
+```
+
+You can globally disable/enable this function with the `BackgroundJob.config.sidekiq.unique_job_active = <true|false>`
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/marcosgz/background_job.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task default: :spec

data/background_job.gemspec

@@ -0,0 +1,39 @@
+require_relative 'lib/background_job/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'background_job'
+  spec.version       = BackgroundJob::VERSION
+  spec.authors       = ['Marcos G. Zimmermann']
+  spec.email         = ['mgzmaster@gmail.com']
+
+  spec.summary       = <<~SUMMARY
+    A generic swappable background job client.
+  SUMMARY
+  spec.description   = <<~DESCRIPTION
+    A generic swappable background job client that allows you to push jobs to different background job services.
+  DESCRIPTION
+
+  spec.homepage      = 'https://github.com/marcosgz/background_job'
+  spec.license       = 'MIT'
+  spec.required_ruby_version = Gem::Requirement.new('>= 2.7.0')
+
+  raise 'RubyGems 2.0 or newer is required to protect against public gem pushes.' unless spec.respond_to?(:metadata)
+  spec.metadata['homepage_uri']      = spec.homepage
+  spec.metadata['bug_tracker_uri']   = 'https://github.com/marcosgz/background_job/issues'
+  spec.metadata['documentation_uri'] = 'https://github.com/marcosgz/background_job'
+  spec.metadata['source_code_uri']   = 'https://github.com/marcosgz/background_job'
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'redis', '>= 0.0.0'
+  spec.add_dependency 'connection_pool', '>= 0.0.0'
+  spec.add_dependency 'multi_json', '>= 0.0.0'
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'dotenv/load'
+require 'pry'
+require 'awesome_print'
+require 'background_job'
+
+BackgroundJob.configure do |config|
+  config.faktory.strict = false
+  config.sidekiq.strict = false
+end
+
+Pry.start

data/bin/setup

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install

data/docker-compose.yml

@@ -0,0 +1,6 @@
+services:
+  redis:
+    image: redis
+    command: redis-server
+    ports:
+      - 6379:6379

data/lib/background-job.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative './background_job'

data/lib/background_job/configuration/base.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+class BackgroundJob::Configuration::Base
+  class << self
+    private
+
+    def attribute_accessor(field, validator: nil, normalizer: nil, default: nil, write: true, read: true)
+      normalizer ||= :"normalize_#{field}"
+      validator ||= :"validate_#{field}"
+
+      define_method(field) do
+        unless instance_variable_defined?(:"@#{field}")
+          fallback = config_from_yaml[field.to_s] || default
+          return if fallback.nil?
+
+          send(:"#{field}=", fallback.respond_to?(:call) ? fallback.call : fallback)
+        end
+        instance_variable_get(:"@#{field}")
+      end if read
+
+      define_method(:"#{field}=") do |value|
+        value = send(normalizer, field, value) if respond_to?(normalizer, true)
+        send(validator, field, value) if respond_to?(validator, true)
+
+        instance_variable_set(:"@#{field}", value)
+      end if write
+    end
+  end
+
+  # Path to the YAML file with configs
+  attr_reader :config_path
+
+  # A Hash with all jobs definitions. The job class name must be the main hash key
+  # Example:
+  #   "FaktoryIndexWorker":
+  #     retry: false
+  #     queue: "indexing"
+  #     adapter: "faktory"
+  #   "FaktoryBatchIndexWorker":
+  #     retry: 5
+  #     queue: "batch_index"
+  #     adapter: "faktory"
+  attribute_accessor :jobs, default: {}
+
+  # Global disable the unique_job_active
+  attribute_accessor :unique_job_active, default: false
+  alias unique_job_active? unique_job_active
+
+  # Does not validate if it's when set to false
+  attribute_accessor :strict, default: true
+  alias strict? strict
+
+  def validate_strict_job!(class_name)
+    class_name = class_name.to_s
+    if strict? && !jobs.key?(class_name)
+      raise BackgroundJob::NotDefinedJobError.new(class_name)
+    end
+    true
+  end
+
+  def config_path=(value)
+    @config_from_yaml = nil
+    @config_path = value
+  end
+
+  def middleware
+    @middleware ||= BackgroundJob::Configuration::MiddlewareChain.new
+    yield @middleware if block_given?
+    @middleware
+  end
+
+  protected
+
+  def normalize_jobs(_, value)
+    return unless value.is_a?(Hash)
+
+    hash = {}
+    value.each do |class_name, opts|
+      hash[class_name.to_s] = deep_symbolize_keys(opts)
+    end
+    hash
+  end
+
+  def deep_symbolize_keys(hash)
+    return hash unless hash.is_a?(Hash)
+
+    hash.each_with_object({}) do |(key, value), memo|
+      memo[key.to_sym] = deep_symbolize_keys(value)
+    end
+  end
+
+  def config_from_yaml
+    return {} unless config_path
+
+    @config_from_yaml ||= if File.exist?(config_path)
+      YAML.load_file(config_path)
+    else
+      raise BackgroundJob::InvalidConfigError, "The file #{config_path} does not exist."
+    end
+    @config_from_yaml || {}
+  end
+end

data/lib/background_job/configuration/faktory.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module BackgroundJob
+  class Configuration::Faktory < Configuration::Base
+  end
+end