background_job 0.0.1.rc1

data/lib/background_job/middleware/unique_job/faktory.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# Server midleware for Faktory
+#
+# @see https://github.com/contribsys/faktory_worker_ruby/wiki/Middleware
+module BackgroundJob
+  module Middleware
+    class UniqueJob
+      module Faktory
+        def self.bootstrap
+          if defined?(::Faktory)
+            ::Faktory.configure_worker do |config|
+              config.worker_middleware do |chain|
+                chain.add Worker
+              end
+            end
+          end
+        end
+
+        # Worker middleware runs around the execution of a job
+        class Worker
+          def call(_jobinst, payload)
+            if payload.is_a?(Hash) && (unique_lock = unique_job_lock(payload))
+              unique_lock.unlock
+            end
+            yield
+          end
+
+          protected
+
+          def unique_job_lock(payload)
+            return unless payload['uniq'].is_a?(Hash)
+
+            unique_job = ::BackgroundJob::UniqueJob.coerce(payload['uniq'])
+            unique_job&.lock
+          end
+        end
+      end
+    end
+  end
+end

data/lib/background_job/middleware/unique_job/sidekiq.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+
+# Provides the Sidekiq middleware that make the unique job control work
+#
+# @see https://github.com/contribsys/faktory_worker_ruby/wiki/Middleware
+module BackgroundJob
+  module Middleware
+    class UniqueJob
+      module Sidekiq
+        def self.bootstrap
+          if defined?(::Sidekiq)
+            ::Sidekiq.configure_worker do |config|
+              config.worker_middleware do |chain|
+                chain.add Worker
+              end
+            end
+          end
+        end
+
+        # Worker middleware runs around the execution of a job
+        class Worker
+          # @param jobinst [Object] the worker/job instance
+          # @param payload [Hash] the full job payload
+          #   * @see https://github.com/mperham/sidekiq/wiki/Job-Format
+          # @param queue [String] the name of the queue the job was pulled from
+          # @yield the next middleware in the chain or worker `perform` method
+          # @return [Void]
+          def call(_jobinst, payload, _queue)
+            if payload.is_a?(Hash) && (unique_lock = unique_job_lock(payload))
+              unique_lock.unlock
+            end
+            yield
+          end
+
+          protected
+
+          def unique_job_lock(job)
+            return unless job['uniq'].is_a?(Hash)
+
+            unique_job = ::BackgroundJob::UniqueJob.coerce(job['uniq'])
+            unique_job&.lock
+          end
+        end
+      end
+    end
+  end
+end

data/lib/background_job/middleware/unique_job.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'background_job/lock'
+require 'background_job/lock_digest'
+
+module BackgroundJob
+  module Middleware
+    # This middleware uses an external redis queue to control duplications. The locking key
+    # is composed of job class and its arguments. Before enqueue new jobs it will check if have a "lock" active.
+    # The TTL of lock is 1 week as default. TTL is important to ensure locks won't last forever.
+    class UniqueJob
+      def self.bootstrap(service:)
+        services = Dir[File.expand_path('../unique_job/*.rb', __FILE__)].map { |f| File.basename(f, '.rb').to_sym }
+        unless services.include?(service)
+          msg = "UniqueJob is not supported for the `%<service>p' service. Supported options are: %<services>s."
+          raise BackgroundJob::Error, format(msg, service: service.to_sym, services: services.map { |s| "`:#{s}'" }.join(', '))
+        end
+        if (require("background_job/middleware/unique_job/#{service}"))
+          class_name = service.to_s.split('_').collect!{ |w| w.capitalize }.join
+          BackgroundJob::Middleware::UniqueJob.const_get(class_name).bootstrap
+        end
+
+        service_config = BackgroundJob.config_for(service)
+        service_config.unique_job_active = true
+        service_config.middleware.add(UniqueJob)
+      end
+
+      def call(job, service)
+        if BackgroundJob.config_for(service).unique_job_active? &&
+            (uniq_lock = unique_job_lock(job: job, service: service))
+          return false if uniq_lock.locked? # Don't push job to server
+
+          # Add unique job information to the job payload
+          job.unique_job.lock = uniq_lock
+          job.payload['uniq'] = job.unique_job.to_hash
+
+          uniq_lock.lock
+        end
+
+        yield
+      end
+
+      protected
+
+      def unique_job_lock(job:, service:)
+        return unless job.unique_job?
+
+        digest = LockDigest.new(
+          *[service || job.options[:service], job.options[:queue]].compact,
+          across: job.unique_job.across,
+        )
+        Lock.new(
+          digest: digest.to_s,
+          lock_id: unique_job_lock_id(job),
+          ttl: job.unique_job.ttl,
+        )
+      end
+
+      def unique_job_lock_id(job)
+        identifier_data = [job.job_class, job.payload.fetch('args'.freeze, [])]
+        Digest::SHA256.hexdigest(
+          MultiJson.dump(identifier_data, mode: :compat),
+        )
+      end
+    end
+  end
+end

data/lib/background_job/mixin/faktory.rb

@@ -0,0 +1,56 @@
+# frizen_string_literal: true
+
+require_relative './shared_interface'
+
+module BackgroundJob
+  module Mixin
+    module Faktory
+      DEFAULT = {
+        queue: "default",
+        retry: 25
+      }.freeze
+
+      def self.background_job_options(job_class_name, strict_check: false)
+        BackgroundJob.config.faktory.validate_strict_job!(job_class_name) if strict_check
+        options = {}
+        BackgroundJob.config.faktory.jobs[job_class_name]&.each do |key, value|
+          options[key] = value
+        end
+        if defined?(::Faktory)
+          ::Faktory.default_job_options.each do |key, value|
+            options[key.to_sym] ||= value
+          end
+        end
+        DEFAULT.each do |key, value|
+          options[key] ||= value
+        end
+        options
+      end
+
+      class Builder < Module
+        def initialize(**options)
+          @runtime_mod = Module.new do
+            define_method(:background_job_user_options) { options }
+          end
+        end
+
+        def extended(base)
+          base.include(::Faktory::Job) if defined?(::Faktory)
+          base.extend BackgroundJob::Mixin::SharedInterface
+          base.extend ClassMethods
+          base.extend @runtime_mod
+        end
+
+        module ClassMethods
+          def background_job_service
+            :faktory
+          end
+
+          def background_job_default_options
+            BackgroundJob::Mixin::Faktory.background_job_options(name)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/background_job/mixin/shared_interface.rb

@@ -0,0 +1,49 @@
+# frizen_string_literal: true
+
+module BackgroundJob
+  module Mixin
+    module SharedInterface
+      def perform_async(*args)
+        build_job.with_args(*args).push
+      end
+
+      def perform_in(interval, *args)
+        build_job.with_args(*args).at(interval).push
+      end
+      alias_method :perform_at, :perform_in
+
+      # This method should be overridden in the including class
+      # @return [Symbol]
+      # @see BackgroundJob::Mixin::Sidekiq::ClassMethods
+      # @see BackgroundJob::Mixin::Faktory::ClassMethods
+      #
+      # @abstract
+      def background_job_service
+        raise NotImplementedError
+      end
+
+      # This method should be overridden in the including class
+      # @return [Hash]
+      # @see BackgroundJob::Mixin::Sidekiq::ClassMethods
+      # @see BackgroundJob::Mixin::Faktory::ClassMethods
+      #
+      # @abstract
+      def background_job_default_options
+        raise NotImplementedError
+      end
+
+      # This method will be defined as a singleton method when the including class is extended
+      # @return [Hash] The hash of options to be passed to the background job
+      # @see BackgroundJob.mixin to see how this method is defined
+      def background_job_user_options
+        raise NotImplementedError
+      end
+
+      protected
+
+      def build_job
+        BackgroundJob.send(background_job_service, name, **background_job_default_options, **background_job_user_options)
+      end
+    end
+  end
+end

data/lib/background_job/mixin/sidekiq.rb

@@ -0,0 +1,61 @@
+# frizen_string_literal: true
+
+require_relative './shared_interface'
+
+module BackgroundJob
+  module Mixin
+    module Sidekiq
+      DEFAULT = {
+        queue: "default",
+        retry: true
+      }.freeze
+
+      def self.background_job_options(job_class_name, strict_check: false)
+        BackgroundJob.config.sidekiq.validate_strict_job!(job_class_name) if strict_check
+        options = {}
+        BackgroundJob.config.sidekiq.jobs[job_class_name]&.each do |key, value|
+          options[key] = value
+        end
+        if defined?(::Sidekiq) && ::Sidekiq.respond_to?(:default_job_options)
+          ::Sidekiq.default_job_options.each do |key, value|
+            options[key.to_sym] ||= value
+          end
+        end
+        if defined?(::Sidekiq) && ::Sidekiq.respond_to?(:default_worker_options)
+          ::Sidekiq.default_worker_options.each do |key, value|
+            options[key.to_sym] ||= value
+          end
+        end
+        DEFAULT.each do |key, value|
+          options[key] ||= value
+        end
+        options
+      end
+
+      class Builder < Module
+        def initialize(**options)
+          @runtime_mod = Module.new do
+            define_method(:background_job_user_options) { options }
+          end
+        end
+
+        def extended(base)
+          base.include(::Sidekiq::Worker) if defined?(::Sidekiq)
+          base.extend BackgroundJob::Mixin::SharedInterface
+          base.extend ClassMethods
+          base.extend @runtime_mod
+        end
+
+        module ClassMethods
+          def background_job_service
+            :sidekiq
+          end
+
+          def background_job_default_options
+            BackgroundJob::Mixin::Sidekiq.background_job_options(name)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/background_job/mixin.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module BackgroundJob
+  module Mixin
+  end
+end

data/lib/background_job/redis_pool.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module BackgroundJob
+  class RedisPool
+    extend Forwardable
+    def_delegator :@connection, :with
+
+    module ConnectionPoolLike
+      def with
+        yield self
+      end
+    end
+
+    def initialize(connection)
+      if connection.respond_to?(:with)
+        @connection = connection
+      elsif connection.is_a?(::Redis)
+        @connection = connection
+        @connection.extend(ConnectionPoolLike)
+      else
+        @connection = connection ? ::Redis.new(connection) : ::Redis.new
+        @connection.extend(ConnectionPoolLike)
+      end
+    end
+  end
+end

data/lib/background_job/testing.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module BackgroundJob
+  class Testing
+    class << self
+      def enable!
+        Thread.current[:background_job_testing] = true
+      end
+
+      def disable!
+        Thread.current[:background_job_testing] = false
+      end
+
+      def enabled?
+        Thread.current[:background_job_testing] == true
+      end
+
+      def disabled?
+        !enabled?
+      end
+    end
+  end
+end
+
+BackgroundJob::Testing.disable!
+
+module BackgroundJob::Jobs
+  class << self
+    def jobs
+      @jobs ||= []
+    end
+
+    def push(job)
+      jobs.push(job)
+    end
+
+    def clear
+      jobs.clear
+    end
+
+    def size
+      jobs.size
+    end
+
+    def jobs_for(service: nil, class_name: nil)
+      filtered = jobs
+      if service
+        filtered = filtered.select do |job|
+          job.class.name.split("::").last.downcase == service.to_s
+        end
+      end
+      if class_name
+        filtered = filtered.select do |job|
+          job.job_class == class_name
+        end
+      end
+      filtered
+    end
+  end
+end
+
+module BackgroundJob::JobsInterceptorAdapter
+  def push
+    return super unless BackgroundJob::Testing.enabled?
+
+    normalize_before_push!
+    BackgroundJob::Jobs.push(self)
+  end
+end
+
+BackgroundJob::SERVICES.each do |service_name, class_name|
+  require_relative "./jobs/#{service_name}"
+
+  klass = Object.const_get("BackgroundJob::Jobs::#{class_name}")
+  klass.prepend(BackgroundJob::JobsInterceptorAdapter)
+end

data/lib/background_job/unique_job.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require_relative './lock'
+
+module BackgroundJob
+  class UniqueJob
+    VALID_OPTIONS = {
+      across: %i[queue systemwide],
+      timeout: 604800, # 1 week
+      unlock_policy: %i[success start],
+    }.freeze
+
+    attr_reader :across, :timeout, :unlock_policy, :lock
+
+    # @options [Hash] Unique definitions
+    # @option [Symbol] :across Valid options are :queue and :systemwide. If jobs should not to be duplicated on
+    #   current queue or the entire system
+    # @option [Integer] :timeout Amount of times in seconds. Timeout decides how long to wait for acquiring the lock.
+    #   A default timeout is defined to 1 week so unique locks won't last forever.
+    # @option [Symbol] :unlock_policy Control when the unique lock is removed. The default value is `success`.
+    #   The job will not unlock until it executes successfully, it will remain locked even if it raises an error and
+    #   goes into the retry queue. The alternative value is `start` the job will unlock right before it starts executing
+    def initialize(across: :queue, timeout: nil, unlock_policy: :success, lock: nil)
+      unless VALID_OPTIONS[:across].include?(across.to_sym)
+        raise Error, format('Invalid `across: %<given>p` option. Only %<expected>p are allowed.',
+          given: across,
+          expected: VALID_OPTIONS[:across])
+      end
+      unless VALID_OPTIONS[:unlock_policy].include?(unlock_policy.to_sym)
+        raise Error, format('Invalid `unlock_policy: %<given>p` option. Only %<expected>p are allowed.',
+          given: unlock_policy,
+          expected: VALID_OPTIONS[:unlock_policy])
+      end
+      timeout = VALID_OPTIONS[:timeout] if timeout.to_i <= 0
+
+      @across = across.to_sym
+      @timeout = timeout.to_i
+      @unlock_policy = unlock_policy.to_sym
+      @lock = lock if lock.is_a?(BackgroundJob::Lock)
+    end
+
+    def self.coerce(value)
+      return unless value.is_a?(Hash)
+
+      new(
+        across: (value['across'] || value[:across] || :queue).to_sym,
+        timeout: (value['timeout'] || value[:timeout] || nil),
+        unlock_policy: (value['unlock_policy'] || value[:unlock_policy] || :success).to_sym,
+      ).tap do |instance|
+        instance.lock = value['lock'] || value[:lock]
+      end
+    end
+
+    def ttl
+      Time.now.to_f + timeout
+    end
+
+    def to_hash
+      {
+        'across' => (across.to_s if across),
+        'timeout' => timeout,
+        'unlock_policy' => (unlock_policy.to_s if unlock_policy),
+      }.tap do |hash|
+        hash['lock'] = lock.to_hash if lock
+      end
+    end
+
+    def eql?(other)
+      return false unless other.is_a?(self.class)
+
+      [across, timeout, unlock_policy] == [other.across, other.timeout, other.unlock_policy]
+    end
+    alias == eql?
+
+    def lock=(value)
+      @lock = case value
+      when BackgroundJob::Lock then value
+      when Hash then BackgroundJob::Lock.coerce(value)
+      else
+        nil
+      end
+    end
+  end
+end

data/lib/background_job/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module BackgroundJob
+  VERSION = '0.0.1.rc1'
+end

data/lib/background_job.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require 'yaml'
+require 'time'
+require 'securerandom'
+require 'multi_json'
+
+require_relative 'background_job/version'
+require_relative 'background_job/errors'
+require_relative "background_job/redis_pool"
+require_relative 'background_job/mixin'
+require_relative 'background_job/jobs'
+require_relative 'background_job/configuration'
+# require_relative 'background_job/adapters/adapter'
+# require_relative 'background_job/adapters/sidekiq'
+# require_relative 'background_job/adapters/faktory'
+
+# This is a central point of our background job enqueuing system.
+# Example:
+#
+# Standard job.
+#   BackgroundJob.sidekiq('UserWorker', queue: 'default')
+#     .with_args(1)
+#     .push
+# Schedule the time when a job will be executed.
+#   BackgroundJob.sidekiq('UserWorker')
+#     .with_args(1)
+#     .at(timestamp)
+#     .push
+#   BackgroundJob.sidekiq('UserWorker')
+#     .with_args(1)
+#     .in(10.minutes)
+#     .push
+#
+# Unique jobs.
+#   BackgroundJob.sidekiq('UserWorker', uniq: { across: :queue, timeout: 1.minute, unlock_policy: :start })
+#     .with_args(1)
+#     .push
+module BackgroundJob
+  SERVICES = {
+    sidekiq: 'Sidekiq',
+    faktory: 'Faktory',
+  }.freeze
+
+  SERVICES.each do |id, name|
+    define_singleton_method(id) do |job_name, **options|
+      Jobs.const_get(name).new(job_name, **options)
+    end
+  end
+
+  def self.mixin(service, **options)
+    service = service.to_sym
+    unless SERVICES.key?(service)
+      raise Error, "Service `#{service}' is not supported. Supported services are: #{SERVICES.keys.join(', ')}"
+    end
+    require_relative "background_job/mixin/#{service}"
+    require_relative "background_job/jobs/#{service}"
+
+    module_name = service.to_s.split(/_/i).collect!{ |w| w.capitalize }.join
+    mod = Mixin.const_get(module_name)
+    mod::Builder.new(**options)
+  end
+
+  def self.jid
+    SecureRandom.hex(12)
+  end
+
+  def self.config
+    @config ||= Configuration.new
+  end
+
+  def self.configure
+    return unless block_given?
+
+    yield(config)
+  end
+
+  def self.config_for(service)
+    service = service.to_sym
+    unless SERVICES.key?(service)
+      raise Error, "Service `#{service}' is not supported. Supported services are: #{SERVICES.keys.join(', ')}"
+    end
+    service_config = config.send(service)
+    yield(service_config) if block_given?
+    service_config
+  end
+end