multi-background-job 0.1.0

data/lib/multi_background_job/adapters/sidekiq.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module MultiBackgroundJob
+  module Adapters
+    # This is a Sidekiq adapter that converts MultiBackgroundJob::Worker object into a sidekiq readable format
+    # and then push the jobs into the service.
+    class Sidekiq < Adapter
+      attr_reader :worker, :queue
+
+      def initialize(worker)
+        @worker = worker
+        @queue = worker.options.fetch(:queue, 'default')
+
+        @payload = worker.payload.merge(
+          'class' => worker.worker_class,
+          'retry' => worker.options.fetch(:retry, true),
+          'queue' => @queue,
+        )
+        @payload['created_at'] ||= Time.now.to_f
+      end
+
+      # Coerces the raw payload into an instance of Worker
+      # @param payload [Hash] The job as json from redis
+      # @options options [Hash] list of options that will be passed along to the Worker instance
+      # @return [MultiBackgroundJob::Worker] and instance of MultiBackgroundJob::Worker
+      def self.coerce_to_worker(payload, **options)
+        raise(Error, 'invalid payload') unless payload.is_a?(Hash)
+        raise(Error, 'invalid payload') unless payload['class'].is_a?(String)
+
+        options[:retry] ||= payload['retry'] if payload.key?('retry')
+        options[:queue] ||= payload['queue'] if payload.key?('queue')
+
+        MultiBackgroundJob[payload['class'], **options].tap do |worker|
+          worker.with_args(*Array(payload['args'])) if payload.key?('args')
+          worker.with_job_jid(payload['jid']) if payload.key?('jid')
+          worker.created_at(payload['created_at']) if payload.key?('created_at')
+          worker.enqueued_at(payload['enqueued_at']) if payload.key?('enqueued_at')
+          worker.at(payload['at']) if payload.key?('at')
+          worker.unique(payload['uniq']) if payload.key?('uniq')
+        end
+      end
+
+      # Initializes adapter and push job into the sidekiq service
+      #
+      # @param worker [MultiBackgroundJob::Worker] An instance of MultiBackgroundJob::Worker
+      # @return [Hash] Job payload
+      # @see push method for more details
+      def self.push(worker)
+        new(worker).push
+      end
+
+      # Push sidekiq to the Sidekiq(Redis actually).
+      #   * If job has the 'at' key. Then schedule it
+      #   * Otherwise enqueue for immediate execution
+      #
+      # @return [Hash] Payload that was sent to redis
+      def push
+        @payload['enqueued_at'] = Time.now.to_f
+        # Optimization to enqueue something now that is scheduled to go out now or in the past
+        if (timestamp = @payload.delete('at')) && (timestamp > Time.now.to_f)
+          MultiBackgroundJob.redis_pool.with do |redis|
+            redis.zadd(scheduled_queue_name, timestamp.to_f.to_s, to_json(@payload))
+          end
+        else
+          MultiBackgroundJob.redis_pool.with do |redis|
+            redis.lpush(immediate_queue_name, to_json(@payload))
+          end
+        end
+        @payload
+      end
+
+      protected
+
+      def namespace
+        MultiBackgroundJob.config.redis_namespace
+      end
+
+      def scheduled_queue_name
+        "#{namespace}:schedule"
+      end
+
+      def immediate_queue_name
+        "#{namespace}:queue:#{queue}"
+      end
+
+      def to_json(value)
+        MultiJson.dump(value, mode: :compat)
+      end
+    end
+  end
+end

data/lib/multi_background_job/config.rb

@@ -0,0 +1,152 @@
+# frizen_string_literal: true
+
+require 'redis'
+require 'connection_pool'
+
+require_relative './middleware_chain'
+
+module MultiBackgroundJob
+  class Config
+    class << self
+      private
+
+      def attribute_accessor(field, validator: nil, normalizer: nil, default: nil)
+        normalizer ||= :"normalize_#{field}"
+        validator ||= :"validate_#{field}"
+
+        define_method(field) do
+          unless instance_variable_defined?(:"@#{field}")
+            fallback = config_from_yaml[field.to_s] || default
+            return if fallback.nil?
+
+            send(:"#{field}=", fallback.respond_to?(:call) ? fallback.call : fallback)
+          end
+          instance_variable_get(:"@#{field}")
+        end
+
+        define_method(:"#{field}=") do |value|
+          value = send(normalizer, field, value) if respond_to?(normalizer, true)
+          send(validator, field, value) if respond_to?(validator, true)
+
+          instance_variable_set(:"@#{field}", value)
+        end
+      end
+    end
+
+    # Path to the YAML file with configs
+    attr_accessor :config_path
+
+    # ConnectionPool options for redis
+    attribute_accessor :redis_pool_size, default: 5, normalizer: :normalize_to_int, validator: :validate_greater_than_zero
+    attribute_accessor :redis_pool_timeout, default: 5, normalizer: :normalize_to_int, validator: :validate_greater_than_zero
+
+    # Namespace used to manage internal data like unique job verification data.
+    attribute_accessor :redis_namespace, default: 'multi-bg'
+
+    # List of configurations to be passed along to the Redis.new
+    attribute_accessor :redis_config, default: {}
+
+    # A Hash with all workers definitions. The worker class name must be the main hash key
+    # Example:
+    #   "Accounts::ConfirmationEmailWorker":
+    #     retry: false
+    #     queue: "mailer"
+    #   "Elastic::BatchIndex":
+    #     retry: 5
+    #     queue: "elasticsearch"
+    #     adapter: "faktory"
+    attribute_accessor :workers, default: {}
+
+    # Does not validate if it's when set to false
+    attribute_accessor :strict, default: true
+    alias strict? strict
+
+    # Global disable the unique_job_active
+    attribute_accessor :unique_job_active, default: false
+    alias unique_job_active? unique_job_active
+
+    def worker_options(class_name)
+      class_name = class_name.to_s
+      if strict? && !workers.key?(class_name)
+        raise NotDefinedWorker.new(class_name)
+      end
+
+      workers.fetch(class_name, {})
+    end
+
+    def redis_pool
+      {
+        size: redis_pool_size,
+        timeout: redis_pool_timeout,
+      }
+    end
+
+    def middleware
+      @middleware ||= MiddlewareChain.new
+      yield @middleware if block_given?
+      @middleware
+    end
+
+    def config_path=(value)
+      @config_from_yaml = nil
+      @config_path = value
+    end
+
+    private
+
+    def normalize_to_int(_attribute, value)
+      value.to_i
+    end
+
+    def validate_greater_than_zero(attribute, value)
+      return if value > 0
+
+      raise InvalidConfig, format(
+        'The %<value>p for %<attr>s is not valid. It must be greater than zero',
+        value: value,
+        attr: attribute,
+      )
+    end
+
+    def normalize_redis_config(_attribute, value)
+      case value
+      when String
+        { url: value }
+      when Hash
+        value.each_with_object({}) { |(k, v), r| r[k.to_sym] = v }
+      else
+        value
+      end
+    end
+
+    def validate_redis_config(attribute, value)
+      return if value.is_a?(Hash)
+
+      raise InvalidConfig, format(
+        'The %<value>p for %<attr>i is not valid. It must be a Hash with the redis initialization options. ' +
+        'See https://github.com/redis/redis-rb for all available options',
+        value: value,
+        attr: attribute,
+      )
+    end
+
+    def normalize_workers(_, value)
+      return unless value.is_a?(Hash)
+
+      hash = {}
+      value.each do |class_name, opts|
+        hash[class_name.to_s] = MultiJson.load(MultiJson.dump(opts), symbolize_names: true)
+      end
+      hash
+    end
+
+    def config_from_yaml
+      @config_from_yaml ||= begin
+        config_path ? YAML.load_file(config_path) : {}
+      rescue Errno::ENOENT, Errno::ESRCH
+        {}
+      end
+    end
+  end
+end
+

data/lib/multi_background_job/errors.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module MultiBackgroundJob
+  class Error < StandardError
+  end
+
+  class InvalidConfig < Error
+  end
+
+  class NotDefinedWorker < Error
+    def initialize(worker_class)
+      @worker_class = worker_class
+    end
+
+    def message
+      format(
+        "The %<worker>p is not defined and the MultiBackgroundJob is configured to work on strict mode.\n" +
+        "it's highly recommended to include this worker class to the list of known workers.\n" +
+        "Example: `MultiBackgroundJob.configure { |config| config.workers = { %<worker>p => {} } }`\n" +
+        'Another option is to set config.strict = false',
+        worker: @worker_class,
+      )
+    end
+  end
+end

data/lib/multi_background_job/lock.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module MultiBackgroundJob
+  # Class Lock provides access to redis "sorted set" used to control unique jobs
+  class Lock
+    attr_reader :digest, :lock_id, :ttl
+
+    # @param :digest [String] It's the uniq string used to group similar jobs
+    # @param :lock_id [String] The uniq job id
+    # @param :ttl [Float] The timestamp related lifietime of the lock before being discarded.
+    def initialize(digest:, lock_id:, ttl:)
+      @digest = digest
+      @lock_id = lock_id
+      @ttl = ttl
+    end
+
+    # Initialize a Lock object from hash
+    #
+    # @param value [Hash] Hash with lock properties
+    # @return [MultiBackgroundJob::Lock, nil]
+    def self.coerce(value)
+      return unless value.is_a?(Hash)
+
+      digest = value[:digest] || value['digest']
+      lock_id = value[:lock_id] || value['lock_id']
+      ttl = value[:ttl] || value['ttl']
+      return if [digest, lock_id, ttl].any?(&:nil?)
+
+      new(digest: digest, lock_id: lock_id, ttl: ttl)
+    end
+
+    # Remove expired locks from redis "sorted set"
+    #
+    # @param [String] digest It's the uniq string used to group similar jobs
+    def self.flush_expired_members(digest, redis: nil)
+      return unless digest
+
+      caller = ->(redis) { redis.zremrangebyscore(digest, '-inf', "(#{now}") }
+
+      if redis
+        caller.(redis)
+      else
+        MultiBackgroundJob.redis_pool.with { |conn| caller.(conn) }
+      end
+    end
+
+    # Remove all locks from redis "sorted set"
+    #
+    # @param [String] digest It's the uniq string used to group similar jobs
+    def self.flush(digest, redis: nil)
+      return unless digest
+
+      caller = ->(conn) { conn.del(digest) }
+
+      if redis
+        caller.(redis)
+      else
+        MultiBackgroundJob.redis_pool.with { |conn| caller.(conn) }
+      end
+    end
+
+    # Number of locks
+    #
+    # @param digest [String] It's the uniq string used to group similar jobs
+    # @option [Number] from The begin of set. Default to 0
+    # @option [Number] to The end of set. Default to the timestamp of 1 week from now
+    # @return Number the amount of entries that within digest
+    def self.count(digest, from: 0, to: nil, redis: nil)
+      to ||= Time.now.to_f + MultiBackgroundJob::UniqueJob::VALID_OPTIONS[:timeout]
+      caller = ->(conn) { conn.zcount(digest, from, to) }
+
+      if redis
+        caller.(redis)
+      else
+        MultiBackgroundJob.redis_pool.with { |conn| caller.(conn) }
+      end
+    end
+
+    def to_hash
+      {
+        'ttl' => ttl,
+        'digest' => (digest.to_s if digest),
+        'lock_id' => (lock_id.to_s if lock_id),
+      }
+    end
+
+    # @return [Float] A float timestamp of current time
+    def self.now
+      Time.now.to_f
+    end
+
+    # Remove lock_id lock from redis
+    # @return [Boolean] Returns true when it's locked or false when there is no lock
+    def unlock
+      MultiBackgroundJob.redis_pool.with do |conn|
+        conn.zrem(digest, lock_id)
+      end
+    end
+
+    # Adds lock_id lock to redis
+    # @return [Boolean] Returns true when it's a fresh lock or false when lock already exists
+    def lock
+      MultiBackgroundJob.redis_pool.with do |conn|
+        conn.zadd(digest, ttl, lock_id)
+      end
+    end
+
+    # Check if the lock_id lock exist
+    # @return [Boolean] true or false when lock exist or not
+    def locked?
+      locked = false
+
+      MultiBackgroundJob.redis_pool.with do |conn|
+        timestamp = conn.zscore(digest, lock_id)
+        return false unless timestamp
+
+        locked = timestamp >= now
+        self.class.flush_expired_members(digest, redis: conn)
+      end
+
+      locked
+    end
+
+    def eql?(other)
+      return false unless other.is_a?(self.class)
+
+      [digest, lock_id, ttl] == [other.digest, other.lock_id, other.ttl]
+    end
+    alias == eql?
+
+    protected
+
+    def now
+      self.class.now
+    end
+  end
+end

data/lib/multi_background_job/lock_digest.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module MultiBackgroundJob
+  # Class Lock generates the uniq digest acording to the uniq config
+  class LockDigest
+    BASE = 'uniqueness'.freeze
+    SEPARATOR = ':'.freeze
+
+    def initialize(*keys, across:)
+      @keys = keys.map { |k| k.to_s.strip.downcase }
+      @across = across.to_sym
+    end
+
+    def to_s
+      case @across
+      when :systemwide
+        build_name(*@keys.slice(0..-2))
+      when :queue
+        build_name(*@keys)
+      else
+        raise Error, format(
+          'Could not resolve the lock digest using across %<across>p. ' +
+          'Valid options are :systemwide and :queue',
+          across: @across,
+        )
+      end
+    end
+
+    private
+
+    def build_name(*segments)
+      [namespace, BASE, *segments].compact.join(SEPARATOR)
+    end
+
+    def namespace
+      MultiBackgroundJob.config.redis_namespace
+    end
+  end
+end

data/lib/multi_background_job/middleware/unique_job.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'multi_background_job/lock'
+require 'multi_background_job/lock_digest'
+
+module MultiBackgroundJob
+  module Middleware
+    # This middleware uses an external redis queue to control duplications. The locking key
+    # is composed of worker class and its arguments. Before enqueue new jobs it will check if have a "lock" active.
+    # The TTL of lock is 1 week as default. TTL is important to ensure locks won't last forever.
+    class UniqueJob
+      def self.bootstrap(service:)
+        services = Dir[File.expand_path('../unique_job/*.rb', __FILE__)].map { |f| File.basename(f, '.rb').to_sym }
+        unless services.include?(service)
+          msg = "UniqueJob is not supported for the `%<service>p' service. Supported options are: %<services>s."
+          raise MultiBackgroundJob::Error, format(msg, service: service.to_sym, services: services.map { |s| "`:#{s}'" }.join(', '))
+        end
+        if (require("multi_background_job/middleware/unique_job/#{service}"))
+          class_name = service.to_s.split('_').collect!{ |w| w.capitalize }.join
+          MultiBackgroundJob::Middleware::UniqueJob.const_get(class_name).bootstrap
+        end
+
+        MultiBackgroundJob.configure do |config|
+          config.unique_job_active = true
+          config.middleware.add(UniqueJob)
+        end
+      end
+
+      def call(worker, service)
+        if MultiBackgroundJob.config.unique_job_active? &&
+            (uniq_lock = unique_job_lock(worker: worker, service: service))
+          return false if uniq_lock.locked? # Don't push job to server
+
+          # Add unique job information to the job payload
+          worker.unique_job.lock = uniq_lock
+          worker.payload['uniq'] = worker.unique_job.to_hash
+
+          uniq_lock.lock
+        end
+
+        yield
+      end
+
+      protected
+
+      def unique_job_lock(worker:, service:)
+        return unless worker.unique_job?
+
+        digest = LockDigest.new(
+          *[service || worker.options[:service], worker.options[:queue]].compact,
+          across: worker.unique_job.across,
+        )
+
+        Lock.new(
+          digest: digest.to_s,
+          lock_id: unique_job_lock_id(worker),
+          ttl: worker.unique_job.ttl,
+        )
+      end
+
+      def unique_job_lock_id(worker)
+        identifier_data = [worker.worker_class, worker.payload.fetch('args'.freeze, [])]
+        Digest::SHA256.hexdigest(
+          MultiJson.dump(identifier_data, mode: :compat),
+        )
+      end
+    end
+  end
+end