solid-jobs 0.1.3 → 0.4.0

data/lib/solid_jobs/fetch.rb

@@ -1,310 +0,0 @@
-# frozen_string_literal: true
-
-require "json"
-require "securerandom"
-require "digest/sha1"
-
-module SolidJobs
-  class Fetch
-    TIMEOUT = 0.25
-    RESERVE = <<~LUA.freeze
-      local payload = redis.call("rpop", KEYS[1])
-      if not payload then
-        return nil
-      end
-
-      redis.call("lpush", KEYS[2], payload)
-      local job = cjson.decode(payload)
-      local attempt = redis.call("hincrby", KEYS[4], job["jid"], 1)
-      local metadata = {
-        job_id = job["jid"],
-        reservation_id = ARGV[1],
-        process_id = ARGV[2],
-        worker_id = ARGV[3],
-        queue = job["queue"],
-        reserved_at = tonumber(ARGV[4]),
-        attempt = attempt
-      }
-      redis.call("hset", KEYS[3], ARGV[3], cjson.encode(metadata))
-      return {payload, attempt}
-    LUA
-    REGISTER = <<~LUA.freeze
-      local attempt = redis.call("hincrby", KEYS[2], ARGV[1], 1)
-      local metadata = cjson.decode(ARGV[3])
-      metadata["attempt"] = attempt
-      redis.call("hset", KEYS[1], ARGV[2], cjson.encode(metadata))
-      return attempt
-    LUA
-    ACK = <<~LUA.freeze
-      local metadata = redis.call("hget", KEYS[2], ARGV[2])
-      if not metadata then
-        return 0
-      end
-      if cjson.decode(metadata)["reservation_id"] ~= ARGV[3] then
-        return 0
-      end
-      local removed = redis.call("lrem", KEYS[1], 1, ARGV[1])
-      redis.call("hdel", KEYS[2], ARGV[2])
-      if removed > 0 then
-        redis.call("hdel", KEYS[3], ARGV[4])
-      end
-      return removed
-    LUA
-    REQUEUE = <<~LUA.freeze
-      local metadata = redis.call("hget", KEYS[2], ARGV[2])
-      if not metadata then
-        return 0
-      end
-      if cjson.decode(metadata)["reservation_id"] ~= ARGV[3] then
-        return 0
-      end
-      local removed = redis.call("lrem", KEYS[1], 1, ARGV[1])
-      if removed > 0 then
-        redis.call("rpush", KEYS[3], ARGV[1])
-      end
-      redis.call("hdel", KEYS[2], ARGV[2])
-      return removed
-    LUA
-    RESERVE_SHA = Digest::SHA1.hexdigest(RESERVE).freeze
-    REGISTER_SHA = Digest::SHA1.hexdigest(REGISTER).freeze
-    ACK_SHA = Digest::SHA1.hexdigest(ACK).freeze
-    REQUEUE_SHA = Digest::SHA1.hexdigest(REQUEUE).freeze
-
-    UnitOfWork = Struct.new(
-      :queue,
-      :payload,
-      :job,
-      :redis_pool,
-      :reserved_key,
-      :metadata_key,
-      :metadata_field,
-      :reservation_id,
-      :attempt,
-      keyword_init: true,
-    ) do
-      def acknowledge
-        if reserved_key
-          removed = begin
-            redis_pool.call(
-              "EVALSHA", Fetch::ACK_SHA, 3,
-              reserved_key, metadata_key, "solid-jobs:attempts",
-              payload, metadata_field, reservation_id, job.fetch("jid"),
-            )
-          rescue SolidRedis::CommandError => error
-            raise unless error.message.include?("NOSCRIPT")
-
-            redis_pool.call(
-              "EVAL", Fetch::ACK, 3,
-              reserved_key, metadata_key, "solid-jobs:attempts",
-              payload, metadata_field, reservation_id, job.fetch("jid"),
-            )
-          end
-          return Integer(removed) == 1
-        end
-        true
-      end
-
-      def requeue
-        if reserved_key
-          destination = "queue:#{queue}"
-          removed = begin
-            redis_pool.call(
-              "EVALSHA", Fetch::REQUEUE_SHA, 3,
-              reserved_key, metadata_key, destination,
-              payload, metadata_field, reservation_id,
-            )
-          rescue SolidRedis::CommandError => error
-            raise unless error.message.include?("NOSCRIPT")
-
-            redis_pool.call(
-              "EVAL", Fetch::REQUEUE, 3,
-              reserved_key, metadata_key, destination,
-              payload, metadata_field, reservation_id,
-            )
-          end
-          return Integer(removed) == 1
-        else
-          redis_pool.call("RPUSH", "queue:#{queue}", payload)
-        end
-        true
-      end
-    end
-
-    def initialize(config, identity: nil, processor_id: nil)
-      @redis_pool = config.redis_pool
-      @queue_mode = config.queue_mode
-      @reliable = config.reliable_fetch
-      @queues = config.queue_entries.flat_map do |name, weight|
-        Array.new(weight, "queue:#{name}")
-      end.freeze
-      @reserved_key = "#{identity}:reserved:#{processor_id}" if @reliable && identity
-      @metadata_key = "#{identity}:reservations" if @reserved_key
-      @identity = identity
-      @processor_id = processor_id
-      @paused = []
-      @paused_refresh_at = 0.0
-      @queue_index = 0
-      @check_reserved = !!@reserved_key
-    end
-
-    def retrieve
-      if @check_reserved
-        @check_reserved = false
-        if (payload = @redis_pool.call("LINDEX", @reserved_key, -1))
-          return existing_work(payload)
-        end
-      end
-
-      refresh_paused
-      queues = @queues.reject { |queue| @paused.include?(queue.delete_prefix("queue:")) }
-      return sleep(0.05) if queues.empty?
-
-      if @reserved_key
-        reserve_reliable(queues)
-      else
-        queues = queues.shuffle if @queue_mode == :random
-        result = @redis_pool.blocking_call(TIMEOUT, "BRPOP", *queues, TIMEOUT)
-        result && work(result.fetch(1), result.fetch(0).delete_prefix("queue:"))
-      end
-    end
-
-    def connection_failed!
-      @check_reserved = !!@reserved_key
-    end
-
-    private
-
-    def work(payload, queue = nil, reservation_id: nil, attempt: nil)
-      job = JSON.parse(payload)
-      queue ||= job.fetch("queue")
-      reservation_id, attempt = register(job, queue) unless reservation_id
-      UnitOfWork.new(
-        queue: queue,
-        payload: payload,
-        job: job,
-        redis_pool: @redis_pool,
-        reserved_key: @reserved_key,
-        metadata_key: @metadata_key,
-        metadata_field: @processor_id.to_s,
-        reservation_id: reservation_id,
-        attempt: attempt,
-      )
-    end
-
-    def existing_work(payload)
-      raw = @redis_pool.call("HGET", @metadata_key, @processor_id)
-      if raw
-        metadata = JSON.parse(raw)
-        job = JSON.parse(payload)
-        if metadata["job_id"] == job["jid"]
-          return UnitOfWork.new(
-            queue: job.fetch("queue"),
-            payload: payload,
-            job: job,
-            redis_pool: @redis_pool,
-            reserved_key: @reserved_key,
-            metadata_key: @metadata_key,
-            metadata_field: @processor_id.to_s,
-            reservation_id: metadata.fetch("reservation_id"),
-            attempt: Integer(metadata.fetch("attempt")),
-          )
-        end
-      end
-      work(payload)
-    end
-
-    def register(job, queue)
-      return [nil, nil] unless @reserved_key
-
-      reservation_id = SecureRandom.hex(12)
-      metadata = JSON.generate(
-        "job_id" => job["jid"],
-        "reservation_id" => reservation_id,
-        "process_id" => @identity,
-        "worker_id" => @processor_id,
-        "queue" => queue,
-        "reserved_at" => Time.now.to_f,
-      )
-      attempt = begin
-        @redis_pool.call(
-          "EVALSHA", REGISTER_SHA, 2,
-          @metadata_key, "solid-jobs:attempts",
-          job["jid"], @processor_id, metadata,
-        )
-      rescue SolidRedis::CommandError => error
-        raise unless error.message.include?("NOSCRIPT")
-
-        @redis_pool.call(
-          "EVAL", REGISTER, 2,
-          @metadata_key, "solid-jobs:attempts",
-          job["jid"], @processor_id, metadata,
-        )
-      end
-      [reservation_id, Integer(attempt)]
-    end
-
-    def next_queue(queues)
-      return queues.sample if @queue_mode == :random
-
-      queue = queues[@queue_index % queues.length]
-      @queue_index += 1
-      queue
-    end
-
-    def reserve_reliable(queues)
-      if @queue_mode == :strict
-        queues.each do |source|
-          unit = reserve_now(source)
-          return unit if unit
-        end
-        source = queues.first
-      else
-        source = next_queue(queues)
-        unit = reserve_now(source)
-        return unit if unit
-      end
-      payload = @redis_pool.blocking_call(
-        TIMEOUT,
-        "BLMOVE", source, @reserved_key, "RIGHT", "LEFT", TIMEOUT,
-      )
-      payload && work(payload, source.delete_prefix("queue:"))
-    end
-
-    def reserve_now(source)
-      reservation_id = SecureRandom.hex(12)
-      reserved_at = Time.now.to_f
-      result = begin
-        @redis_pool.call(
-          "EVALSHA", RESERVE_SHA, 4,
-          source, @reserved_key, @metadata_key, "solid-jobs:attempts",
-          reservation_id, @identity, @processor_id, reserved_at,
-        )
-      rescue SolidRedis::CommandError => error
-        raise unless error.message.include?("NOSCRIPT")
-
-        @redis_pool.call(
-          "EVAL", RESERVE, 4,
-          source, @reserved_key, @metadata_key, "solid-jobs:attempts",
-          reservation_id, @identity, @processor_id, reserved_at,
-        )
-      end
-      return unless result
-
-      payload, attempt = result
-      work(
-        payload,
-        source.delete_prefix("queue:"),
-        reservation_id: reservation_id,
-        attempt: Integer(attempt),
-      )
-    end
-
-    def refresh_paused
-      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-      return if now < @paused_refresh_at
-
-      @paused = Array(@redis_pool.call("SMEMBERS", "paused"))
-      @paused_refresh_at = now + 5
-    end
-  end
-end

data/lib/solid_jobs/job.rb

@@ -1,156 +0,0 @@
-# frozen_string_literal: true
-
-module SolidJobs
-  module Job
-    def self.included(base)
-      base.extend(ClassMethods)
-      base.include(InstanceMethods)
-      base.solid_jobs_options
-    end
-
-    module InstanceMethods
-      attr_accessor :jid
-      attr_writer :_context
-
-      def _context
-        @_context ||= {}
-      end
-
-      def logger
-        SolidJobs.config.logger
-      end
-
-      def interrupted?
-        _context[:interrupted]&.call || false
-      end
-    end
-
-    module ClassMethods
-      def solid_jobs_options(options = nil)
-        if options
-          merged = get_solid_jobs_options.merge(Utilities.stringify_keys(options))
-          @solid_jobs_options = Utilities.shareable_copy(merged)
-        else
-          @solid_jobs_options ||= Utilities.shareable_copy(SolidJobs.config.default_job_options)
-        end
-      end
-      alias_method :sidekiq_options, :solid_jobs_options
-
-      def get_solid_jobs_options
-        solid_jobs_options
-      end
-      alias_method :get_sidekiq_options, :get_solid_jobs_options
-
-      def solid_jobs_retry_in(&block)
-        define_singleton_method(:solid_jobs_retry_in_callback, &block) if block
-        method(:solid_jobs_retry_in_callback) if respond_to?(:solid_jobs_retry_in_callback)
-      end
-      alias_method :sidekiq_retry_in, :solid_jobs_retry_in
-
-      def solid_jobs_retries_exhausted(&block)
-        define_singleton_method(:solid_jobs_retries_exhausted_callback, &block) if block
-        method(:solid_jobs_retries_exhausted_callback) if respond_to?(:solid_jobs_retries_exhausted_callback)
-      end
-      alias_method :sidekiq_retries_exhausted, :solid_jobs_retries_exhausted
-
-      def perform_async(*args)
-        client_push("class" => self, "args" => args)
-      end
-
-      def perform_bulk(args, batch_size: Client::DEFAULT_BATCH_SIZE)
-        Client.new.push_bulk(
-          get_solid_jobs_options.merge(
-            "class" => self,
-            "args" => args,
-            "batch_size" => batch_size,
-          ),
-        )
-      end
-
-      def perform_in(interval, *args)
-        timestamp = interval.is_a?(Time) ? interval.to_f : Float(interval)
-        timestamp += Time.now.to_f if timestamp < 1_000_000_000
-        item = {"class" => self, "args" => args}
-        item["at"] = timestamp if timestamp > Time.now.to_f
-        client_push(item)
-      end
-
-      def perform_at(timestamp, *args)
-        perform_in(timestamp, *args)
-      end
-
-      def perform_inline(*args)
-        payload = get_solid_jobs_options.merge(
-          "class" => self,
-          "args" => args,
-        )
-        SolidJobs::Testing.execute_inline(payload)
-      end
-      alias_method :perform_sync, :perform_inline
-
-      def set(options)
-        Setter.new(self, options)
-      end
-
-      def client_push(item)
-        Client.new.push(get_solid_jobs_options.merge(item))
-      end
-
-      def jobs
-        Testing.jobs_for(self)
-      end
-
-      def clear
-        jobs.clear
-      end
-
-      def perform_one
-        payload = jobs.shift
-        raise EmptyQueueError, "No jobs for #{name}" unless payload
-
-        Testing.execute_inline(payload)
-      end
-    end
-
-    class Setter
-      def initialize(job_class, options)
-        @job_class = job_class
-        @options = Utilities.stringify_keys(options)
-      end
-
-      def perform_async(*args)
-        if @options.delete("sync")
-          @job_class.perform_inline(*args)
-        elsif (wait = @options.delete("wait"))
-          perform_in(wait, *args)
-        elsif (wait_until = @options.delete("wait_until"))
-          perform_at(wait_until, *args)
-        else
-          push(args)
-        end
-      end
-
-      def perform_in(interval, *args)
-        timestamp = interval.is_a?(Time) ? interval.to_f : Float(interval)
-        timestamp += Time.now.to_f if timestamp < 1_000_000_000
-        push(args, "at" => timestamp)
-      end
-
-      def perform_at(timestamp, *args)
-        perform_in(timestamp, *args)
-      end
-
-      private
-
-      def push(args, extra = {})
-        Client.new.push(@job_class.get_solid_jobs_options.merge(@options).merge(
-          extra,
-          "class" => @job_class,
-          "args" => args,
-        ))
-      end
-    end
-  end
-
-  class EmptyQueueError < Error; end
-end

data/lib/solid_jobs/middleware/chain.rb

@@ -1,104 +0,0 @@
-# frozen_string_literal: true
-
-module SolidJobs
-  module Middleware
-    class Chain
-      Entry = Struct.new(:klass, :args, keyword_init: true)
-
-      include Enumerable
-
-      def initialize(copy = nil)
-        @entries = copy ? copy.map { |entry| Entry.new(klass: entry.klass, args: entry.args.dup) } : []
-      end
-
-      def each(&block)
-        @entries.each(&block)
-      end
-
-      def add(klass, *args)
-        remove(klass)
-        @entries << Entry.new(klass: klass, args: args)
-        self
-      end
-
-      def prepend(klass, *args)
-        remove(klass)
-        @entries.unshift(Entry.new(klass: klass, args: args))
-        self
-      end
-
-      def remove(klass)
-        @entries.delete_if { |entry| entry.klass == klass }
-        self
-      end
-
-      def insert_before(oldklass, newklass, *args)
-        insert_relative(oldklass, newklass, args, 0)
-      end
-
-      def insert_after(oldklass, newklass, *args)
-        insert_relative(oldklass, newklass, args, 1)
-      end
-
-      def exists?(klass)
-        @entries.any? { |entry| entry.klass == klass }
-      end
-      alias_method :include?, :exists?
-
-      def clear
-        @entries.clear
-        self
-      end
-
-      def empty?
-        @entries.empty?
-      end
-
-      def retrieve
-        @entries.map { |entry| entry.klass.new(*entry.args) }
-      end
-
-      def snapshot
-        Utilities.shareable_copy(
-          @entries.map { |entry| [entry.klass, entry.args] },
-        )
-      end
-
-      def restore(snapshot)
-        clear
-        snapshot.each { |klass, args| add(klass, *args) }
-        self
-      end
-
-      def invoke(*arguments, &final)
-        return yield if empty?
-
-        stack = retrieve
-        traverse(stack, 0, arguments, &final)
-      end
-
-      def dup
-        self.class.new(@entries)
-      end
-
-      private
-
-      def traverse(stack, index, arguments, &final)
-        return final.call if index >= stack.length
-
-        stack[index].call(*arguments) do
-          traverse(stack, index + 1, arguments, &final)
-        end
-      end
-
-      def insert_relative(oldklass, newklass, args, offset)
-        remove(newklass)
-        index = @entries.index { |entry| entry.klass == oldklass }
-        raise ArgumentError, "#{oldklass} is not in the middleware chain" unless index
-
-        @entries.insert(index + offset, Entry.new(klass: newklass, args: args))
-        self
-      end
-    end
-  end
-end

data/lib/solid_jobs/retry_service.rb

@@ -1,100 +0,0 @@
-# frozen_string_literal: true
-
-require "json"
-
-module SolidJobs
-  class RetryService < Service::Base
-    DEFAULT_MAX_RETRIES = 25
-
-    def call
-      payload = @payload.dup
-      now = Time.now
-      now_ms = Utilities.realtime_milliseconds
-      retry_count = payload.key?("retry_count") ? Integer(payload["retry_count"]) + 1 : 0
-      payload["error_message"] = @error.message.to_s
-      payload["error_class"] = @error.class.name
-      payload["failed_at"] ||= now_ms
-      payload["retried_at"] = now_ms if retry_count.positive?
-      payload["retry_count"] = retry_count
-      payload["queue"] = payload["retry_queue"] if payload["retry_queue"]
-
-      setting = payload["retry"]
-      if setting && retry_allowed?(payload, setting, retry_count, now)
-        decision = retry_decision(payload, retry_count)
-        return nil if decision == :discard
-        return move_to_dead(payload, now.to_f) if decision == :kill
-
-        retry_at = now.to_f + (decision || retry_delay(retry_count))
-        @config.redis_pool.call("ZADD", "retry", retry_at, JSON.generate(payload))
-        append_message("retry")
-        retry_at
-      else
-        invoke_exhausted(payload)
-        move_to_dead(payload, now.to_f)
-        append_message("dead")
-        nil
-      end
-    end
-
-    private
-
-    def retry_allowed?(payload, setting, retry_count, now)
-      if payload["retry_for"]
-        first_failure = Float(payload["failed_at"]) / 1_000
-        now.to_f - first_failure < Float(payload["retry_for"])
-      else
-        maximum = setting == true ? DEFAULT_MAX_RETRIES : Integer(setting)
-        retry_count < maximum
-      end
-    end
-
-    def retry_delay(retry_count)
-      ceiling = [
-        Float(@config.retry_base_delay) * (2**retry_count),
-        Float(@config.retry_max_delay),
-      ].min
-      (ceiling / 2) + (rand * ceiling / 2)
-    end
-
-    def retry_decision(payload, retry_count)
-      job_class = resolve_job_class(payload)
-      return unless job_class.respond_to?(:solid_jobs_retry_in_callback)
-
-      value = job_class.solid_jobs_retry_in_callback(retry_count, @error, payload)
-      return value if value.is_a?(Numeric) || %i[discard kill].include?(value)
-
-      nil
-    end
-
-    def invoke_exhausted(payload)
-      job_class = resolve_job_class(payload)
-      return unless job_class.respond_to?(:solid_jobs_retries_exhausted_callback)
-
-      job_class.solid_jobs_retries_exhausted_callback(payload, @error)
-    end
-
-    def resolve_job_class(payload)
-      Utilities.constantize(payload.fetch("class"))
-    rescue NameError
-      nil
-    end
-
-    def move_to_dead(payload, timestamp)
-      @config.redis_pool.pipelined do |pipeline|
-        pipeline.call("ZADD", "dead", timestamp, JSON.generate(payload))
-        pipeline.call(
-          "ZREMRANGEBYSCORE",
-          "dead",
-          "-inf",
-          timestamp - @config.dead_timeout,
-        )
-        pipeline.call(
-          "ZREMRANGEBYRANK",
-          "dead",
-          0,
-          -(@config.dead_max_jobs + 1),
-        )
-      end
-    end
-  end
-end

data/lib/solid_jobs/worker.rb

@@ -1,35 +0,0 @@
-# frozen_string_literal: true
-
-require "json"
-
-module SolidJobs
-  class Worker
-    attr_reader :config
-
-    def initialize(redis_config:, config: nil)
-      @config = config || Config.new(redis: redis_config)
-    end
-
-    def perform(payload)
-      payload = JSON.parse(payload) if payload.is_a?(String)
-      raise InvalidJobError, "Job payload must be a Hash" unless payload.is_a?(Hash)
-
-      class_name = payload["class"] || payload["job_class"]
-      raise InvalidJobError, "Job must include a non-empty class" if class_name.nil? || class_name.empty?
-      raise InvalidJobError, "Job args must be an Array" unless payload["args"].is_a?(Array)
-      raise InvalidJobError, "Job queue must be present" if payload["queue"].nil? || payload["queue"].empty?
-
-      job_class = Utilities.constantize(class_name)
-      job = job_class.new
-      job.jid = payload["jid"] if job.respond_to?(:jid=)
-
-      if config.server_middleware.empty?
-        job.perform(*payload.fetch("args"))
-      else
-        config.server_middleware.invoke(job, payload, payload.fetch("queue")) do
-          job.perform(*payload.fetch("args"))
-        end
-      end
-    end
-  end
-end