solid-jobs 0.1.0

data/lib/solid_jobs/client.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+require "json"
+require "securerandom"
+
+module SolidJobs
+  class Client
+    DEFAULT_BATCH_SIZE = 1_000
+    SCHEDULED_BATCH_SIZE = 100
+
+    def self.push(item)
+      new.push(item)
+    end
+
+    def self.push_bulk(item)
+      new.push_bulk(item)
+    end
+
+    def initialize(config: nil, configuration: nil)
+      selected = config || configuration || SolidJobs.config
+      @config = selected.is_a?(SolidRedis::Config) ? Config.new(redis: selected) : selected
+    end
+
+    def push(item)
+      payload = normalize_item(item)
+      payload = invoke_middleware(payload)
+      return unless payload
+
+      verify_json!(payload)
+      raw_push([payload])
+      payload["jid"]
+    end
+
+    def push_bulk(item)
+      raise InvalidJobError, "Job payload must be a Hash" unless item.is_a?(Hash)
+
+      source = Utilities.stringify_hash_keys(item)
+      args = source.delete("args")
+      raise InvalidJobError, "Job 'args' must be an Array or Enumerable" unless args.respond_to?(:each)
+
+      ats = source.delete("at")
+      if ats && !ats.is_a?(Numeric) && !ats.is_a?(Array)
+        raise InvalidJobError, "Job 'at' must be a Numeric or an Array"
+      end
+      if ats.is_a?(Array) && args.respond_to?(:size) && ats.size != args.size
+        raise InvalidJobError, "Job 'at' Array must have the same size as 'args'"
+      end
+      batch_size = Integer(source.delete("batch_size") || (ats ? SCHEDULED_BATCH_SIZE : DEFAULT_BATCH_SIZE))
+      raise ArgumentError, "batch_size must be positive" unless batch_size.positive?
+
+      explicit_jid = source.delete("jid")
+      if explicit_jid && args.respond_to?(:size) && args.size > 1
+        raise InvalidJobError, "Explicit jid is only supported for one bulk job"
+      end
+      base = normalize_item(source.merge("args" => []))
+      base.delete("jid")
+      result = []
+      args.each_slice(batch_size).with_index do |batch, batch_index|
+        payloads = []
+        batch.each_with_index do |job_args, index|
+          raise InvalidJobError, "Bulk job arguments must be Arrays" unless job_args.is_a?(Array)
+
+          payload = base.merge(
+            "args" => job_args,
+            "jid" => explicit_jid || SecureRandom.hex(12),
+          )
+          payload["at"] = ats.is_a?(Array) ? ats[batch_index * batch_size + index] : ats if ats
+          normalized = invoke_middleware(payload)
+          if normalized
+            verify_json!(normalized)
+            payloads << normalized
+            result << normalized["jid"]
+          else
+            result << nil
+          end
+        end
+        raw_push(payloads)
+      end
+      result
+    end
+
+    def raw_push(payloads)
+      return true if payloads.empty?
+
+      if defined?(Testing)
+        case Testing.mode
+        when :fake
+          payloads.each do |payload|
+            Testing.jobs_for(Utilities.constantize(payload["class"])) << payload
+          end
+          return true
+        when :inline
+          payloads.each { |payload| Testing.execute_inline(payload) }
+          return true
+        end
+      end
+
+      @config.redis_pool.pipelined do |pipeline|
+        if payloads.all? { |payload| payload["at"] }
+          payloads.each do |payload|
+            score = payload.delete("at")
+            pipeline.call("ZADD", "schedule", score, JSON.generate(payload))
+          end
+        elsif payloads.none? { |payload| payload["at"] }
+          push_immediate(pipeline, payloads)
+        else
+          scheduled, immediate = payloads.partition { |payload| payload["at"] }
+          scheduled.each do |payload|
+            score = payload.delete("at")
+            pipeline.call("ZADD", "schedule", score, JSON.generate(payload))
+          end
+          push_immediate(pipeline, immediate)
+        end
+      end
+      true
+    end
+
+    private
+
+    def normalize_item(item)
+      raise InvalidJobError, "Job payload must be a Hash" unless item.is_a?(Hash)
+
+      payload = @config.default_job_options.merge(Utilities.stringify_hash_keys(item))
+      payload["class"] ||= payload.delete("job_class")
+      payload["jid"] ||= payload.delete("job_id")
+      job_class = payload["class"]
+      args = payload["args"]
+      unless job_class.is_a?(Class) || (job_class.is_a?(String) && !job_class.empty?)
+        raise InvalidJobError, "Job must include a non-empty class"
+      end
+      raise InvalidJobError, "Job args must be an Array" unless args.is_a?(Array)
+
+      payload["class"] = job_class.name if job_class.is_a?(Class)
+      payload["queue"] = String(payload.fetch("queue"))
+      payload["jid"] ||= SecureRandom.hex(12)
+      payload["created_at"] ||= Utilities.realtime_milliseconds
+      payload
+    end
+
+    def invoke_middleware(payload)
+      job_class = payload["class"]
+      @config.client_middleware.invoke(job_class, payload, payload["queue"], @config.redis_pool) do
+        payload
+      end
+    end
+
+    def push_immediate(pipeline, payloads)
+      return if payloads.empty?
+
+      groups = if payloads.all? { |payload| payload["queue"] == payloads.first["queue"] }
+        [[payloads.first["queue"], payloads]]
+      else
+        payloads.group_by { |payload| payload["queue"] }
+      end
+      groups.each do |queue, jobs|
+        pipeline.call("SADD", "queues", queue)
+        now = Utilities.realtime_milliseconds
+        encoded = jobs.map do |payload|
+          payload["enqueued_at"] = now
+          JSON.generate(payload)
+        end
+        pipeline.call("LPUSH", "queue:#{queue}", *encoded)
+      end
+    end
+
+    def verify_json!(payload)
+      invalid = find_invalid_json_value(payload["args"])
+      return unless invalid
+
+      message = "Job arguments must contain only JSON-native values with string hash keys; found #{invalid.inspect}"
+      case @config.on_complex_arguments
+      when :raise
+        raise InvalidArgumentError, message
+      when :warn
+        @config.logger.warn(message)
+      end
+    end
+
+    def find_invalid_json_value(value)
+      case value
+      when nil, true, false, String, Integer, Float
+        nil
+      when Array
+        value.each do |nested|
+          invalid = find_invalid_json_value(nested)
+          return invalid if invalid
+        end
+        nil
+      when Hash
+        value.each do |key, nested|
+          return key unless key.is_a?(String)
+
+          invalid = find_invalid_json_value(nested)
+          return invalid if invalid
+        end
+        nil
+      else
+        value
+      end
+    end
+  end
+end

data/lib/solid_jobs/config.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+require "logger"
+
+module SolidJobs
+  class Config
+    DEFAULT_JOB_OPTIONS = {
+      "queue" => "default",
+      "retry" => true,
+    }.freeze
+
+    attr_accessor :concurrency, :dead_max_jobs, :dead_timeout, :logger,
+      :on_complex_arguments, :poll_interval_average, :shutdown_timeout,
+      :reliable_fetch, :retry_base_delay, :retry_max_delay
+    attr_reader :client_middleware, :default_job_options, :error_handlers,
+      :redis_config, :server_middleware
+
+    def initialize(redis: nil, concurrency: 5, queues: ["default"])
+      @redis_config = redis || SolidRedis.config(url: ENV.fetch("REDIS_URL", "redis://127.0.0.1:6379/0"))
+      @concurrency = Integer(concurrency)
+      @queues = normalize_queues(queues)
+      @default_job_options = DEFAULT_JOB_OPTIONS
+      @client_middleware = Middleware::Chain.new
+      @server_middleware = Middleware::Chain.new
+      @error_handlers = []
+      @lifecycle_callbacks = Hash.new { |hash, event| hash[event] = [] }
+      @on_complex_arguments = :raise
+      @poll_interval_average = 5.0
+      @shutdown_timeout = 25.0
+      @queue_mode = :weighted
+      @reliable_fetch = true
+      @retry_base_delay = 15.0
+      @retry_max_delay = 3_600.0
+      @dead_max_jobs = 10_000
+      @dead_timeout = 180 * 24 * 60 * 60
+      @logger = Logger.new($stdout)
+      @redis_pool = nil
+    end
+
+    def redis=(configuration)
+      close
+      @redis_config = configuration
+    end
+
+    def redis_pool
+      @redis_pool ||= redis_config.new_pool(size: concurrency + 2, timeout: 1.0)
+    end
+
+    def redis
+      redis_pool.with { |connection| yield connection }
+    end
+
+    def queues
+      @queues.map(&:first)
+    end
+
+    def queues=(values)
+      @queues = normalize_queues(values)
+    end
+
+    def queue_entries
+      @queues.dup
+    end
+
+    def queue_mode
+      @queue_mode
+    end
+
+    def queue_mode=(mode)
+      mode = mode.to_sym
+      unless %i[strict weighted random].include?(mode)
+        raise ArgumentError, "queue_mode must be :strict, :weighted, or :random"
+      end
+
+      @queue_mode = mode
+    end
+
+    def strict
+      queue_mode == :strict
+    end
+
+    def strict=(value)
+      self.queue_mode = value ? :strict : :weighted
+    end
+
+    def default_job_options=(options)
+      @default_job_options = Utilities.shareable_copy(
+        DEFAULT_JOB_OPTIONS.merge(Utilities.stringify_keys(options)),
+      )
+    end
+
+    def on(event, &block)
+      raise ArgumentError, "A lifecycle callback block is required" unless block
+
+      @lifecycle_callbacks[event.to_sym] << CallbackCollection.new do |callbacks|
+        callbacks.public_send(event, &block)
+      end
+      self
+    end
+
+    def fire(event, *arguments)
+      @lifecycle_callbacks[event.to_sym].each do |callbacks|
+        callbacks.respond_with(event, *arguments)
+      end
+    end
+
+    def close
+      @redis_pool&.close
+      @redis_pool = nil
+      self
+    end
+
+    def inspect
+      "#<#{self.class.name} concurrency=#{concurrency} queues=#{queues.inspect}>"
+    end
+
+    def ractor_snapshot
+      Utilities.shareable_copy(
+        redis_config: redis_config,
+        concurrency: concurrency,
+        queues: queue_entries,
+        queue_mode: queue_mode,
+        reliable_fetch: reliable_fetch,
+        default_job_options: default_job_options,
+        on_complex_arguments: on_complex_arguments,
+        poll_interval_average: poll_interval_average,
+        shutdown_timeout: shutdown_timeout,
+        dead_max_jobs: dead_max_jobs,
+        dead_timeout: dead_timeout,
+        retry_base_delay: retry_base_delay,
+        retry_max_delay: retry_max_delay,
+        client_middleware: client_middleware.snapshot,
+        server_middleware: server_middleware.snapshot,
+      )
+    end
+
+    def self.from_ractor_snapshot(snapshot)
+      config = new(
+        redis: snapshot.fetch(:redis_config),
+        concurrency: snapshot.fetch(:concurrency),
+        queues: snapshot.fetch(:queues),
+      )
+      config.queue_mode = snapshot.fetch(:queue_mode)
+      config.reliable_fetch = snapshot.fetch(:reliable_fetch)
+      config.default_job_options = snapshot.fetch(:default_job_options)
+      config.on_complex_arguments = snapshot.fetch(:on_complex_arguments)
+      config.poll_interval_average = snapshot.fetch(:poll_interval_average)
+      config.shutdown_timeout = snapshot.fetch(:shutdown_timeout)
+      config.dead_max_jobs = snapshot.fetch(:dead_max_jobs)
+      config.dead_timeout = snapshot.fetch(:dead_timeout)
+      config.retry_base_delay = snapshot.fetch(:retry_base_delay)
+      config.retry_max_delay = snapshot.fetch(:retry_max_delay)
+      config.client_middleware.restore(snapshot.fetch(:client_middleware))
+      config.server_middleware.restore(snapshot.fetch(:server_middleware))
+      config
+    end
+
+    private
+
+    def normalize_queues(values)
+      Array(values).map do |value|
+        name, weight = value.is_a?(Array) ? value : [value, 1]
+        name = String(name)
+        weight = Integer(weight)
+        raise ArgumentError, "Queue weight must be positive" unless weight.positive?
+
+        [name.freeze, weight].freeze
+      end
+    end
+  end
+end

data/lib/solid_jobs/errors.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module SolidJobs
+  class Error < StandardError; end
+  class InvalidJobError < ArgumentError; end
+  class InvalidArgumentError < InvalidJobError; end
+  class ConfigurationError < Error; end
+  class Shutdown < Error; end
+end

data/lib/solid_jobs/fetch.rb

@@ -0,0 +1,310 @@
+# frozen_string_literal: true
+
+require "json"
+require "securerandom"
+require "digest/sha1"
+
+module SolidJobs
+  class Fetch
+    TIMEOUT = 0.25
+    RESERVE = <<~LUA.freeze
+      local payload = redis.call("rpop", KEYS[1])
+      if not payload then
+        return nil
+      end
+
+      redis.call("lpush", KEYS[2], payload)
+      local job = cjson.decode(payload)
+      local attempt = redis.call("hincrby", KEYS[4], job["jid"], 1)
+      local metadata = {
+        job_id = job["jid"],
+        reservation_id = ARGV[1],
+        process_id = ARGV[2],
+        worker_id = ARGV[3],
+        queue = job["queue"],
+        reserved_at = tonumber(ARGV[4]),
+        attempt = attempt
+      }
+      redis.call("hset", KEYS[3], ARGV[3], cjson.encode(metadata))
+      return {payload, attempt}
+    LUA
+    REGISTER = <<~LUA.freeze
+      local attempt = redis.call("hincrby", KEYS[2], ARGV[1], 1)
+      local metadata = cjson.decode(ARGV[3])
+      metadata["attempt"] = attempt
+      redis.call("hset", KEYS[1], ARGV[2], cjson.encode(metadata))
+      return attempt
+    LUA
+    ACK = <<~LUA.freeze
+      local metadata = redis.call("hget", KEYS[2], ARGV[2])
+      if not metadata then
+        return 0
+      end
+      if cjson.decode(metadata)["reservation_id"] ~= ARGV[3] then
+        return 0
+      end
+      local removed = redis.call("lrem", KEYS[1], 1, ARGV[1])
+      redis.call("hdel", KEYS[2], ARGV[2])
+      if removed > 0 then
+        redis.call("hdel", KEYS[3], ARGV[4])
+      end
+      return removed
+    LUA
+    REQUEUE = <<~LUA.freeze
+      local metadata = redis.call("hget", KEYS[2], ARGV[2])
+      if not metadata then
+        return 0
+      end
+      if cjson.decode(metadata)["reservation_id"] ~= ARGV[3] then
+        return 0
+      end
+      local removed = redis.call("lrem", KEYS[1], 1, ARGV[1])
+      if removed > 0 then
+        redis.call("rpush", KEYS[3], ARGV[1])
+      end
+      redis.call("hdel", KEYS[2], ARGV[2])
+      return removed
+    LUA
+    RESERVE_SHA = Digest::SHA1.hexdigest(RESERVE).freeze
+    REGISTER_SHA = Digest::SHA1.hexdigest(REGISTER).freeze
+    ACK_SHA = Digest::SHA1.hexdigest(ACK).freeze
+    REQUEUE_SHA = Digest::SHA1.hexdigest(REQUEUE).freeze
+
+    UnitOfWork = Struct.new(
+      :queue,
+      :payload,
+      :job,
+      :redis_pool,
+      :reserved_key,
+      :metadata_key,
+      :metadata_field,
+      :reservation_id,
+      :attempt,
+      keyword_init: true,
+    ) do
+      def acknowledge
+        if reserved_key
+          removed = begin
+            redis_pool.call(
+              "EVALSHA", Fetch::ACK_SHA, 3,
+              reserved_key, metadata_key, "solid-jobs:attempts",
+              payload, metadata_field, reservation_id, job.fetch("jid"),
+            )
+          rescue SolidRedis::CommandError => error
+            raise unless error.message.include?("NOSCRIPT")
+
+            redis_pool.call(
+              "EVAL", Fetch::ACK, 3,
+              reserved_key, metadata_key, "solid-jobs:attempts",
+              payload, metadata_field, reservation_id, job.fetch("jid"),
+            )
+          end
+          return Integer(removed) == 1
+        end
+        true
+      end
+
+      def requeue
+        if reserved_key
+          destination = "queue:#{queue}"
+          removed = begin
+            redis_pool.call(
+              "EVALSHA", Fetch::REQUEUE_SHA, 3,
+              reserved_key, metadata_key, destination,
+              payload, metadata_field, reservation_id,
+            )
+          rescue SolidRedis::CommandError => error
+            raise unless error.message.include?("NOSCRIPT")
+
+            redis_pool.call(
+              "EVAL", Fetch::REQUEUE, 3,
+              reserved_key, metadata_key, destination,
+              payload, metadata_field, reservation_id,
+            )
+          end
+          return Integer(removed) == 1
+        else
+          redis_pool.call("RPUSH", "queue:#{queue}", payload)
+        end
+        true
+      end
+    end
+
+    def initialize(config, identity: nil, processor_id: nil)
+      @redis_pool = config.redis_pool
+      @queue_mode = config.queue_mode
+      @reliable = config.reliable_fetch
+      @queues = config.queue_entries.flat_map do |name, weight|
+        Array.new(weight, "queue:#{name}")
+      end.freeze
+      @reserved_key = "#{identity}:reserved:#{processor_id}" if @reliable && identity
+      @metadata_key = "#{identity}:reservations" if @reserved_key
+      @identity = identity
+      @processor_id = processor_id
+      @paused = []
+      @paused_refresh_at = 0.0
+      @queue_index = 0
+      @check_reserved = !!@reserved_key
+    end
+
+    def retrieve
+      if @check_reserved
+        @check_reserved = false
+        if (payload = @redis_pool.call("LINDEX", @reserved_key, -1))
+          return existing_work(payload)
+        end
+      end
+
+      refresh_paused
+      queues = @queues.reject { |queue| @paused.include?(queue.delete_prefix("queue:")) }
+      return sleep(0.05) if queues.empty?
+
+      if @reserved_key
+        reserve_reliable(queues)
+      else
+        queues = queues.shuffle if @queue_mode == :random
+        result = @redis_pool.blocking_call(TIMEOUT, "BRPOP", *queues, TIMEOUT)
+        result && work(result.fetch(1), result.fetch(0).delete_prefix("queue:"))
+      end
+    end
+
+    def connection_failed!
+      @check_reserved = !!@reserved_key
+    end
+
+    private
+
+    def work(payload, queue = nil, reservation_id: nil, attempt: nil)
+      job = JSON.parse(payload)
+      queue ||= job.fetch("queue")
+      reservation_id, attempt = register(job, queue) unless reservation_id
+      UnitOfWork.new(
+        queue: queue,
+        payload: payload,
+        job: job,
+        redis_pool: @redis_pool,
+        reserved_key: @reserved_key,
+        metadata_key: @metadata_key,
+        metadata_field: @processor_id.to_s,
+        reservation_id: reservation_id,
+        attempt: attempt,
+      )
+    end
+
+    def existing_work(payload)
+      raw = @redis_pool.call("HGET", @metadata_key, @processor_id)
+      if raw
+        metadata = JSON.parse(raw)
+        job = JSON.parse(payload)
+        if metadata["job_id"] == job["jid"]
+          return UnitOfWork.new(
+            queue: job.fetch("queue"),
+            payload: payload,
+            job: job,
+            redis_pool: @redis_pool,
+            reserved_key: @reserved_key,
+            metadata_key: @metadata_key,
+            metadata_field: @processor_id.to_s,
+            reservation_id: metadata.fetch("reservation_id"),
+            attempt: Integer(metadata.fetch("attempt")),
+          )
+        end
+      end
+      work(payload)
+    end
+
+    def register(job, queue)
+      return [nil, nil] unless @reserved_key
+
+      reservation_id = SecureRandom.hex(12)
+      metadata = JSON.generate(
+        "job_id" => job["jid"],
+        "reservation_id" => reservation_id,
+        "process_id" => @identity,
+        "worker_id" => @processor_id,
+        "queue" => queue,
+        "reserved_at" => Time.now.to_f,
+      )
+      attempt = begin
+        @redis_pool.call(
+          "EVALSHA", REGISTER_SHA, 2,
+          @metadata_key, "solid-jobs:attempts",
+          job["jid"], @processor_id, metadata,
+        )
+      rescue SolidRedis::CommandError => error
+        raise unless error.message.include?("NOSCRIPT")
+
+        @redis_pool.call(
+          "EVAL", REGISTER, 2,
+          @metadata_key, "solid-jobs:attempts",
+          job["jid"], @processor_id, metadata,
+        )
+      end
+      [reservation_id, Integer(attempt)]
+    end
+
+    def next_queue(queues)
+      return queues.sample if @queue_mode == :random
+
+      queue = queues[@queue_index % queues.length]
+      @queue_index += 1
+      queue
+    end
+
+    def reserve_reliable(queues)
+      if @queue_mode == :strict
+        queues.each do |source|
+          unit = reserve_now(source)
+          return unit if unit
+        end
+        source = queues.first
+      else
+        source = next_queue(queues)
+        unit = reserve_now(source)
+        return unit if unit
+      end
+      payload = @redis_pool.blocking_call(
+        TIMEOUT,
+        "BLMOVE", source, @reserved_key, "RIGHT", "LEFT", TIMEOUT,
+      )
+      payload && work(payload, source.delete_prefix("queue:"))
+    end
+
+    def reserve_now(source)
+      reservation_id = SecureRandom.hex(12)
+      reserved_at = Time.now.to_f
+      result = begin
+        @redis_pool.call(
+          "EVALSHA", RESERVE_SHA, 4,
+          source, @reserved_key, @metadata_key, "solid-jobs:attempts",
+          reservation_id, @identity, @processor_id, reserved_at,
+        )
+      rescue SolidRedis::CommandError => error
+        raise unless error.message.include?("NOSCRIPT")
+
+        @redis_pool.call(
+          "EVAL", RESERVE, 4,
+          source, @reserved_key, @metadata_key, "solid-jobs:attempts",
+          reservation_id, @identity, @processor_id, reserved_at,
+        )
+      end
+      return unless result
+
+      payload, attempt = result
+      work(
+        payload,
+        source.delete_prefix("queue:"),
+        reservation_id: reservation_id,
+        attempt: Integer(attempt),
+      )
+    end
+
+    def refresh_paused
+      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      return if now < @paused_refresh_at
+
+      @paused = Array(@redis_pool.call("SMEMBERS", "paused"))
+      @paused_refresh_at = now + 5
+    end
+  end
+end