solid-jobs 0.1.2 → 0.2.0

data/lib/solid_jobs/claim.rb

@@ -0,0 +1,315 @@
+# frozen_string_literal: true
+
+require "json"
+require "securerandom"
+require "digest/sha1"
+
+module SolidJobs
+  class Claim
+    TIMEOUT = 0.25
+    RESERVE = <<~LUA.freeze
+      local payload = redis.call("rpop", KEYS[1])
+      if not payload then
+        return nil
+      end
+
+      redis.call("lpush", KEYS[2], payload)
+      local job = cjson.decode(payload)
+      local attempt = redis.call("hincrby", KEYS[4], job["id"], 1)
+      local metadata = {
+        task_id = job["id"],
+        claim_token = ARGV[1],
+        node_id = ARGV[2],
+        executor_id = ARGV[3],
+        channel = job["channel"],
+        claimed_at = tonumber(ARGV[4]),
+        attempt = attempt
+      }
+      redis.call("hset", KEYS[3], ARGV[3], cjson.encode(metadata))
+      return {payload, attempt}
+    LUA
+    REGISTER = <<~LUA.freeze
+      local attempt = redis.call("hincrby", KEYS[2], ARGV[1], 1)
+      local metadata = cjson.decode(ARGV[3])
+      metadata["attempt"] = attempt
+      redis.call("hset", KEYS[1], ARGV[2], cjson.encode(metadata))
+      return attempt
+    LUA
+    ACK = <<~LUA.freeze
+      local metadata = redis.call("hget", KEYS[2], ARGV[2])
+      if not metadata then
+        return 0
+      end
+      if cjson.decode(metadata)["claim_token"] ~= ARGV[3] then
+        return 0
+      end
+      local removed = redis.call("lrem", KEYS[1], 1, ARGV[1])
+      redis.call("hdel", KEYS[2], ARGV[2])
+      if removed > 0 then
+        redis.call("hdel", KEYS[3], ARGV[4])
+      end
+      return removed
+    LUA
+    REQUEUE = <<~LUA.freeze
+      local metadata = redis.call("hget", KEYS[2], ARGV[2])
+      if not metadata then
+        return 0
+      end
+      if cjson.decode(metadata)["claim_token"] ~= ARGV[3] then
+        return 0
+      end
+      local removed = redis.call("lrem", KEYS[1], 1, ARGV[1])
+      if removed > 0 then
+        redis.call("rpush", KEYS[3], ARGV[1])
+      end
+      redis.call("hdel", KEYS[2], ARGV[2])
+      return removed
+    LUA
+    RESERVE_SHA = Digest::SHA1.hexdigest(RESERVE).freeze
+    REGISTER_SHA = Digest::SHA1.hexdigest(REGISTER).freeze
+    ACK_SHA = Digest::SHA1.hexdigest(ACK).freeze
+    REQUEUE_SHA = Digest::SHA1.hexdigest(REQUEUE).freeze
+
+    ClaimRecord = Struct.new(
+      :channel,
+      :payload,
+      :envelope,
+      :redis_pool,
+      :claimed_key,
+      :claims_key,
+      :executor_field,
+      :claim_token,
+      :attempt,
+      keyword_init: true,
+    ) do
+      def complete
+        if claimed_key
+          removed = begin
+            redis_pool.call(
+              "EVALSHA", Claim::ACK_SHA, 3,
+              claimed_key, claims_key, Keyspace::ATTEMPTS,
+              payload, executor_field, claim_token, envelope.fetch("id"),
+            )
+          rescue SolidRedis::CommandError => error
+            raise unless error.message.include?("NOSCRIPT")
+
+            redis_pool.call(
+              "EVAL", Claim::ACK, 3,
+              claimed_key, claims_key, Keyspace::ATTEMPTS,
+              payload, executor_field, claim_token, envelope.fetch("id"),
+            )
+          end
+          return Integer(removed) == 1
+        end
+        true
+      end
+
+      def requeue
+        if claimed_key
+          destination = Keyspace.channel(channel)
+          removed = begin
+            redis_pool.call(
+              "EVALSHA", Claim::REQUEUE_SHA, 3,
+              claimed_key, claims_key, destination,
+              payload, executor_field, claim_token,
+            )
+          rescue SolidRedis::CommandError => error
+            raise unless error.message.include?("NOSCRIPT")
+
+            redis_pool.call(
+              "EVAL", Claim::REQUEUE, 3,
+              claimed_key, claims_key, destination,
+              payload, executor_field, claim_token,
+            )
+          end
+          return Integer(removed) == 1
+        else
+          redis_pool.call("RPUSH", Keyspace.channel(channel), payload)
+        end
+        true
+      end
+    end
+
+    def initialize(config, identity: nil, processor_id: nil)
+      @redis_pool = config.redis_pool
+      @channel_order = config.channel_order
+      @reliable = config.reliable_fetch
+      @channels = config.channel_entries.flat_map do |name, weight|
+        Array.new(weight, Keyspace.channel(name))
+      end.freeze
+      @claimed_key = Keyspace.claimed(identity, processor_id) if @reliable && identity
+      @claims_key = Keyspace.claims(identity) if @claimed_key
+      @identity = identity
+      @processor_id = processor_id
+      @paused = []
+      @paused_refresh_at = 0.0
+      @channel_index = 0
+      @check_claimed = !!@claimed_key
+    end
+
+    def next
+      if @check_claimed
+        @check_claimed = false
+        if (payload = @redis_pool.call("LINDEX", @claimed_key, -1))
+          return existing_claim(payload)
+        end
+      end
+
+      refresh_paused
+      channels = @channels.reject do |key|
+        @paused.include?(key.delete_prefix("#{Keyspace::PREFIX}:channel:"))
+      end
+      return sleep(0.05) if channels.empty?
+
+      if @claimed_key
+        claim_reliable(channels)
+      else
+        channels = channels.shuffle if @channel_order == :shuffle
+        result = @redis_pool.blocking_call(TIMEOUT, "BRPOP", *channels, TIMEOUT)
+        result && record(
+          result.fetch(1),
+          result.fetch(0).delete_prefix("#{Keyspace::PREFIX}:channel:"),
+        )
+      end
+    end
+
+    def connection_failed!
+      @check_claimed = !!@claimed_key
+    end
+
+    private
+
+    def record(payload, channel = nil, claim_token: nil, attempt: nil)
+      envelope = JSON.parse(payload)
+      channel ||= envelope.fetch("channel")
+      claim_token, attempt = register(envelope, channel) unless claim_token
+      ClaimRecord.new(
+        channel: channel,
+        payload: payload,
+        envelope: envelope,
+        redis_pool: @redis_pool,
+        claimed_key: @claimed_key,
+        claims_key: @claims_key,
+        executor_field: @processor_id.to_s,
+        claim_token: claim_token,
+        attempt: attempt,
+      )
+    end
+
+    def existing_claim(payload)
+      raw = @redis_pool.call("HGET", @claims_key, @processor_id)
+      if raw
+        metadata = JSON.parse(raw)
+        envelope = JSON.parse(payload)
+        if metadata["task_id"] == envelope["id"]
+          return ClaimRecord.new(
+            channel: envelope.fetch("channel"),
+            payload: payload,
+            envelope: envelope,
+            redis_pool: @redis_pool,
+            claimed_key: @claimed_key,
+            claims_key: @claims_key,
+            executor_field: @processor_id.to_s,
+            claim_token: metadata.fetch("claim_token"),
+            attempt: Integer(metadata.fetch("attempt")),
+          )
+        end
+      end
+      record(payload)
+    end
+
+    def register(envelope, channel)
+      return [nil, nil] unless @claimed_key
+
+      claim_token = SecureRandom.uuid
+      metadata = JSON.generate(
+        "task_id" => envelope["id"],
+        "claim_token" => claim_token,
+        "node_id" => @identity,
+        "executor_id" => @processor_id,
+        "channel" => channel,
+        "claimed_at" => Time.now.to_f,
+      )
+      attempt = begin
+        @redis_pool.call(
+          "EVALSHA", REGISTER_SHA, 2,
+          @claims_key, Keyspace::ATTEMPTS,
+          envelope["id"], @processor_id, metadata,
+        )
+      rescue SolidRedis::CommandError => error
+        raise unless error.message.include?("NOSCRIPT")
+
+        @redis_pool.call(
+          "EVAL", REGISTER, 2,
+          @claims_key, Keyspace::ATTEMPTS,
+          envelope["id"], @processor_id, metadata,
+        )
+      end
+      [claim_token, Integer(attempt)]
+    end
+
+    def next_channel(channels)
+      return channels.sample if @channel_order == :shuffle
+
+      channel = channels[@channel_index % channels.length]
+      @channel_index += 1
+      channel
+    end
+
+    def claim_reliable(channels)
+      if @channel_order == :priority
+        channels.each do |source|
+          claim = claim_now(source)
+          return claim if claim
+        end
+        source = channels.first
+      else
+        source = next_channel(channels)
+        claim = claim_now(source)
+        return claim if claim
+      end
+      payload = @redis_pool.blocking_call(
+        TIMEOUT,
+        "BLMOVE", source, @claimed_key, "RIGHT", "LEFT", TIMEOUT,
+      )
+      payload && record(payload, source.delete_prefix("#{Keyspace::PREFIX}:channel:"))
+    end
+
+    def claim_now(source)
+      claim_token = SecureRandom.uuid
+      claimed_at = Time.now.to_f
+      result = begin
+        @redis_pool.call(
+          "EVALSHA", RESERVE_SHA, 4,
+          source, @claimed_key, @claims_key, Keyspace::ATTEMPTS,
+          claim_token, @identity, @processor_id, claimed_at,
+        )
+      rescue SolidRedis::CommandError => error
+        raise unless error.message.include?("NOSCRIPT")
+
+        @redis_pool.call(
+          "EVAL", RESERVE, 4,
+          source, @claimed_key, @claims_key, Keyspace::ATTEMPTS,
+          claim_token, @identity, @processor_id, claimed_at,
+        )
+      end
+      return unless result
+
+      payload, attempt = result
+      record(
+        payload,
+        source.delete_prefix("#{Keyspace::PREFIX}:channel:"),
+        claim_token: claim_token,
+        attempt: Integer(attempt),
+      )
+    end
+
+    def refresh_paused
+      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      return if now < @paused_refresh_at
+
+      @paused = Array(@redis_pool.call("SMEMBERS", Keyspace::PAUSED_CHANNELS))
+      @paused_refresh_at = now + 5
+    end
+  end
+end

data/lib/solid_jobs/cli.rb

@@ -8,7 +8,7 @@ module SolidJobs
     SIGNALS = %w[INT TERM TSTP TTIN INFO].freeze
 
     def initialize
-      @options = {queues: []}
+      @options = {channels: []}
     end
 
     def run(arguments = ARGV)
@@ -39,7 +39,9 @@ module SolidJobs
         options.on("-r", "--require PATH", "Require an application file") { |value| @options[:require] = value }
         options.on("-C", "--config PATH", "Load YAML configuration") { |value| @options[:config] = value }
         options.on("-c", "--concurrency N", Integer, "Processor Ractor count") { |value| @options[:concurrency] = value }
-        options.on("-q", "--queue QUEUE", "Queue name or name,weight") { |value| @options[:queues] << queue(value) }
+        options.on("--channel CHANNEL", "Channel name or name,weight") do |value|
+          @options[:channels] << channel(value)
+        end
         options.on("-e", "--environment NAME", "Application environment") { |value| @options[:environment] = value }
         options.on("-t", "--timeout SECONDS", Float, "Graceful shutdown timeout") { |value| @options[:timeout] = value }
         options.on("-v", "--version", "Print version") do
@@ -62,7 +64,9 @@ module SolidJobs
       values = values.fetch(@options[:environment], values) if @options[:environment]
       @options[:concurrency] ||= values["concurrency"]
       @options[:timeout] ||= values["timeout"]
-      @options[:queues] = Array(values["queues"]).map { |value| queue(value) } if @options[:queues].empty?
+      if @options[:channels].empty?
+        @options[:channels] = Array(values["channels"]).map { |value| channel(value) }
+      end
       if (url = values.dig("redis", "url") || values["redis_url"])
         SolidJobs.config.redis = SolidRedis::Config.new(url: url)
       end
@@ -77,7 +81,7 @@ module SolidJobs
     def apply_options
       ENV["RAILS_ENV"] = ENV["RACK_ENV"] = @options[:environment] if @options[:environment]
       SolidJobs.config.concurrency = @options[:concurrency] if @options[:concurrency]
-      SolidJobs.config.queues = @options[:queues] unless @options[:queues].empty?
+      SolidJobs.config.channels = @options[:channels] unless @options[:channels].empty?
       SolidJobs.config.shutdown_timeout = @options[:timeout] if @options[:timeout]
     end
 
@@ -126,10 +130,9 @@ module SolidJobs
       end
     end
 
-    def queue(value)
+    def channel(value)
       name, weight = value.to_s.split(",", 2)
       weight ? [name, Integer(weight)] : name
     end
   end
 end
-

data/lib/solid_jobs/config.rb

@@ -4,35 +4,35 @@ require "logger"
 
 module SolidJobs
   class Config
-    DEFAULT_JOB_OPTIONS = {
-      "queue" => "default",
-      "retry" => true,
+    DEFAULT_TASK_OPTIONS = {
+      "channel" => "default",
+      "max_failures" => 25,
     }.freeze
 
-    attr_accessor :concurrency, :dead_max_jobs, :dead_timeout, :logger,
+    attr_accessor :concurrency, :discarded_limit, :discarded_retention, :logger,
       :on_complex_arguments, :poll_interval_average, :shutdown_timeout,
       :reliable_fetch, :retry_base_delay, :retry_max_delay
-    attr_reader :client_middleware, :default_job_options, :error_handlers,
-      :redis_config, :server_middleware
+    attr_reader :default_task_options, :error_handlers, :execute_interceptors,
+      :publish_interceptors, :redis_config
 
-    def initialize(redis: nil, concurrency: 5, queues: ["default"])
+    def initialize(redis: nil, concurrency: 5, channels: ["default"])
       @redis_config = redis || SolidRedis.config(url: ENV.fetch("REDIS_URL", "redis://127.0.0.1:6379/0"))
       @concurrency = Integer(concurrency)
-      @queues = normalize_queues(queues)
-      @default_job_options = DEFAULT_JOB_OPTIONS
-      @client_middleware = Middleware::Chain.new
-      @server_middleware = Middleware::Chain.new
+      @channels = normalize_channels(channels)
+      @default_task_options = DEFAULT_TASK_OPTIONS
+      @publish_interceptors = InterceptorRegistry.new
+      @execute_interceptors = InterceptorRegistry.new
       @error_handlers = []
       @lifecycle_callbacks = Hash.new { |hash, event| hash[event] = [] }
       @on_complex_arguments = :raise
       @poll_interval_average = 5.0
       @shutdown_timeout = 25.0
-      @queue_mode = :weighted
+      @channel_order = :weighted
       @reliable_fetch = true
       @retry_base_delay = 15.0
       @retry_max_delay = 3_600.0
-      @dead_max_jobs = 10_000
-      @dead_timeout = 180 * 24 * 60 * 60
+      @discarded_limit = 10_000
+      @discarded_retention = 180 * 24 * 60 * 60
       @logger = Logger.new($stdout)
       @redis_pool = nil
     end
@@ -50,42 +50,42 @@ module SolidJobs
       redis_pool.with { |connection| yield connection }
     end
 
-    def queues
-      @queues.map(&:first)
+    def channels
+      @channels.map(&:first)
     end
 
-    def queues=(values)
-      @queues = normalize_queues(values)
+    def channels=(values)
+      @channels = normalize_channels(values)
     end
 
-    def queue_entries
-      @queues.dup
+    def channel_entries
+      @channels.dup
     end
 
-    def queue_mode
-      @queue_mode
+    def channel_order
+      @channel_order
     end
 
-    def queue_mode=(mode)
+    def channel_order=(mode)
       mode = mode.to_sym
-      unless %i[strict weighted random].include?(mode)
-        raise ArgumentError, "queue_mode must be :strict, :weighted, or :random"
+      unless %i[priority weighted shuffle].include?(mode)
+        raise ArgumentError, "channel_order must be :priority, :weighted, or :shuffle"
       end
 
-      @queue_mode = mode
+      @channel_order = mode
     end
 
-    def strict
-      queue_mode == :strict
+    def prioritized?
+      channel_order == :priority
     end
 
-    def strict=(value)
-      self.queue_mode = value ? :strict : :weighted
+    def prioritized=(value)
+      self.channel_order = value ? :priority : :weighted
     end
 
-    def default_job_options=(options)
-      @default_job_options = Utilities.shareable_copy(
-        DEFAULT_JOB_OPTIONS.merge(Utilities.stringify_keys(options)),
+    def default_task_options=(options)
+      @default_task_options = Utilities.shareable_copy(
+        DEFAULT_TASK_OPTIONS.merge(Utilities.stringify_keys(options)),
       )
     end
 
@@ -111,26 +111,26 @@ module SolidJobs
     end
 
     def inspect
-      "#<#{self.class.name} concurrency=#{concurrency} queues=#{queues.inspect}>"
+      "#<#{self.class.name} concurrency=#{concurrency} channels=#{channels.inspect}>"
     end
 
     def ractor_snapshot
       Utilities.shareable_copy(
         redis_config: redis_config,
         concurrency: concurrency,
-        queues: queue_entries,
-        queue_mode: queue_mode,
+        channels: channel_entries,
+        channel_order: channel_order,
         reliable_fetch: reliable_fetch,
-        default_job_options: default_job_options,
+        default_task_options: default_task_options,
         on_complex_arguments: on_complex_arguments,
         poll_interval_average: poll_interval_average,
         shutdown_timeout: shutdown_timeout,
-        dead_max_jobs: dead_max_jobs,
-        dead_timeout: dead_timeout,
+        discarded_limit: discarded_limit,
+        discarded_retention: discarded_retention,
         retry_base_delay: retry_base_delay,
         retry_max_delay: retry_max_delay,
-        client_middleware: client_middleware.snapshot,
-        server_middleware: server_middleware.snapshot,
+        publish_interceptors: publish_interceptors.export,
+        execute_interceptors: execute_interceptors.export,
       )
     end
 
@@ -138,31 +138,31 @@ module SolidJobs
       config = new(
         redis: snapshot.fetch(:redis_config),
         concurrency: snapshot.fetch(:concurrency),
-        queues: snapshot.fetch(:queues),
+        channels: snapshot.fetch(:channels),
       )
-      config.queue_mode = snapshot.fetch(:queue_mode)
+      config.channel_order = snapshot.fetch(:channel_order)
       config.reliable_fetch = snapshot.fetch(:reliable_fetch)
-      config.default_job_options = snapshot.fetch(:default_job_options)
+      config.default_task_options = snapshot.fetch(:default_task_options)
       config.on_complex_arguments = snapshot.fetch(:on_complex_arguments)
       config.poll_interval_average = snapshot.fetch(:poll_interval_average)
       config.shutdown_timeout = snapshot.fetch(:shutdown_timeout)
-      config.dead_max_jobs = snapshot.fetch(:dead_max_jobs)
-      config.dead_timeout = snapshot.fetch(:dead_timeout)
+      config.discarded_limit = snapshot.fetch(:discarded_limit)
+      config.discarded_retention = snapshot.fetch(:discarded_retention)
       config.retry_base_delay = snapshot.fetch(:retry_base_delay)
       config.retry_max_delay = snapshot.fetch(:retry_max_delay)
-      config.client_middleware.restore(snapshot.fetch(:client_middleware))
-      config.server_middleware.restore(snapshot.fetch(:server_middleware))
+      config.publish_interceptors.import(snapshot.fetch(:publish_interceptors))
+      config.execute_interceptors.import(snapshot.fetch(:execute_interceptors))
       config
     end
 
     private
 
-    def normalize_queues(values)
+    def normalize_channels(values)
       Array(values).map do |value|
         name, weight = value.is_a?(Array) ? value : [value, 1]
         name = String(name)
         weight = Integer(weight)
-        raise ArgumentError, "Queue weight must be positive" unless weight.positive?
+        raise ArgumentError, "Channel weight must be positive" unless weight.positive?
 
         [name.freeze, weight].freeze
       end

data/lib/solid_jobs/executor.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "json"
+
+module SolidJobs
+  class Executor
+    Execution = Data.define(:task, :envelope)
+
+    attr_reader :config
+
+    def initialize(redis_config:, config: nil)
+      @config = config || Config.new(redis: redis_config)
+    end
+
+    def execute(envelope)
+      envelope = JSON.parse(envelope) if envelope.is_a?(String)
+      raise InvalidJobError, "Task envelope must be a Hash" unless envelope.is_a?(Hash)
+
+      task_name = envelope["task"]
+      raise InvalidJobError, "Task must be present" unless task_name.is_a?(String) && !task_name.empty?
+      raise InvalidJobError, "Task arguments must be an Array" unless envelope["arguments"].is_a?(Array)
+      unless envelope["channel"].is_a?(String) && !envelope["channel"].empty?
+        raise InvalidJobError, "Task channel must be present"
+      end
+
+      task = Utilities.constantize(task_name).new
+      task.task_id = envelope["id"] if task.respond_to?(:task_id=)
+      operation = -> { task.perform(*envelope.fetch("arguments")) }
+      return operation.call if config.execute_interceptors.empty?
+
+      config.execute_interceptors.call(Execution.new(task, envelope), &operation)
+    end
+  end
+end

data/lib/solid_jobs/failure_policy.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require "json"
+
+module SolidJobs
+  class FailurePolicy < Service::Base
+
+    def call
+      payload = @payload.dup
+      now = Time.now
+      now_ms = Utilities.realtime_milliseconds
+      failure_count = Integer(payload.fetch("failure_count", 0)) + 1
+      payload["exception_message"] = @error.message.to_s
+      payload["exception_type"] = @error.class.name
+      payload["first_failed_ms"] ||= now_ms
+      payload["last_failed_ms"] = now_ms
+      payload["failure_count"] = failure_count
+      payload["channel"] = payload["retry_channel"] if payload["retry_channel"]
+
+      if retry_allowed?(payload, failure_count, now)
+        decision = retry_decision(payload, failure_count)
+        return nil if decision == :drop
+        return archive(payload, now.to_f) if decision == :archive
+
+        next_attempt = now.to_f + (decision || retry_delay(failure_count))
+        @config.redis_pool.call("ZADD", Keyspace::RETRIES, next_attempt, JSON.generate(payload))
+        append_message("retry scheduled")
+        next_attempt
+      else
+        invoke_final_failure(payload)
+        archive(payload, now.to_f)
+        append_message("task discarded")
+        nil
+      end
+    end
+
+    private
+
+    def retry_allowed?(payload, failure_count, now)
+      if payload["retry_within"]
+        first_failure = Float(payload["first_failed_ms"]) / 1_000
+        now.to_f - first_failure < Float(payload["retry_within"])
+      else
+        maximum = Integer(payload.fetch("max_failures", 0))
+        failure_count <= maximum
+      end
+    end
+
+    def retry_delay(failure_count)
+      ceiling = [
+        Float(@config.retry_base_delay) * (2**(failure_count - 1)),
+        Float(@config.retry_max_delay),
+      ].min
+      (ceiling / 2) + (rand * ceiling / 2)
+    end
+
+    def retry_decision(payload, failure_count)
+      task = resolve_task(payload)
+      return unless task.respond_to?(:retry_delay_callback)
+
+      value = task.retry_delay_callback(failure_count, @error, payload)
+      return value if value.is_a?(Numeric) || %i[drop archive].include?(value)
+
+      nil
+    end
+
+    def invoke_final_failure(payload)
+      task = resolve_task(payload)
+      return unless task.respond_to?(:final_failure_callback)
+
+      task.final_failure_callback(payload, @error)
+    end
+
+    def resolve_task(payload)
+      Utilities.constantize(payload.fetch("task"))
+    rescue NameError
+      nil
+    end
+
+    def archive(payload, timestamp)
+      @config.redis_pool.pipelined do |pipeline|
+        pipeline.call("ZADD", Keyspace::DISCARDED, timestamp, JSON.generate(payload))
+        pipeline.call(
+          "ZREMRANGEBYSCORE",
+          Keyspace::DISCARDED,
+          "-inf",
+          timestamp - @config.discarded_retention,
+        )
+        pipeline.call(
+          "ZREMRANGEBYRANK",
+          Keyspace::DISCARDED,
+          0,
+          -(@config.discarded_limit + 1),
+        )
+      end
+    end
+  end
+end