solid-jobs 0.1.3 → 0.4.0

data/lib/solid_jobs/claim.rb

@@ -0,0 +1,315 @@
+# frozen_string_literal: true
+
+require "json"
+require "securerandom"
+require "digest/sha1"
+
+module SolidJobs
+  class Claim
+    TIMEOUT = 0.25
+    RESERVE = <<~LUA.freeze
+      local payload = redis.call("rpop", KEYS[1])
+      if not payload then
+        return nil
+      end
+
+      redis.call("lpush", KEYS[2], payload)
+      local job = cjson.decode(payload)
+      local attempt = redis.call("hincrby", KEYS[4], job["id"], 1)
+      local metadata = {
+        task_id = job["id"],
+        claim_token = ARGV[1],
+        node_id = ARGV[2],
+        executor_id = ARGV[3],
+        channel = job["channel"],
+        claimed_at = tonumber(ARGV[4]),
+        attempt = attempt
+      }
+      redis.call("hset", KEYS[3], ARGV[3], cjson.encode(metadata))
+      return {payload, attempt}
+    LUA
+    REGISTER = <<~LUA.freeze
+      local attempt = redis.call("hincrby", KEYS[2], ARGV[1], 1)
+      local metadata = cjson.decode(ARGV[3])
+      metadata["attempt"] = attempt
+      redis.call("hset", KEYS[1], ARGV[2], cjson.encode(metadata))
+      return attempt
+    LUA
+    ACK = <<~LUA.freeze
+      local metadata = redis.call("hget", KEYS[2], ARGV[2])
+      if not metadata then
+        return 0
+      end
+      if cjson.decode(metadata)["claim_token"] ~= ARGV[3] then
+        return 0
+      end
+      local removed = redis.call("lrem", KEYS[1], 1, ARGV[1])
+      redis.call("hdel", KEYS[2], ARGV[2])
+      if removed > 0 then
+        redis.call("hdel", KEYS[3], ARGV[4])
+      end
+      return removed
+    LUA
+    REQUEUE = <<~LUA.freeze
+      local metadata = redis.call("hget", KEYS[2], ARGV[2])
+      if not metadata then
+        return 0
+      end
+      if cjson.decode(metadata)["claim_token"] ~= ARGV[3] then
+        return 0
+      end
+      local removed = redis.call("lrem", KEYS[1], 1, ARGV[1])
+      if removed > 0 then
+        redis.call("rpush", KEYS[3], ARGV[1])
+      end
+      redis.call("hdel", KEYS[2], ARGV[2])
+      return removed
+    LUA
+    RESERVE_SHA = Digest::SHA1.hexdigest(RESERVE).freeze
+    REGISTER_SHA = Digest::SHA1.hexdigest(REGISTER).freeze
+    ACK_SHA = Digest::SHA1.hexdigest(ACK).freeze
+    REQUEUE_SHA = Digest::SHA1.hexdigest(REQUEUE).freeze
+
+    ClaimRecord = Struct.new(
+      :channel,
+      :payload,
+      :envelope,
+      :redis_pool,
+      :claimed_key,
+      :claims_key,
+      :executor_field,
+      :claim_token,
+      :attempt,
+      keyword_init: true,
+    ) do
+      def complete
+        if claimed_key
+          removed = begin
+            redis_pool.call(
+              "EVALSHA", Claim::ACK_SHA, 3,
+              claimed_key, claims_key, Keyspace::ATTEMPTS,
+              payload, executor_field, claim_token, envelope.fetch("id"),
+            )
+          rescue SolidRedis::CommandError => error
+            raise unless error.message.include?("NOSCRIPT")
+
+            redis_pool.call(
+              "EVAL", Claim::ACK, 3,
+              claimed_key, claims_key, Keyspace::ATTEMPTS,
+              payload, executor_field, claim_token, envelope.fetch("id"),
+            )
+          end
+          return Integer(removed) == 1
+        end
+        true
+      end
+
+      def requeue
+        if claimed_key
+          destination = Keyspace.channel(channel)
+          removed = begin
+            redis_pool.call(
+              "EVALSHA", Claim::REQUEUE_SHA, 3,
+              claimed_key, claims_key, destination,
+              payload, executor_field, claim_token,
+            )
+          rescue SolidRedis::CommandError => error
+            raise unless error.message.include?("NOSCRIPT")
+
+            redis_pool.call(
+              "EVAL", Claim::REQUEUE, 3,
+              claimed_key, claims_key, destination,
+              payload, executor_field, claim_token,
+            )
+          end
+          return Integer(removed) == 1
+        else
+          redis_pool.call("RPUSH", Keyspace.channel(channel), payload)
+        end
+        true
+      end
+    end
+
+    def initialize(config, identity: nil, processor_id: nil)
+      @redis_pool = config.redis_pool
+      @channel_order = config.channel_order
+      @reliable = config.reliable_fetch
+      @channels = config.channel_entries.flat_map do |name, weight|
+        Array.new(weight, Keyspace.channel(name))
+      end.freeze
+      @claimed_key = Keyspace.claimed(identity, processor_id) if @reliable && identity
+      @claims_key = Keyspace.claims(identity) if @claimed_key
+      @identity = identity
+      @processor_id = processor_id
+      @paused = []
+      @paused_refresh_at = 0.0
+      @channel_index = 0
+      @check_claimed = !!@claimed_key
+    end
+
+    def next
+      if @check_claimed
+        @check_claimed = false
+        if (payload = @redis_pool.call("LINDEX", @claimed_key, -1))
+          return existing_claim(payload)
+        end
+      end
+
+      refresh_paused
+      channels = @channels.reject do |key|
+        @paused.include?(key.delete_prefix("#{Keyspace::PREFIX}:channel:"))
+      end
+      return sleep(0.05) if channels.empty?
+
+      if @claimed_key
+        claim_reliable(channels)
+      else
+        channels = channels.shuffle if @channel_order == :shuffle
+        result = @redis_pool.blocking_call(TIMEOUT, "BRPOP", *channels, TIMEOUT)
+        result && record(
+          result.fetch(1),
+          result.fetch(0).delete_prefix("#{Keyspace::PREFIX}:channel:"),
+        )
+      end
+    end
+
+    def connection_failed!
+      @check_claimed = !!@claimed_key
+    end
+
+    private
+
+    def record(payload, channel = nil, claim_token: nil, attempt: nil)
+      envelope = JSON.parse(payload)
+      channel ||= envelope.fetch("channel")
+      claim_token, attempt = register(envelope, channel) unless claim_token
+      ClaimRecord.new(
+        channel: channel,
+        payload: payload,
+        envelope: envelope,
+        redis_pool: @redis_pool,
+        claimed_key: @claimed_key,
+        claims_key: @claims_key,
+        executor_field: @processor_id.to_s,
+        claim_token: claim_token,
+        attempt: attempt,
+      )
+    end
+
+    def existing_claim(payload)
+      raw = @redis_pool.call("HGET", @claims_key, @processor_id)
+      if raw
+        metadata = JSON.parse(raw)
+        envelope = JSON.parse(payload)
+        if metadata["task_id"] == envelope["id"]
+          return ClaimRecord.new(
+            channel: envelope.fetch("channel"),
+            payload: payload,
+            envelope: envelope,
+            redis_pool: @redis_pool,
+            claimed_key: @claimed_key,
+            claims_key: @claims_key,
+            executor_field: @processor_id.to_s,
+            claim_token: metadata.fetch("claim_token"),
+            attempt: Integer(metadata.fetch("attempt")),
+          )
+        end
+      end
+      record(payload)
+    end
+
+    def register(envelope, channel)
+      return [nil, nil] unless @claimed_key
+
+      claim_token = SecureRandom.uuid
+      metadata = JSON.generate(
+        "task_id" => envelope["id"],
+        "claim_token" => claim_token,
+        "node_id" => @identity,
+        "executor_id" => @processor_id,
+        "channel" => channel,
+        "claimed_at" => Time.now.to_f,
+      )
+      attempt = begin
+        @redis_pool.call(
+          "EVALSHA", REGISTER_SHA, 2,
+          @claims_key, Keyspace::ATTEMPTS,
+          envelope["id"], @processor_id, metadata,
+        )
+      rescue SolidRedis::CommandError => error
+        raise unless error.message.include?("NOSCRIPT")
+
+        @redis_pool.call(
+          "EVAL", REGISTER, 2,
+          @claims_key, Keyspace::ATTEMPTS,
+          envelope["id"], @processor_id, metadata,
+        )
+      end
+      [claim_token, Integer(attempt)]
+    end
+
+    def next_channel(channels)
+      return channels.sample if @channel_order == :shuffle
+
+      channel = channels[@channel_index % channels.length]
+      @channel_index += 1
+      channel
+    end
+
+    def claim_reliable(channels)
+      if @channel_order == :priority
+        channels.each do |source|
+          claim = claim_now(source)
+          return claim if claim
+        end
+        source = channels.first
+      else
+        source = next_channel(channels)
+        claim = claim_now(source)
+        return claim if claim
+      end
+      payload = @redis_pool.blocking_call(
+        TIMEOUT,
+        "BLMOVE", source, @claimed_key, "RIGHT", "LEFT", TIMEOUT,
+      )
+      payload && record(payload, source.delete_prefix("#{Keyspace::PREFIX}:channel:"))
+    end
+
+    def claim_now(source)
+      claim_token = SecureRandom.uuid
+      claimed_at = Time.now.to_f
+      result = begin
+        @redis_pool.call(
+          "EVALSHA", RESERVE_SHA, 4,
+          source, @claimed_key, @claims_key, Keyspace::ATTEMPTS,
+          claim_token, @identity, @processor_id, claimed_at,
+        )
+      rescue SolidRedis::CommandError => error
+        raise unless error.message.include?("NOSCRIPT")
+
+        @redis_pool.call(
+          "EVAL", RESERVE, 4,
+          source, @claimed_key, @claims_key, Keyspace::ATTEMPTS,
+          claim_token, @identity, @processor_id, claimed_at,
+        )
+      end
+      return unless result
+
+      payload, attempt = result
+      record(
+        payload,
+        source.delete_prefix("#{Keyspace::PREFIX}:channel:"),
+        claim_token: claim_token,
+        attempt: Integer(attempt),
+      )
+    end
+
+    def refresh_paused
+      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      return if now < @paused_refresh_at
+
+      @paused = Array(@redis_pool.call("SMEMBERS", Keyspace::PAUSED_CHANNELS))
+      @paused_refresh_at = now + 5
+    end
+  end
+end

data/lib/solid_jobs/{server.rb → conductor.rb}

@@ -5,7 +5,7 @@ require "socket"
 require "timeout"
 
 module SolidJobs
-  class Server
+  class Conductor
     # Extra time granted beyond shutdown_timeout before a component is
     # considered stuck, and how many times :stop is re-sent before giving up.
     STOP_GRACE = 5.0
@@ -25,7 +25,7 @@ module SolidJobs
     end
 
     def start
-      raise Error, "Server is already running" if @started
+      raise Error, "Conductor is already running" if @started
 
       warn_ruby34_multi_ractor
       snapshot = Utilities.shareable_copy(
@@ -38,7 +38,7 @@ module SolidJobs
         local_config = heartbeat = nil
         begin
           started = SolidJobs::StartupBarrier.boot(ready) do
-            local_config = SolidJobs::Config.from_ractor_snapshot(settings)
+            local_config = SolidJobs::Blueprint.from_ractor_snapshot(settings)
             SolidJobs.use_config(local_config)
             heartbeat = SolidJobs::Heartbeat.new(
               local_config,
@@ -119,9 +119,9 @@ module SolidJobs
           local_config = processor = nil
           begin
             started = SolidJobs::StartupBarrier.boot(ready_port) do
-              local_config = SolidJobs::Config.from_ractor_snapshot(settings)
+              local_config = SolidJobs::Blueprint.from_ractor_snapshot(settings)
               SolidJobs.use_config(local_config)
-              processor = SolidJobs::Processor.new(
+              processor = SolidJobs::Engine.new(
                 local_config,
                 identity: settings.fetch(:identity),
                 processor_id: id,
@@ -161,9 +161,9 @@ module SolidJobs
         local_config = scheduler = nil
         begin
           started = SolidJobs::StartupBarrier.boot(ready) do
-            local_config = SolidJobs::Config.from_ractor_snapshot(settings)
+            local_config = SolidJobs::Blueprint.from_ractor_snapshot(settings)
             SolidJobs.use_config(local_config)
-            scheduler = SolidJobs::Scheduler.new(local_config)
+            scheduler = SolidJobs::Timer.new(local_config)
             local_config.redis_pool.call("PING")
           end
           next :aborted unless started
@@ -227,8 +227,8 @@ module SolidJobs
 
     RUBY34_MULTI_RACTOR_WARNING =
       "SolidJobs: Ruby %s can deadlock the whole VM on a Ractor GC barrier " \
-      "under multi-Processor load (see docs/reliability.md). Run multi-Ractor " \
-      "servers on Ruby >= 4.0, or use concurrency: 1 (one process per Processor)."
+      "under multi-Engine load (see docs/reliability.md). Run multi-Ractor " \
+      "servers on Ruby >= 4.0, or use concurrency: 1 (one process per Engine)."
 
     def warn_ruby34_multi_ractor
       return if RUBY_VERSION >= "4" || config.concurrency <= 1
@@ -265,7 +265,7 @@ module SolidJobs
     end
 
     def remote_signal
-      config.redis_pool.call("RPOP", "#{identity}-signals")
+      config.redis_pool.call("RPOP", Keyspace.node_signals(identity))
     end
 
   end

data/lib/solid_jobs/{cli.rb → console.rb}

@@ -4,11 +4,11 @@ require "optparse"
 require "yaml"
 
 module SolidJobs
-  class CLI
+  class Console
     SIGNALS = %w[INT TERM TSTP TTIN INFO].freeze
 
     def initialize
-      @options = {queues: []}
+      @options = {channels: []}
     end
 
     def run(arguments = ARGV)
@@ -16,7 +16,7 @@ module SolidJobs
       load_configuration
       load_application
       apply_options
-      server = Server.new(config: SolidJobs.config).start
+      server = Conductor.new(config: SolidJobs.config).start
       read_io, write_io = IO.pipe
       handlers = install_signal_handlers(write_io)
       wait_for_shutdown(server, read_io)
@@ -38,8 +38,10 @@ module SolidJobs
         options.banner = "Usage: solid-jobs [options]"
         options.on("-r", "--require PATH", "Require an application file") { |value| @options[:require] = value }
         options.on("-C", "--config PATH", "Load YAML configuration") { |value| @options[:config] = value }
-        options.on("-c", "--concurrency N", Integer, "Processor Ractor count") { |value| @options[:concurrency] = value }
-        options.on("-q", "--queue QUEUE", "Queue name or name,weight") { |value| @options[:queues] << queue(value) }
+        options.on("-c", "--concurrency N", Integer, "Engine Ractor count") { |value| @options[:concurrency] = value }
+        options.on("--channel CHANNEL", "Channel name or name,weight") do |value|
+          @options[:channels] << channel(value)
+        end
         options.on("-e", "--environment NAME", "Application environment") { |value| @options[:environment] = value }
         options.on("-t", "--timeout SECONDS", Float, "Graceful shutdown timeout") { |value| @options[:timeout] = value }
         options.on("-v", "--version", "Print version") do
@@ -62,7 +64,9 @@ module SolidJobs
       values = values.fetch(@options[:environment], values) if @options[:environment]
       @options[:concurrency] ||= values["concurrency"]
       @options[:timeout] ||= values["timeout"]
-      @options[:queues] = Array(values["queues"]).map { |value| queue(value) } if @options[:queues].empty?
+      if @options[:channels].empty?
+        @options[:channels] = Array(values["channels"]).map { |value| channel(value) }
+      end
       if (url = values.dig("redis", "url") || values["redis_url"])
         SolidJobs.config.redis = SolidRedis::Config.new(url: url)
       end
@@ -77,7 +81,7 @@ module SolidJobs
     def apply_options
       ENV["RAILS_ENV"] = ENV["RACK_ENV"] = @options[:environment] if @options[:environment]
       SolidJobs.config.concurrency = @options[:concurrency] if @options[:concurrency]
-      SolidJobs.config.queues = @options[:queues] unless @options[:queues].empty?
+      SolidJobs.config.channels = @options[:channels] unless @options[:channels].empty?
       SolidJobs.config.shutdown_timeout = @options[:timeout] if @options[:timeout]
     end
 
@@ -126,10 +130,9 @@ module SolidJobs
       end
     end
 
-    def queue(value)
+    def channel(value)
       name, weight = value.to_s.split(",", 2)
       weight ? [name, Integer(weight)] : name
     end
   end
 end
-

data/lib/solid_jobs/{processor.rb → engine.rb}

@@ -3,11 +3,11 @@
 require "json"
 
 module SolidJobs
-  class Processor
+  class Engine
     def initialize(config, identity: nil, processor_id: nil, heartbeat: nil)
       @config = config
-      @fetcher = Fetch.new(config, identity: identity, processor_id: processor_id)
-      @worker = Worker.new(redis_config: config.redis_config, config: config)
+      @claims = Claim.new(config, identity: identity, processor_id: processor_id)
+      @executor = Executor.new(redis_config: config.redis_config, config: config)
       @identity = identity
       @processor_id = processor_id
       @heartbeat = heartbeat
@@ -29,12 +29,12 @@ module SolidJobs
           next
         end
 
-        unit = retrieve
-        next unless unit
+        claim = retrieve
+        next unless claim
 
         receive_commands(control)
         if @stopping || @quiet
-          requeue(unit)
+          requeue(claim)
           break if @stopping
 
           next
@@ -42,19 +42,19 @@ module SolidJobs
 
         begin
           @state_mutex.synchronize { @busy = true }
-          payload = unit.job
-          register_work(unit, payload)
+          envelope = claim.envelope
+          register_work(claim, envelope)
           begin
-            @worker.perform(payload)
-          rescue Shutdown
-            requeue(unit)
+            @executor.execute(envelope)
+          rescue ExecutionHalt
+            requeue(claim)
             break
           rescue StandardError => error
-            RetryService.call(config: @config, payload: payload, error: error)
-            acknowledge(unit)
+            FailurePolicy.call(config: @config, payload: envelope, error: error)
+            complete(claim)
             failed += 1
           else
-            processed += 1 if acknowledge(unit)
+            processed += 1 if complete(claim)
           end
         ensure
           @state_mutex.synchronize { @busy = false }
@@ -73,55 +73,55 @@ module SolidJobs
 
     def interrupt_current
       runner = @state_mutex.synchronize { @runner if @busy }
-      runner&.raise(Shutdown, "SolidJobs shutdown timeout exceeded")
+      runner&.raise(ExecutionHalt, "SolidJobs shutdown timeout exceeded")
     end
 
     private
 
-    def acknowledge(unit)
-      acknowledged = unit.acknowledge
-      unless acknowledged
+    def complete(claim)
+      completed = claim.complete
+      unless completed
         @config.logger.warn(
-          "Reservation fencing rejected stale ACK: #{unit.reservation_id}",
+          "Claim fencing rejected stale completion: #{claim.claim_token}",
         )
       end
-      acknowledged
+      completed
     rescue SolidRedis::ConnectionError => error
       @fetcher.connection_failed!
       @config.logger.warn(
-        "Redis ACK failed; reserved job will be replayed: #{error.class}: #{error.message}",
+        "Redis completion failed; claimed task will be replayed: #{error.class}: #{error.message}",
       )
       false
     end
 
-    def requeue(unit)
-      requeued = unit.requeue
+    def requeue(claim)
+      requeued = claim.requeue
       unless requeued
         @config.logger.warn(
-          "Reservation fencing rejected stale requeue: #{unit.reservation_id}",
+          "Claim fencing rejected stale requeue: #{claim.claim_token}",
         )
       end
       requeued
     end
 
     def retrieve
-      @fetcher.retrieve
+      @claims.next
     rescue SolidRedis::ConnectionError => error
-      @fetcher.connection_failed!
-      @config.logger.warn("Redis fetch failed, retrying: #{error.class}: #{error.message}")
+      @claims.connection_failed!
+      @config.logger.warn("Redis claim failed, retrying: #{error.class}: #{error.message}")
       sleep 0.1
       nil
     end
 
-    def register_work(unit, payload)
+    def register_work(claim, envelope)
       return unless @identity
 
       work = JSON.generate(
-        "queue" => unit.queue,
-        "payload" => payload,
-        "run_at" => Time.now.to_f,
-        "reservation_id" => unit.reservation_id,
-        "attempt" => unit.attempt,
+        "channel" => claim.channel,
+        "envelope" => envelope,
+        "started_at" => Time.now.to_f,
+        "claim_token" => claim.claim_token,
+        "attempt" => claim.attempt,
       )
       send_heartbeat(:work, @processor_id, work)
     end

data/lib/solid_jobs/errors.rb

@@ -5,5 +5,5 @@ module SolidJobs
   class InvalidJobError < ArgumentError; end
   class InvalidArgumentError < InvalidJobError; end
   class ConfigurationError < Error; end
-  class Shutdown < Error; end
+  class ExecutionHalt < Error; end
 end

data/lib/solid_jobs/executor.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "json"
+
+module SolidJobs
+  class Executor
+    Execution = Data.define(:task, :envelope)
+
+    attr_reader :config
+
+    def initialize(redis_config:, config: nil)
+      @config = config || Blueprint.new(redis: redis_config)
+    end
+
+    def execute(envelope)
+      envelope = JSON.parse(envelope) if envelope.is_a?(String)
+      raise InvalidJobError, "Task envelope must be a Hash" unless envelope.is_a?(Hash)
+
+      task_name = envelope["task"]
+      raise InvalidJobError, "Task must be present" unless task_name.is_a?(String) && !task_name.empty?
+      raise InvalidJobError, "Task arguments must be an Array" unless envelope["arguments"].is_a?(Array)
+      unless envelope["channel"].is_a?(String) && !envelope["channel"].empty?
+        raise InvalidJobError, "Task channel must be present"
+      end
+
+      task = Utilities.constantize(task_name).new
+      task.task_id = envelope["id"] if task.respond_to?(:task_id=)
+      operation = -> { task.execute_task(*envelope.fetch("arguments")) }
+      return operation.call if config.execute_interceptors.empty?
+
+      config.execute_interceptors.call(Execution.new(task, envelope), &operation)
+    end
+  end
+end