solid-jobs 0.4.2 → 0.6.0

data/lib/solid_jobs/blueprint.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require "logger"
+require "securerandom"
+require "socket"
 
 module SolidJobs
   class Blueprint
@@ -11,9 +13,10 @@ module SolidJobs
 
     attr_accessor :concurrency, :discarded_limit, :discarded_retention, :logger,
       :on_complex_arguments, :poll_interval_average, :shutdown_timeout,
-      :reliable_fetch, :retry_base_delay, :retry_max_delay
+      :reliable_fetch, :retry_base_delay, :retry_max_delay, :instrumenter,
+      :backend, :url
     attr_reader :default_task_options, :error_handlers, :execute_interceptors,
-      :publish_interceptors, :redis_config
+      :publish_interceptors, :redis_config, :identity
 
     def initialize(redis: nil, concurrency: 5, channels: ["default"])
       @redis_config = redis || SolidRedis.config(url: ENV.fetch("REDIS_URL", "redis://127.0.0.1:6379/0"))
@@ -34,7 +37,103 @@ module SolidJobs
       @discarded_limit = 10_000
       @discarded_retention = 180 * 24 * 60 * 60
       @logger = Logger.new($stdout)
+      @identity = "#{Socket.gethostname}:#{::Process.pid}:#{SecureRandom.hex(6)}".freeze
+      @instrumenter = NullInstrumenter
       @redis_pool = nil
+      @backend = :redis
+      @url = nil
+      @backends = {}
+      @execution_mode = :ractor
+      @channel_backends = {}
+      @backend_adapters = {}
+    end
+
+    REDIS_ADAPTERS = %i[redis solid_redis].freeze
+
+    EXECUTION_MODES = %i[ractor thread].freeze
+
+    attr_reader :backends, :channel_backends, :execution_mode
+
+    # How non-Redis backend workers run: :ractor (parallel, one client per
+    # Ractor) or :thread (GVL-bound). Redis channels always use Ractors.
+    def execution_mode=(mode)
+      mode = mode.to_sym
+      raise ArgumentError, "execution_mode must be :ractor or :thread" unless EXECUTION_MODES.include?(mode)
+
+      @execution_mode = mode
+    end
+
+    # Named backend definitions: { name => { adapter:, url:, ... } }.
+    def backends=(definitions)
+      close_adapters
+      @backends = definitions.to_h { |name, spec| [name.to_sym, spec.transform_keys(&:to_sym)] }
+    end
+
+    # Per-channel overrides of the default backend: { "bulk" => :solid_engine }.
+    def channel_backends=(mapping)
+      @channel_backends = mapping.to_h { |channel, name| [channel.to_s, name.to_sym] }
+    end
+
+    def backend_name_for(channel)
+      (channel_backends[channel.to_s] || backend).to_sym
+    end
+
+    def redis_backend?(name)
+      REDIS_ADAPTERS.include?((backends.dig(name.to_sym, :adapter) || name).to_sym)
+    end
+
+    # True when any channel can leave the default Redis path.
+    def routed?
+      !redis_backend?(backend) || !channel_backends.empty?
+    end
+
+    def backend_adapter_for(channel)
+      backend_adapter(backend_name_for(channel))
+    end
+
+    def backend_adapter(name)
+      @backend_adapters[name.to_sym] ||= build_backend(name)
+    end
+
+    # Channels served by Redis workers, and those served by other backends.
+    def redis_channel_entries
+      channel_entries.select { |channel, _| redis_backend?(backend_name_for(channel)) }
+    end
+
+    def engine_channel_entries
+      channel_entries - redis_channel_entries
+    end
+
+    # Adapter for :solid_engine (nil when it is not configured).
+    def engine_backend
+      name = (backends.keys + [backend, *channel_backends.values]).uniq.find do |candidate|
+        !redis_backend?(candidate)
+      end
+      backend_adapter(name) if name
+    end
+
+    # A fresh, unshared adapter instance (one connection per worker thread).
+    def build_backend(name)
+      name = name.to_sym
+      spec = backends[name] || {}
+      adapter = (spec[:adapter] || name).to_sym
+      case adapter
+      when :redis, :solid_redis
+        return Backends::Redis.new(config: self) unless spec[:url]
+
+        derived = Blueprint.new(redis: SolidRedis.config(url: spec[:url]), concurrency: concurrency)
+        derived.logger = logger
+        derived.instrumenter = instrumenter
+        Backends::Redis.new(config: derived, owned: true)
+      when :solid_engine
+        options = spec.except(:adapter)
+        options[:url] ||= url || ENV.fetch("SOLID_ENGINE_URL") do
+          raise ConfigurationError, "SolidEngine backend #{name.inspect} needs a url"
+        end
+        Backends::SolidEngine.new(**options)
+      else
+        raise ConfigurationError, "Unknown backend adapter #{adapter.inspect} for #{name.inspect}"
+      end
     end
 
     def redis=(configuration)
@@ -107,6 +206,7 @@ module SolidJobs
     def close
       @redis_pool&.close
       @redis_pool = nil
+      close_adapters
       self
     end
 
@@ -115,10 +215,14 @@ module SolidJobs
     end
 
     def ractor_snapshot
+      unless Ractor.shareable?(instrumenter)
+        raise ArgumentError, "SolidJobs instrumenter must be Ractor-shareable"
+      end
+
       Utilities.shareable_copy(
         redis_config: redis_config,
         concurrency: concurrency,
-        channels: channel_entries,
+        channels: redis_channel_entries,
         channel_order: channel_order,
         reliable_fetch: reliable_fetch,
         default_task_options: default_task_options,
@@ -131,6 +235,13 @@ module SolidJobs
         retry_max_delay: retry_max_delay,
         publish_interceptors: publish_interceptors.export,
         execute_interceptors: execute_interceptors.export,
+        identity: identity,
+        instrumenter: instrumenter,
+        backend: backend,
+        url: url,
+        backends: backends,
+        channel_backends: channel_backends,
+        execution_mode: execution_mode,
       )
     end
 
@@ -152,11 +263,23 @@ module SolidJobs
       config.retry_max_delay = snapshot.fetch(:retry_max_delay)
       config.publish_interceptors.import(snapshot.fetch(:publish_interceptors))
       config.execute_interceptors.import(snapshot.fetch(:execute_interceptors))
+      config.instance_variable_set(:@identity, snapshot.fetch(:identity))
+      config.instrumenter = snapshot.fetch(:instrumenter)
+      config.backend = snapshot.fetch(:backend)
+      config.url = snapshot.fetch(:url)
+      config.backends = snapshot.fetch(:backends)
+      config.channel_backends = snapshot.fetch(:channel_backends)
+      config.execution_mode = snapshot.fetch(:execution_mode)
       config
     end
 
     private
 
+    def close_adapters
+      @backend_adapters.each_value(&:close)
+      @backend_adapters = {}
+    end
+
     def normalize_channels(values)
       Array(values).map do |value|
         name, weight = value.is_a?(Array) ? value : [value, 1]

data/lib/solid_jobs/claim.rb

@@ -12,9 +12,14 @@ module SolidJobs
       if not payload then
         return nil
       end
+      local job = cjson.decode(payload)
+      local publication = job["#{PublicationBarrier::FIELD}"]
+      if publication and redis.call("exists", ARGV[5] .. publication) == 1 then
+        redis.call("rpush", KEYS[1], payload)
+        return nil
+      end
 
       redis.call("lpush", KEYS[2], payload)
-      local job = cjson.decode(payload)
       local attempt = redis.call("hincrby", KEYS[4], job["id"], 1)
       local metadata = {
         task_id = job["id"],
@@ -28,6 +33,19 @@ module SolidJobs
       redis.call("hset", KEYS[3], ARGV[3], cjson.encode(metadata))
       return {payload, attempt}
     LUA
+    POP_READY = <<~LUA.freeze
+      local payload = redis.call("rpop", KEYS[1])
+      if not payload then
+        return nil
+      end
+      local job = cjson.decode(payload)
+      local publication = job["#{PublicationBarrier::FIELD}"]
+      if publication and redis.call("exists", ARGV[1] .. publication) == 1 then
+        redis.call("rpush", KEYS[1], payload)
+        return nil
+      end
+      return payload
+    LUA
     REGISTER = <<~LUA.freeze
       local attempt = redis.call("hincrby", KEYS[2], ARGV[1], 1)
       local metadata = cjson.decode(ARGV[3])
@@ -66,6 +84,7 @@ module SolidJobs
       return removed
     LUA
     RESERVE_SHA = Digest::SHA1.hexdigest(RESERVE).freeze
+    POP_READY_SHA = Digest::SHA1.hexdigest(POP_READY).freeze
     REGISTER_SHA = Digest::SHA1.hexdigest(REGISTER).freeze
     ACK_SHA = Digest::SHA1.hexdigest(ACK).freeze
     REQUEUE_SHA = Digest::SHA1.hexdigest(REQUEUE).freeze
@@ -131,6 +150,7 @@ module SolidJobs
     end
 
     def initialize(config, identity: nil, processor_id: nil)
+      @config = config
       @redis_pool = config.redis_pool
       @channel_order = config.channel_order
       @reliable = config.reliable_fetch
@@ -167,12 +187,7 @@ module SolidJobs
       if @claimed_key
         claim_reliable(channels)
       else
-        channels = channels.shuffle if @channel_order == :shuffle
-        result = @redis_pool.blocking_call(TIMEOUT, "BRPOP", *channels, TIMEOUT)
-        result && record(
-          result.fetch(1),
-          result.fetch(0).delete_prefix("#{Keyspace::PREFIX}:channel:"),
-        )
+        claim_unreliable(channels)
       end
     end
 
@@ -183,10 +198,10 @@ module SolidJobs
     private
 
     def record(payload, channel = nil, claim_token: nil, attempt: nil)
-      envelope = JSON.parse(payload)
+      envelope = PublicationBarrier.strip(JSON.parse(payload))
       channel ||= envelope.fetch("channel")
       claim_token, attempt = register(envelope, channel) unless claim_token
-      ClaimRecord.new(
+      claim = ClaimRecord.new(
         channel: channel,
         payload: payload,
         envelope: envelope,
@@ -197,13 +212,22 @@ module SolidJobs
         claim_token: claim_token,
         attempt: attempt,
       )
+      Instrumentation.emit(
+        @config,
+        :reserved,
+        envelope,
+        reservation_id: claim_token,
+        attempt: attempt,
+        **Instrumentation.worker_attributes(@processor_id),
+      )
+      claim
     end
 
     def existing_claim(payload)
       raw = @redis_pool.call("HGET", @claims_key, @processor_id)
       if raw
         metadata = JSON.parse(raw)
-        envelope = JSON.parse(payload)
+        envelope = PublicationBarrier.strip(JSON.parse(payload))
         if metadata["task_id"] == envelope["id"]
           return ClaimRecord.new(
             channel: envelope.fetch("channel"),
@@ -260,22 +284,33 @@ module SolidJobs
     end
 
     def claim_reliable(channels)
-      if @channel_order == :priority
-        channels.each do |source|
-          claim = claim_now(source)
-          return claim if claim
-        end
-        source = channels.first
-      else
-        source = next_channel(channels)
+      ordered_sources(channels).each do |source|
         claim = claim_now(source)
         return claim if claim
       end
-      payload = @redis_pool.blocking_call(
-        TIMEOUT,
-        "BLMOVE", source, @claimed_key, "RIGHT", "LEFT", TIMEOUT,
-      )
-      payload && record(payload, @channel_names.fetch(source))
+      sleep(TIMEOUT)
+      nil
+    end
+
+    def claim_unreliable(channels)
+      ordered_sources(channels).each do |source|
+        payload = pop_ready(source)
+        return record(payload, @channel_names.fetch(source)) if payload
+      end
+      sleep(TIMEOUT)
+      nil
+    end
+
+    def ordered_sources(channels)
+      case @channel_order
+      when :priority
+        channels.uniq
+      when :shuffle
+        channels.shuffle.uniq
+      else
+        preferred = next_channel(channels)
+        ([preferred] + channels).uniq
+      end
     end
 
     def claim_now(source)
@@ -285,7 +320,7 @@ module SolidJobs
         @redis_pool.call(
           "EVALSHA", RESERVE_SHA, 4,
           source, @claimed_key, @claims_key, Keyspace::ATTEMPTS,
-          claim_token, @identity, @processor_id, claimed_at,
+          claim_token, @identity, @processor_id, claimed_at, Keyspace::PUBLICATION_PREFIX,
         )
       rescue SolidRedis::CommandError => error
         raise unless error.message.include?("NOSCRIPT")
@@ -293,7 +328,7 @@ module SolidJobs
         @redis_pool.call(
           "EVAL", RESERVE, 4,
           source, @claimed_key, @claims_key, Keyspace::ATTEMPTS,
-          claim_token, @identity, @processor_id, claimed_at,
+          claim_token, @identity, @processor_id, claimed_at, Keyspace::PUBLICATION_PREFIX,
         )
       end
       return unless result
@@ -307,6 +342,20 @@ module SolidJobs
       )
     end
 
+    def pop_ready(source)
+      @redis_pool.call(
+        "EVALSHA", POP_READY_SHA, 1,
+        source, Keyspace::PUBLICATION_PREFIX,
+      )
+    rescue SolidRedis::CommandError => error
+      raise unless error.message.include?("NOSCRIPT")
+
+      @redis_pool.call(
+        "EVAL", POP_READY, 1,
+        source, Keyspace::PUBLICATION_PREFIX,
+      )
+    end
+
     def refresh_paused
       now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       return if now < @paused_refresh_at

data/lib/solid_jobs/conductor.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "securerandom"
-require "socket"
 require "timeout"
 
 module SolidJobs
@@ -20,16 +18,23 @@ module SolidJobs
       @scheduler = nil
       @heartbeat = nil
       @started = false
-      @identity = "#{Socket.gethostname}:#{::Process.pid}:#{SecureRandom.hex(6)}".freeze
+      @backend_threads = []
+      @backend_workers = []
+      @backend_ractors = []
+      @backend_stop = false
+      @backend_paused = false
+      @identity = config.identity
       @started_at = Time.now.to_f
     end
 
     def start
       raise Error, "Conductor is already running" if @started
 
+      return start_backend_only if config.redis_channel_entries.empty?
+
       warn_ruby34_multi_ractor
       snapshot = Utilities.shareable_copy(
-        config.ractor_snapshot.merge(identity: identity, started_at: @started_at),
+        config.ractor_snapshot.merge(started_at: @started_at),
       )
       Recovery.call(config: config)
       barrier = StartupBarrier.new(config.concurrency + 2)
@@ -60,6 +65,7 @@ module SolidJobs
           quiet = false
           stopping = false
           work = {}
+          redis = {}
           processed = failed = 0
           next_beat = 0.0
           until stopping
@@ -68,6 +74,7 @@ module SolidJobs
                 case (message = messages.pop(true))
                 when :quiet
                   quiet = true
+                  heartbeat.observe(quiet: true, work: work, redis: redis)
                 when :stop
                   stopping = true
                 else
@@ -75,11 +82,21 @@ module SolidJobs
                   case event
                   when :work
                     work[arguments[0].to_s] = arguments[1]
+                    heartbeat.observe_ractor(
+                      arguments[0],
+                      arguments[1],
+                      job_id: arguments[2],
+                      quiet: quiet,
+                    )
                   when :done
                     work.delete(arguments[0].to_s)
+                    heartbeat.observe_ractor(arguments[0], nil, quiet: quiet)
                   when :stats
                     processed += arguments[0]
                     failed += arguments[1]
+                  when :redis
+                    redis[arguments[0].to_s] = arguments[1]
+                    heartbeat.observe_redis(arguments[0], arguments[1])
                   end
                 end
               end
@@ -89,7 +106,13 @@ module SolidJobs
             now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
             if now >= next_beat
               begin
-                heartbeat.beat(quiet: quiet, work: work, processed: processed, failed: failed)
+                heartbeat.beat(
+                  quiet: quiet,
+                  work: work,
+                  processed: processed,
+                  failed: failed,
+                  redis: redis,
+                )
                 processed = failed = 0
               rescue SolidRedis::ConnectionError => error
                 local_config.logger.warn("Redis heartbeat failed, retrying: #{error.class}: #{error.message}")
@@ -99,7 +122,15 @@ module SolidJobs
             sleep 0.05 unless stopping
           end
           begin
-            heartbeat.beat(quiet: quiet, work: work, processed: processed, failed: failed)
+            heartbeat.observe(quiet: quiet, work: work, redis: redis, phase: :stopping)
+            heartbeat.beat(
+              quiet: quiet,
+              work: work,
+              processed: processed,
+              failed: failed,
+              redis: redis,
+              phase: :stopped,
+            )
             heartbeat.cleanup
           rescue SolidRedis::ConnectionError => error
             local_config.logger.warn("Redis heartbeat cleanup failed: #{error.class}: #{error.message}")
@@ -181,6 +212,7 @@ module SolidJobs
       end
       barrier.register!(@scheduler, scheduler_ready)
       barrier.run!
+      start_backend_workers
       @started = true
       config.fire(:startup, self)
       self
@@ -194,6 +226,8 @@ module SolidJobs
     def quiet
       return self unless @started
 
+      @backend_paused = true
+      @backend_ractors.each { |ractor| ractor.send(:quiet) }
       @ractors.each { |ractor| ractor.send(:quiet) }
       @heartbeat&.send(:quiet)
       config.fire(:quiet, self)
@@ -203,6 +237,7 @@ module SolidJobs
     def stop
       return [] unless @started
 
+      @backend_stop = true
       @ractors.each { |ractor| ractor.send(:stop) }
       @scheduler&.send(:stop)
       results = @ractors.each_with_index.map do |ractor, index|
@@ -211,6 +246,7 @@ module SolidJobs
       await_termination(@scheduler, "scheduler") if @scheduler
       @heartbeat&.send(:stop)
       await_termination(@heartbeat, "heartbeat") if @heartbeat
+      results.concat(stop_backend_workers)
       @ractors = []
       @scheduler = nil
       @heartbeat = nil
@@ -225,6 +261,109 @@ module SolidJobs
 
     private
 
+    # Channels routed to a non-Redis backend run in Ractors (one client each)
+    # or, with execution_mode = :thread, in threads of this process.
+    def start_backend_workers
+      channels = config.engine_channel_entries
+      return if channels.empty?
+
+      @backend_stop = false
+      @backend_paused = false
+      groups = channels.group_by { |channel, _| config.backend_name_for(channel) }
+      if config.execution_mode == :ractor
+        start_backend_ractors(groups)
+      else
+        start_backend_threads(groups)
+      end
+    end
+
+    def start_backend_threads(groups)
+      groups.each do |name, scoped|
+        config.concurrency.times do |index|
+          worker = Backends::Worker.new(config: config, backend_name: name, channels: scoped, processor_id: index)
+          @backend_workers << worker
+          @backend_threads << Thread.new do
+            worker.run(stop: -> { @backend_stop }, paused: -> { @backend_paused })
+          end
+        end
+      end
+    end
+
+    def start_backend_ractors(groups)
+      snapshot = Utilities.shareable_copy(config.ractor_snapshot.merge(started_at: @started_at))
+      barrier = StartupBarrier.new(groups.size * config.concurrency)
+      groups.each do |name, scoped|
+        shared_channels = Utilities.shareable_copy(scoped)
+        config.concurrency.times do |index|
+          ready = barrier.channel
+          ractor = Ractor.new(snapshot, name, shared_channels, index, ready) do |settings, backend, entries, id, ready_port|
+            local_config = worker = nil
+            begin
+              started = SolidJobs::StartupBarrier.boot(ready_port) do
+                local_config = SolidJobs::Blueprint.from_ractor_snapshot(settings)
+                SolidJobs.use_config(local_config)
+                worker = SolidJobs::Backends::Worker.new(
+                  config: local_config, backend_name: backend, channels: entries, processor_id: id,
+                )
+              end
+              next :aborted unless started
+
+              control = ::Queue.new
+              Thread.new do
+                loop do
+                  command = Ractor.receive
+                  control << command
+                  break if command == :stop
+                end
+              end
+              state = {stop: false, paused: false}
+              drain = lambda do
+                until control.empty?
+                  case control.pop(true)
+                  when :stop then state[:stop] = true
+                  when :quiet then state[:paused] = true
+                  end
+                end
+              end
+              worker.run(stop: -> { drain.call; state[:stop] }, paused: -> { state[:paused] })
+              {processed: worker.processed, failed: worker.failed}
+            ensure
+              local_config&.close
+            end
+          end
+          barrier.register!(ractor, ready)
+          @backend_ractors << ractor
+        end
+      end
+      barrier.run!
+    rescue StandardError
+      barrier&.abort!
+      @backend_ractors = []
+      raise
+    end
+
+    # Without Redis channels no Ractor, heartbeat or Redis scheduler is needed.
+    def start_backend_only
+      start_backend_workers
+      @started = true
+      config.fire(:startup, self)
+      self
+    end
+
+    def stop_backend_workers
+      @backend_ractors.each { |ractor| ractor.send(:stop) }
+      ractor_results = @backend_ractors.each_with_index.map do |ractor, index|
+        await_termination(ractor, "backend worker #{index}")
+      end
+      @backend_ractors = []
+      deadline = config.shutdown_timeout + @stop_grace
+      @backend_threads.each { |thread| thread.join(deadline) || thread.kill }
+      results = @backend_workers.map { |worker| {processed: worker.processed, failed: worker.failed} }
+      @backend_threads = []
+      @backend_workers = []
+      ractor_results.compact + results
+    end
+
     RUBY34_MULTI_RACTOR_WARNING =
       "SolidJobs: Ruby %s can deadlock the whole VM on a Ractor GC barrier " \
       "under multi-Engine load (see docs/reliability.md). Run multi-Ractor " \