solid-jobs 0.5.0 → 0.6.0

data/lib/solid_jobs/blueprint.rb

@@ -13,7 +13,8 @@ module SolidJobs
 
     attr_accessor :concurrency, :discarded_limit, :discarded_retention, :logger,
       :on_complex_arguments, :poll_interval_average, :shutdown_timeout,
-      :reliable_fetch, :retry_base_delay, :retry_max_delay, :instrumenter
+      :reliable_fetch, :retry_base_delay, :retry_max_delay, :instrumenter,
+      :backend, :url
     attr_reader :default_task_options, :error_handlers, :execute_interceptors,
       :publish_interceptors, :redis_config, :identity
 
@@ -39,6 +40,100 @@ module SolidJobs
       @identity = "#{Socket.gethostname}:#{::Process.pid}:#{SecureRandom.hex(6)}".freeze
       @instrumenter = NullInstrumenter
       @redis_pool = nil
+      @backend = :redis
+      @url = nil
+      @backends = {}
+      @execution_mode = :ractor
+      @channel_backends = {}
+      @backend_adapters = {}
+    end
+
+    REDIS_ADAPTERS = %i[redis solid_redis].freeze
+
+    EXECUTION_MODES = %i[ractor thread].freeze
+
+    attr_reader :backends, :channel_backends, :execution_mode
+
+    # How non-Redis backend workers run: :ractor (parallel, one client per
+    # Ractor) or :thread (GVL-bound). Redis channels always use Ractors.
+    def execution_mode=(mode)
+      mode = mode.to_sym
+      raise ArgumentError, "execution_mode must be :ractor or :thread" unless EXECUTION_MODES.include?(mode)
+
+      @execution_mode = mode
+    end
+
+    # Named backend definitions: { name => { adapter:, url:, ... } }.
+    def backends=(definitions)
+      close_adapters
+      @backends = definitions.to_h { |name, spec| [name.to_sym, spec.transform_keys(&:to_sym)] }
+    end
+
+    # Per-channel overrides of the default backend: { "bulk" => :solid_engine }.
+    def channel_backends=(mapping)
+      @channel_backends = mapping.to_h { |channel, name| [channel.to_s, name.to_sym] }
+    end
+
+    def backend_name_for(channel)
+      (channel_backends[channel.to_s] || backend).to_sym
+    end
+
+    def redis_backend?(name)
+      REDIS_ADAPTERS.include?((backends.dig(name.to_sym, :adapter) || name).to_sym)
+    end
+
+    # True when any channel can leave the default Redis path.
+    def routed?
+      !redis_backend?(backend) || !channel_backends.empty?
+    end
+
+    def backend_adapter_for(channel)
+      backend_adapter(backend_name_for(channel))
+    end
+
+    def backend_adapter(name)
+      @backend_adapters[name.to_sym] ||= build_backend(name)
+    end
+
+    # Channels served by Redis workers, and those served by other backends.
+    def redis_channel_entries
+      channel_entries.select { |channel, _| redis_backend?(backend_name_for(channel)) }
+    end
+
+    def engine_channel_entries
+      channel_entries - redis_channel_entries
+    end
+
+    # Adapter for :solid_engine (nil when it is not configured).
+    def engine_backend
+      name = (backends.keys + [backend, *channel_backends.values]).uniq.find do |candidate|
+        !redis_backend?(candidate)
+      end
+      backend_adapter(name) if name
+    end
+
+    # A fresh, unshared adapter instance (one connection per worker thread).
+    def build_backend(name)
+      name = name.to_sym
+      spec = backends[name] || {}
+      adapter = (spec[:adapter] || name).to_sym
+      case adapter
+      when :redis, :solid_redis
+        return Backends::Redis.new(config: self) unless spec[:url]
+
+        derived = Blueprint.new(redis: SolidRedis.config(url: spec[:url]), concurrency: concurrency)
+        derived.logger = logger
+        derived.instrumenter = instrumenter
+        Backends::Redis.new(config: derived, owned: true)
+      when :solid_engine
+        options = spec.except(:adapter)
+        options[:url] ||= url || ENV.fetch("SOLID_ENGINE_URL") do
+          raise ConfigurationError, "SolidEngine backend #{name.inspect} needs a url"
+        end
+        Backends::SolidEngine.new(**options)
+      else
+        raise ConfigurationError, "Unknown backend adapter #{adapter.inspect} for #{name.inspect}"
+      end
     end
 
     def redis=(configuration)
@@ -111,6 +206,7 @@ module SolidJobs
     def close
       @redis_pool&.close
       @redis_pool = nil
+      close_adapters
       self
     end
 
@@ -126,7 +222,7 @@ module SolidJobs
       Utilities.shareable_copy(
         redis_config: redis_config,
         concurrency: concurrency,
-        channels: channel_entries,
+        channels: redis_channel_entries,
         channel_order: channel_order,
         reliable_fetch: reliable_fetch,
         default_task_options: default_task_options,
@@ -141,6 +237,11 @@ module SolidJobs
         execute_interceptors: execute_interceptors.export,
         identity: identity,
         instrumenter: instrumenter,
+        backend: backend,
+        url: url,
+        backends: backends,
+        channel_backends: channel_backends,
+        execution_mode: execution_mode,
       )
     end
 
@@ -164,11 +265,21 @@ module SolidJobs
       config.execute_interceptors.import(snapshot.fetch(:execute_interceptors))
       config.instance_variable_set(:@identity, snapshot.fetch(:identity))
       config.instrumenter = snapshot.fetch(:instrumenter)
+      config.backend = snapshot.fetch(:backend)
+      config.url = snapshot.fetch(:url)
+      config.backends = snapshot.fetch(:backends)
+      config.channel_backends = snapshot.fetch(:channel_backends)
+      config.execution_mode = snapshot.fetch(:execution_mode)
       config
     end
 
     private
 
+    def close_adapters
+      @backend_adapters.each_value(&:close)
+      @backend_adapters = {}
+    end
+
     def normalize_channels(values)
       Array(values).map do |value|
         name, weight = value.is_a?(Array) ? value : [value, 1]

data/lib/solid_jobs/conductor.rb

@@ -18,6 +18,11 @@ module SolidJobs
       @scheduler = nil
       @heartbeat = nil
       @started = false
+      @backend_threads = []
+      @backend_workers = []
+      @backend_ractors = []
+      @backend_stop = false
+      @backend_paused = false
       @identity = config.identity
       @started_at = Time.now.to_f
     end
@@ -25,6 +30,8 @@ module SolidJobs
     def start
       raise Error, "Conductor is already running" if @started
 
+      return start_backend_only if config.redis_channel_entries.empty?
+
       warn_ruby34_multi_ractor
       snapshot = Utilities.shareable_copy(
         config.ractor_snapshot.merge(started_at: @started_at),
@@ -205,6 +212,7 @@ module SolidJobs
       end
       barrier.register!(@scheduler, scheduler_ready)
       barrier.run!
+      start_backend_workers
       @started = true
       config.fire(:startup, self)
       self
@@ -218,6 +226,8 @@ module SolidJobs
     def quiet
       return self unless @started
 
+      @backend_paused = true
+      @backend_ractors.each { |ractor| ractor.send(:quiet) }
       @ractors.each { |ractor| ractor.send(:quiet) }
       @heartbeat&.send(:quiet)
       config.fire(:quiet, self)
@@ -227,6 +237,7 @@ module SolidJobs
     def stop
       return [] unless @started
 
+      @backend_stop = true
       @ractors.each { |ractor| ractor.send(:stop) }
       @scheduler&.send(:stop)
       results = @ractors.each_with_index.map do |ractor, index|
@@ -235,6 +246,7 @@ module SolidJobs
       await_termination(@scheduler, "scheduler") if @scheduler
       @heartbeat&.send(:stop)
       await_termination(@heartbeat, "heartbeat") if @heartbeat
+      results.concat(stop_backend_workers)
       @ractors = []
       @scheduler = nil
       @heartbeat = nil
@@ -249,6 +261,109 @@ module SolidJobs
 
     private
 
+    # Channels routed to a non-Redis backend run in Ractors (one client each)
+    # or, with execution_mode = :thread, in threads of this process.
+    def start_backend_workers
+      channels = config.engine_channel_entries
+      return if channels.empty?
+
+      @backend_stop = false
+      @backend_paused = false
+      groups = channels.group_by { |channel, _| config.backend_name_for(channel) }
+      if config.execution_mode == :ractor
+        start_backend_ractors(groups)
+      else
+        start_backend_threads(groups)
+      end
+    end
+
+    def start_backend_threads(groups)
+      groups.each do |name, scoped|
+        config.concurrency.times do |index|
+          worker = Backends::Worker.new(config: config, backend_name: name, channels: scoped, processor_id: index)
+          @backend_workers << worker
+          @backend_threads << Thread.new do
+            worker.run(stop: -> { @backend_stop }, paused: -> { @backend_paused })
+          end
+        end
+      end
+    end
+
+    def start_backend_ractors(groups)
+      snapshot = Utilities.shareable_copy(config.ractor_snapshot.merge(started_at: @started_at))
+      barrier = StartupBarrier.new(groups.size * config.concurrency)
+      groups.each do |name, scoped|
+        shared_channels = Utilities.shareable_copy(scoped)
+        config.concurrency.times do |index|
+          ready = barrier.channel
+          ractor = Ractor.new(snapshot, name, shared_channels, index, ready) do |settings, backend, entries, id, ready_port|
+            local_config = worker = nil
+            begin
+              started = SolidJobs::StartupBarrier.boot(ready_port) do
+                local_config = SolidJobs::Blueprint.from_ractor_snapshot(settings)
+                SolidJobs.use_config(local_config)
+                worker = SolidJobs::Backends::Worker.new(
+                  config: local_config, backend_name: backend, channels: entries, processor_id: id,
+                )
+              end
+              next :aborted unless started
+
+              control = ::Queue.new
+              Thread.new do
+                loop do
+                  command = Ractor.receive
+                  control << command
+                  break if command == :stop
+                end
+              end
+              state = {stop: false, paused: false}
+              drain = lambda do
+                until control.empty?
+                  case control.pop(true)
+                  when :stop then state[:stop] = true
+                  when :quiet then state[:paused] = true
+                  end
+                end
+              end
+              worker.run(stop: -> { drain.call; state[:stop] }, paused: -> { state[:paused] })
+              {processed: worker.processed, failed: worker.failed}
+            ensure
+              local_config&.close
+            end
+          end
+          barrier.register!(ractor, ready)
+          @backend_ractors << ractor
+        end
+      end
+      barrier.run!
+    rescue StandardError
+      barrier&.abort!
+      @backend_ractors = []
+      raise
+    end
+
+    # Without Redis channels no Ractor, heartbeat or Redis scheduler is needed.
+    def start_backend_only
+      start_backend_workers
+      @started = true
+      config.fire(:startup, self)
+      self
+    end
+
+    def stop_backend_workers
+      @backend_ractors.each { |ractor| ractor.send(:stop) }
+      ractor_results = @backend_ractors.each_with_index.map do |ractor, index|
+        await_termination(ractor, "backend worker #{index}")
+      end
+      @backend_ractors = []
+      deadline = config.shutdown_timeout + @stop_grace
+      @backend_threads.each { |thread| thread.join(deadline) || thread.kill }
+      results = @backend_workers.map { |worker| {processed: worker.processed, failed: worker.failed} }
+      @backend_threads = []
+      @backend_workers = []
+      ractor_results.compact + results
+    end
+
     RUBY34_MULTI_RACTOR_WARNING =
       "SolidJobs: Ruby %s can deadlock the whole VM on a Ractor GC barrier " \
       "under multi-Engine load (see docs/reliability.md). Run multi-Ractor " \

data/lib/solid_jobs/publisher.rb

@@ -103,6 +103,22 @@ module SolidJobs
         end
       end
 
+      return persist_redis(envelopes) unless @config.routed?
+
+      envelopes.group_by { |envelope| @config.backend_name_for(envelope.fetch("channel")) }
+        .flat_map do |name, group|
+          adapter = @config.backend_adapter(name)
+          if adapter.is_a?(Backends::Redis) && adapter.config.equal?(@config)
+            persist_redis(group)
+          else
+            adapter.enqueue_many(group.each { |envelope| envelope["queued_ms"] = Utilities.realtime_milliseconds })
+          end
+        end
+    end
+
+    def persist_redis(envelopes)
+      return [] if envelopes.empty?
+
       planned, ready = envelopes.partition { |envelope| envelope["run_at"] }
       barriers = []
       @config.redis_pool.pipelined do |pipeline|

data/lib/solid_jobs/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SolidJobs
-  VERSION = "0.5.0"
+  VERSION = "0.6.0"
 end

data/lib/solid_jobs.rb

@@ -13,7 +13,10 @@ require_relative "solid_jobs/startup_barrier"
 require_relative "solid_jobs/interceptor_registry"
 require_relative "solid_jobs/instrumentation"
 require_relative "solid_jobs/publication_barrier"
+require_relative "solid_jobs/backends/base"
+require_relative "solid_jobs/backends/solid_engine"
 require_relative "solid_jobs/blueprint"
+require_relative "solid_jobs/backends/redis"
 require_relative "solid_jobs/publisher"
 require_relative "solid_jobs/lab"
 require_relative "solid_jobs/task"
@@ -26,6 +29,7 @@ require_relative "solid_jobs/observations"
 require_relative "solid_jobs/heartbeat"
 require_relative "solid_jobs/recovery"
 require_relative "solid_jobs/integrity_check"
+require_relative "solid_jobs/backends/worker"
 require_relative "solid_jobs/conductor"
 require_relative "solid_jobs/catalog"
 require_relative "solid_jobs/rails_adapter" if defined?(::ActiveJob::Base)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: solid-jobs
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Nicolas Vandenbogaerde
@@ -119,6 +119,10 @@ files:
 - exe/solid-jobs
 - lib/active_job/queue_adapters/solid_jobs_adapter.rb
 - lib/solid_jobs.rb
+- lib/solid_jobs/backends/base.rb
+- lib/solid_jobs/backends/redis.rb
+- lib/solid_jobs/backends/solid_engine.rb
+- lib/solid_jobs/backends/worker.rb
 - lib/solid_jobs/blueprint.rb
 - lib/solid_jobs/catalog.rb
 - lib/solid_jobs/claim.rb