solid-jobs 0.1.0

data/lib/solid_jobs/rails.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require_relative "active_job"
+
+if defined?(Rails::Railtie)
+  module SolidJobs
+    class Railtie < Rails::Railtie
+      initializer "solid_jobs.active_job" do
+        ActiveSupport.on_load(:active_job) do
+          require "active_job/queue_adapters/solid_jobs_adapter"
+        end
+      end
+    end
+  end
+end
+

data/lib/solid_jobs/recovery.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "json"
+require "socket"
+
+module SolidJobs
+  class Recovery < Service::Base
+    RESERVED_PATTERN = "*:reserved:*"
+    RESTORE_ONE = <<~LUA.freeze
+      local payload = redis.call("rpop", KEYS[1])
+      if payload then
+        local job = cjson.decode(payload)
+        redis.call("rpush", "queue:" .. job["queue"], payload)
+        return payload
+      end
+    LUA
+
+    def call
+      recovered = 0
+      scan_keys.each do |key|
+        identity = key.split(":reserved:", 2).first
+        next if alive?(identity)
+
+        while @config.redis_pool.call("EVAL", RESTORE_ONE, 1, key)
+          recovered += 1
+        end
+        @config.redis_pool.call("DEL", key)
+        @config.redis_pool.call("DEL", "#{identity}:reservations")
+      end
+      recovered
+    end
+
+    private
+
+    def alive?(identity)
+      hostname, pid = identity.split(":", 3)
+      return false if hostname == Socket.gethostname && !process_alive?(Integer(pid))
+
+      beat = @config.redis_pool.call("HGET", identity, "beat")
+      beat && Time.now.to_f - Float(beat) < Heartbeat::TTL
+    end
+
+    def process_alive?(pid)
+      ::Process.kill(0, pid)
+      true
+    rescue Errno::ESRCH
+      false
+    rescue Errno::EPERM
+      true
+    end
+
+    def scan_keys
+      cursor = "0"
+      keys = []
+      loop do
+        cursor, found = @config.redis_pool.call(
+          "SCAN", cursor, "MATCH", RESERVED_PATTERN, "COUNT", 100,
+        )
+        keys.concat(found)
+        break if cursor == "0"
+      end
+      keys
+    end
+  end
+end

data/lib/solid_jobs/retry_service.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require "json"
+
+module SolidJobs
+  class RetryService < Service::Base
+    DEFAULT_MAX_RETRIES = 25
+
+    def call
+      payload = @payload.dup
+      now = Time.now
+      now_ms = Utilities.realtime_milliseconds
+      retry_count = payload.key?("retry_count") ? Integer(payload["retry_count"]) + 1 : 0
+      payload["error_message"] = @error.message.to_s
+      payload["error_class"] = @error.class.name
+      payload["failed_at"] ||= now_ms
+      payload["retried_at"] = now_ms if retry_count.positive?
+      payload["retry_count"] = retry_count
+      payload["queue"] = payload["retry_queue"] if payload["retry_queue"]
+
+      setting = payload["retry"]
+      if setting && retry_allowed?(payload, setting, retry_count, now)
+        decision = retry_decision(payload, retry_count)
+        return nil if decision == :discard
+        return move_to_dead(payload, now.to_f) if decision == :kill
+
+        retry_at = now.to_f + (decision || retry_delay(retry_count))
+        @config.redis_pool.call("ZADD", "retry", retry_at, JSON.generate(payload))
+        append_message("retry")
+        retry_at
+      else
+        invoke_exhausted(payload)
+        move_to_dead(payload, now.to_f)
+        append_message("dead")
+        nil
+      end
+    end
+
+    private
+
+    def retry_allowed?(payload, setting, retry_count, now)
+      if payload["retry_for"]
+        first_failure = Float(payload["failed_at"]) / 1_000
+        now.to_f - first_failure < Float(payload["retry_for"])
+      else
+        maximum = setting == true ? DEFAULT_MAX_RETRIES : Integer(setting)
+        retry_count < maximum
+      end
+    end
+
+    def retry_delay(retry_count)
+      ceiling = [
+        Float(@config.retry_base_delay) * (2**retry_count),
+        Float(@config.retry_max_delay),
+      ].min
+      (ceiling / 2) + (rand * ceiling / 2)
+    end
+
+    def retry_decision(payload, retry_count)
+      job_class = resolve_job_class(payload)
+      return unless job_class.respond_to?(:solid_jobs_retry_in_callback)
+
+      value = job_class.solid_jobs_retry_in_callback(retry_count, @error, payload)
+      return value if value.is_a?(Numeric) || %i[discard kill].include?(value)
+
+      nil
+    end
+
+    def invoke_exhausted(payload)
+      job_class = resolve_job_class(payload)
+      return unless job_class.respond_to?(:solid_jobs_retries_exhausted_callback)
+
+      job_class.solid_jobs_retries_exhausted_callback(payload, @error)
+    end
+
+    def resolve_job_class(payload)
+      Utilities.constantize(payload.fetch("class"))
+    rescue NameError
+      nil
+    end
+
+    def move_to_dead(payload, timestamp)
+      @config.redis_pool.pipelined do |pipeline|
+        pipeline.call("ZADD", "dead", timestamp, JSON.generate(payload))
+        pipeline.call(
+          "ZREMRANGEBYSCORE",
+          "dead",
+          "-inf",
+          timestamp - @config.dead_timeout,
+        )
+        pipeline.call(
+          "ZREMRANGEBYRANK",
+          "dead",
+          0,
+          -(@config.dead_max_jobs + 1),
+        )
+      end
+    end
+  end
+end

data/lib/solid_jobs/scheduler.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "json"
+
+module SolidJobs
+  class Scheduler
+    SETS = %w[retry schedule].freeze
+    POP_DUE = <<~LUA.freeze
+      local jobs = redis.call("zrange", KEYS[1], "-inf", ARGV[1], "byscore", "limit", 0, 1)
+      if jobs[1] then
+        redis.call("zrem", KEYS[1], jobs[1])
+        return jobs[1]
+      end
+    LUA
+
+    def initialize(config)
+      @config = config
+      @client = Client.new(config: config)
+    end
+
+    def enqueue_due(now = Time.now.to_f)
+      count = 0
+      SETS.each do |set|
+        while (raw = pop_due(set, now))
+          @client.push(JSON.parse(raw))
+          count += 1
+        end
+      end
+      count
+    end
+
+    def run(stop:)
+      until stop.call
+        begin
+          Recovery.call(config: @config)
+          enqueue_due
+        rescue SolidRedis::ConnectionError => error
+          @config.logger.warn("Redis scheduler poll failed, retrying: #{error.class}: #{error.message}")
+        end
+        interruptible_sleep(randomized_interval, stop)
+      end
+    ensure
+      @config.close
+    end
+
+    private
+
+    def pop_due(set, now)
+      @config.redis_pool.call("EVAL", POP_DUE, 1, set, now)
+    end
+
+    def randomized_interval
+      average = Float(@config.poll_interval_average)
+      average * (0.5 + rand)
+    end
+
+    def interruptible_sleep(duration, stop)
+      deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + duration
+      until stop.call || ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) >= deadline
+        sleep 0.05
+      end
+    end
+  end
+end

data/lib/solid_jobs/server.rb

@@ -0,0 +1,221 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "socket"
+
+module SolidJobs
+  class Server
+    attr_reader :config, :identity, :ractors
+
+    def initialize(config: SolidJobs.config)
+      @config = config
+      @ractors = []
+      @scheduler = nil
+      @heartbeat = nil
+      @started = false
+      @identity = "#{Socket.gethostname}:#{::Process.pid}:#{SecureRandom.hex(6)}".freeze
+      @started_at = Time.now.to_f
+    end
+
+    def start
+      raise Error, "Server is already running" if @started
+
+      snapshot = Utilities.shareable_copy(
+        config.ractor_snapshot.merge(identity: identity, started_at: @started_at),
+      )
+      Recovery.call(config: config)
+      barrier = StartupBarrier.new(config.concurrency + 2)
+      heartbeat_ready = barrier.channel
+      @heartbeat = Ractor.new(snapshot, heartbeat_ready) do |settings, ready|
+        local_config = heartbeat = nil
+        begin
+          started = SolidJobs::StartupBarrier.boot(ready) do
+            local_config = SolidJobs::Config.from_ractor_snapshot(settings)
+            SolidJobs.use_config(local_config)
+            heartbeat = SolidJobs::Heartbeat.new(
+              local_config,
+              identity: settings.fetch(:identity),
+              started_at: settings.fetch(:started_at),
+              concurrency: settings.fetch(:concurrency),
+            )
+            local_config.redis_pool.call("PING")
+          end
+          next :aborted unless started
+          messages = ::Queue.new
+          listener = Thread.new do
+            loop do
+              message = Ractor.receive
+              messages << message
+              break if message == :stop
+            end
+          end
+          quiet = false
+          stopping = false
+          work = {}
+          processed = failed = 0
+          next_beat = 0.0
+          until stopping
+            begin
+              until messages.empty?
+                case (message = messages.pop(true))
+                when :quiet
+                  quiet = true
+                when :stop
+                  stopping = true
+                else
+                  event, *arguments = message
+                  case event
+                  when :work
+                    work[arguments[0].to_s] = arguments[1]
+                  when :done
+                    work.delete(arguments[0].to_s)
+                  when :stats
+                    processed += arguments[0]
+                    failed += arguments[1]
+                  end
+                end
+              end
+            rescue ThreadError
+              nil
+            end
+            now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+            if now >= next_beat
+              begin
+                heartbeat.beat(quiet: quiet, work: work, processed: processed, failed: failed)
+                processed = failed = 0
+              rescue SolidRedis::ConnectionError => error
+                local_config.logger.warn("Redis heartbeat failed, retrying: #{error.class}: #{error.message}")
+              end
+              next_beat = now + SolidJobs::Heartbeat::INTERVAL
+            end
+            sleep 0.05 unless stopping
+          end
+          begin
+            heartbeat.beat(quiet: quiet, work: work, processed: processed, failed: failed)
+            heartbeat.cleanup
+          rescue SolidRedis::ConnectionError => error
+            local_config.logger.warn("Redis heartbeat cleanup failed: #{error.class}: #{error.message}")
+          end
+          listener.join
+          :stopped
+        ensure
+          local_config&.close
+        end
+      end
+      barrier.register!(@heartbeat, heartbeat_ready)
+      @ractors = Array.new(config.concurrency) do |processor_id|
+        ready = barrier.channel
+        ractor = Ractor.new(
+          snapshot, processor_id, @heartbeat, ready,
+        ) do |settings, id, heartbeat_port, ready_port|
+          local_config = processor = nil
+          begin
+            started = SolidJobs::StartupBarrier.boot(ready_port) do
+              local_config = SolidJobs::Config.from_ractor_snapshot(settings)
+              SolidJobs.use_config(local_config)
+              processor = SolidJobs::Processor.new(
+                local_config,
+                identity: settings.fetch(:identity),
+                processor_id: id,
+                heartbeat: heartbeat_port,
+              )
+              local_config.redis_pool.call("PING")
+            end
+            next :aborted unless started
+            control = ::Queue.new
+            listener = Thread.new do
+              loop do
+                command = Ractor.receive
+                control << command
+                next unless command == :stop
+
+                if processor.busy?
+                  sleep settings.fetch(:shutdown_timeout)
+                  processor.interrupt_current
+                end
+                break
+              end
+            end
+            result = processor.run(control: control)
+            listener.join
+            result
+          ensure
+            local_config&.close
+          end
+        end
+        barrier.register!(ractor, ready)
+        ractor
+      end
+      scheduler_ready = barrier.channel
+      @scheduler = Ractor.new(
+        snapshot, scheduler_ready,
+      ) do |settings, ready|
+        local_config = scheduler = nil
+        begin
+          started = SolidJobs::StartupBarrier.boot(ready) do
+            local_config = SolidJobs::Config.from_ractor_snapshot(settings)
+            SolidJobs.use_config(local_config)
+            scheduler = SolidJobs::Scheduler.new(local_config)
+            local_config.redis_pool.call("PING")
+          end
+          next :aborted unless started
+          control = ::Queue.new
+          listener = Thread.new do
+            command = Ractor.receive
+            control << command
+          end
+          scheduler.run(stop: -> { !control.empty? })
+          listener.join
+          :stopped
+        ensure
+          local_config&.close
+        end
+      end
+      barrier.register!(@scheduler, scheduler_ready)
+      barrier.run!
+      @started = true
+      config.fire(:startup, self)
+      self
+    rescue StandardError
+      barrier&.abort!
+      @ractors = []
+      @scheduler = @heartbeat = nil
+      raise
+    end
+
+    def quiet
+      return self unless @started
+
+      @ractors.each { |ractor| ractor.send(:quiet) }
+      @heartbeat&.send(:quiet)
+      config.fire(:quiet, self)
+      self
+    end
+
+    def stop
+      return [] unless @started
+
+      @ractors.each { |ractor| ractor.send(:stop) }
+      @scheduler&.send(:stop)
+      results = @ractors.map { |ractor| RactorSupport.value(ractor) }
+      RactorSupport.value(@scheduler) if @scheduler
+      @heartbeat&.send(:stop)
+      RactorSupport.value(@heartbeat) if @heartbeat
+      @ractors = []
+      @scheduler = nil
+      @heartbeat = nil
+      @started = false
+      config.fire(:shutdown, self)
+      results
+    end
+
+    def running?
+      @started
+    end
+
+    def remote_signal
+      config.redis_pool.call("RPOP", "#{identity}-signals")
+    end
+
+  end
+end

data/lib/solid_jobs/startup_barrier.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require "timeout"
+
+module SolidJobs
+  class StartupBarrier
+    STATES = %i[booting all_ready running boot_failed].freeze
+    Component = Struct.new(:ractor, :start_channel, keyword_init: true)
+
+    attr_reader :expected_count, :ready_count, :state
+
+    def initialize(expected_count, timeout: 10)
+      @expected_count = Integer(expected_count)
+      raise ArgumentError, "expected_count must be positive" unless @expected_count.positive?
+
+      @timeout = Float(timeout)
+      raise ArgumentError, "timeout must be positive" unless @timeout.positive?
+
+      @components = []
+      @registered = {}
+      @ready_count = 0
+      @state = :booting
+      @cleaned = false
+    end
+
+    def channel
+      ensure_state!(:booting)
+      Ractor::Port.new if defined?(Ractor::Port)
+    end
+
+    def register!(ractor, channel)
+      ensure_state!(:booting)
+      identity = ractor.object_id
+      raise Error, "Ractor announced READY more than once" if @registered.key?(identity)
+
+      signal, start_channel = wait_for_signal(ractor, channel)
+      unless signal == :ready
+        RactorSupport.value(ractor)
+        raise Error, "Ractor failed during boot"
+      end
+
+      @registered[identity] = true
+      @components << Component.new(ractor: ractor, start_channel: start_channel)
+      @ready_count += 1
+      @state = :all_ready if @ready_count == expected_count
+      ractor
+    rescue Timeout::Error
+      boot_failed!(Error.new("Ractor boot timed out after #{@timeout}s"))
+    rescue StandardError => error
+      boot_failed!(error)
+    end
+
+    def run!
+      ensure_state!(:all_ready)
+      @components.each { |component| signal(component, :start) }
+      @state = :running
+      @components.map(&:ractor)
+    end
+
+    def abort!
+      return self if @cleaned || state == :running
+
+      @state = :boot_failed
+      @cleaned = true
+      @components.each { |component| signal(component, :abort) }
+      @components.each do |component|
+        RactorSupport.value(component.ractor)
+      rescue Ractor::RemoteError, Ractor::ClosedError
+        nil
+      end
+      self
+    end
+
+    def self.boot(channel)
+      announced = false
+      yield
+      command = if channel
+        start_channel = Ractor::Port.new
+        channel << [:ready, start_channel]
+        announced = true
+        start_channel.receive
+      else
+        Ractor.yield(:ready)
+        announced = true
+        Ractor.receive
+      end
+      return false if command == :abort
+      raise Error, "Invalid startup command: #{command.inspect}" unless command == :start
+
+      true
+    rescue StandardError
+      channel << [:boot_error, nil] if channel && !announced
+      raise
+    end
+
+    private
+
+    def boot_failed!(error)
+      @state = :boot_failed
+      abort!
+      raise error
+    end
+
+    def wait_for_signal(ractor, channel)
+      Timeout.timeout(@timeout) do
+        channel ? channel.receive : [ractor.take, nil]
+      end
+    end
+
+    def signal(component, command)
+      if component.start_channel
+        component.start_channel << command
+      else
+        component.ractor.send(command)
+      end
+    rescue Ractor::ClosedError
+      nil
+    end
+
+    def ensure_state!(required)
+      return if state == required
+
+      raise Error, "StartupBarrier is #{state}, expected #{required}"
+    end
+  end
+end

data/lib/solid_jobs/testing.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module SolidJobs
+  module Testing
+    MODES = %i[disable fake inline].freeze
+    STORAGE_KEY = :solid_jobs_testing_jobs
+    MODE_KEY = :solid_jobs_testing_mode
+
+    module_function
+
+    def mode
+      Thread.current[MODE_KEY] || :disable
+    end
+
+    def testing!(new_mode)
+      new_mode = new_mode.to_sym
+      raise ArgumentError, "Unknown testing mode: #{new_mode.inspect}" unless MODES.include?(new_mode)
+
+      if block_given?
+        previous = mode
+        Thread.current[MODE_KEY] = new_mode
+        begin
+          yield
+        ensure
+          Thread.current[MODE_KEY] = previous
+        end
+      else
+        Thread.current[MODE_KEY] = new_mode
+      end
+    end
+
+    def fake!
+      testing!(:fake) { yield } if block_given?
+      testing!(:fake) unless block_given?
+    end
+
+    def inline!
+      testing!(:inline) { yield } if block_given?
+      testing!(:inline) unless block_given?
+    end
+
+    def disable!
+      testing!(:disable) { yield } if block_given?
+      testing!(:disable) unless block_given?
+    end
+
+    def jobs_for(job_class)
+      storage[job_class.name] ||= []
+    end
+
+    def clear_all
+      storage.clear
+    end
+
+    def execute_inline(payload)
+      job_class = payload["class"]
+      job_class = Utilities.constantize(job_class) if job_class.is_a?(String)
+      instance = job_class.new
+      instance.jid = payload["jid"] if instance.respond_to?(:jid=)
+      SolidJobs.config.server_middleware.invoke(instance, payload, payload.fetch("queue", "default")) do
+        instance.perform(*payload.fetch("args"))
+      end
+    end
+
+    def storage
+      Thread.current[STORAGE_KEY] ||= {}
+    end
+  end
+end