solid-jobs 0.1.0

data/lib/solid_jobs/heartbeat.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "json"
+require "socket"
+
+module SolidJobs
+  class Heartbeat
+    INTERVAL = 5
+    TTL = 60
+
+    def initialize(config, identity:, started_at:, concurrency:)
+      @config = config
+      @identity = identity
+      @started_at = started_at
+      @concurrency = concurrency
+    end
+
+    def beat(quiet: false, work: {}, processed: 0, failed: 0)
+      now = Time.now.to_f
+      info = {
+        "hostname" => Socket.gethostname,
+        "pid" => ::Process.pid,
+        "started_at" => @started_at,
+        "queues" => @config.queues,
+        "labels" => [],
+        "identity" => @identity,
+      }
+      busy = @config.redis_pool.call("HLEN", "#{@identity}:work")
+      @config.redis_pool.pipelined do |pipeline|
+        pipeline.call("INCRBY", "stat:processed", processed) if processed.positive?
+        pipeline.call("INCRBY", "stat:failed", failed) if failed.positive?
+        pipeline.call("SADD", "processes", @identity)
+        pipeline.call(
+          "HSET",
+          @identity,
+          "info", JSON.generate(info),
+          "beat", now,
+          "busy", busy,
+          "quiet", quiet ? "true" : "false",
+          "concurrency", @concurrency,
+        )
+        pipeline.call("EXPIRE", @identity, TTL)
+        pipeline.call("DEL", "#{@identity}:work")
+        pipeline.call("HSET", "#{@identity}:work", *work.flatten(1)) unless work.empty?
+        pipeline.call("EXPIRE", "#{@identity}:work", TTL)
+      end
+      now
+    end
+
+    def cleanup
+      @config.redis_pool.pipelined do |pipeline|
+        pipeline.call("SREM", "processes", @identity)
+        pipeline.call("DEL", @identity)
+        pipeline.call("DEL", "#{@identity}:work")
+      end
+    ensure
+      @config.close
+    end
+  end
+end

data/lib/solid_jobs/integrity_check.rb

@@ -0,0 +1,222 @@
+# frozen_string_literal: true
+
+require "json"
+require "set"
+
+module SolidJobs
+  class IntegrityCheck
+    ACTIVE_SETS = {
+      "schedule" => "SCHEDULED",
+      "retry" => "RETRY",
+      "dead" => "DEAD",
+    }.freeze
+
+    Report = Struct.new(
+      :lost,
+      :orphaned,
+      :invalid_reservations,
+      :dangling_indexes,
+      :duplicates,
+      :malformed_payloads,
+      :states,
+      keyword_init: true,
+    ) do
+      def ok?
+        lost.empty? &&
+          orphaned.empty? &&
+          invalid_reservations.empty? &&
+          dangling_indexes.empty? &&
+          duplicates.empty? &&
+          malformed_payloads.empty?
+      end
+    end
+
+    def self.call(expected_job_ids:, acked_job_ids: [], acked_key: nil, config: SolidJobs.config)
+      new(
+        expected_job_ids: expected_job_ids,
+        acked_job_ids: acked_job_ids,
+        acked_key: acked_key,
+        config: config,
+      ).call
+    end
+
+    def initialize(expected_job_ids:, acked_job_ids:, acked_key:, config:)
+      @expected = expected_job_ids.map(&:to_s).to_set
+      @acked = acked_job_ids.map(&:to_s).to_set
+      @acked.merge(Array(config.redis_pool.call("SMEMBERS", acked_key))) if acked_key
+      @redis = config.redis_pool
+      @states = Hash.new { |hash, job_id| hash[job_id] = [] }
+      @malformed_payloads = []
+      @invalid_reservations = []
+    end
+
+    def call
+      read_queues
+      read_sorted_sets
+      reserved = read_reservations
+      validate_reservation_metadata(reserved)
+      @acked.each { |job_id| add_state(job_id, "ACKED", "acknowledged") }
+
+      observed = @states.keys.to_set
+      Report.new(
+        lost: (@expected - observed).sort,
+        orphaned: (observed - @expected).sort,
+        invalid_reservations: @invalid_reservations,
+        dangling_indexes: dangling_attempts,
+        duplicates: duplicates,
+        malformed_payloads: @malformed_payloads,
+        states: @states.to_h,
+      )
+    end
+
+    private
+
+    def read_queues
+      scan("queue:*").each do |key|
+        Array(@redis.call("LRANGE", key, 0, -1)).each_with_index do |raw, index|
+          add_payload(raw, "READY", "#{key}[#{index}]")
+        end
+      end
+    end
+
+    def read_sorted_sets
+      ACTIVE_SETS.each do |key, state|
+        Array(@redis.call("ZRANGE", key, 0, -1)).each_with_index do |raw, index|
+          add_payload(raw, state, "#{key}[#{index}]")
+        end
+      end
+    end
+
+    def read_reservations
+      scan("*:reserved:*").to_h do |key|
+        records = Array(@redis.call("LRANGE", key, 0, -1)).each_with_index.map do |raw, index|
+          payload = parse_payload(raw, "#{key}[#{index}]")
+          add_state(payload["jid"], "RESERVED", "#{key}[#{index}]") if payload
+          payload
+        end.compact
+        [key, records]
+      end
+    end
+
+    def validate_reservation_metadata(reserved)
+      metadata = reservation_metadata
+      reserved.each do |key, payloads|
+        identity, worker_id = key.split(":reserved:", 2)
+        entry = metadata.delete([identity, worker_id])
+        if payloads.empty?
+          @invalid_reservations << issue(key, "empty_reserved_list")
+        elsif payloads.length > 1
+          @invalid_reservations << issue(key, "multiple_payloads_for_worker")
+        end
+        payload = payloads.first
+        unless entry
+          @invalid_reservations << issue(key, "missing_metadata", job_id: payload&.[]("jid"))
+          next
+        end
+        next unless payload
+
+        attributes = entry.fetch(:attributes)
+        {
+          "job_id" => payload["jid"],
+          "process_id" => identity,
+          "worker_id" => worker_id,
+          "queue" => payload["queue"],
+        }.each do |field, expected|
+          next if attributes[field].to_s == expected.to_s
+
+          @invalid_reservations << issue(
+            key,
+            "metadata_mismatch",
+            field: field,
+            expected: expected,
+            actual: attributes[field],
+          )
+        end
+      end
+      metadata.each_value do |entry|
+        @invalid_reservations << issue(
+          entry.fetch(:key),
+          "metadata_without_payload",
+          field: entry.fetch(:field),
+          job_id: entry.fetch(:attributes)["job_id"],
+        )
+      end
+    end
+
+    def reservation_metadata
+      scan("*:reservations").each_with_object({}) do |key, entries|
+        identity = key.delete_suffix(":reservations")
+        hash_entries(key).each do |field, raw|
+          attributes = parse_json(raw, "#{key}[#{field}]")
+          next unless attributes
+
+          entries[[identity, field]] = {
+            key: key,
+            field: field,
+            attributes: attributes,
+          }
+        end
+      end
+    end
+
+    def dangling_attempts
+      hash_entries("solid-jobs:attempts").filter_map do |job_id, attempt|
+        next if @states[job_id]&.any? { |location| location.fetch(:state) != "ACKED" }
+
+        {job_id: job_id, attempt: Integer(attempt), index: "solid-jobs:attempts"}
+      end
+    end
+
+    def duplicates
+      @states.filter_map do |job_id, locations|
+        next unless locations.length > 1
+
+        {job_id: job_id, locations: locations}
+      end
+    end
+
+    def add_payload(raw, state, location)
+      payload = parse_payload(raw, location)
+      add_state(payload["jid"], state, location) if payload
+    end
+
+    def parse_payload(raw, location)
+      payload = parse_json(raw, location)
+      return unless payload
+      return payload if payload["jid"]
+
+      @malformed_payloads << {location: location, error: "missing jid", raw: raw}
+      nil
+    end
+
+    def parse_json(raw, location)
+      JSON.parse(raw)
+    rescue JSON::ParserError => error
+      @malformed_payloads << {location: location, error: error.message, raw: raw}
+      nil
+    end
+
+    def add_state(job_id, state, location)
+      @states[job_id.to_s] << {state: state, location: location}
+    end
+
+    def hash_entries(key)
+      Array(@redis.call("HGETALL", key)).each_slice(2).to_h
+    end
+
+    def scan(pattern)
+      cursor = "0"
+      keys = []
+      loop do
+        cursor, batch = @redis.call("SCAN", cursor, "MATCH", pattern, "COUNT", 100)
+        keys.concat(batch)
+        break if cursor == "0"
+      end
+      keys
+    end
+
+    def issue(key, reason, **details)
+      {key: key, reason: reason}.merge(details)
+    end
+  end
+end

data/lib/solid_jobs/job.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+module SolidJobs
+  module Job
+    def self.included(base)
+      base.extend(ClassMethods)
+      base.include(InstanceMethods)
+      base.solid_jobs_options
+    end
+
+    module InstanceMethods
+      attr_accessor :jid
+      attr_writer :_context
+
+      def _context
+        @_context ||= {}
+      end
+
+      def logger
+        SolidJobs.config.logger
+      end
+
+      def interrupted?
+        _context[:interrupted]&.call || false
+      end
+    end
+
+    module ClassMethods
+      def solid_jobs_options(options = nil)
+        if options
+          merged = get_solid_jobs_options.merge(Utilities.stringify_keys(options))
+          @solid_jobs_options = Utilities.shareable_copy(merged)
+        else
+          @solid_jobs_options ||= Utilities.shareable_copy(SolidJobs.config.default_job_options)
+        end
+      end
+      alias_method :sidekiq_options, :solid_jobs_options
+
+      def get_solid_jobs_options
+        solid_jobs_options
+      end
+      alias_method :get_sidekiq_options, :get_solid_jobs_options
+
+      def solid_jobs_retry_in(&block)
+        define_singleton_method(:solid_jobs_retry_in_callback, &block) if block
+        method(:solid_jobs_retry_in_callback) if respond_to?(:solid_jobs_retry_in_callback)
+      end
+      alias_method :sidekiq_retry_in, :solid_jobs_retry_in
+
+      def solid_jobs_retries_exhausted(&block)
+        define_singleton_method(:solid_jobs_retries_exhausted_callback, &block) if block
+        method(:solid_jobs_retries_exhausted_callback) if respond_to?(:solid_jobs_retries_exhausted_callback)
+      end
+      alias_method :sidekiq_retries_exhausted, :solid_jobs_retries_exhausted
+
+      def perform_async(*args)
+        client_push("class" => self, "args" => args)
+      end
+
+      def perform_bulk(args, batch_size: Client::DEFAULT_BATCH_SIZE)
+        Client.new.push_bulk(
+          get_solid_jobs_options.merge(
+            "class" => self,
+            "args" => args,
+            "batch_size" => batch_size,
+          ),
+        )
+      end
+
+      def perform_in(interval, *args)
+        timestamp = interval.is_a?(Time) ? interval.to_f : Float(interval)
+        timestamp += Time.now.to_f if timestamp < 1_000_000_000
+        item = {"class" => self, "args" => args}
+        item["at"] = timestamp if timestamp > Time.now.to_f
+        client_push(item)
+      end
+
+      def perform_at(timestamp, *args)
+        perform_in(timestamp, *args)
+      end
+
+      def perform_inline(*args)
+        payload = get_solid_jobs_options.merge(
+          "class" => self,
+          "args" => args,
+        )
+        SolidJobs::Testing.execute_inline(payload)
+      end
+      alias_method :perform_sync, :perform_inline
+
+      def set(options)
+        Setter.new(self, options)
+      end
+
+      def client_push(item)
+        Client.new.push(get_solid_jobs_options.merge(item))
+      end
+
+      def jobs
+        Testing.jobs_for(self)
+      end
+
+      def clear
+        jobs.clear
+      end
+
+      def perform_one
+        payload = jobs.shift
+        raise EmptyQueueError, "No jobs for #{name}" unless payload
+
+        Testing.execute_inline(payload)
+      end
+    end
+
+    class Setter
+      def initialize(job_class, options)
+        @job_class = job_class
+        @options = Utilities.stringify_keys(options)
+      end
+
+      def perform_async(*args)
+        if @options.delete("sync")
+          @job_class.perform_inline(*args)
+        elsif (wait = @options.delete("wait"))
+          perform_in(wait, *args)
+        elsif (wait_until = @options.delete("wait_until"))
+          perform_at(wait_until, *args)
+        else
+          push(args)
+        end
+      end
+
+      def perform_in(interval, *args)
+        timestamp = interval.is_a?(Time) ? interval.to_f : Float(interval)
+        timestamp += Time.now.to_f if timestamp < 1_000_000_000
+        push(args, "at" => timestamp)
+      end
+
+      def perform_at(timestamp, *args)
+        perform_in(timestamp, *args)
+      end
+
+      private
+
+      def push(args, extra = {})
+        Client.new.push(@job_class.get_solid_jobs_options.merge(@options).merge(
+          extra,
+          "class" => @job_class,
+          "args" => args,
+        ))
+      end
+    end
+  end
+
+  class EmptyQueueError < Error; end
+end

data/lib/solid_jobs/middleware/chain.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module SolidJobs
+  module Middleware
+    class Chain
+      Entry = Struct.new(:klass, :args, keyword_init: true)
+
+      include Enumerable
+
+      def initialize(copy = nil)
+        @entries = copy ? copy.map { |entry| Entry.new(klass: entry.klass, args: entry.args.dup) } : []
+      end
+
+      def each(&block)
+        @entries.each(&block)
+      end
+
+      def add(klass, *args)
+        remove(klass)
+        @entries << Entry.new(klass: klass, args: args)
+        self
+      end
+
+      def prepend(klass, *args)
+        remove(klass)
+        @entries.unshift(Entry.new(klass: klass, args: args))
+        self
+      end
+
+      def remove(klass)
+        @entries.delete_if { |entry| entry.klass == klass }
+        self
+      end
+
+      def insert_before(oldklass, newklass, *args)
+        insert_relative(oldklass, newklass, args, 0)
+      end
+
+      def insert_after(oldklass, newklass, *args)
+        insert_relative(oldklass, newklass, args, 1)
+      end
+
+      def exists?(klass)
+        @entries.any? { |entry| entry.klass == klass }
+      end
+      alias_method :include?, :exists?
+
+      def clear
+        @entries.clear
+        self
+      end
+
+      def empty?
+        @entries.empty?
+      end
+
+      def retrieve
+        @entries.map { |entry| entry.klass.new(*entry.args) }
+      end
+
+      def snapshot
+        Utilities.shareable_copy(
+          @entries.map { |entry| [entry.klass, entry.args] },
+        )
+      end
+
+      def restore(snapshot)
+        clear
+        snapshot.each { |klass, args| add(klass, *args) }
+        self
+      end
+
+      def invoke(*arguments, &final)
+        return yield if empty?
+
+        stack = retrieve
+        traverse(stack, 0, arguments, &final)
+      end
+
+      def dup
+        self.class.new(@entries)
+      end
+
+      private
+
+      def traverse(stack, index, arguments, &final)
+        return final.call if index >= stack.length
+
+        stack[index].call(*arguments) do
+          traverse(stack, index + 1, arguments, &final)
+        end
+      end
+
+      def insert_relative(oldklass, newklass, args, offset)
+        remove(newklass)
+        index = @entries.index { |entry| entry.klass == oldklass }
+        raise ArgumentError, "#{oldklass} is not in the middleware chain" unless index
+
+        @entries.insert(index + offset, Entry.new(klass: newklass, args: args))
+        self
+      end
+    end
+  end
+end

data/lib/solid_jobs/processor.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+require "json"
+
+module SolidJobs
+  class Processor
+    def initialize(config, identity: nil, processor_id: nil, heartbeat: nil)
+      @config = config
+      @fetcher = Fetch.new(config, identity: identity, processor_id: processor_id)
+      @worker = Worker.new(redis_config: config.redis_config, config: config)
+      @identity = identity
+      @processor_id = processor_id
+      @heartbeat = heartbeat
+      @state_mutex = Mutex.new
+      @busy = false
+    end
+
+    def run(control:)
+      @state_mutex.synchronize { @runner = Thread.current }
+      processed = 0
+      failed = 0
+      @quiet = false
+      @stopping = false
+      loop do
+        receive_commands(control)
+        break if @stopping
+        if @quiet
+          sleep 0.05
+          next
+        end
+
+        unit = retrieve
+        next unless unit
+
+        receive_commands(control)
+        if @stopping || @quiet
+          requeue(unit)
+          break if @stopping
+
+          next
+        end
+
+        begin
+          @state_mutex.synchronize { @busy = true }
+          payload = unit.job
+          register_work(unit, payload)
+          begin
+            @worker.perform(payload)
+          rescue Shutdown
+            requeue(unit)
+            break
+          rescue StandardError => error
+            RetryService.call(config: @config, payload: payload, error: error)
+            acknowledge(unit)
+            failed += 1
+          else
+            processed += 1 if acknowledge(unit)
+          end
+        ensure
+          @state_mutex.synchronize { @busy = false }
+          clear_work
+        end
+      end
+      {processed: processed, failed: failed}
+    ensure
+      send_heartbeat(:stats, processed || 0, failed || 0)
+      @config.close
+    end
+
+    def busy?
+      @state_mutex.synchronize { @busy }
+    end
+
+    def interrupt_current
+      runner = @state_mutex.synchronize { @runner if @busy }
+      runner&.raise(Shutdown, "SolidJobs shutdown timeout exceeded")
+    end
+
+    private
+
+    def acknowledge(unit)
+      acknowledged = unit.acknowledge
+      unless acknowledged
+        @config.logger.warn(
+          "Reservation fencing rejected stale ACK: #{unit.reservation_id}",
+        )
+      end
+      acknowledged
+    rescue SolidRedis::ConnectionError => error
+      @fetcher.connection_failed!
+      @config.logger.warn(
+        "Redis ACK failed; reserved job will be replayed: #{error.class}: #{error.message}",
+      )
+      false
+    end
+
+    def requeue(unit)
+      requeued = unit.requeue
+      unless requeued
+        @config.logger.warn(
+          "Reservation fencing rejected stale requeue: #{unit.reservation_id}",
+        )
+      end
+      requeued
+    end
+
+    def retrieve
+      @fetcher.retrieve
+    rescue SolidRedis::ConnectionError => error
+      @fetcher.connection_failed!
+      @config.logger.warn("Redis fetch failed, retrying: #{error.class}: #{error.message}")
+      sleep 0.1
+      nil
+    end
+
+    def register_work(unit, payload)
+      return unless @identity
+
+      work = JSON.generate(
+        "queue" => unit.queue,
+        "payload" => payload,
+        "run_at" => Time.now.to_f,
+        "reservation_id" => unit.reservation_id,
+        "attempt" => unit.attempt,
+      )
+      send_heartbeat(:work, @processor_id, work)
+    end
+
+    def clear_work
+      return unless @identity
+
+      send_heartbeat(:done, @processor_id)
+    end
+
+    def send_heartbeat(*message)
+      @heartbeat&.send(message, move: true)
+    end
+
+    def receive_commands(control)
+      until control.empty?
+        case control.pop(true)
+        when :quiet
+          @quiet = true
+        when :stop
+          @stopping = true
+        end
+      end
+    rescue ThreadError
+      nil
+    end
+  end
+end

data/lib/solid_jobs/ractor_support.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module SolidJobs
+  module RactorSupport
+    module_function
+
+    def value(ractor)
+      ractor.respond_to?(:value) ? ractor.value : ractor.take
+    end
+  end
+end