solid-jobs 0.1.3 → 0.4.0

data/lib/solid_jobs/api.rb

@@ -1,457 +0,0 @@
-# frozen_string_literal: true
-
-require "json"
-
-module SolidJobs
-  class Stats
-    attr_reader :config
-
-    def initialize(config: SolidJobs.config)
-      @config = config
-    end
-
-    def processed
-      integer("stat:processed")
-    end
-
-    def failed
-      integer("stat:failed")
-    end
-
-    def enqueued
-      queues.sum(&:size)
-    end
-
-    def scheduled_size
-      integer_command("ZCARD", "schedule")
-    end
-
-    def retry_size
-      integer_command("ZCARD", "retry")
-    end
-
-    def dead_size
-      integer_command("ZCARD", "dead")
-    end
-
-    def processes_size
-      integer_command("SCARD", "processes")
-    end
-
-    def workers_size
-      ProcessSet.new(config: config).sum(&:busy)
-    end
-
-    def reset
-      config.redis_pool.call("DEL", "stat:processed", "stat:failed")
-      self
-    end
-
-    private
-
-    def queues
-      Queue.all(config: config)
-    end
-
-    def integer(key)
-      Integer(config.redis_pool.call("GET", key) || 0)
-    end
-
-    def integer_command(*command)
-      Integer(config.redis_pool.call(*command))
-    end
-  end
-
-  class Queue
-    include Enumerable
-
-    attr_reader :name
-
-    def self.all(config: SolidJobs.config)
-      Array(config.redis_pool.call("SMEMBERS", "queues")).sort.map do |name|
-        new(name, config: config)
-      end
-    end
-
-    def initialize(name, config: SolidJobs.config)
-      @name = String(name)
-      @config = config
-    end
-
-    def each
-      return enum_for(:each) unless block_given?
-
-      Array(@config.redis_pool.call("LRANGE", key, 0, -1)).reverse_each do |raw|
-        yield JobRecord.new(raw, config: @config, container: self)
-      end
-    end
-
-    def size
-      Integer(@config.redis_pool.call("LLEN", key))
-    end
-    alias_method :length, :size
-
-    def latency
-      raw = @config.redis_pool.call("LINDEX", key, -1)
-      return 0.0 unless raw
-
-      payload = JSON.parse(raw)
-      enqueued_at = payload["enqueued_at"] || payload["created_at"]
-      return 0.0 unless enqueued_at
-
-      [Time.now.to_f - Float(enqueued_at) / 1_000, 0.0].max
-    end
-
-    def clear
-      @config.redis_pool.call("DEL", key)
-      self
-    end
-
-    def delete
-      @config.redis_pool.pipelined do |pipeline|
-        pipeline.call("DEL", key)
-        pipeline.call("SREM", "queues", name)
-      end
-      self
-    end
-
-    def paused?
-      Array(@config.redis_pool.call("SISMEMBER", "paused", name)).first.to_i == 1
-    end
-
-    def pause
-      @config.redis_pool.call("SADD", "paused", name)
-      self
-    end
-
-    def unpause
-      @config.redis_pool.call("SREM", "paused", name)
-      self
-    end
-
-    private
-
-    def key
-      "queue:#{name}"
-    end
-  end
-
-  class JobRecord
-    attr_reader :item, :score
-
-    def initialize(raw, score: nil, config: SolidJobs.config, container: nil)
-      @raw = raw
-      @item = JSON.parse(raw)
-      @score = Float(score) if score
-      @config = config
-      @container = container
-    end
-
-    def jid
-      item["jid"]
-    end
-
-    def queue
-      item["queue"]
-    end
-
-    def klass
-      item["class"]
-    end
-
-    def args
-      item["args"]
-    end
-
-    def created_at
-      milliseconds_time(item["created_at"])
-    end
-
-    def enqueued_at
-      milliseconds_time(item["enqueued_at"])
-    end
-
-    def at
-      Time.at(score) if score
-    end
-
-    def delete
-      case @container
-      when SortedSet
-        @config.redis_pool.call("ZREM", @container.name, @raw)
-      when Queue
-        @config.redis_pool.call("LREM", "queue:#{@container.name}", 1, @raw)
-      else
-        false
-      end
-    end
-
-    def retry
-      delete
-      Client.new(config: @config).push(item)
-    end
-
-    private
-
-    def milliseconds_time(value)
-      Time.at(Float(value) / 1_000) if value
-    end
-  end
-
-  class SortedSet
-    include Enumerable
-
-    attr_reader :name
-
-    def initialize(name, config: SolidJobs.config)
-      @name = name
-      @config = config
-    end
-
-    def each
-      return enum_for(:each) unless block_given?
-
-      values = Array(@config.redis_pool.call("ZRANGE", name, 0, -1, "WITHSCORES"))
-      values.each_slice(2) do |raw, score|
-        yield JobRecord.new(raw, score: score, config: @config, container: self)
-      end
-    end
-
-    def size
-      Integer(@config.redis_pool.call("ZCARD", name))
-    end
-    alias_method :length, :size
-
-    def clear
-      @config.redis_pool.call("DEL", name)
-      self
-    end
-
-    def fetch(*jids)
-      wanted = jids.flatten
-      select { |job| wanted.include?(job.jid) }
-    end
-  end
-
-  class ScheduledSet < SortedSet
-    def initialize(config: SolidJobs.config)
-      super("schedule", config: config)
-    end
-  end
-
-  class RetrySet < SortedSet
-    def initialize(config: SolidJobs.config)
-      super("retry", config: config)
-    end
-
-    def retry_all
-      each.to_a.each(&:retry)
-    end
-  end
-
-  class DeadSet < SortedSet
-    def initialize(config: SolidJobs.config)
-      super("dead", config: config)
-    end
-
-    def retry_all
-      each.to_a.each(&:retry)
-    end
-  end
-
-  class Process
-    attr_reader :identity
-
-    def initialize(identity, config: SolidJobs.config)
-      @identity = identity
-      @config = config
-      fields = Array(@config.redis_pool.call("HGETALL", identity))
-      @attributes = fields.each_slice(2).to_h
-      @info = JSON.parse(@attributes.fetch("info", "{}"))
-    end
-
-    def hostname
-      @info["hostname"]
-    end
-
-    def pid
-      Integer(@info["pid"] || 0)
-    end
-
-    def queues
-      @info["queues"] || []
-    end
-
-    def busy
-      Integer(@attributes["busy"] || 0)
-    end
-
-    def concurrency
-      Integer(@attributes["concurrency"] || 0)
-    end
-
-    def quiet?
-      @attributes["quiet"] == "true"
-    end
-
-    def beat
-      Time.at(Float(@attributes["beat"])) if @attributes["beat"]
-    end
-
-    def quiet!
-      signal("TSTP")
-    end
-
-    def stop!
-      signal("TERM")
-    end
-
-    def dump_threads
-      signal("TTIN")
-    end
-
-    def workers
-      fields = Array(@config.redis_pool.call("HGETALL", "#{identity}:work"))
-      fields.each_slice(2).map do |processor_id, raw|
-        Work.new(identity, processor_id, raw)
-      end
-    end
-
-    private
-
-    def signal(name)
-      @config.redis_pool.call("LPUSH", "#{identity}-signals", name)
-      true
-    end
-  end
-
-  class ProcessSet
-    include Enumerable
-
-    def initialize(config: SolidJobs.config)
-      @config = config
-    end
-
-    def each
-      return enum_for(:each) unless block_given?
-
-      Array(@config.redis_pool.call("SMEMBERS", "processes")).each do |identity|
-        process = Process.new(identity, config: @config)
-        if process.beat
-          yield process
-        else
-          @config.redis_pool.call("SREM", "processes", identity)
-        end
-      end
-    end
-
-    def size
-      count
-    end
-  end
-
-  class Work
-    attr_reader :process_id, :processor_id
-
-    def initialize(process_id, processor_id, raw)
-      @process_id = process_id
-      @processor_id = processor_id
-      @attributes = JSON.parse(raw)
-    end
-
-    def queue
-      @attributes["queue"]
-    end
-
-    def payload
-      @attributes["payload"]
-    end
-
-    def run_at
-      Time.at(Float(@attributes["run_at"]))
-    end
-
-    def reservation_id
-      @attributes["reservation_id"]
-    end
-
-    def attempt
-      Integer(@attributes["attempt"] || 1)
-    end
-  end
-
-  class Reservation
-    attr_reader :attributes
-
-    def initialize(raw)
-      @attributes = JSON.parse(raw)
-    end
-
-    %w[job_id reservation_id process_id worker_id queue].each do |name|
-      define_method(name) { attributes[name] }
-    end
-
-    def attempt
-      Integer(attributes["attempt"])
-    end
-
-    def reserved_at
-      Time.at(Float(attributes["reserved_at"]))
-    end
-  end
-
-  class ReservationSet
-    include Enumerable
-
-    def initialize(config: SolidJobs.config)
-      @config = config
-    end
-
-    def each
-      return enum_for(:each) unless block_given?
-
-      keys.each do |key|
-        Array(@config.redis_pool.call("HVALS", key)).each do |raw|
-          yield Reservation.new(raw)
-        end
-      end
-    end
-
-    def find_job(job_id)
-      find { |reservation| reservation.job_id == job_id }
-    end
-
-    private
-
-    def keys
-      cursor = "0"
-      found = []
-      loop do
-        cursor, batch = @config.redis_pool.call(
-          "SCAN", cursor, "MATCH", "*:reservations", "COUNT", 100,
-        )
-        found.concat(batch)
-        break if cursor == "0"
-      end
-      found
-    end
-  end
-
-  class Workers
-    include Enumerable
-
-    def initialize(config: SolidJobs.config)
-      @processes = ProcessSet.new(config: config)
-    end
-
-    def each
-      return enum_for(:each) unless block_given?
-
-      @processes.each { |process| process.workers.each { |work| yield work } }
-    end
-
-    def size
-      count
-    end
-  end
-end

data/lib/solid_jobs/client.rb

@@ -1,202 +0,0 @@
-# frozen_string_literal: true
-
-require "json"
-require "securerandom"
-
-module SolidJobs
-  class Client
-    DEFAULT_BATCH_SIZE = 1_000
-    SCHEDULED_BATCH_SIZE = 100
-
-    def self.push(item)
-      new.push(item)
-    end
-
-    def self.push_bulk(item)
-      new.push_bulk(item)
-    end
-
-    def initialize(config: nil, configuration: nil)
-      selected = config || configuration || SolidJobs.config
-      @config = selected.is_a?(SolidRedis::Config) ? Config.new(redis: selected) : selected
-    end
-
-    def push(item)
-      payload = normalize_item(item)
-      payload = invoke_middleware(payload)
-      return unless payload
-
-      verify_json!(payload)
-      raw_push([payload])
-      payload["jid"]
-    end
-
-    def push_bulk(item)
-      raise InvalidJobError, "Job payload must be a Hash" unless item.is_a?(Hash)
-
-      source = Utilities.stringify_hash_keys(item)
-      args = source.delete("args")
-      raise InvalidJobError, "Job 'args' must be an Array or Enumerable" unless args.respond_to?(:each)
-
-      ats = source.delete("at")
-      if ats && !ats.is_a?(Numeric) && !ats.is_a?(Array)
-        raise InvalidJobError, "Job 'at' must be a Numeric or an Array"
-      end
-      if ats.is_a?(Array) && args.respond_to?(:size) && ats.size != args.size
-        raise InvalidJobError, "Job 'at' Array must have the same size as 'args'"
-      end
-      batch_size = Integer(source.delete("batch_size") || (ats ? SCHEDULED_BATCH_SIZE : DEFAULT_BATCH_SIZE))
-      raise ArgumentError, "batch_size must be positive" unless batch_size.positive?
-
-      explicit_jid = source.delete("jid")
-      if explicit_jid && args.respond_to?(:size) && args.size > 1
-        raise InvalidJobError, "Explicit jid is only supported for one bulk job"
-      end
-      base = normalize_item(source.merge("args" => []))
-      base.delete("jid")
-      result = []
-      args.each_slice(batch_size).with_index do |batch, batch_index|
-        payloads = []
-        batch.each_with_index do |job_args, index|
-          raise InvalidJobError, "Bulk job arguments must be Arrays" unless job_args.is_a?(Array)
-
-          payload = base.merge(
-            "args" => job_args,
-            "jid" => explicit_jid || SecureRandom.hex(12),
-          )
-          payload["at"] = ats.is_a?(Array) ? ats[batch_index * batch_size + index] : ats if ats
-          normalized = invoke_middleware(payload)
-          if normalized
-            verify_json!(normalized)
-            payloads << normalized
-            result << normalized["jid"]
-          else
-            result << nil
-          end
-        end
-        raw_push(payloads)
-      end
-      result
-    end
-
-    def raw_push(payloads)
-      return true if payloads.empty?
-
-      if defined?(Testing)
-        case Testing.mode
-        when :fake
-          payloads.each do |payload|
-            Testing.jobs_for(Utilities.constantize(payload["class"])) << payload
-          end
-          return true
-        when :inline
-          payloads.each { |payload| Testing.execute_inline(payload) }
-          return true
-        end
-      end
-
-      @config.redis_pool.pipelined do |pipeline|
-        if payloads.all? { |payload| payload["at"] }
-          payloads.each do |payload|
-            score = payload.delete("at")
-            pipeline.call("ZADD", "schedule", score, JSON.generate(payload))
-          end
-        elsif payloads.none? { |payload| payload["at"] }
-          push_immediate(pipeline, payloads)
-        else
-          scheduled, immediate = payloads.partition { |payload| payload["at"] }
-          scheduled.each do |payload|
-            score = payload.delete("at")
-            pipeline.call("ZADD", "schedule", score, JSON.generate(payload))
-          end
-          push_immediate(pipeline, immediate)
-        end
-      end
-      true
-    end
-
-    private
-
-    def normalize_item(item)
-      raise InvalidJobError, "Job payload must be a Hash" unless item.is_a?(Hash)
-
-      payload = @config.default_job_options.merge(Utilities.stringify_hash_keys(item))
-      payload["class"] ||= payload.delete("job_class")
-      payload["jid"] ||= payload.delete("job_id")
-      job_class = payload["class"]
-      args = payload["args"]
-      unless job_class.is_a?(Class) || (job_class.is_a?(String) && !job_class.empty?)
-        raise InvalidJobError, "Job must include a non-empty class"
-      end
-      raise InvalidJobError, "Job args must be an Array" unless args.is_a?(Array)
-
-      payload["class"] = job_class.name if job_class.is_a?(Class)
-      payload["queue"] = String(payload.fetch("queue"))
-      payload["jid"] ||= SecureRandom.hex(12)
-      payload["created_at"] ||= Utilities.realtime_milliseconds
-      payload
-    end
-
-    def invoke_middleware(payload)
-      job_class = payload["class"]
-      @config.client_middleware.invoke(job_class, payload, payload["queue"], @config.redis_pool) do
-        payload
-      end
-    end
-
-    def push_immediate(pipeline, payloads)
-      return if payloads.empty?
-
-      groups = if payloads.all? { |payload| payload["queue"] == payloads.first["queue"] }
-        [[payloads.first["queue"], payloads]]
-      else
-        payloads.group_by { |payload| payload["queue"] }
-      end
-      groups.each do |queue, jobs|
-        pipeline.call("SADD", "queues", queue)
-        now = Utilities.realtime_milliseconds
-        encoded = jobs.map do |payload|
-          payload["enqueued_at"] = now
-          JSON.generate(payload)
-        end
-        pipeline.call("LPUSH", "queue:#{queue}", *encoded)
-      end
-    end
-
-    def verify_json!(payload)
-      invalid = find_invalid_json_value(payload["args"])
-      return unless invalid
-
-      message = "Job arguments must contain only JSON-native values with string hash keys; found #{invalid.inspect}"
-      case @config.on_complex_arguments
-      when :raise
-        raise InvalidArgumentError, message
-      when :warn
-        @config.logger.warn(message)
-      end
-    end
-
-    def find_invalid_json_value(value)
-      case value
-      when nil, true, false, String, Integer, Float
-        nil
-      when Array
-        value.each do |nested|
-          invalid = find_invalid_json_value(nested)
-          return invalid if invalid
-        end
-        nil
-      when Hash
-        value.each do |key, nested|
-          return key unless key.is_a?(String)
-
-          invalid = find_invalid_json_value(nested)
-          return invalid if invalid
-        end
-        nil
-      else
-        value
-      end
-    end
-  end
-end